1 /* memcontrol.c - Memory Controller 2 * 3 * Copyright IBM Corporation, 2007 4 * Author Balbir Singh <balbir@linux.vnet.ibm.com> 5 * 6 * Copyright 2007 OpenVZ SWsoft Inc 7 * Author: Pavel Emelianov <xemul@openvz.org> 8 * 9 * Memory thresholds 10 * Copyright (C) 2009 Nokia Corporation 11 * Author: Kirill A. Shutemov 12 * 13 * Kernel Memory Controller 14 * Copyright (C) 2012 Parallels Inc. and Google Inc. 15 * Authors: Glauber Costa and Suleiman Souhlal 16 * 17 * This program is free software; you can redistribute it and/or modify 18 * it under the terms of the GNU General Public License as published by 19 * the Free Software Foundation; either version 2 of the License, or 20 * (at your option) any later version. 21 * 22 * This program is distributed in the hope that it will be useful, 23 * but WITHOUT ANY WARRANTY; without even the implied warranty of 24 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 25 * GNU General Public License for more details. 26 */ 27 28 #include <linux/res_counter.h> 29 #include <linux/memcontrol.h> 30 #include <linux/cgroup.h> 31 #include <linux/mm.h> 32 #include <linux/hugetlb.h> 33 #include <linux/pagemap.h> 34 #include <linux/smp.h> 35 #include <linux/page-flags.h> 36 #include <linux/backing-dev.h> 37 #include <linux/bit_spinlock.h> 38 #include <linux/rcupdate.h> 39 #include <linux/limits.h> 40 #include <linux/export.h> 41 #include <linux/mutex.h> 42 #include <linux/rbtree.h> 43 #include <linux/slab.h> 44 #include <linux/swap.h> 45 #include <linux/swapops.h> 46 #include <linux/spinlock.h> 47 #include <linux/eventfd.h> 48 #include <linux/poll.h> 49 #include <linux/sort.h> 50 #include <linux/fs.h> 51 #include <linux/seq_file.h> 52 #include <linux/vmpressure.h> 53 #include <linux/mm_inline.h> 54 #include <linux/page_cgroup.h> 55 #include <linux/cpu.h> 56 #include <linux/oom.h> 57 #include <linux/lockdep.h> 58 #include <linux/file.h> 59 #include "internal.h" 60 #include <net/sock.h> 61 #include <net/ip.h> 62 #include <net/tcp_memcontrol.h> 63 #include "slab.h" 64 65 #include <asm/uaccess.h> 66 67 #include <trace/events/vmscan.h> 68 69 struct cgroup_subsys memory_cgrp_subsys __read_mostly; 70 EXPORT_SYMBOL(memory_cgrp_subsys); 71 72 #define MEM_CGROUP_RECLAIM_RETRIES 5 73 static struct mem_cgroup *root_mem_cgroup __read_mostly; 74 75 #ifdef CONFIG_MEMCG_SWAP 76 /* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */ 77 int do_swap_account __read_mostly; 78 79 /* for remember boot option*/ 80 #ifdef CONFIG_MEMCG_SWAP_ENABLED 81 static int really_do_swap_account __initdata = 1; 82 #else 83 static int really_do_swap_account __initdata; 84 #endif 85 86 #else 87 #define do_swap_account 0 88 #endif 89 90 91 static const char * const mem_cgroup_stat_names[] = { 92 "cache", 93 "rss", 94 "rss_huge", 95 "mapped_file", 96 "writeback", 97 "swap", 98 }; 99 100 enum mem_cgroup_events_index { 101 MEM_CGROUP_EVENTS_PGPGIN, /* # of pages paged in */ 102 MEM_CGROUP_EVENTS_PGPGOUT, /* # of pages paged out */ 103 MEM_CGROUP_EVENTS_PGFAULT, /* # of page-faults */ 104 MEM_CGROUP_EVENTS_PGMAJFAULT, /* # of major page-faults */ 105 MEM_CGROUP_EVENTS_NSTATS, 106 }; 107 108 static const char * const mem_cgroup_events_names[] = { 109 "pgpgin", 110 "pgpgout", 111 "pgfault", 112 "pgmajfault", 113 }; 114 115 static const char * const mem_cgroup_lru_names[] = { 116 "inactive_anon", 117 "active_anon", 118 "inactive_file", 119 "active_file", 120 "unevictable", 121 }; 122 123 /* 124 * Per memcg event counter is incremented at every pagein/pageout. With THP, 125 * it will be incremated by the number of pages. This counter is used for 126 * for trigger some periodic events. This is straightforward and better 127 * than using jiffies etc. to handle periodic memcg event. 128 */ 129 enum mem_cgroup_events_target { 130 MEM_CGROUP_TARGET_THRESH, 131 MEM_CGROUP_TARGET_SOFTLIMIT, 132 MEM_CGROUP_TARGET_NUMAINFO, 133 MEM_CGROUP_NTARGETS, 134 }; 135 #define THRESHOLDS_EVENTS_TARGET 128 136 #define SOFTLIMIT_EVENTS_TARGET 1024 137 #define NUMAINFO_EVENTS_TARGET 1024 138 139 struct mem_cgroup_stat_cpu { 140 long count[MEM_CGROUP_STAT_NSTATS]; 141 unsigned long events[MEM_CGROUP_EVENTS_NSTATS]; 142 unsigned long nr_page_events; 143 unsigned long targets[MEM_CGROUP_NTARGETS]; 144 }; 145 146 struct mem_cgroup_reclaim_iter { 147 /* 148 * last scanned hierarchy member. Valid only if last_dead_count 149 * matches memcg->dead_count of the hierarchy root group. 150 */ 151 struct mem_cgroup *last_visited; 152 int last_dead_count; 153 154 /* scan generation, increased every round-trip */ 155 unsigned int generation; 156 }; 157 158 /* 159 * per-zone information in memory controller. 160 */ 161 struct mem_cgroup_per_zone { 162 struct lruvec lruvec; 163 unsigned long lru_size[NR_LRU_LISTS]; 164 165 struct mem_cgroup_reclaim_iter reclaim_iter[DEF_PRIORITY + 1]; 166 167 struct rb_node tree_node; /* RB tree node */ 168 unsigned long long usage_in_excess;/* Set to the value by which */ 169 /* the soft limit is exceeded*/ 170 bool on_tree; 171 struct mem_cgroup *memcg; /* Back pointer, we cannot */ 172 /* use container_of */ 173 }; 174 175 struct mem_cgroup_per_node { 176 struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES]; 177 }; 178 179 /* 180 * Cgroups above their limits are maintained in a RB-Tree, independent of 181 * their hierarchy representation 182 */ 183 184 struct mem_cgroup_tree_per_zone { 185 struct rb_root rb_root; 186 spinlock_t lock; 187 }; 188 189 struct mem_cgroup_tree_per_node { 190 struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES]; 191 }; 192 193 struct mem_cgroup_tree { 194 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES]; 195 }; 196 197 static struct mem_cgroup_tree soft_limit_tree __read_mostly; 198 199 struct mem_cgroup_threshold { 200 struct eventfd_ctx *eventfd; 201 u64 threshold; 202 }; 203 204 /* For threshold */ 205 struct mem_cgroup_threshold_ary { 206 /* An array index points to threshold just below or equal to usage. */ 207 int current_threshold; 208 /* Size of entries[] */ 209 unsigned int size; 210 /* Array of thresholds */ 211 struct mem_cgroup_threshold entries[0]; 212 }; 213 214 struct mem_cgroup_thresholds { 215 /* Primary thresholds array */ 216 struct mem_cgroup_threshold_ary *primary; 217 /* 218 * Spare threshold array. 219 * This is needed to make mem_cgroup_unregister_event() "never fail". 220 * It must be able to store at least primary->size - 1 entries. 221 */ 222 struct mem_cgroup_threshold_ary *spare; 223 }; 224 225 /* for OOM */ 226 struct mem_cgroup_eventfd_list { 227 struct list_head list; 228 struct eventfd_ctx *eventfd; 229 }; 230 231 /* 232 * cgroup_event represents events which userspace want to receive. 233 */ 234 struct mem_cgroup_event { 235 /* 236 * memcg which the event belongs to. 237 */ 238 struct mem_cgroup *memcg; 239 /* 240 * eventfd to signal userspace about the event. 241 */ 242 struct eventfd_ctx *eventfd; 243 /* 244 * Each of these stored in a list by the cgroup. 245 */ 246 struct list_head list; 247 /* 248 * register_event() callback will be used to add new userspace 249 * waiter for changes related to this event. Use eventfd_signal() 250 * on eventfd to send notification to userspace. 251 */ 252 int (*register_event)(struct mem_cgroup *memcg, 253 struct eventfd_ctx *eventfd, const char *args); 254 /* 255 * unregister_event() callback will be called when userspace closes 256 * the eventfd or on cgroup removing. This callback must be set, 257 * if you want provide notification functionality. 258 */ 259 void (*unregister_event)(struct mem_cgroup *memcg, 260 struct eventfd_ctx *eventfd); 261 /* 262 * All fields below needed to unregister event when 263 * userspace closes eventfd. 264 */ 265 poll_table pt; 266 wait_queue_head_t *wqh; 267 wait_queue_t wait; 268 struct work_struct remove; 269 }; 270 271 static void mem_cgroup_threshold(struct mem_cgroup *memcg); 272 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg); 273 274 /* 275 * The memory controller data structure. The memory controller controls both 276 * page cache and RSS per cgroup. We would eventually like to provide 277 * statistics based on the statistics developed by Rik Van Riel for clock-pro, 278 * to help the administrator determine what knobs to tune. 279 * 280 * TODO: Add a water mark for the memory controller. Reclaim will begin when 281 * we hit the water mark. May be even add a low water mark, such that 282 * no reclaim occurs from a cgroup at it's low water mark, this is 283 * a feature that will be implemented much later in the future. 284 */ 285 struct mem_cgroup { 286 struct cgroup_subsys_state css; 287 /* 288 * the counter to account for memory usage 289 */ 290 struct res_counter res; 291 292 /* vmpressure notifications */ 293 struct vmpressure vmpressure; 294 295 /* 296 * the counter to account for mem+swap usage. 297 */ 298 struct res_counter memsw; 299 300 /* 301 * the counter to account for kernel memory usage. 302 */ 303 struct res_counter kmem; 304 /* 305 * Should the accounting and control be hierarchical, per subtree? 306 */ 307 bool use_hierarchy; 308 unsigned long kmem_account_flags; /* See KMEM_ACCOUNTED_*, below */ 309 310 bool oom_lock; 311 atomic_t under_oom; 312 atomic_t oom_wakeups; 313 314 int swappiness; 315 /* OOM-Killer disable */ 316 int oom_kill_disable; 317 318 /* set when res.limit == memsw.limit */ 319 bool memsw_is_minimum; 320 321 /* protect arrays of thresholds */ 322 struct mutex thresholds_lock; 323 324 /* thresholds for memory usage. RCU-protected */ 325 struct mem_cgroup_thresholds thresholds; 326 327 /* thresholds for mem+swap usage. RCU-protected */ 328 struct mem_cgroup_thresholds memsw_thresholds; 329 330 /* For oom notifier event fd */ 331 struct list_head oom_notify; 332 333 /* 334 * Should we move charges of a task when a task is moved into this 335 * mem_cgroup ? And what type of charges should we move ? 336 */ 337 unsigned long move_charge_at_immigrate; 338 /* 339 * set > 0 if pages under this cgroup are moving to other cgroup. 340 */ 341 atomic_t moving_account; 342 /* taken only while moving_account > 0 */ 343 spinlock_t move_lock; 344 /* 345 * percpu counter. 346 */ 347 struct mem_cgroup_stat_cpu __percpu *stat; 348 /* 349 * used when a cpu is offlined or other synchronizations 350 * See mem_cgroup_read_stat(). 351 */ 352 struct mem_cgroup_stat_cpu nocpu_base; 353 spinlock_t pcp_counter_lock; 354 355 atomic_t dead_count; 356 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_INET) 357 struct cg_proto tcp_mem; 358 #endif 359 #if defined(CONFIG_MEMCG_KMEM) 360 /* analogous to slab_common's slab_caches list, but per-memcg; 361 * protected by memcg_slab_mutex */ 362 struct list_head memcg_slab_caches; 363 /* Index in the kmem_cache->memcg_params->memcg_caches array */ 364 int kmemcg_id; 365 #endif 366 367 int last_scanned_node; 368 #if MAX_NUMNODES > 1 369 nodemask_t scan_nodes; 370 atomic_t numainfo_events; 371 atomic_t numainfo_updating; 372 #endif 373 374 /* List of events which userspace want to receive */ 375 struct list_head event_list; 376 spinlock_t event_list_lock; 377 378 struct mem_cgroup_per_node *nodeinfo[0]; 379 /* WARNING: nodeinfo must be the last member here */ 380 }; 381 382 /* internal only representation about the status of kmem accounting. */ 383 enum { 384 KMEM_ACCOUNTED_ACTIVE, /* accounted by this cgroup itself */ 385 KMEM_ACCOUNTED_DEAD, /* dead memcg with pending kmem charges */ 386 }; 387 388 #ifdef CONFIG_MEMCG_KMEM 389 static inline void memcg_kmem_set_active(struct mem_cgroup *memcg) 390 { 391 set_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags); 392 } 393 394 static bool memcg_kmem_is_active(struct mem_cgroup *memcg) 395 { 396 return test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags); 397 } 398 399 static void memcg_kmem_mark_dead(struct mem_cgroup *memcg) 400 { 401 /* 402 * Our caller must use css_get() first, because memcg_uncharge_kmem() 403 * will call css_put() if it sees the memcg is dead. 404 */ 405 smp_wmb(); 406 if (test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags)) 407 set_bit(KMEM_ACCOUNTED_DEAD, &memcg->kmem_account_flags); 408 } 409 410 static bool memcg_kmem_test_and_clear_dead(struct mem_cgroup *memcg) 411 { 412 return test_and_clear_bit(KMEM_ACCOUNTED_DEAD, 413 &memcg->kmem_account_flags); 414 } 415 #endif 416 417 /* Stuffs for move charges at task migration. */ 418 /* 419 * Types of charges to be moved. "move_charge_at_immitgrate" and 420 * "immigrate_flags" are treated as a left-shifted bitmap of these types. 421 */ 422 enum move_type { 423 MOVE_CHARGE_TYPE_ANON, /* private anonymous page and swap of it */ 424 MOVE_CHARGE_TYPE_FILE, /* file page(including tmpfs) and swap of it */ 425 NR_MOVE_TYPE, 426 }; 427 428 /* "mc" and its members are protected by cgroup_mutex */ 429 static struct move_charge_struct { 430 spinlock_t lock; /* for from, to */ 431 struct mem_cgroup *from; 432 struct mem_cgroup *to; 433 unsigned long immigrate_flags; 434 unsigned long precharge; 435 unsigned long moved_charge; 436 unsigned long moved_swap; 437 struct task_struct *moving_task; /* a task moving charges */ 438 wait_queue_head_t waitq; /* a waitq for other context */ 439 } mc = { 440 .lock = __SPIN_LOCK_UNLOCKED(mc.lock), 441 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq), 442 }; 443 444 static bool move_anon(void) 445 { 446 return test_bit(MOVE_CHARGE_TYPE_ANON, &mc.immigrate_flags); 447 } 448 449 static bool move_file(void) 450 { 451 return test_bit(MOVE_CHARGE_TYPE_FILE, &mc.immigrate_flags); 452 } 453 454 /* 455 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft 456 * limit reclaim to prevent infinite loops, if they ever occur. 457 */ 458 #define MEM_CGROUP_MAX_RECLAIM_LOOPS 100 459 #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2 460 461 enum charge_type { 462 MEM_CGROUP_CHARGE_TYPE_CACHE = 0, 463 MEM_CGROUP_CHARGE_TYPE_ANON, 464 MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */ 465 MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */ 466 NR_CHARGE_TYPE, 467 }; 468 469 /* for encoding cft->private value on file */ 470 enum res_type { 471 _MEM, 472 _MEMSWAP, 473 _OOM_TYPE, 474 _KMEM, 475 }; 476 477 #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val)) 478 #define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff) 479 #define MEMFILE_ATTR(val) ((val) & 0xffff) 480 /* Used for OOM nofiier */ 481 #define OOM_CONTROL (0) 482 483 /* 484 * Reclaim flags for mem_cgroup_hierarchical_reclaim 485 */ 486 #define MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0 487 #define MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT) 488 #define MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1 489 #define MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT) 490 491 /* 492 * The memcg_create_mutex will be held whenever a new cgroup is created. 493 * As a consequence, any change that needs to protect against new child cgroups 494 * appearing has to hold it as well. 495 */ 496 static DEFINE_MUTEX(memcg_create_mutex); 497 498 struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *s) 499 { 500 return s ? container_of(s, struct mem_cgroup, css) : NULL; 501 } 502 503 /* Some nice accessors for the vmpressure. */ 504 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg) 505 { 506 if (!memcg) 507 memcg = root_mem_cgroup; 508 return &memcg->vmpressure; 509 } 510 511 struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr) 512 { 513 return &container_of(vmpr, struct mem_cgroup, vmpressure)->css; 514 } 515 516 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg) 517 { 518 return (memcg == root_mem_cgroup); 519 } 520 521 /* 522 * We restrict the id in the range of [1, 65535], so it can fit into 523 * an unsigned short. 524 */ 525 #define MEM_CGROUP_ID_MAX USHRT_MAX 526 527 static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg) 528 { 529 return memcg->css.id; 530 } 531 532 static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id) 533 { 534 struct cgroup_subsys_state *css; 535 536 css = css_from_id(id, &memory_cgrp_subsys); 537 return mem_cgroup_from_css(css); 538 } 539 540 /* Writing them here to avoid exposing memcg's inner layout */ 541 #if defined(CONFIG_INET) && defined(CONFIG_MEMCG_KMEM) 542 543 void sock_update_memcg(struct sock *sk) 544 { 545 if (mem_cgroup_sockets_enabled) { 546 struct mem_cgroup *memcg; 547 struct cg_proto *cg_proto; 548 549 BUG_ON(!sk->sk_prot->proto_cgroup); 550 551 /* Socket cloning can throw us here with sk_cgrp already 552 * filled. It won't however, necessarily happen from 553 * process context. So the test for root memcg given 554 * the current task's memcg won't help us in this case. 555 * 556 * Respecting the original socket's memcg is a better 557 * decision in this case. 558 */ 559 if (sk->sk_cgrp) { 560 BUG_ON(mem_cgroup_is_root(sk->sk_cgrp->memcg)); 561 css_get(&sk->sk_cgrp->memcg->css); 562 return; 563 } 564 565 rcu_read_lock(); 566 memcg = mem_cgroup_from_task(current); 567 cg_proto = sk->sk_prot->proto_cgroup(memcg); 568 if (!mem_cgroup_is_root(memcg) && 569 memcg_proto_active(cg_proto) && 570 css_tryget_online(&memcg->css)) { 571 sk->sk_cgrp = cg_proto; 572 } 573 rcu_read_unlock(); 574 } 575 } 576 EXPORT_SYMBOL(sock_update_memcg); 577 578 void sock_release_memcg(struct sock *sk) 579 { 580 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) { 581 struct mem_cgroup *memcg; 582 WARN_ON(!sk->sk_cgrp->memcg); 583 memcg = sk->sk_cgrp->memcg; 584 css_put(&sk->sk_cgrp->memcg->css); 585 } 586 } 587 588 struct cg_proto *tcp_proto_cgroup(struct mem_cgroup *memcg) 589 { 590 if (!memcg || mem_cgroup_is_root(memcg)) 591 return NULL; 592 593 return &memcg->tcp_mem; 594 } 595 EXPORT_SYMBOL(tcp_proto_cgroup); 596 597 static void disarm_sock_keys(struct mem_cgroup *memcg) 598 { 599 if (!memcg_proto_activated(&memcg->tcp_mem)) 600 return; 601 static_key_slow_dec(&memcg_socket_limit_enabled); 602 } 603 #else 604 static void disarm_sock_keys(struct mem_cgroup *memcg) 605 { 606 } 607 #endif 608 609 #ifdef CONFIG_MEMCG_KMEM 610 /* 611 * This will be the memcg's index in each cache's ->memcg_params->memcg_caches. 612 * The main reason for not using cgroup id for this: 613 * this works better in sparse environments, where we have a lot of memcgs, 614 * but only a few kmem-limited. Or also, if we have, for instance, 200 615 * memcgs, and none but the 200th is kmem-limited, we'd have to have a 616 * 200 entry array for that. 617 * 618 * The current size of the caches array is stored in 619 * memcg_limited_groups_array_size. It will double each time we have to 620 * increase it. 621 */ 622 static DEFINE_IDA(kmem_limited_groups); 623 int memcg_limited_groups_array_size; 624 625 /* 626 * MIN_SIZE is different than 1, because we would like to avoid going through 627 * the alloc/free process all the time. In a small machine, 4 kmem-limited 628 * cgroups is a reasonable guess. In the future, it could be a parameter or 629 * tunable, but that is strictly not necessary. 630 * 631 * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get 632 * this constant directly from cgroup, but it is understandable that this is 633 * better kept as an internal representation in cgroup.c. In any case, the 634 * cgrp_id space is not getting any smaller, and we don't have to necessarily 635 * increase ours as well if it increases. 636 */ 637 #define MEMCG_CACHES_MIN_SIZE 4 638 #define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX 639 640 /* 641 * A lot of the calls to the cache allocation functions are expected to be 642 * inlined by the compiler. Since the calls to memcg_kmem_get_cache are 643 * conditional to this static branch, we'll have to allow modules that does 644 * kmem_cache_alloc and the such to see this symbol as well 645 */ 646 struct static_key memcg_kmem_enabled_key; 647 EXPORT_SYMBOL(memcg_kmem_enabled_key); 648 649 static void disarm_kmem_keys(struct mem_cgroup *memcg) 650 { 651 if (memcg_kmem_is_active(memcg)) { 652 static_key_slow_dec(&memcg_kmem_enabled_key); 653 ida_simple_remove(&kmem_limited_groups, memcg->kmemcg_id); 654 } 655 /* 656 * This check can't live in kmem destruction function, 657 * since the charges will outlive the cgroup 658 */ 659 WARN_ON(res_counter_read_u64(&memcg->kmem, RES_USAGE) != 0); 660 } 661 #else 662 static void disarm_kmem_keys(struct mem_cgroup *memcg) 663 { 664 } 665 #endif /* CONFIG_MEMCG_KMEM */ 666 667 static void disarm_static_keys(struct mem_cgroup *memcg) 668 { 669 disarm_sock_keys(memcg); 670 disarm_kmem_keys(memcg); 671 } 672 673 static void drain_all_stock_async(struct mem_cgroup *memcg); 674 675 static struct mem_cgroup_per_zone * 676 mem_cgroup_zone_zoneinfo(struct mem_cgroup *memcg, struct zone *zone) 677 { 678 int nid = zone_to_nid(zone); 679 int zid = zone_idx(zone); 680 681 return &memcg->nodeinfo[nid]->zoneinfo[zid]; 682 } 683 684 struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg) 685 { 686 return &memcg->css; 687 } 688 689 static struct mem_cgroup_per_zone * 690 mem_cgroup_page_zoneinfo(struct mem_cgroup *memcg, struct page *page) 691 { 692 int nid = page_to_nid(page); 693 int zid = page_zonenum(page); 694 695 return &memcg->nodeinfo[nid]->zoneinfo[zid]; 696 } 697 698 static struct mem_cgroup_tree_per_zone * 699 soft_limit_tree_node_zone(int nid, int zid) 700 { 701 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid]; 702 } 703 704 static struct mem_cgroup_tree_per_zone * 705 soft_limit_tree_from_page(struct page *page) 706 { 707 int nid = page_to_nid(page); 708 int zid = page_zonenum(page); 709 710 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid]; 711 } 712 713 static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_zone *mz, 714 struct mem_cgroup_tree_per_zone *mctz, 715 unsigned long long new_usage_in_excess) 716 { 717 struct rb_node **p = &mctz->rb_root.rb_node; 718 struct rb_node *parent = NULL; 719 struct mem_cgroup_per_zone *mz_node; 720 721 if (mz->on_tree) 722 return; 723 724 mz->usage_in_excess = new_usage_in_excess; 725 if (!mz->usage_in_excess) 726 return; 727 while (*p) { 728 parent = *p; 729 mz_node = rb_entry(parent, struct mem_cgroup_per_zone, 730 tree_node); 731 if (mz->usage_in_excess < mz_node->usage_in_excess) 732 p = &(*p)->rb_left; 733 /* 734 * We can't avoid mem cgroups that are over their soft 735 * limit by the same amount 736 */ 737 else if (mz->usage_in_excess >= mz_node->usage_in_excess) 738 p = &(*p)->rb_right; 739 } 740 rb_link_node(&mz->tree_node, parent, p); 741 rb_insert_color(&mz->tree_node, &mctz->rb_root); 742 mz->on_tree = true; 743 } 744 745 static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_zone *mz, 746 struct mem_cgroup_tree_per_zone *mctz) 747 { 748 if (!mz->on_tree) 749 return; 750 rb_erase(&mz->tree_node, &mctz->rb_root); 751 mz->on_tree = false; 752 } 753 754 static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_zone *mz, 755 struct mem_cgroup_tree_per_zone *mctz) 756 { 757 spin_lock(&mctz->lock); 758 __mem_cgroup_remove_exceeded(mz, mctz); 759 spin_unlock(&mctz->lock); 760 } 761 762 763 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page) 764 { 765 unsigned long long excess; 766 struct mem_cgroup_per_zone *mz; 767 struct mem_cgroup_tree_per_zone *mctz; 768 769 mctz = soft_limit_tree_from_page(page); 770 /* 771 * Necessary to update all ancestors when hierarchy is used. 772 * because their event counter is not touched. 773 */ 774 for (; memcg; memcg = parent_mem_cgroup(memcg)) { 775 mz = mem_cgroup_page_zoneinfo(memcg, page); 776 excess = res_counter_soft_limit_excess(&memcg->res); 777 /* 778 * We have to update the tree if mz is on RB-tree or 779 * mem is over its softlimit. 780 */ 781 if (excess || mz->on_tree) { 782 spin_lock(&mctz->lock); 783 /* if on-tree, remove it */ 784 if (mz->on_tree) 785 __mem_cgroup_remove_exceeded(mz, mctz); 786 /* 787 * Insert again. mz->usage_in_excess will be updated. 788 * If excess is 0, no tree ops. 789 */ 790 __mem_cgroup_insert_exceeded(mz, mctz, excess); 791 spin_unlock(&mctz->lock); 792 } 793 } 794 } 795 796 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg) 797 { 798 struct mem_cgroup_tree_per_zone *mctz; 799 struct mem_cgroup_per_zone *mz; 800 int nid, zid; 801 802 for_each_node(nid) { 803 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 804 mz = &memcg->nodeinfo[nid]->zoneinfo[zid]; 805 mctz = soft_limit_tree_node_zone(nid, zid); 806 mem_cgroup_remove_exceeded(mz, mctz); 807 } 808 } 809 } 810 811 static struct mem_cgroup_per_zone * 812 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz) 813 { 814 struct rb_node *rightmost = NULL; 815 struct mem_cgroup_per_zone *mz; 816 817 retry: 818 mz = NULL; 819 rightmost = rb_last(&mctz->rb_root); 820 if (!rightmost) 821 goto done; /* Nothing to reclaim from */ 822 823 mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node); 824 /* 825 * Remove the node now but someone else can add it back, 826 * we will to add it back at the end of reclaim to its correct 827 * position in the tree. 828 */ 829 __mem_cgroup_remove_exceeded(mz, mctz); 830 if (!res_counter_soft_limit_excess(&mz->memcg->res) || 831 !css_tryget_online(&mz->memcg->css)) 832 goto retry; 833 done: 834 return mz; 835 } 836 837 static struct mem_cgroup_per_zone * 838 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz) 839 { 840 struct mem_cgroup_per_zone *mz; 841 842 spin_lock(&mctz->lock); 843 mz = __mem_cgroup_largest_soft_limit_node(mctz); 844 spin_unlock(&mctz->lock); 845 return mz; 846 } 847 848 /* 849 * Implementation Note: reading percpu statistics for memcg. 850 * 851 * Both of vmstat[] and percpu_counter has threshold and do periodic 852 * synchronization to implement "quick" read. There are trade-off between 853 * reading cost and precision of value. Then, we may have a chance to implement 854 * a periodic synchronizion of counter in memcg's counter. 855 * 856 * But this _read() function is used for user interface now. The user accounts 857 * memory usage by memory cgroup and he _always_ requires exact value because 858 * he accounts memory. Even if we provide quick-and-fuzzy read, we always 859 * have to visit all online cpus and make sum. So, for now, unnecessary 860 * synchronization is not implemented. (just implemented for cpu hotplug) 861 * 862 * If there are kernel internal actions which can make use of some not-exact 863 * value, and reading all cpu value can be performance bottleneck in some 864 * common workload, threashold and synchonization as vmstat[] should be 865 * implemented. 866 */ 867 static long mem_cgroup_read_stat(struct mem_cgroup *memcg, 868 enum mem_cgroup_stat_index idx) 869 { 870 long val = 0; 871 int cpu; 872 873 get_online_cpus(); 874 for_each_online_cpu(cpu) 875 val += per_cpu(memcg->stat->count[idx], cpu); 876 #ifdef CONFIG_HOTPLUG_CPU 877 spin_lock(&memcg->pcp_counter_lock); 878 val += memcg->nocpu_base.count[idx]; 879 spin_unlock(&memcg->pcp_counter_lock); 880 #endif 881 put_online_cpus(); 882 return val; 883 } 884 885 static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg, 886 bool charge) 887 { 888 int val = (charge) ? 1 : -1; 889 this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAP], val); 890 } 891 892 static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg, 893 enum mem_cgroup_events_index idx) 894 { 895 unsigned long val = 0; 896 int cpu; 897 898 get_online_cpus(); 899 for_each_online_cpu(cpu) 900 val += per_cpu(memcg->stat->events[idx], cpu); 901 #ifdef CONFIG_HOTPLUG_CPU 902 spin_lock(&memcg->pcp_counter_lock); 903 val += memcg->nocpu_base.events[idx]; 904 spin_unlock(&memcg->pcp_counter_lock); 905 #endif 906 put_online_cpus(); 907 return val; 908 } 909 910 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg, 911 struct page *page, 912 bool anon, int nr_pages) 913 { 914 /* 915 * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is 916 * counted as CACHE even if it's on ANON LRU. 917 */ 918 if (anon) 919 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS], 920 nr_pages); 921 else 922 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE], 923 nr_pages); 924 925 if (PageTransHuge(page)) 926 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], 927 nr_pages); 928 929 /* pagein of a big page is an event. So, ignore page size */ 930 if (nr_pages > 0) 931 __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]); 932 else { 933 __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]); 934 nr_pages = -nr_pages; /* for event */ 935 } 936 937 __this_cpu_add(memcg->stat->nr_page_events, nr_pages); 938 } 939 940 unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru) 941 { 942 struct mem_cgroup_per_zone *mz; 943 944 mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec); 945 return mz->lru_size[lru]; 946 } 947 948 static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg, 949 int nid, 950 unsigned int lru_mask) 951 { 952 unsigned long nr = 0; 953 int zid; 954 955 VM_BUG_ON((unsigned)nid >= nr_node_ids); 956 957 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 958 struct mem_cgroup_per_zone *mz; 959 enum lru_list lru; 960 961 for_each_lru(lru) { 962 if (!(BIT(lru) & lru_mask)) 963 continue; 964 mz = &memcg->nodeinfo[nid]->zoneinfo[zid]; 965 nr += mz->lru_size[lru]; 966 } 967 } 968 return nr; 969 } 970 971 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg, 972 unsigned int lru_mask) 973 { 974 unsigned long nr = 0; 975 int nid; 976 977 for_each_node_state(nid, N_MEMORY) 978 nr += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask); 979 return nr; 980 } 981 982 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg, 983 enum mem_cgroup_events_target target) 984 { 985 unsigned long val, next; 986 987 val = __this_cpu_read(memcg->stat->nr_page_events); 988 next = __this_cpu_read(memcg->stat->targets[target]); 989 /* from time_after() in jiffies.h */ 990 if ((long)next - (long)val < 0) { 991 switch (target) { 992 case MEM_CGROUP_TARGET_THRESH: 993 next = val + THRESHOLDS_EVENTS_TARGET; 994 break; 995 case MEM_CGROUP_TARGET_SOFTLIMIT: 996 next = val + SOFTLIMIT_EVENTS_TARGET; 997 break; 998 case MEM_CGROUP_TARGET_NUMAINFO: 999 next = val + NUMAINFO_EVENTS_TARGET; 1000 break; 1001 default: 1002 break; 1003 } 1004 __this_cpu_write(memcg->stat->targets[target], next); 1005 return true; 1006 } 1007 return false; 1008 } 1009 1010 /* 1011 * Check events in order. 1012 * 1013 */ 1014 static void memcg_check_events(struct mem_cgroup *memcg, struct page *page) 1015 { 1016 preempt_disable(); 1017 /* threshold event is triggered in finer grain than soft limit */ 1018 if (unlikely(mem_cgroup_event_ratelimit(memcg, 1019 MEM_CGROUP_TARGET_THRESH))) { 1020 bool do_softlimit; 1021 bool do_numainfo __maybe_unused; 1022 1023 do_softlimit = mem_cgroup_event_ratelimit(memcg, 1024 MEM_CGROUP_TARGET_SOFTLIMIT); 1025 #if MAX_NUMNODES > 1 1026 do_numainfo = mem_cgroup_event_ratelimit(memcg, 1027 MEM_CGROUP_TARGET_NUMAINFO); 1028 #endif 1029 preempt_enable(); 1030 1031 mem_cgroup_threshold(memcg); 1032 if (unlikely(do_softlimit)) 1033 mem_cgroup_update_tree(memcg, page); 1034 #if MAX_NUMNODES > 1 1035 if (unlikely(do_numainfo)) 1036 atomic_inc(&memcg->numainfo_events); 1037 #endif 1038 } else 1039 preempt_enable(); 1040 } 1041 1042 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) 1043 { 1044 /* 1045 * mm_update_next_owner() may clear mm->owner to NULL 1046 * if it races with swapoff, page migration, etc. 1047 * So this can be called with p == NULL. 1048 */ 1049 if (unlikely(!p)) 1050 return NULL; 1051 1052 return mem_cgroup_from_css(task_css(p, memory_cgrp_id)); 1053 } 1054 1055 static struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm) 1056 { 1057 struct mem_cgroup *memcg = NULL; 1058 1059 rcu_read_lock(); 1060 do { 1061 /* 1062 * Page cache insertions can happen withou an 1063 * actual mm context, e.g. during disk probing 1064 * on boot, loopback IO, acct() writes etc. 1065 */ 1066 if (unlikely(!mm)) 1067 memcg = root_mem_cgroup; 1068 else { 1069 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); 1070 if (unlikely(!memcg)) 1071 memcg = root_mem_cgroup; 1072 } 1073 } while (!css_tryget_online(&memcg->css)); 1074 rcu_read_unlock(); 1075 return memcg; 1076 } 1077 1078 /* 1079 * Returns a next (in a pre-order walk) alive memcg (with elevated css 1080 * ref. count) or NULL if the whole root's subtree has been visited. 1081 * 1082 * helper function to be used by mem_cgroup_iter 1083 */ 1084 static struct mem_cgroup *__mem_cgroup_iter_next(struct mem_cgroup *root, 1085 struct mem_cgroup *last_visited) 1086 { 1087 struct cgroup_subsys_state *prev_css, *next_css; 1088 1089 prev_css = last_visited ? &last_visited->css : NULL; 1090 skip_node: 1091 next_css = css_next_descendant_pre(prev_css, &root->css); 1092 1093 /* 1094 * Even if we found a group we have to make sure it is 1095 * alive. css && !memcg means that the groups should be 1096 * skipped and we should continue the tree walk. 1097 * last_visited css is safe to use because it is 1098 * protected by css_get and the tree walk is rcu safe. 1099 * 1100 * We do not take a reference on the root of the tree walk 1101 * because we might race with the root removal when it would 1102 * be the only node in the iterated hierarchy and mem_cgroup_iter 1103 * would end up in an endless loop because it expects that at 1104 * least one valid node will be returned. Root cannot disappear 1105 * because caller of the iterator should hold it already so 1106 * skipping css reference should be safe. 1107 */ 1108 if (next_css) { 1109 if ((next_css == &root->css) || 1110 ((next_css->flags & CSS_ONLINE) && 1111 css_tryget_online(next_css))) 1112 return mem_cgroup_from_css(next_css); 1113 1114 prev_css = next_css; 1115 goto skip_node; 1116 } 1117 1118 return NULL; 1119 } 1120 1121 static void mem_cgroup_iter_invalidate(struct mem_cgroup *root) 1122 { 1123 /* 1124 * When a group in the hierarchy below root is destroyed, the 1125 * hierarchy iterator can no longer be trusted since it might 1126 * have pointed to the destroyed group. Invalidate it. 1127 */ 1128 atomic_inc(&root->dead_count); 1129 } 1130 1131 static struct mem_cgroup * 1132 mem_cgroup_iter_load(struct mem_cgroup_reclaim_iter *iter, 1133 struct mem_cgroup *root, 1134 int *sequence) 1135 { 1136 struct mem_cgroup *position = NULL; 1137 /* 1138 * A cgroup destruction happens in two stages: offlining and 1139 * release. They are separated by a RCU grace period. 1140 * 1141 * If the iterator is valid, we may still race with an 1142 * offlining. The RCU lock ensures the object won't be 1143 * released, tryget will fail if we lost the race. 1144 */ 1145 *sequence = atomic_read(&root->dead_count); 1146 if (iter->last_dead_count == *sequence) { 1147 smp_rmb(); 1148 position = iter->last_visited; 1149 1150 /* 1151 * We cannot take a reference to root because we might race 1152 * with root removal and returning NULL would end up in 1153 * an endless loop on the iterator user level when root 1154 * would be returned all the time. 1155 */ 1156 if (position && position != root && 1157 !css_tryget_online(&position->css)) 1158 position = NULL; 1159 } 1160 return position; 1161 } 1162 1163 static void mem_cgroup_iter_update(struct mem_cgroup_reclaim_iter *iter, 1164 struct mem_cgroup *last_visited, 1165 struct mem_cgroup *new_position, 1166 struct mem_cgroup *root, 1167 int sequence) 1168 { 1169 /* root reference counting symmetric to mem_cgroup_iter_load */ 1170 if (last_visited && last_visited != root) 1171 css_put(&last_visited->css); 1172 /* 1173 * We store the sequence count from the time @last_visited was 1174 * loaded successfully instead of rereading it here so that we 1175 * don't lose destruction events in between. We could have 1176 * raced with the destruction of @new_position after all. 1177 */ 1178 iter->last_visited = new_position; 1179 smp_wmb(); 1180 iter->last_dead_count = sequence; 1181 } 1182 1183 /** 1184 * mem_cgroup_iter - iterate over memory cgroup hierarchy 1185 * @root: hierarchy root 1186 * @prev: previously returned memcg, NULL on first invocation 1187 * @reclaim: cookie for shared reclaim walks, NULL for full walks 1188 * 1189 * Returns references to children of the hierarchy below @root, or 1190 * @root itself, or %NULL after a full round-trip. 1191 * 1192 * Caller must pass the return value in @prev on subsequent 1193 * invocations for reference counting, or use mem_cgroup_iter_break() 1194 * to cancel a hierarchy walk before the round-trip is complete. 1195 * 1196 * Reclaimers can specify a zone and a priority level in @reclaim to 1197 * divide up the memcgs in the hierarchy among all concurrent 1198 * reclaimers operating on the same zone and priority. 1199 */ 1200 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root, 1201 struct mem_cgroup *prev, 1202 struct mem_cgroup_reclaim_cookie *reclaim) 1203 { 1204 struct mem_cgroup *memcg = NULL; 1205 struct mem_cgroup *last_visited = NULL; 1206 1207 if (mem_cgroup_disabled()) 1208 return NULL; 1209 1210 if (!root) 1211 root = root_mem_cgroup; 1212 1213 if (prev && !reclaim) 1214 last_visited = prev; 1215 1216 if (!root->use_hierarchy && root != root_mem_cgroup) { 1217 if (prev) 1218 goto out_css_put; 1219 return root; 1220 } 1221 1222 rcu_read_lock(); 1223 while (!memcg) { 1224 struct mem_cgroup_reclaim_iter *uninitialized_var(iter); 1225 int uninitialized_var(seq); 1226 1227 if (reclaim) { 1228 struct mem_cgroup_per_zone *mz; 1229 1230 mz = mem_cgroup_zone_zoneinfo(root, reclaim->zone); 1231 iter = &mz->reclaim_iter[reclaim->priority]; 1232 if (prev && reclaim->generation != iter->generation) { 1233 iter->last_visited = NULL; 1234 goto out_unlock; 1235 } 1236 1237 last_visited = mem_cgroup_iter_load(iter, root, &seq); 1238 } 1239 1240 memcg = __mem_cgroup_iter_next(root, last_visited); 1241 1242 if (reclaim) { 1243 mem_cgroup_iter_update(iter, last_visited, memcg, root, 1244 seq); 1245 1246 if (!memcg) 1247 iter->generation++; 1248 else if (!prev && memcg) 1249 reclaim->generation = iter->generation; 1250 } 1251 1252 if (prev && !memcg) 1253 goto out_unlock; 1254 } 1255 out_unlock: 1256 rcu_read_unlock(); 1257 out_css_put: 1258 if (prev && prev != root) 1259 css_put(&prev->css); 1260 1261 return memcg; 1262 } 1263 1264 /** 1265 * mem_cgroup_iter_break - abort a hierarchy walk prematurely 1266 * @root: hierarchy root 1267 * @prev: last visited hierarchy member as returned by mem_cgroup_iter() 1268 */ 1269 void mem_cgroup_iter_break(struct mem_cgroup *root, 1270 struct mem_cgroup *prev) 1271 { 1272 if (!root) 1273 root = root_mem_cgroup; 1274 if (prev && prev != root) 1275 css_put(&prev->css); 1276 } 1277 1278 /* 1279 * Iteration constructs for visiting all cgroups (under a tree). If 1280 * loops are exited prematurely (break), mem_cgroup_iter_break() must 1281 * be used for reference counting. 1282 */ 1283 #define for_each_mem_cgroup_tree(iter, root) \ 1284 for (iter = mem_cgroup_iter(root, NULL, NULL); \ 1285 iter != NULL; \ 1286 iter = mem_cgroup_iter(root, iter, NULL)) 1287 1288 #define for_each_mem_cgroup(iter) \ 1289 for (iter = mem_cgroup_iter(NULL, NULL, NULL); \ 1290 iter != NULL; \ 1291 iter = mem_cgroup_iter(NULL, iter, NULL)) 1292 1293 void __mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx) 1294 { 1295 struct mem_cgroup *memcg; 1296 1297 rcu_read_lock(); 1298 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); 1299 if (unlikely(!memcg)) 1300 goto out; 1301 1302 switch (idx) { 1303 case PGFAULT: 1304 this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT]); 1305 break; 1306 case PGMAJFAULT: 1307 this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT]); 1308 break; 1309 default: 1310 BUG(); 1311 } 1312 out: 1313 rcu_read_unlock(); 1314 } 1315 EXPORT_SYMBOL(__mem_cgroup_count_vm_event); 1316 1317 /** 1318 * mem_cgroup_zone_lruvec - get the lru list vector for a zone and memcg 1319 * @zone: zone of the wanted lruvec 1320 * @memcg: memcg of the wanted lruvec 1321 * 1322 * Returns the lru list vector holding pages for the given @zone and 1323 * @mem. This can be the global zone lruvec, if the memory controller 1324 * is disabled. 1325 */ 1326 struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone, 1327 struct mem_cgroup *memcg) 1328 { 1329 struct mem_cgroup_per_zone *mz; 1330 struct lruvec *lruvec; 1331 1332 if (mem_cgroup_disabled()) { 1333 lruvec = &zone->lruvec; 1334 goto out; 1335 } 1336 1337 mz = mem_cgroup_zone_zoneinfo(memcg, zone); 1338 lruvec = &mz->lruvec; 1339 out: 1340 /* 1341 * Since a node can be onlined after the mem_cgroup was created, 1342 * we have to be prepared to initialize lruvec->zone here; 1343 * and if offlined then reonlined, we need to reinitialize it. 1344 */ 1345 if (unlikely(lruvec->zone != zone)) 1346 lruvec->zone = zone; 1347 return lruvec; 1348 } 1349 1350 /* 1351 * Following LRU functions are allowed to be used without PCG_LOCK. 1352 * Operations are called by routine of global LRU independently from memcg. 1353 * What we have to take care of here is validness of pc->mem_cgroup. 1354 * 1355 * Changes to pc->mem_cgroup happens when 1356 * 1. charge 1357 * 2. moving account 1358 * In typical case, "charge" is done before add-to-lru. Exception is SwapCache. 1359 * It is added to LRU before charge. 1360 * If PCG_USED bit is not set, page_cgroup is not added to this private LRU. 1361 * When moving account, the page is not on LRU. It's isolated. 1362 */ 1363 1364 /** 1365 * mem_cgroup_page_lruvec - return lruvec for adding an lru page 1366 * @page: the page 1367 * @zone: zone of the page 1368 */ 1369 struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct zone *zone) 1370 { 1371 struct mem_cgroup_per_zone *mz; 1372 struct mem_cgroup *memcg; 1373 struct page_cgroup *pc; 1374 struct lruvec *lruvec; 1375 1376 if (mem_cgroup_disabled()) { 1377 lruvec = &zone->lruvec; 1378 goto out; 1379 } 1380 1381 pc = lookup_page_cgroup(page); 1382 memcg = pc->mem_cgroup; 1383 1384 /* 1385 * Surreptitiously switch any uncharged offlist page to root: 1386 * an uncharged page off lru does nothing to secure 1387 * its former mem_cgroup from sudden removal. 1388 * 1389 * Our caller holds lru_lock, and PageCgroupUsed is updated 1390 * under page_cgroup lock: between them, they make all uses 1391 * of pc->mem_cgroup safe. 1392 */ 1393 if (!PageLRU(page) && !PageCgroupUsed(pc) && memcg != root_mem_cgroup) 1394 pc->mem_cgroup = memcg = root_mem_cgroup; 1395 1396 mz = mem_cgroup_page_zoneinfo(memcg, page); 1397 lruvec = &mz->lruvec; 1398 out: 1399 /* 1400 * Since a node can be onlined after the mem_cgroup was created, 1401 * we have to be prepared to initialize lruvec->zone here; 1402 * and if offlined then reonlined, we need to reinitialize it. 1403 */ 1404 if (unlikely(lruvec->zone != zone)) 1405 lruvec->zone = zone; 1406 return lruvec; 1407 } 1408 1409 /** 1410 * mem_cgroup_update_lru_size - account for adding or removing an lru page 1411 * @lruvec: mem_cgroup per zone lru vector 1412 * @lru: index of lru list the page is sitting on 1413 * @nr_pages: positive when adding or negative when removing 1414 * 1415 * This function must be called when a page is added to or removed from an 1416 * lru list. 1417 */ 1418 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, 1419 int nr_pages) 1420 { 1421 struct mem_cgroup_per_zone *mz; 1422 unsigned long *lru_size; 1423 1424 if (mem_cgroup_disabled()) 1425 return; 1426 1427 mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec); 1428 lru_size = mz->lru_size + lru; 1429 *lru_size += nr_pages; 1430 VM_BUG_ON((long)(*lru_size) < 0); 1431 } 1432 1433 /* 1434 * Checks whether given mem is same or in the root_mem_cgroup's 1435 * hierarchy subtree 1436 */ 1437 bool __mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg, 1438 struct mem_cgroup *memcg) 1439 { 1440 if (root_memcg == memcg) 1441 return true; 1442 if (!root_memcg->use_hierarchy || !memcg) 1443 return false; 1444 return cgroup_is_descendant(memcg->css.cgroup, root_memcg->css.cgroup); 1445 } 1446 1447 static bool mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg, 1448 struct mem_cgroup *memcg) 1449 { 1450 bool ret; 1451 1452 rcu_read_lock(); 1453 ret = __mem_cgroup_same_or_subtree(root_memcg, memcg); 1454 rcu_read_unlock(); 1455 return ret; 1456 } 1457 1458 bool task_in_mem_cgroup(struct task_struct *task, 1459 const struct mem_cgroup *memcg) 1460 { 1461 struct mem_cgroup *curr = NULL; 1462 struct task_struct *p; 1463 bool ret; 1464 1465 p = find_lock_task_mm(task); 1466 if (p) { 1467 curr = get_mem_cgroup_from_mm(p->mm); 1468 task_unlock(p); 1469 } else { 1470 /* 1471 * All threads may have already detached their mm's, but the oom 1472 * killer still needs to detect if they have already been oom 1473 * killed to prevent needlessly killing additional tasks. 1474 */ 1475 rcu_read_lock(); 1476 curr = mem_cgroup_from_task(task); 1477 if (curr) 1478 css_get(&curr->css); 1479 rcu_read_unlock(); 1480 } 1481 /* 1482 * We should check use_hierarchy of "memcg" not "curr". Because checking 1483 * use_hierarchy of "curr" here make this function true if hierarchy is 1484 * enabled in "curr" and "curr" is a child of "memcg" in *cgroup* 1485 * hierarchy(even if use_hierarchy is disabled in "memcg"). 1486 */ 1487 ret = mem_cgroup_same_or_subtree(memcg, curr); 1488 css_put(&curr->css); 1489 return ret; 1490 } 1491 1492 int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec) 1493 { 1494 unsigned long inactive_ratio; 1495 unsigned long inactive; 1496 unsigned long active; 1497 unsigned long gb; 1498 1499 inactive = mem_cgroup_get_lru_size(lruvec, LRU_INACTIVE_ANON); 1500 active = mem_cgroup_get_lru_size(lruvec, LRU_ACTIVE_ANON); 1501 1502 gb = (inactive + active) >> (30 - PAGE_SHIFT); 1503 if (gb) 1504 inactive_ratio = int_sqrt(10 * gb); 1505 else 1506 inactive_ratio = 1; 1507 1508 return inactive * inactive_ratio < active; 1509 } 1510 1511 #define mem_cgroup_from_res_counter(counter, member) \ 1512 container_of(counter, struct mem_cgroup, member) 1513 1514 /** 1515 * mem_cgroup_margin - calculate chargeable space of a memory cgroup 1516 * @memcg: the memory cgroup 1517 * 1518 * Returns the maximum amount of memory @mem can be charged with, in 1519 * pages. 1520 */ 1521 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg) 1522 { 1523 unsigned long long margin; 1524 1525 margin = res_counter_margin(&memcg->res); 1526 if (do_swap_account) 1527 margin = min(margin, res_counter_margin(&memcg->memsw)); 1528 return margin >> PAGE_SHIFT; 1529 } 1530 1531 int mem_cgroup_swappiness(struct mem_cgroup *memcg) 1532 { 1533 /* root ? */ 1534 if (mem_cgroup_disabled() || !memcg->css.parent) 1535 return vm_swappiness; 1536 1537 return memcg->swappiness; 1538 } 1539 1540 /* 1541 * memcg->moving_account is used for checking possibility that some thread is 1542 * calling move_account(). When a thread on CPU-A starts moving pages under 1543 * a memcg, other threads should check memcg->moving_account under 1544 * rcu_read_lock(), like this: 1545 * 1546 * CPU-A CPU-B 1547 * rcu_read_lock() 1548 * memcg->moving_account+1 if (memcg->mocing_account) 1549 * take heavy locks. 1550 * synchronize_rcu() update something. 1551 * rcu_read_unlock() 1552 * start move here. 1553 */ 1554 1555 /* for quick checking without looking up memcg */ 1556 atomic_t memcg_moving __read_mostly; 1557 1558 static void mem_cgroup_start_move(struct mem_cgroup *memcg) 1559 { 1560 atomic_inc(&memcg_moving); 1561 atomic_inc(&memcg->moving_account); 1562 synchronize_rcu(); 1563 } 1564 1565 static void mem_cgroup_end_move(struct mem_cgroup *memcg) 1566 { 1567 /* 1568 * Now, mem_cgroup_clear_mc() may call this function with NULL. 1569 * We check NULL in callee rather than caller. 1570 */ 1571 if (memcg) { 1572 atomic_dec(&memcg_moving); 1573 atomic_dec(&memcg->moving_account); 1574 } 1575 } 1576 1577 /* 1578 * A routine for checking "mem" is under move_account() or not. 1579 * 1580 * Checking a cgroup is mc.from or mc.to or under hierarchy of 1581 * moving cgroups. This is for waiting at high-memory pressure 1582 * caused by "move". 1583 */ 1584 static bool mem_cgroup_under_move(struct mem_cgroup *memcg) 1585 { 1586 struct mem_cgroup *from; 1587 struct mem_cgroup *to; 1588 bool ret = false; 1589 /* 1590 * Unlike task_move routines, we access mc.to, mc.from not under 1591 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead. 1592 */ 1593 spin_lock(&mc.lock); 1594 from = mc.from; 1595 to = mc.to; 1596 if (!from) 1597 goto unlock; 1598 1599 ret = mem_cgroup_same_or_subtree(memcg, from) 1600 || mem_cgroup_same_or_subtree(memcg, to); 1601 unlock: 1602 spin_unlock(&mc.lock); 1603 return ret; 1604 } 1605 1606 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg) 1607 { 1608 if (mc.moving_task && current != mc.moving_task) { 1609 if (mem_cgroup_under_move(memcg)) { 1610 DEFINE_WAIT(wait); 1611 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE); 1612 /* moving charge context might have finished. */ 1613 if (mc.moving_task) 1614 schedule(); 1615 finish_wait(&mc.waitq, &wait); 1616 return true; 1617 } 1618 } 1619 return false; 1620 } 1621 1622 /* 1623 * Take this lock when 1624 * - a code tries to modify page's memcg while it's USED. 1625 * - a code tries to modify page state accounting in a memcg. 1626 */ 1627 static void move_lock_mem_cgroup(struct mem_cgroup *memcg, 1628 unsigned long *flags) 1629 { 1630 spin_lock_irqsave(&memcg->move_lock, *flags); 1631 } 1632 1633 static void move_unlock_mem_cgroup(struct mem_cgroup *memcg, 1634 unsigned long *flags) 1635 { 1636 spin_unlock_irqrestore(&memcg->move_lock, *flags); 1637 } 1638 1639 #define K(x) ((x) << (PAGE_SHIFT-10)) 1640 /** 1641 * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller. 1642 * @memcg: The memory cgroup that went over limit 1643 * @p: Task that is going to be killed 1644 * 1645 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is 1646 * enabled 1647 */ 1648 void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p) 1649 { 1650 /* oom_info_lock ensures that parallel ooms do not interleave */ 1651 static DEFINE_MUTEX(oom_info_lock); 1652 struct mem_cgroup *iter; 1653 unsigned int i; 1654 1655 if (!p) 1656 return; 1657 1658 mutex_lock(&oom_info_lock); 1659 rcu_read_lock(); 1660 1661 pr_info("Task in "); 1662 pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id)); 1663 pr_info(" killed as a result of limit of "); 1664 pr_cont_cgroup_path(memcg->css.cgroup); 1665 pr_info("\n"); 1666 1667 rcu_read_unlock(); 1668 1669 pr_info("memory: usage %llukB, limit %llukB, failcnt %llu\n", 1670 res_counter_read_u64(&memcg->res, RES_USAGE) >> 10, 1671 res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10, 1672 res_counter_read_u64(&memcg->res, RES_FAILCNT)); 1673 pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %llu\n", 1674 res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10, 1675 res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10, 1676 res_counter_read_u64(&memcg->memsw, RES_FAILCNT)); 1677 pr_info("kmem: usage %llukB, limit %llukB, failcnt %llu\n", 1678 res_counter_read_u64(&memcg->kmem, RES_USAGE) >> 10, 1679 res_counter_read_u64(&memcg->kmem, RES_LIMIT) >> 10, 1680 res_counter_read_u64(&memcg->kmem, RES_FAILCNT)); 1681 1682 for_each_mem_cgroup_tree(iter, memcg) { 1683 pr_info("Memory cgroup stats for "); 1684 pr_cont_cgroup_path(iter->css.cgroup); 1685 pr_cont(":"); 1686 1687 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) { 1688 if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account) 1689 continue; 1690 pr_cont(" %s:%ldKB", mem_cgroup_stat_names[i], 1691 K(mem_cgroup_read_stat(iter, i))); 1692 } 1693 1694 for (i = 0; i < NR_LRU_LISTS; i++) 1695 pr_cont(" %s:%luKB", mem_cgroup_lru_names[i], 1696 K(mem_cgroup_nr_lru_pages(iter, BIT(i)))); 1697 1698 pr_cont("\n"); 1699 } 1700 mutex_unlock(&oom_info_lock); 1701 } 1702 1703 /* 1704 * This function returns the number of memcg under hierarchy tree. Returns 1705 * 1(self count) if no children. 1706 */ 1707 static int mem_cgroup_count_children(struct mem_cgroup *memcg) 1708 { 1709 int num = 0; 1710 struct mem_cgroup *iter; 1711 1712 for_each_mem_cgroup_tree(iter, memcg) 1713 num++; 1714 return num; 1715 } 1716 1717 /* 1718 * Return the memory (and swap, if configured) limit for a memcg. 1719 */ 1720 static u64 mem_cgroup_get_limit(struct mem_cgroup *memcg) 1721 { 1722 u64 limit; 1723 1724 limit = res_counter_read_u64(&memcg->res, RES_LIMIT); 1725 1726 /* 1727 * Do not consider swap space if we cannot swap due to swappiness 1728 */ 1729 if (mem_cgroup_swappiness(memcg)) { 1730 u64 memsw; 1731 1732 limit += total_swap_pages << PAGE_SHIFT; 1733 memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 1734 1735 /* 1736 * If memsw is finite and limits the amount of swap space 1737 * available to this memcg, return that limit. 1738 */ 1739 limit = min(limit, memsw); 1740 } 1741 1742 return limit; 1743 } 1744 1745 static void mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask, 1746 int order) 1747 { 1748 struct mem_cgroup *iter; 1749 unsigned long chosen_points = 0; 1750 unsigned long totalpages; 1751 unsigned int points = 0; 1752 struct task_struct *chosen = NULL; 1753 1754 /* 1755 * If current has a pending SIGKILL or is exiting, then automatically 1756 * select it. The goal is to allow it to allocate so that it may 1757 * quickly exit and free its memory. 1758 */ 1759 if (fatal_signal_pending(current) || current->flags & PF_EXITING) { 1760 set_thread_flag(TIF_MEMDIE); 1761 return; 1762 } 1763 1764 check_panic_on_oom(CONSTRAINT_MEMCG, gfp_mask, order, NULL); 1765 totalpages = mem_cgroup_get_limit(memcg) >> PAGE_SHIFT ? : 1; 1766 for_each_mem_cgroup_tree(iter, memcg) { 1767 struct css_task_iter it; 1768 struct task_struct *task; 1769 1770 css_task_iter_start(&iter->css, &it); 1771 while ((task = css_task_iter_next(&it))) { 1772 switch (oom_scan_process_thread(task, totalpages, NULL, 1773 false)) { 1774 case OOM_SCAN_SELECT: 1775 if (chosen) 1776 put_task_struct(chosen); 1777 chosen = task; 1778 chosen_points = ULONG_MAX; 1779 get_task_struct(chosen); 1780 /* fall through */ 1781 case OOM_SCAN_CONTINUE: 1782 continue; 1783 case OOM_SCAN_ABORT: 1784 css_task_iter_end(&it); 1785 mem_cgroup_iter_break(memcg, iter); 1786 if (chosen) 1787 put_task_struct(chosen); 1788 return; 1789 case OOM_SCAN_OK: 1790 break; 1791 }; 1792 points = oom_badness(task, memcg, NULL, totalpages); 1793 if (!points || points < chosen_points) 1794 continue; 1795 /* Prefer thread group leaders for display purposes */ 1796 if (points == chosen_points && 1797 thread_group_leader(chosen)) 1798 continue; 1799 1800 if (chosen) 1801 put_task_struct(chosen); 1802 chosen = task; 1803 chosen_points = points; 1804 get_task_struct(chosen); 1805 } 1806 css_task_iter_end(&it); 1807 } 1808 1809 if (!chosen) 1810 return; 1811 points = chosen_points * 1000 / totalpages; 1812 oom_kill_process(chosen, gfp_mask, order, points, totalpages, memcg, 1813 NULL, "Memory cgroup out of memory"); 1814 } 1815 1816 static unsigned long mem_cgroup_reclaim(struct mem_cgroup *memcg, 1817 gfp_t gfp_mask, 1818 unsigned long flags) 1819 { 1820 unsigned long total = 0; 1821 bool noswap = false; 1822 int loop; 1823 1824 if (flags & MEM_CGROUP_RECLAIM_NOSWAP) 1825 noswap = true; 1826 if (!(flags & MEM_CGROUP_RECLAIM_SHRINK) && memcg->memsw_is_minimum) 1827 noswap = true; 1828 1829 for (loop = 0; loop < MEM_CGROUP_MAX_RECLAIM_LOOPS; loop++) { 1830 if (loop) 1831 drain_all_stock_async(memcg); 1832 total += try_to_free_mem_cgroup_pages(memcg, gfp_mask, noswap); 1833 /* 1834 * Allow limit shrinkers, which are triggered directly 1835 * by userspace, to catch signals and stop reclaim 1836 * after minimal progress, regardless of the margin. 1837 */ 1838 if (total && (flags & MEM_CGROUP_RECLAIM_SHRINK)) 1839 break; 1840 if (mem_cgroup_margin(memcg)) 1841 break; 1842 /* 1843 * If nothing was reclaimed after two attempts, there 1844 * may be no reclaimable pages in this hierarchy. 1845 */ 1846 if (loop && !total) 1847 break; 1848 } 1849 return total; 1850 } 1851 1852 /** 1853 * test_mem_cgroup_node_reclaimable 1854 * @memcg: the target memcg 1855 * @nid: the node ID to be checked. 1856 * @noswap : specify true here if the user wants flle only information. 1857 * 1858 * This function returns whether the specified memcg contains any 1859 * reclaimable pages on a node. Returns true if there are any reclaimable 1860 * pages in the node. 1861 */ 1862 static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg, 1863 int nid, bool noswap) 1864 { 1865 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE)) 1866 return true; 1867 if (noswap || !total_swap_pages) 1868 return false; 1869 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON)) 1870 return true; 1871 return false; 1872 1873 } 1874 #if MAX_NUMNODES > 1 1875 1876 /* 1877 * Always updating the nodemask is not very good - even if we have an empty 1878 * list or the wrong list here, we can start from some node and traverse all 1879 * nodes based on the zonelist. So update the list loosely once per 10 secs. 1880 * 1881 */ 1882 static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg) 1883 { 1884 int nid; 1885 /* 1886 * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET 1887 * pagein/pageout changes since the last update. 1888 */ 1889 if (!atomic_read(&memcg->numainfo_events)) 1890 return; 1891 if (atomic_inc_return(&memcg->numainfo_updating) > 1) 1892 return; 1893 1894 /* make a nodemask where this memcg uses memory from */ 1895 memcg->scan_nodes = node_states[N_MEMORY]; 1896 1897 for_each_node_mask(nid, node_states[N_MEMORY]) { 1898 1899 if (!test_mem_cgroup_node_reclaimable(memcg, nid, false)) 1900 node_clear(nid, memcg->scan_nodes); 1901 } 1902 1903 atomic_set(&memcg->numainfo_events, 0); 1904 atomic_set(&memcg->numainfo_updating, 0); 1905 } 1906 1907 /* 1908 * Selecting a node where we start reclaim from. Because what we need is just 1909 * reducing usage counter, start from anywhere is O,K. Considering 1910 * memory reclaim from current node, there are pros. and cons. 1911 * 1912 * Freeing memory from current node means freeing memory from a node which 1913 * we'll use or we've used. So, it may make LRU bad. And if several threads 1914 * hit limits, it will see a contention on a node. But freeing from remote 1915 * node means more costs for memory reclaim because of memory latency. 1916 * 1917 * Now, we use round-robin. Better algorithm is welcomed. 1918 */ 1919 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg) 1920 { 1921 int node; 1922 1923 mem_cgroup_may_update_nodemask(memcg); 1924 node = memcg->last_scanned_node; 1925 1926 node = next_node(node, memcg->scan_nodes); 1927 if (node == MAX_NUMNODES) 1928 node = first_node(memcg->scan_nodes); 1929 /* 1930 * We call this when we hit limit, not when pages are added to LRU. 1931 * No LRU may hold pages because all pages are UNEVICTABLE or 1932 * memcg is too small and all pages are not on LRU. In that case, 1933 * we use curret node. 1934 */ 1935 if (unlikely(node == MAX_NUMNODES)) 1936 node = numa_node_id(); 1937 1938 memcg->last_scanned_node = node; 1939 return node; 1940 } 1941 1942 /* 1943 * Check all nodes whether it contains reclaimable pages or not. 1944 * For quick scan, we make use of scan_nodes. This will allow us to skip 1945 * unused nodes. But scan_nodes is lazily updated and may not cotain 1946 * enough new information. We need to do double check. 1947 */ 1948 static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap) 1949 { 1950 int nid; 1951 1952 /* 1953 * quick check...making use of scan_node. 1954 * We can skip unused nodes. 1955 */ 1956 if (!nodes_empty(memcg->scan_nodes)) { 1957 for (nid = first_node(memcg->scan_nodes); 1958 nid < MAX_NUMNODES; 1959 nid = next_node(nid, memcg->scan_nodes)) { 1960 1961 if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap)) 1962 return true; 1963 } 1964 } 1965 /* 1966 * Check rest of nodes. 1967 */ 1968 for_each_node_state(nid, N_MEMORY) { 1969 if (node_isset(nid, memcg->scan_nodes)) 1970 continue; 1971 if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap)) 1972 return true; 1973 } 1974 return false; 1975 } 1976 1977 #else 1978 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg) 1979 { 1980 return 0; 1981 } 1982 1983 static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap) 1984 { 1985 return test_mem_cgroup_node_reclaimable(memcg, 0, noswap); 1986 } 1987 #endif 1988 1989 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg, 1990 struct zone *zone, 1991 gfp_t gfp_mask, 1992 unsigned long *total_scanned) 1993 { 1994 struct mem_cgroup *victim = NULL; 1995 int total = 0; 1996 int loop = 0; 1997 unsigned long excess; 1998 unsigned long nr_scanned; 1999 struct mem_cgroup_reclaim_cookie reclaim = { 2000 .zone = zone, 2001 .priority = 0, 2002 }; 2003 2004 excess = res_counter_soft_limit_excess(&root_memcg->res) >> PAGE_SHIFT; 2005 2006 while (1) { 2007 victim = mem_cgroup_iter(root_memcg, victim, &reclaim); 2008 if (!victim) { 2009 loop++; 2010 if (loop >= 2) { 2011 /* 2012 * If we have not been able to reclaim 2013 * anything, it might because there are 2014 * no reclaimable pages under this hierarchy 2015 */ 2016 if (!total) 2017 break; 2018 /* 2019 * We want to do more targeted reclaim. 2020 * excess >> 2 is not to excessive so as to 2021 * reclaim too much, nor too less that we keep 2022 * coming back to reclaim from this cgroup 2023 */ 2024 if (total >= (excess >> 2) || 2025 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) 2026 break; 2027 } 2028 continue; 2029 } 2030 if (!mem_cgroup_reclaimable(victim, false)) 2031 continue; 2032 total += mem_cgroup_shrink_node_zone(victim, gfp_mask, false, 2033 zone, &nr_scanned); 2034 *total_scanned += nr_scanned; 2035 if (!res_counter_soft_limit_excess(&root_memcg->res)) 2036 break; 2037 } 2038 mem_cgroup_iter_break(root_memcg, victim); 2039 return total; 2040 } 2041 2042 #ifdef CONFIG_LOCKDEP 2043 static struct lockdep_map memcg_oom_lock_dep_map = { 2044 .name = "memcg_oom_lock", 2045 }; 2046 #endif 2047 2048 static DEFINE_SPINLOCK(memcg_oom_lock); 2049 2050 /* 2051 * Check OOM-Killer is already running under our hierarchy. 2052 * If someone is running, return false. 2053 */ 2054 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg) 2055 { 2056 struct mem_cgroup *iter, *failed = NULL; 2057 2058 spin_lock(&memcg_oom_lock); 2059 2060 for_each_mem_cgroup_tree(iter, memcg) { 2061 if (iter->oom_lock) { 2062 /* 2063 * this subtree of our hierarchy is already locked 2064 * so we cannot give a lock. 2065 */ 2066 failed = iter; 2067 mem_cgroup_iter_break(memcg, iter); 2068 break; 2069 } else 2070 iter->oom_lock = true; 2071 } 2072 2073 if (failed) { 2074 /* 2075 * OK, we failed to lock the whole subtree so we have 2076 * to clean up what we set up to the failing subtree 2077 */ 2078 for_each_mem_cgroup_tree(iter, memcg) { 2079 if (iter == failed) { 2080 mem_cgroup_iter_break(memcg, iter); 2081 break; 2082 } 2083 iter->oom_lock = false; 2084 } 2085 } else 2086 mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_); 2087 2088 spin_unlock(&memcg_oom_lock); 2089 2090 return !failed; 2091 } 2092 2093 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg) 2094 { 2095 struct mem_cgroup *iter; 2096 2097 spin_lock(&memcg_oom_lock); 2098 mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_); 2099 for_each_mem_cgroup_tree(iter, memcg) 2100 iter->oom_lock = false; 2101 spin_unlock(&memcg_oom_lock); 2102 } 2103 2104 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg) 2105 { 2106 struct mem_cgroup *iter; 2107 2108 for_each_mem_cgroup_tree(iter, memcg) 2109 atomic_inc(&iter->under_oom); 2110 } 2111 2112 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg) 2113 { 2114 struct mem_cgroup *iter; 2115 2116 /* 2117 * When a new child is created while the hierarchy is under oom, 2118 * mem_cgroup_oom_lock() may not be called. We have to use 2119 * atomic_add_unless() here. 2120 */ 2121 for_each_mem_cgroup_tree(iter, memcg) 2122 atomic_add_unless(&iter->under_oom, -1, 0); 2123 } 2124 2125 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq); 2126 2127 struct oom_wait_info { 2128 struct mem_cgroup *memcg; 2129 wait_queue_t wait; 2130 }; 2131 2132 static int memcg_oom_wake_function(wait_queue_t *wait, 2133 unsigned mode, int sync, void *arg) 2134 { 2135 struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg; 2136 struct mem_cgroup *oom_wait_memcg; 2137 struct oom_wait_info *oom_wait_info; 2138 2139 oom_wait_info = container_of(wait, struct oom_wait_info, wait); 2140 oom_wait_memcg = oom_wait_info->memcg; 2141 2142 /* 2143 * Both of oom_wait_info->memcg and wake_memcg are stable under us. 2144 * Then we can use css_is_ancestor without taking care of RCU. 2145 */ 2146 if (!mem_cgroup_same_or_subtree(oom_wait_memcg, wake_memcg) 2147 && !mem_cgroup_same_or_subtree(wake_memcg, oom_wait_memcg)) 2148 return 0; 2149 return autoremove_wake_function(wait, mode, sync, arg); 2150 } 2151 2152 static void memcg_wakeup_oom(struct mem_cgroup *memcg) 2153 { 2154 atomic_inc(&memcg->oom_wakeups); 2155 /* for filtering, pass "memcg" as argument. */ 2156 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg); 2157 } 2158 2159 static void memcg_oom_recover(struct mem_cgroup *memcg) 2160 { 2161 if (memcg && atomic_read(&memcg->under_oom)) 2162 memcg_wakeup_oom(memcg); 2163 } 2164 2165 static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order) 2166 { 2167 if (!current->memcg_oom.may_oom) 2168 return; 2169 /* 2170 * We are in the middle of the charge context here, so we 2171 * don't want to block when potentially sitting on a callstack 2172 * that holds all kinds of filesystem and mm locks. 2173 * 2174 * Also, the caller may handle a failed allocation gracefully 2175 * (like optional page cache readahead) and so an OOM killer 2176 * invocation might not even be necessary. 2177 * 2178 * That's why we don't do anything here except remember the 2179 * OOM context and then deal with it at the end of the page 2180 * fault when the stack is unwound, the locks are released, 2181 * and when we know whether the fault was overall successful. 2182 */ 2183 css_get(&memcg->css); 2184 current->memcg_oom.memcg = memcg; 2185 current->memcg_oom.gfp_mask = mask; 2186 current->memcg_oom.order = order; 2187 } 2188 2189 /** 2190 * mem_cgroup_oom_synchronize - complete memcg OOM handling 2191 * @handle: actually kill/wait or just clean up the OOM state 2192 * 2193 * This has to be called at the end of a page fault if the memcg OOM 2194 * handler was enabled. 2195 * 2196 * Memcg supports userspace OOM handling where failed allocations must 2197 * sleep on a waitqueue until the userspace task resolves the 2198 * situation. Sleeping directly in the charge context with all kinds 2199 * of locks held is not a good idea, instead we remember an OOM state 2200 * in the task and mem_cgroup_oom_synchronize() has to be called at 2201 * the end of the page fault to complete the OOM handling. 2202 * 2203 * Returns %true if an ongoing memcg OOM situation was detected and 2204 * completed, %false otherwise. 2205 */ 2206 bool mem_cgroup_oom_synchronize(bool handle) 2207 { 2208 struct mem_cgroup *memcg = current->memcg_oom.memcg; 2209 struct oom_wait_info owait; 2210 bool locked; 2211 2212 /* OOM is global, do not handle */ 2213 if (!memcg) 2214 return false; 2215 2216 if (!handle) 2217 goto cleanup; 2218 2219 owait.memcg = memcg; 2220 owait.wait.flags = 0; 2221 owait.wait.func = memcg_oom_wake_function; 2222 owait.wait.private = current; 2223 INIT_LIST_HEAD(&owait.wait.task_list); 2224 2225 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE); 2226 mem_cgroup_mark_under_oom(memcg); 2227 2228 locked = mem_cgroup_oom_trylock(memcg); 2229 2230 if (locked) 2231 mem_cgroup_oom_notify(memcg); 2232 2233 if (locked && !memcg->oom_kill_disable) { 2234 mem_cgroup_unmark_under_oom(memcg); 2235 finish_wait(&memcg_oom_waitq, &owait.wait); 2236 mem_cgroup_out_of_memory(memcg, current->memcg_oom.gfp_mask, 2237 current->memcg_oom.order); 2238 } else { 2239 schedule(); 2240 mem_cgroup_unmark_under_oom(memcg); 2241 finish_wait(&memcg_oom_waitq, &owait.wait); 2242 } 2243 2244 if (locked) { 2245 mem_cgroup_oom_unlock(memcg); 2246 /* 2247 * There is no guarantee that an OOM-lock contender 2248 * sees the wakeups triggered by the OOM kill 2249 * uncharges. Wake any sleepers explicitely. 2250 */ 2251 memcg_oom_recover(memcg); 2252 } 2253 cleanup: 2254 current->memcg_oom.memcg = NULL; 2255 css_put(&memcg->css); 2256 return true; 2257 } 2258 2259 /* 2260 * Used to update mapped file or writeback or other statistics. 2261 * 2262 * Notes: Race condition 2263 * 2264 * We usually use lock_page_cgroup() for accessing page_cgroup member but 2265 * it tends to be costly. But considering some conditions, we doesn't need 2266 * to do so _always_. 2267 * 2268 * Considering "charge", lock_page_cgroup() is not required because all 2269 * file-stat operations happen after a page is attached to radix-tree. There 2270 * are no race with "charge". 2271 * 2272 * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup 2273 * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even 2274 * if there are race with "uncharge". Statistics itself is properly handled 2275 * by flags. 2276 * 2277 * Considering "move", this is an only case we see a race. To make the race 2278 * small, we check memcg->moving_account and detect there are possibility 2279 * of race or not. If there is, we take a lock. 2280 */ 2281 2282 void __mem_cgroup_begin_update_page_stat(struct page *page, 2283 bool *locked, unsigned long *flags) 2284 { 2285 struct mem_cgroup *memcg; 2286 struct page_cgroup *pc; 2287 2288 pc = lookup_page_cgroup(page); 2289 again: 2290 memcg = pc->mem_cgroup; 2291 if (unlikely(!memcg || !PageCgroupUsed(pc))) 2292 return; 2293 /* 2294 * If this memory cgroup is not under account moving, we don't 2295 * need to take move_lock_mem_cgroup(). Because we already hold 2296 * rcu_read_lock(), any calls to move_account will be delayed until 2297 * rcu_read_unlock(). 2298 */ 2299 VM_BUG_ON(!rcu_read_lock_held()); 2300 if (atomic_read(&memcg->moving_account) <= 0) 2301 return; 2302 2303 move_lock_mem_cgroup(memcg, flags); 2304 if (memcg != pc->mem_cgroup || !PageCgroupUsed(pc)) { 2305 move_unlock_mem_cgroup(memcg, flags); 2306 goto again; 2307 } 2308 *locked = true; 2309 } 2310 2311 void __mem_cgroup_end_update_page_stat(struct page *page, unsigned long *flags) 2312 { 2313 struct page_cgroup *pc = lookup_page_cgroup(page); 2314 2315 /* 2316 * It's guaranteed that pc->mem_cgroup never changes while 2317 * lock is held because a routine modifies pc->mem_cgroup 2318 * should take move_lock_mem_cgroup(). 2319 */ 2320 move_unlock_mem_cgroup(pc->mem_cgroup, flags); 2321 } 2322 2323 void mem_cgroup_update_page_stat(struct page *page, 2324 enum mem_cgroup_stat_index idx, int val) 2325 { 2326 struct mem_cgroup *memcg; 2327 struct page_cgroup *pc = lookup_page_cgroup(page); 2328 unsigned long uninitialized_var(flags); 2329 2330 if (mem_cgroup_disabled()) 2331 return; 2332 2333 VM_BUG_ON(!rcu_read_lock_held()); 2334 memcg = pc->mem_cgroup; 2335 if (unlikely(!memcg || !PageCgroupUsed(pc))) 2336 return; 2337 2338 this_cpu_add(memcg->stat->count[idx], val); 2339 } 2340 2341 /* 2342 * size of first charge trial. "32" comes from vmscan.c's magic value. 2343 * TODO: maybe necessary to use big numbers in big irons. 2344 */ 2345 #define CHARGE_BATCH 32U 2346 struct memcg_stock_pcp { 2347 struct mem_cgroup *cached; /* this never be root cgroup */ 2348 unsigned int nr_pages; 2349 struct work_struct work; 2350 unsigned long flags; 2351 #define FLUSHING_CACHED_CHARGE 0 2352 }; 2353 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock); 2354 static DEFINE_MUTEX(percpu_charge_mutex); 2355 2356 /** 2357 * consume_stock: Try to consume stocked charge on this cpu. 2358 * @memcg: memcg to consume from. 2359 * @nr_pages: how many pages to charge. 2360 * 2361 * The charges will only happen if @memcg matches the current cpu's memcg 2362 * stock, and at least @nr_pages are available in that stock. Failure to 2363 * service an allocation will refill the stock. 2364 * 2365 * returns true if successful, false otherwise. 2366 */ 2367 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) 2368 { 2369 struct memcg_stock_pcp *stock; 2370 bool ret = true; 2371 2372 if (nr_pages > CHARGE_BATCH) 2373 return false; 2374 2375 stock = &get_cpu_var(memcg_stock); 2376 if (memcg == stock->cached && stock->nr_pages >= nr_pages) 2377 stock->nr_pages -= nr_pages; 2378 else /* need to call res_counter_charge */ 2379 ret = false; 2380 put_cpu_var(memcg_stock); 2381 return ret; 2382 } 2383 2384 /* 2385 * Returns stocks cached in percpu to res_counter and reset cached information. 2386 */ 2387 static void drain_stock(struct memcg_stock_pcp *stock) 2388 { 2389 struct mem_cgroup *old = stock->cached; 2390 2391 if (stock->nr_pages) { 2392 unsigned long bytes = stock->nr_pages * PAGE_SIZE; 2393 2394 res_counter_uncharge(&old->res, bytes); 2395 if (do_swap_account) 2396 res_counter_uncharge(&old->memsw, bytes); 2397 stock->nr_pages = 0; 2398 } 2399 stock->cached = NULL; 2400 } 2401 2402 /* 2403 * This must be called under preempt disabled or must be called by 2404 * a thread which is pinned to local cpu. 2405 */ 2406 static void drain_local_stock(struct work_struct *dummy) 2407 { 2408 struct memcg_stock_pcp *stock = this_cpu_ptr(&memcg_stock); 2409 drain_stock(stock); 2410 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags); 2411 } 2412 2413 static void __init memcg_stock_init(void) 2414 { 2415 int cpu; 2416 2417 for_each_possible_cpu(cpu) { 2418 struct memcg_stock_pcp *stock = 2419 &per_cpu(memcg_stock, cpu); 2420 INIT_WORK(&stock->work, drain_local_stock); 2421 } 2422 } 2423 2424 /* 2425 * Cache charges(val) which is from res_counter, to local per_cpu area. 2426 * This will be consumed by consume_stock() function, later. 2427 */ 2428 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) 2429 { 2430 struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock); 2431 2432 if (stock->cached != memcg) { /* reset if necessary */ 2433 drain_stock(stock); 2434 stock->cached = memcg; 2435 } 2436 stock->nr_pages += nr_pages; 2437 put_cpu_var(memcg_stock); 2438 } 2439 2440 /* 2441 * Drains all per-CPU charge caches for given root_memcg resp. subtree 2442 * of the hierarchy under it. sync flag says whether we should block 2443 * until the work is done. 2444 */ 2445 static void drain_all_stock(struct mem_cgroup *root_memcg, bool sync) 2446 { 2447 int cpu, curcpu; 2448 2449 /* Notify other cpus that system-wide "drain" is running */ 2450 get_online_cpus(); 2451 curcpu = get_cpu(); 2452 for_each_online_cpu(cpu) { 2453 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); 2454 struct mem_cgroup *memcg; 2455 2456 memcg = stock->cached; 2457 if (!memcg || !stock->nr_pages) 2458 continue; 2459 if (!mem_cgroup_same_or_subtree(root_memcg, memcg)) 2460 continue; 2461 if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) { 2462 if (cpu == curcpu) 2463 drain_local_stock(&stock->work); 2464 else 2465 schedule_work_on(cpu, &stock->work); 2466 } 2467 } 2468 put_cpu(); 2469 2470 if (!sync) 2471 goto out; 2472 2473 for_each_online_cpu(cpu) { 2474 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); 2475 if (test_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) 2476 flush_work(&stock->work); 2477 } 2478 out: 2479 put_online_cpus(); 2480 } 2481 2482 /* 2483 * Tries to drain stocked charges in other cpus. This function is asynchronous 2484 * and just put a work per cpu for draining localy on each cpu. Caller can 2485 * expects some charges will be back to res_counter later but cannot wait for 2486 * it. 2487 */ 2488 static void drain_all_stock_async(struct mem_cgroup *root_memcg) 2489 { 2490 /* 2491 * If someone calls draining, avoid adding more kworker runs. 2492 */ 2493 if (!mutex_trylock(&percpu_charge_mutex)) 2494 return; 2495 drain_all_stock(root_memcg, false); 2496 mutex_unlock(&percpu_charge_mutex); 2497 } 2498 2499 /* This is a synchronous drain interface. */ 2500 static void drain_all_stock_sync(struct mem_cgroup *root_memcg) 2501 { 2502 /* called when force_empty is called */ 2503 mutex_lock(&percpu_charge_mutex); 2504 drain_all_stock(root_memcg, true); 2505 mutex_unlock(&percpu_charge_mutex); 2506 } 2507 2508 /* 2509 * This function drains percpu counter value from DEAD cpu and 2510 * move it to local cpu. Note that this function can be preempted. 2511 */ 2512 static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *memcg, int cpu) 2513 { 2514 int i; 2515 2516 spin_lock(&memcg->pcp_counter_lock); 2517 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) { 2518 long x = per_cpu(memcg->stat->count[i], cpu); 2519 2520 per_cpu(memcg->stat->count[i], cpu) = 0; 2521 memcg->nocpu_base.count[i] += x; 2522 } 2523 for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) { 2524 unsigned long x = per_cpu(memcg->stat->events[i], cpu); 2525 2526 per_cpu(memcg->stat->events[i], cpu) = 0; 2527 memcg->nocpu_base.events[i] += x; 2528 } 2529 spin_unlock(&memcg->pcp_counter_lock); 2530 } 2531 2532 static int memcg_cpu_hotplug_callback(struct notifier_block *nb, 2533 unsigned long action, 2534 void *hcpu) 2535 { 2536 int cpu = (unsigned long)hcpu; 2537 struct memcg_stock_pcp *stock; 2538 struct mem_cgroup *iter; 2539 2540 if (action == CPU_ONLINE) 2541 return NOTIFY_OK; 2542 2543 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN) 2544 return NOTIFY_OK; 2545 2546 for_each_mem_cgroup(iter) 2547 mem_cgroup_drain_pcp_counter(iter, cpu); 2548 2549 stock = &per_cpu(memcg_stock, cpu); 2550 drain_stock(stock); 2551 return NOTIFY_OK; 2552 } 2553 2554 2555 /* See mem_cgroup_try_charge() for details */ 2556 enum { 2557 CHARGE_OK, /* success */ 2558 CHARGE_RETRY, /* need to retry but retry is not bad */ 2559 CHARGE_NOMEM, /* we can't do more. return -ENOMEM */ 2560 CHARGE_WOULDBLOCK, /* GFP_WAIT wasn't set and no enough res. */ 2561 }; 2562 2563 static int mem_cgroup_do_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, 2564 unsigned int nr_pages, unsigned int min_pages, 2565 bool invoke_oom) 2566 { 2567 unsigned long csize = nr_pages * PAGE_SIZE; 2568 struct mem_cgroup *mem_over_limit; 2569 struct res_counter *fail_res; 2570 unsigned long flags = 0; 2571 int ret; 2572 2573 ret = res_counter_charge(&memcg->res, csize, &fail_res); 2574 2575 if (likely(!ret)) { 2576 if (!do_swap_account) 2577 return CHARGE_OK; 2578 ret = res_counter_charge(&memcg->memsw, csize, &fail_res); 2579 if (likely(!ret)) 2580 return CHARGE_OK; 2581 2582 res_counter_uncharge(&memcg->res, csize); 2583 mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw); 2584 flags |= MEM_CGROUP_RECLAIM_NOSWAP; 2585 } else 2586 mem_over_limit = mem_cgroup_from_res_counter(fail_res, res); 2587 /* 2588 * Never reclaim on behalf of optional batching, retry with a 2589 * single page instead. 2590 */ 2591 if (nr_pages > min_pages) 2592 return CHARGE_RETRY; 2593 2594 if (!(gfp_mask & __GFP_WAIT)) 2595 return CHARGE_WOULDBLOCK; 2596 2597 if (gfp_mask & __GFP_NORETRY) 2598 return CHARGE_NOMEM; 2599 2600 ret = mem_cgroup_reclaim(mem_over_limit, gfp_mask, flags); 2601 if (mem_cgroup_margin(mem_over_limit) >= nr_pages) 2602 return CHARGE_RETRY; 2603 /* 2604 * Even though the limit is exceeded at this point, reclaim 2605 * may have been able to free some pages. Retry the charge 2606 * before killing the task. 2607 * 2608 * Only for regular pages, though: huge pages are rather 2609 * unlikely to succeed so close to the limit, and we fall back 2610 * to regular pages anyway in case of failure. 2611 */ 2612 if (nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER) && ret) 2613 return CHARGE_RETRY; 2614 2615 /* 2616 * At task move, charge accounts can be doubly counted. So, it's 2617 * better to wait until the end of task_move if something is going on. 2618 */ 2619 if (mem_cgroup_wait_acct_move(mem_over_limit)) 2620 return CHARGE_RETRY; 2621 2622 if (invoke_oom) 2623 mem_cgroup_oom(mem_over_limit, gfp_mask, get_order(csize)); 2624 2625 return CHARGE_NOMEM; 2626 } 2627 2628 /** 2629 * mem_cgroup_try_charge - try charging a memcg 2630 * @memcg: memcg to charge 2631 * @nr_pages: number of pages to charge 2632 * @oom: trigger OOM if reclaim fails 2633 * 2634 * Returns 0 if @memcg was charged successfully, -EINTR if the charge 2635 * was bypassed to root_mem_cgroup, and -ENOMEM if the charge failed. 2636 */ 2637 static int mem_cgroup_try_charge(struct mem_cgroup *memcg, 2638 gfp_t gfp_mask, 2639 unsigned int nr_pages, 2640 bool oom) 2641 { 2642 unsigned int batch = max(CHARGE_BATCH, nr_pages); 2643 int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES; 2644 int ret; 2645 2646 if (mem_cgroup_is_root(memcg)) 2647 goto done; 2648 /* 2649 * Unlike in global OOM situations, memcg is not in a physical 2650 * memory shortage. Allow dying and OOM-killed tasks to 2651 * bypass the last charges so that they can exit quickly and 2652 * free their memory. 2653 */ 2654 if (unlikely(test_thread_flag(TIF_MEMDIE) || 2655 fatal_signal_pending(current) || 2656 current->flags & PF_EXITING)) 2657 goto bypass; 2658 2659 if (unlikely(task_in_memcg_oom(current))) 2660 goto nomem; 2661 2662 if (gfp_mask & __GFP_NOFAIL) 2663 oom = false; 2664 again: 2665 if (consume_stock(memcg, nr_pages)) 2666 goto done; 2667 2668 do { 2669 bool invoke_oom = oom && !nr_oom_retries; 2670 2671 /* If killed, bypass charge */ 2672 if (fatal_signal_pending(current)) 2673 goto bypass; 2674 2675 ret = mem_cgroup_do_charge(memcg, gfp_mask, batch, 2676 nr_pages, invoke_oom); 2677 switch (ret) { 2678 case CHARGE_OK: 2679 break; 2680 case CHARGE_RETRY: /* not in OOM situation but retry */ 2681 batch = nr_pages; 2682 goto again; 2683 case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */ 2684 goto nomem; 2685 case CHARGE_NOMEM: /* OOM routine works */ 2686 if (!oom || invoke_oom) 2687 goto nomem; 2688 nr_oom_retries--; 2689 break; 2690 } 2691 } while (ret != CHARGE_OK); 2692 2693 if (batch > nr_pages) 2694 refill_stock(memcg, batch - nr_pages); 2695 done: 2696 return 0; 2697 nomem: 2698 if (!(gfp_mask & __GFP_NOFAIL)) 2699 return -ENOMEM; 2700 bypass: 2701 return -EINTR; 2702 } 2703 2704 /** 2705 * mem_cgroup_try_charge_mm - try charging a mm 2706 * @mm: mm_struct to charge 2707 * @nr_pages: number of pages to charge 2708 * @oom: trigger OOM if reclaim fails 2709 * 2710 * Returns the charged mem_cgroup associated with the given mm_struct or 2711 * NULL the charge failed. 2712 */ 2713 static struct mem_cgroup *mem_cgroup_try_charge_mm(struct mm_struct *mm, 2714 gfp_t gfp_mask, 2715 unsigned int nr_pages, 2716 bool oom) 2717 2718 { 2719 struct mem_cgroup *memcg; 2720 int ret; 2721 2722 memcg = get_mem_cgroup_from_mm(mm); 2723 ret = mem_cgroup_try_charge(memcg, gfp_mask, nr_pages, oom); 2724 css_put(&memcg->css); 2725 if (ret == -EINTR) 2726 memcg = root_mem_cgroup; 2727 else if (ret) 2728 memcg = NULL; 2729 2730 return memcg; 2731 } 2732 2733 /* 2734 * Somemtimes we have to undo a charge we got by try_charge(). 2735 * This function is for that and do uncharge, put css's refcnt. 2736 * gotten by try_charge(). 2737 */ 2738 static void __mem_cgroup_cancel_charge(struct mem_cgroup *memcg, 2739 unsigned int nr_pages) 2740 { 2741 if (!mem_cgroup_is_root(memcg)) { 2742 unsigned long bytes = nr_pages * PAGE_SIZE; 2743 2744 res_counter_uncharge(&memcg->res, bytes); 2745 if (do_swap_account) 2746 res_counter_uncharge(&memcg->memsw, bytes); 2747 } 2748 } 2749 2750 /* 2751 * Cancel chrages in this cgroup....doesn't propagate to parent cgroup. 2752 * This is useful when moving usage to parent cgroup. 2753 */ 2754 static void __mem_cgroup_cancel_local_charge(struct mem_cgroup *memcg, 2755 unsigned int nr_pages) 2756 { 2757 unsigned long bytes = nr_pages * PAGE_SIZE; 2758 2759 if (mem_cgroup_is_root(memcg)) 2760 return; 2761 2762 res_counter_uncharge_until(&memcg->res, memcg->res.parent, bytes); 2763 if (do_swap_account) 2764 res_counter_uncharge_until(&memcg->memsw, 2765 memcg->memsw.parent, bytes); 2766 } 2767 2768 /* 2769 * A helper function to get mem_cgroup from ID. must be called under 2770 * rcu_read_lock(). The caller is responsible for calling 2771 * css_tryget_online() if the mem_cgroup is used for charging. (dropping 2772 * refcnt from swap can be called against removed memcg.) 2773 */ 2774 static struct mem_cgroup *mem_cgroup_lookup(unsigned short id) 2775 { 2776 /* ID 0 is unused ID */ 2777 if (!id) 2778 return NULL; 2779 return mem_cgroup_from_id(id); 2780 } 2781 2782 struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page) 2783 { 2784 struct mem_cgroup *memcg = NULL; 2785 struct page_cgroup *pc; 2786 unsigned short id; 2787 swp_entry_t ent; 2788 2789 VM_BUG_ON_PAGE(!PageLocked(page), page); 2790 2791 pc = lookup_page_cgroup(page); 2792 lock_page_cgroup(pc); 2793 if (PageCgroupUsed(pc)) { 2794 memcg = pc->mem_cgroup; 2795 if (memcg && !css_tryget_online(&memcg->css)) 2796 memcg = NULL; 2797 } else if (PageSwapCache(page)) { 2798 ent.val = page_private(page); 2799 id = lookup_swap_cgroup_id(ent); 2800 rcu_read_lock(); 2801 memcg = mem_cgroup_lookup(id); 2802 if (memcg && !css_tryget_online(&memcg->css)) 2803 memcg = NULL; 2804 rcu_read_unlock(); 2805 } 2806 unlock_page_cgroup(pc); 2807 return memcg; 2808 } 2809 2810 static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg, 2811 struct page *page, 2812 unsigned int nr_pages, 2813 enum charge_type ctype, 2814 bool lrucare) 2815 { 2816 struct page_cgroup *pc = lookup_page_cgroup(page); 2817 struct zone *uninitialized_var(zone); 2818 struct lruvec *lruvec; 2819 bool was_on_lru = false; 2820 bool anon; 2821 2822 lock_page_cgroup(pc); 2823 VM_BUG_ON_PAGE(PageCgroupUsed(pc), page); 2824 /* 2825 * we don't need page_cgroup_lock about tail pages, becase they are not 2826 * accessed by any other context at this point. 2827 */ 2828 2829 /* 2830 * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page 2831 * may already be on some other mem_cgroup's LRU. Take care of it. 2832 */ 2833 if (lrucare) { 2834 zone = page_zone(page); 2835 spin_lock_irq(&zone->lru_lock); 2836 if (PageLRU(page)) { 2837 lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup); 2838 ClearPageLRU(page); 2839 del_page_from_lru_list(page, lruvec, page_lru(page)); 2840 was_on_lru = true; 2841 } 2842 } 2843 2844 pc->mem_cgroup = memcg; 2845 /* 2846 * We access a page_cgroup asynchronously without lock_page_cgroup(). 2847 * Especially when a page_cgroup is taken from a page, pc->mem_cgroup 2848 * is accessed after testing USED bit. To make pc->mem_cgroup visible 2849 * before USED bit, we need memory barrier here. 2850 * See mem_cgroup_add_lru_list(), etc. 2851 */ 2852 smp_wmb(); 2853 SetPageCgroupUsed(pc); 2854 2855 if (lrucare) { 2856 if (was_on_lru) { 2857 lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup); 2858 VM_BUG_ON_PAGE(PageLRU(page), page); 2859 SetPageLRU(page); 2860 add_page_to_lru_list(page, lruvec, page_lru(page)); 2861 } 2862 spin_unlock_irq(&zone->lru_lock); 2863 } 2864 2865 if (ctype == MEM_CGROUP_CHARGE_TYPE_ANON) 2866 anon = true; 2867 else 2868 anon = false; 2869 2870 mem_cgroup_charge_statistics(memcg, page, anon, nr_pages); 2871 unlock_page_cgroup(pc); 2872 2873 /* 2874 * "charge_statistics" updated event counter. Then, check it. 2875 * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree. 2876 * if they exceeds softlimit. 2877 */ 2878 memcg_check_events(memcg, page); 2879 } 2880 2881 static DEFINE_MUTEX(set_limit_mutex); 2882 2883 #ifdef CONFIG_MEMCG_KMEM 2884 /* 2885 * The memcg_slab_mutex is held whenever a per memcg kmem cache is created or 2886 * destroyed. It protects memcg_caches arrays and memcg_slab_caches lists. 2887 */ 2888 static DEFINE_MUTEX(memcg_slab_mutex); 2889 2890 static DEFINE_MUTEX(activate_kmem_mutex); 2891 2892 static inline bool memcg_can_account_kmem(struct mem_cgroup *memcg) 2893 { 2894 return !mem_cgroup_disabled() && !mem_cgroup_is_root(memcg) && 2895 memcg_kmem_is_active(memcg); 2896 } 2897 2898 /* 2899 * This is a bit cumbersome, but it is rarely used and avoids a backpointer 2900 * in the memcg_cache_params struct. 2901 */ 2902 static struct kmem_cache *memcg_params_to_cache(struct memcg_cache_params *p) 2903 { 2904 struct kmem_cache *cachep; 2905 2906 VM_BUG_ON(p->is_root_cache); 2907 cachep = p->root_cache; 2908 return cache_from_memcg_idx(cachep, memcg_cache_id(p->memcg)); 2909 } 2910 2911 #ifdef CONFIG_SLABINFO 2912 static int mem_cgroup_slabinfo_read(struct seq_file *m, void *v) 2913 { 2914 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 2915 struct memcg_cache_params *params; 2916 2917 if (!memcg_can_account_kmem(memcg)) 2918 return -EIO; 2919 2920 print_slabinfo_header(m); 2921 2922 mutex_lock(&memcg_slab_mutex); 2923 list_for_each_entry(params, &memcg->memcg_slab_caches, list) 2924 cache_show(memcg_params_to_cache(params), m); 2925 mutex_unlock(&memcg_slab_mutex); 2926 2927 return 0; 2928 } 2929 #endif 2930 2931 static int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp, u64 size) 2932 { 2933 struct res_counter *fail_res; 2934 int ret = 0; 2935 2936 ret = res_counter_charge(&memcg->kmem, size, &fail_res); 2937 if (ret) 2938 return ret; 2939 2940 ret = mem_cgroup_try_charge(memcg, gfp, size >> PAGE_SHIFT, 2941 oom_gfp_allowed(gfp)); 2942 if (ret == -EINTR) { 2943 /* 2944 * mem_cgroup_try_charge() chosed to bypass to root due to 2945 * OOM kill or fatal signal. Since our only options are to 2946 * either fail the allocation or charge it to this cgroup, do 2947 * it as a temporary condition. But we can't fail. From a 2948 * kmem/slab perspective, the cache has already been selected, 2949 * by mem_cgroup_kmem_get_cache(), so it is too late to change 2950 * our minds. 2951 * 2952 * This condition will only trigger if the task entered 2953 * memcg_charge_kmem in a sane state, but was OOM-killed during 2954 * mem_cgroup_try_charge() above. Tasks that were already 2955 * dying when the allocation triggers should have been already 2956 * directed to the root cgroup in memcontrol.h 2957 */ 2958 res_counter_charge_nofail(&memcg->res, size, &fail_res); 2959 if (do_swap_account) 2960 res_counter_charge_nofail(&memcg->memsw, size, 2961 &fail_res); 2962 ret = 0; 2963 } else if (ret) 2964 res_counter_uncharge(&memcg->kmem, size); 2965 2966 return ret; 2967 } 2968 2969 static void memcg_uncharge_kmem(struct mem_cgroup *memcg, u64 size) 2970 { 2971 res_counter_uncharge(&memcg->res, size); 2972 if (do_swap_account) 2973 res_counter_uncharge(&memcg->memsw, size); 2974 2975 /* Not down to 0 */ 2976 if (res_counter_uncharge(&memcg->kmem, size)) 2977 return; 2978 2979 /* 2980 * Releases a reference taken in kmem_cgroup_css_offline in case 2981 * this last uncharge is racing with the offlining code or it is 2982 * outliving the memcg existence. 2983 * 2984 * The memory barrier imposed by test&clear is paired with the 2985 * explicit one in memcg_kmem_mark_dead(). 2986 */ 2987 if (memcg_kmem_test_and_clear_dead(memcg)) 2988 css_put(&memcg->css); 2989 } 2990 2991 /* 2992 * helper for acessing a memcg's index. It will be used as an index in the 2993 * child cache array in kmem_cache, and also to derive its name. This function 2994 * will return -1 when this is not a kmem-limited memcg. 2995 */ 2996 int memcg_cache_id(struct mem_cgroup *memcg) 2997 { 2998 return memcg ? memcg->kmemcg_id : -1; 2999 } 3000 3001 static size_t memcg_caches_array_size(int num_groups) 3002 { 3003 ssize_t size; 3004 if (num_groups <= 0) 3005 return 0; 3006 3007 size = 2 * num_groups; 3008 if (size < MEMCG_CACHES_MIN_SIZE) 3009 size = MEMCG_CACHES_MIN_SIZE; 3010 else if (size > MEMCG_CACHES_MAX_SIZE) 3011 size = MEMCG_CACHES_MAX_SIZE; 3012 3013 return size; 3014 } 3015 3016 /* 3017 * We should update the current array size iff all caches updates succeed. This 3018 * can only be done from the slab side. The slab mutex needs to be held when 3019 * calling this. 3020 */ 3021 void memcg_update_array_size(int num) 3022 { 3023 if (num > memcg_limited_groups_array_size) 3024 memcg_limited_groups_array_size = memcg_caches_array_size(num); 3025 } 3026 3027 int memcg_update_cache_size(struct kmem_cache *s, int num_groups) 3028 { 3029 struct memcg_cache_params *cur_params = s->memcg_params; 3030 3031 VM_BUG_ON(!is_root_cache(s)); 3032 3033 if (num_groups > memcg_limited_groups_array_size) { 3034 int i; 3035 struct memcg_cache_params *new_params; 3036 ssize_t size = memcg_caches_array_size(num_groups); 3037 3038 size *= sizeof(void *); 3039 size += offsetof(struct memcg_cache_params, memcg_caches); 3040 3041 new_params = kzalloc(size, GFP_KERNEL); 3042 if (!new_params) 3043 return -ENOMEM; 3044 3045 new_params->is_root_cache = true; 3046 3047 /* 3048 * There is the chance it will be bigger than 3049 * memcg_limited_groups_array_size, if we failed an allocation 3050 * in a cache, in which case all caches updated before it, will 3051 * have a bigger array. 3052 * 3053 * But if that is the case, the data after 3054 * memcg_limited_groups_array_size is certainly unused 3055 */ 3056 for (i = 0; i < memcg_limited_groups_array_size; i++) { 3057 if (!cur_params->memcg_caches[i]) 3058 continue; 3059 new_params->memcg_caches[i] = 3060 cur_params->memcg_caches[i]; 3061 } 3062 3063 /* 3064 * Ideally, we would wait until all caches succeed, and only 3065 * then free the old one. But this is not worth the extra 3066 * pointer per-cache we'd have to have for this. 3067 * 3068 * It is not a big deal if some caches are left with a size 3069 * bigger than the others. And all updates will reset this 3070 * anyway. 3071 */ 3072 rcu_assign_pointer(s->memcg_params, new_params); 3073 if (cur_params) 3074 kfree_rcu(cur_params, rcu_head); 3075 } 3076 return 0; 3077 } 3078 3079 int memcg_alloc_cache_params(struct mem_cgroup *memcg, struct kmem_cache *s, 3080 struct kmem_cache *root_cache) 3081 { 3082 size_t size; 3083 3084 if (!memcg_kmem_enabled()) 3085 return 0; 3086 3087 if (!memcg) { 3088 size = offsetof(struct memcg_cache_params, memcg_caches); 3089 size += memcg_limited_groups_array_size * sizeof(void *); 3090 } else 3091 size = sizeof(struct memcg_cache_params); 3092 3093 s->memcg_params = kzalloc(size, GFP_KERNEL); 3094 if (!s->memcg_params) 3095 return -ENOMEM; 3096 3097 if (memcg) { 3098 s->memcg_params->memcg = memcg; 3099 s->memcg_params->root_cache = root_cache; 3100 css_get(&memcg->css); 3101 } else 3102 s->memcg_params->is_root_cache = true; 3103 3104 return 0; 3105 } 3106 3107 void memcg_free_cache_params(struct kmem_cache *s) 3108 { 3109 if (!s->memcg_params) 3110 return; 3111 if (!s->memcg_params->is_root_cache) 3112 css_put(&s->memcg_params->memcg->css); 3113 kfree(s->memcg_params); 3114 } 3115 3116 static void memcg_register_cache(struct mem_cgroup *memcg, 3117 struct kmem_cache *root_cache) 3118 { 3119 static char memcg_name_buf[NAME_MAX + 1]; /* protected by 3120 memcg_slab_mutex */ 3121 struct kmem_cache *cachep; 3122 int id; 3123 3124 lockdep_assert_held(&memcg_slab_mutex); 3125 3126 id = memcg_cache_id(memcg); 3127 3128 /* 3129 * Since per-memcg caches are created asynchronously on first 3130 * allocation (see memcg_kmem_get_cache()), several threads can try to 3131 * create the same cache, but only one of them may succeed. 3132 */ 3133 if (cache_from_memcg_idx(root_cache, id)) 3134 return; 3135 3136 cgroup_name(memcg->css.cgroup, memcg_name_buf, NAME_MAX + 1); 3137 cachep = memcg_create_kmem_cache(memcg, root_cache, memcg_name_buf); 3138 /* 3139 * If we could not create a memcg cache, do not complain, because 3140 * that's not critical at all as we can always proceed with the root 3141 * cache. 3142 */ 3143 if (!cachep) 3144 return; 3145 3146 list_add(&cachep->memcg_params->list, &memcg->memcg_slab_caches); 3147 3148 /* 3149 * Since readers won't lock (see cache_from_memcg_idx()), we need a 3150 * barrier here to ensure nobody will see the kmem_cache partially 3151 * initialized. 3152 */ 3153 smp_wmb(); 3154 3155 BUG_ON(root_cache->memcg_params->memcg_caches[id]); 3156 root_cache->memcg_params->memcg_caches[id] = cachep; 3157 } 3158 3159 static void memcg_unregister_cache(struct kmem_cache *cachep) 3160 { 3161 struct kmem_cache *root_cache; 3162 struct mem_cgroup *memcg; 3163 int id; 3164 3165 lockdep_assert_held(&memcg_slab_mutex); 3166 3167 BUG_ON(is_root_cache(cachep)); 3168 3169 root_cache = cachep->memcg_params->root_cache; 3170 memcg = cachep->memcg_params->memcg; 3171 id = memcg_cache_id(memcg); 3172 3173 BUG_ON(root_cache->memcg_params->memcg_caches[id] != cachep); 3174 root_cache->memcg_params->memcg_caches[id] = NULL; 3175 3176 list_del(&cachep->memcg_params->list); 3177 3178 kmem_cache_destroy(cachep); 3179 } 3180 3181 /* 3182 * During the creation a new cache, we need to disable our accounting mechanism 3183 * altogether. This is true even if we are not creating, but rather just 3184 * enqueing new caches to be created. 3185 * 3186 * This is because that process will trigger allocations; some visible, like 3187 * explicit kmallocs to auxiliary data structures, name strings and internal 3188 * cache structures; some well concealed, like INIT_WORK() that can allocate 3189 * objects during debug. 3190 * 3191 * If any allocation happens during memcg_kmem_get_cache, we will recurse back 3192 * to it. This may not be a bounded recursion: since the first cache creation 3193 * failed to complete (waiting on the allocation), we'll just try to create the 3194 * cache again, failing at the same point. 3195 * 3196 * memcg_kmem_get_cache is prepared to abort after seeing a positive count of 3197 * memcg_kmem_skip_account. So we enclose anything that might allocate memory 3198 * inside the following two functions. 3199 */ 3200 static inline void memcg_stop_kmem_account(void) 3201 { 3202 VM_BUG_ON(!current->mm); 3203 current->memcg_kmem_skip_account++; 3204 } 3205 3206 static inline void memcg_resume_kmem_account(void) 3207 { 3208 VM_BUG_ON(!current->mm); 3209 current->memcg_kmem_skip_account--; 3210 } 3211 3212 int __memcg_cleanup_cache_params(struct kmem_cache *s) 3213 { 3214 struct kmem_cache *c; 3215 int i, failed = 0; 3216 3217 mutex_lock(&memcg_slab_mutex); 3218 for_each_memcg_cache_index(i) { 3219 c = cache_from_memcg_idx(s, i); 3220 if (!c) 3221 continue; 3222 3223 memcg_unregister_cache(c); 3224 3225 if (cache_from_memcg_idx(s, i)) 3226 failed++; 3227 } 3228 mutex_unlock(&memcg_slab_mutex); 3229 return failed; 3230 } 3231 3232 static void memcg_unregister_all_caches(struct mem_cgroup *memcg) 3233 { 3234 struct kmem_cache *cachep; 3235 struct memcg_cache_params *params, *tmp; 3236 3237 if (!memcg_kmem_is_active(memcg)) 3238 return; 3239 3240 mutex_lock(&memcg_slab_mutex); 3241 list_for_each_entry_safe(params, tmp, &memcg->memcg_slab_caches, list) { 3242 cachep = memcg_params_to_cache(params); 3243 kmem_cache_shrink(cachep); 3244 if (atomic_read(&cachep->memcg_params->nr_pages) == 0) 3245 memcg_unregister_cache(cachep); 3246 } 3247 mutex_unlock(&memcg_slab_mutex); 3248 } 3249 3250 struct memcg_register_cache_work { 3251 struct mem_cgroup *memcg; 3252 struct kmem_cache *cachep; 3253 struct work_struct work; 3254 }; 3255 3256 static void memcg_register_cache_func(struct work_struct *w) 3257 { 3258 struct memcg_register_cache_work *cw = 3259 container_of(w, struct memcg_register_cache_work, work); 3260 struct mem_cgroup *memcg = cw->memcg; 3261 struct kmem_cache *cachep = cw->cachep; 3262 3263 mutex_lock(&memcg_slab_mutex); 3264 memcg_register_cache(memcg, cachep); 3265 mutex_unlock(&memcg_slab_mutex); 3266 3267 css_put(&memcg->css); 3268 kfree(cw); 3269 } 3270 3271 /* 3272 * Enqueue the creation of a per-memcg kmem_cache. 3273 */ 3274 static void __memcg_schedule_register_cache(struct mem_cgroup *memcg, 3275 struct kmem_cache *cachep) 3276 { 3277 struct memcg_register_cache_work *cw; 3278 3279 cw = kmalloc(sizeof(*cw), GFP_NOWAIT); 3280 if (cw == NULL) { 3281 css_put(&memcg->css); 3282 return; 3283 } 3284 3285 cw->memcg = memcg; 3286 cw->cachep = cachep; 3287 3288 INIT_WORK(&cw->work, memcg_register_cache_func); 3289 schedule_work(&cw->work); 3290 } 3291 3292 static void memcg_schedule_register_cache(struct mem_cgroup *memcg, 3293 struct kmem_cache *cachep) 3294 { 3295 /* 3296 * We need to stop accounting when we kmalloc, because if the 3297 * corresponding kmalloc cache is not yet created, the first allocation 3298 * in __memcg_schedule_register_cache will recurse. 3299 * 3300 * However, it is better to enclose the whole function. Depending on 3301 * the debugging options enabled, INIT_WORK(), for instance, can 3302 * trigger an allocation. This too, will make us recurse. Because at 3303 * this point we can't allow ourselves back into memcg_kmem_get_cache, 3304 * the safest choice is to do it like this, wrapping the whole function. 3305 */ 3306 memcg_stop_kmem_account(); 3307 __memcg_schedule_register_cache(memcg, cachep); 3308 memcg_resume_kmem_account(); 3309 } 3310 3311 int __memcg_charge_slab(struct kmem_cache *cachep, gfp_t gfp, int order) 3312 { 3313 int res; 3314 3315 res = memcg_charge_kmem(cachep->memcg_params->memcg, gfp, 3316 PAGE_SIZE << order); 3317 if (!res) 3318 atomic_add(1 << order, &cachep->memcg_params->nr_pages); 3319 return res; 3320 } 3321 3322 void __memcg_uncharge_slab(struct kmem_cache *cachep, int order) 3323 { 3324 memcg_uncharge_kmem(cachep->memcg_params->memcg, PAGE_SIZE << order); 3325 atomic_sub(1 << order, &cachep->memcg_params->nr_pages); 3326 } 3327 3328 /* 3329 * Return the kmem_cache we're supposed to use for a slab allocation. 3330 * We try to use the current memcg's version of the cache. 3331 * 3332 * If the cache does not exist yet, if we are the first user of it, 3333 * we either create it immediately, if possible, or create it asynchronously 3334 * in a workqueue. 3335 * In the latter case, we will let the current allocation go through with 3336 * the original cache. 3337 * 3338 * Can't be called in interrupt context or from kernel threads. 3339 * This function needs to be called with rcu_read_lock() held. 3340 */ 3341 struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep, 3342 gfp_t gfp) 3343 { 3344 struct mem_cgroup *memcg; 3345 struct kmem_cache *memcg_cachep; 3346 3347 VM_BUG_ON(!cachep->memcg_params); 3348 VM_BUG_ON(!cachep->memcg_params->is_root_cache); 3349 3350 if (!current->mm || current->memcg_kmem_skip_account) 3351 return cachep; 3352 3353 rcu_read_lock(); 3354 memcg = mem_cgroup_from_task(rcu_dereference(current->mm->owner)); 3355 3356 if (!memcg_can_account_kmem(memcg)) 3357 goto out; 3358 3359 memcg_cachep = cache_from_memcg_idx(cachep, memcg_cache_id(memcg)); 3360 if (likely(memcg_cachep)) { 3361 cachep = memcg_cachep; 3362 goto out; 3363 } 3364 3365 /* The corresponding put will be done in the workqueue. */ 3366 if (!css_tryget_online(&memcg->css)) 3367 goto out; 3368 rcu_read_unlock(); 3369 3370 /* 3371 * If we are in a safe context (can wait, and not in interrupt 3372 * context), we could be be predictable and return right away. 3373 * This would guarantee that the allocation being performed 3374 * already belongs in the new cache. 3375 * 3376 * However, there are some clashes that can arrive from locking. 3377 * For instance, because we acquire the slab_mutex while doing 3378 * memcg_create_kmem_cache, this means no further allocation 3379 * could happen with the slab_mutex held. So it's better to 3380 * defer everything. 3381 */ 3382 memcg_schedule_register_cache(memcg, cachep); 3383 return cachep; 3384 out: 3385 rcu_read_unlock(); 3386 return cachep; 3387 } 3388 3389 /* 3390 * We need to verify if the allocation against current->mm->owner's memcg is 3391 * possible for the given order. But the page is not allocated yet, so we'll 3392 * need a further commit step to do the final arrangements. 3393 * 3394 * It is possible for the task to switch cgroups in this mean time, so at 3395 * commit time, we can't rely on task conversion any longer. We'll then use 3396 * the handle argument to return to the caller which cgroup we should commit 3397 * against. We could also return the memcg directly and avoid the pointer 3398 * passing, but a boolean return value gives better semantics considering 3399 * the compiled-out case as well. 3400 * 3401 * Returning true means the allocation is possible. 3402 */ 3403 bool 3404 __memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **_memcg, int order) 3405 { 3406 struct mem_cgroup *memcg; 3407 int ret; 3408 3409 *_memcg = NULL; 3410 3411 /* 3412 * Disabling accounting is only relevant for some specific memcg 3413 * internal allocations. Therefore we would initially not have such 3414 * check here, since direct calls to the page allocator that are 3415 * accounted to kmemcg (alloc_kmem_pages and friends) only happen 3416 * outside memcg core. We are mostly concerned with cache allocations, 3417 * and by having this test at memcg_kmem_get_cache, we are already able 3418 * to relay the allocation to the root cache and bypass the memcg cache 3419 * altogether. 3420 * 3421 * There is one exception, though: the SLUB allocator does not create 3422 * large order caches, but rather service large kmallocs directly from 3423 * the page allocator. Therefore, the following sequence when backed by 3424 * the SLUB allocator: 3425 * 3426 * memcg_stop_kmem_account(); 3427 * kmalloc(<large_number>) 3428 * memcg_resume_kmem_account(); 3429 * 3430 * would effectively ignore the fact that we should skip accounting, 3431 * since it will drive us directly to this function without passing 3432 * through the cache selector memcg_kmem_get_cache. Such large 3433 * allocations are extremely rare but can happen, for instance, for the 3434 * cache arrays. We bring this test here. 3435 */ 3436 if (!current->mm || current->memcg_kmem_skip_account) 3437 return true; 3438 3439 memcg = get_mem_cgroup_from_mm(current->mm); 3440 3441 if (!memcg_can_account_kmem(memcg)) { 3442 css_put(&memcg->css); 3443 return true; 3444 } 3445 3446 ret = memcg_charge_kmem(memcg, gfp, PAGE_SIZE << order); 3447 if (!ret) 3448 *_memcg = memcg; 3449 3450 css_put(&memcg->css); 3451 return (ret == 0); 3452 } 3453 3454 void __memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg, 3455 int order) 3456 { 3457 struct page_cgroup *pc; 3458 3459 VM_BUG_ON(mem_cgroup_is_root(memcg)); 3460 3461 /* The page allocation failed. Revert */ 3462 if (!page) { 3463 memcg_uncharge_kmem(memcg, PAGE_SIZE << order); 3464 return; 3465 } 3466 3467 pc = lookup_page_cgroup(page); 3468 lock_page_cgroup(pc); 3469 pc->mem_cgroup = memcg; 3470 SetPageCgroupUsed(pc); 3471 unlock_page_cgroup(pc); 3472 } 3473 3474 void __memcg_kmem_uncharge_pages(struct page *page, int order) 3475 { 3476 struct mem_cgroup *memcg = NULL; 3477 struct page_cgroup *pc; 3478 3479 3480 pc = lookup_page_cgroup(page); 3481 /* 3482 * Fast unlocked return. Theoretically might have changed, have to 3483 * check again after locking. 3484 */ 3485 if (!PageCgroupUsed(pc)) 3486 return; 3487 3488 lock_page_cgroup(pc); 3489 if (PageCgroupUsed(pc)) { 3490 memcg = pc->mem_cgroup; 3491 ClearPageCgroupUsed(pc); 3492 } 3493 unlock_page_cgroup(pc); 3494 3495 /* 3496 * We trust that only if there is a memcg associated with the page, it 3497 * is a valid allocation 3498 */ 3499 if (!memcg) 3500 return; 3501 3502 VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page); 3503 memcg_uncharge_kmem(memcg, PAGE_SIZE << order); 3504 } 3505 #else 3506 static inline void memcg_unregister_all_caches(struct mem_cgroup *memcg) 3507 { 3508 } 3509 #endif /* CONFIG_MEMCG_KMEM */ 3510 3511 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3512 3513 #define PCGF_NOCOPY_AT_SPLIT (1 << PCG_LOCK | 1 << PCG_MIGRATION) 3514 /* 3515 * Because tail pages are not marked as "used", set it. We're under 3516 * zone->lru_lock, 'splitting on pmd' and compound_lock. 3517 * charge/uncharge will be never happen and move_account() is done under 3518 * compound_lock(), so we don't have to take care of races. 3519 */ 3520 void mem_cgroup_split_huge_fixup(struct page *head) 3521 { 3522 struct page_cgroup *head_pc = lookup_page_cgroup(head); 3523 struct page_cgroup *pc; 3524 struct mem_cgroup *memcg; 3525 int i; 3526 3527 if (mem_cgroup_disabled()) 3528 return; 3529 3530 memcg = head_pc->mem_cgroup; 3531 for (i = 1; i < HPAGE_PMD_NR; i++) { 3532 pc = head_pc + i; 3533 pc->mem_cgroup = memcg; 3534 smp_wmb();/* see __commit_charge() */ 3535 pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT; 3536 } 3537 __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], 3538 HPAGE_PMD_NR); 3539 } 3540 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 3541 3542 /** 3543 * mem_cgroup_move_account - move account of the page 3544 * @page: the page 3545 * @nr_pages: number of regular pages (>1 for huge pages) 3546 * @pc: page_cgroup of the page. 3547 * @from: mem_cgroup which the page is moved from. 3548 * @to: mem_cgroup which the page is moved to. @from != @to. 3549 * 3550 * The caller must confirm following. 3551 * - page is not on LRU (isolate_page() is useful.) 3552 * - compound_lock is held when nr_pages > 1 3553 * 3554 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge" 3555 * from old cgroup. 3556 */ 3557 static int mem_cgroup_move_account(struct page *page, 3558 unsigned int nr_pages, 3559 struct page_cgroup *pc, 3560 struct mem_cgroup *from, 3561 struct mem_cgroup *to) 3562 { 3563 unsigned long flags; 3564 int ret; 3565 bool anon = PageAnon(page); 3566 3567 VM_BUG_ON(from == to); 3568 VM_BUG_ON_PAGE(PageLRU(page), page); 3569 /* 3570 * The page is isolated from LRU. So, collapse function 3571 * will not handle this page. But page splitting can happen. 3572 * Do this check under compound_page_lock(). The caller should 3573 * hold it. 3574 */ 3575 ret = -EBUSY; 3576 if (nr_pages > 1 && !PageTransHuge(page)) 3577 goto out; 3578 3579 lock_page_cgroup(pc); 3580 3581 ret = -EINVAL; 3582 if (!PageCgroupUsed(pc) || pc->mem_cgroup != from) 3583 goto unlock; 3584 3585 move_lock_mem_cgroup(from, &flags); 3586 3587 if (!anon && page_mapped(page)) { 3588 __this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED], 3589 nr_pages); 3590 __this_cpu_add(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED], 3591 nr_pages); 3592 } 3593 3594 if (PageWriteback(page)) { 3595 __this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_WRITEBACK], 3596 nr_pages); 3597 __this_cpu_add(to->stat->count[MEM_CGROUP_STAT_WRITEBACK], 3598 nr_pages); 3599 } 3600 3601 mem_cgroup_charge_statistics(from, page, anon, -nr_pages); 3602 3603 /* caller should have done css_get */ 3604 pc->mem_cgroup = to; 3605 mem_cgroup_charge_statistics(to, page, anon, nr_pages); 3606 move_unlock_mem_cgroup(from, &flags); 3607 ret = 0; 3608 unlock: 3609 unlock_page_cgroup(pc); 3610 /* 3611 * check events 3612 */ 3613 memcg_check_events(to, page); 3614 memcg_check_events(from, page); 3615 out: 3616 return ret; 3617 } 3618 3619 /** 3620 * mem_cgroup_move_parent - moves page to the parent group 3621 * @page: the page to move 3622 * @pc: page_cgroup of the page 3623 * @child: page's cgroup 3624 * 3625 * move charges to its parent or the root cgroup if the group has no 3626 * parent (aka use_hierarchy==0). 3627 * Although this might fail (get_page_unless_zero, isolate_lru_page or 3628 * mem_cgroup_move_account fails) the failure is always temporary and 3629 * it signals a race with a page removal/uncharge or migration. In the 3630 * first case the page is on the way out and it will vanish from the LRU 3631 * on the next attempt and the call should be retried later. 3632 * Isolation from the LRU fails only if page has been isolated from 3633 * the LRU since we looked at it and that usually means either global 3634 * reclaim or migration going on. The page will either get back to the 3635 * LRU or vanish. 3636 * Finaly mem_cgroup_move_account fails only if the page got uncharged 3637 * (!PageCgroupUsed) or moved to a different group. The page will 3638 * disappear in the next attempt. 3639 */ 3640 static int mem_cgroup_move_parent(struct page *page, 3641 struct page_cgroup *pc, 3642 struct mem_cgroup *child) 3643 { 3644 struct mem_cgroup *parent; 3645 unsigned int nr_pages; 3646 unsigned long uninitialized_var(flags); 3647 int ret; 3648 3649 VM_BUG_ON(mem_cgroup_is_root(child)); 3650 3651 ret = -EBUSY; 3652 if (!get_page_unless_zero(page)) 3653 goto out; 3654 if (isolate_lru_page(page)) 3655 goto put; 3656 3657 nr_pages = hpage_nr_pages(page); 3658 3659 parent = parent_mem_cgroup(child); 3660 /* 3661 * If no parent, move charges to root cgroup. 3662 */ 3663 if (!parent) 3664 parent = root_mem_cgroup; 3665 3666 if (nr_pages > 1) { 3667 VM_BUG_ON_PAGE(!PageTransHuge(page), page); 3668 flags = compound_lock_irqsave(page); 3669 } 3670 3671 ret = mem_cgroup_move_account(page, nr_pages, 3672 pc, child, parent); 3673 if (!ret) 3674 __mem_cgroup_cancel_local_charge(child, nr_pages); 3675 3676 if (nr_pages > 1) 3677 compound_unlock_irqrestore(page, flags); 3678 putback_lru_page(page); 3679 put: 3680 put_page(page); 3681 out: 3682 return ret; 3683 } 3684 3685 int mem_cgroup_charge_anon(struct page *page, 3686 struct mm_struct *mm, gfp_t gfp_mask) 3687 { 3688 unsigned int nr_pages = 1; 3689 struct mem_cgroup *memcg; 3690 bool oom = true; 3691 3692 if (mem_cgroup_disabled()) 3693 return 0; 3694 3695 VM_BUG_ON_PAGE(page_mapped(page), page); 3696 VM_BUG_ON_PAGE(page->mapping && !PageAnon(page), page); 3697 VM_BUG_ON(!mm); 3698 3699 if (PageTransHuge(page)) { 3700 nr_pages <<= compound_order(page); 3701 VM_BUG_ON_PAGE(!PageTransHuge(page), page); 3702 /* 3703 * Never OOM-kill a process for a huge page. The 3704 * fault handler will fall back to regular pages. 3705 */ 3706 oom = false; 3707 } 3708 3709 memcg = mem_cgroup_try_charge_mm(mm, gfp_mask, nr_pages, oom); 3710 if (!memcg) 3711 return -ENOMEM; 3712 __mem_cgroup_commit_charge(memcg, page, nr_pages, 3713 MEM_CGROUP_CHARGE_TYPE_ANON, false); 3714 return 0; 3715 } 3716 3717 /* 3718 * While swap-in, try_charge -> commit or cancel, the page is locked. 3719 * And when try_charge() successfully returns, one refcnt to memcg without 3720 * struct page_cgroup is acquired. This refcnt will be consumed by 3721 * "commit()" or removed by "cancel()" 3722 */ 3723 static int __mem_cgroup_try_charge_swapin(struct mm_struct *mm, 3724 struct page *page, 3725 gfp_t mask, 3726 struct mem_cgroup **memcgp) 3727 { 3728 struct mem_cgroup *memcg = NULL; 3729 struct page_cgroup *pc; 3730 int ret; 3731 3732 pc = lookup_page_cgroup(page); 3733 /* 3734 * Every swap fault against a single page tries to charge the 3735 * page, bail as early as possible. shmem_unuse() encounters 3736 * already charged pages, too. The USED bit is protected by 3737 * the page lock, which serializes swap cache removal, which 3738 * in turn serializes uncharging. 3739 */ 3740 if (PageCgroupUsed(pc)) 3741 goto out; 3742 if (do_swap_account) 3743 memcg = try_get_mem_cgroup_from_page(page); 3744 if (!memcg) 3745 memcg = get_mem_cgroup_from_mm(mm); 3746 ret = mem_cgroup_try_charge(memcg, mask, 1, true); 3747 css_put(&memcg->css); 3748 if (ret == -EINTR) 3749 memcg = root_mem_cgroup; 3750 else if (ret) 3751 return ret; 3752 out: 3753 *memcgp = memcg; 3754 return 0; 3755 } 3756 3757 int mem_cgroup_try_charge_swapin(struct mm_struct *mm, struct page *page, 3758 gfp_t gfp_mask, struct mem_cgroup **memcgp) 3759 { 3760 if (mem_cgroup_disabled()) { 3761 *memcgp = NULL; 3762 return 0; 3763 } 3764 /* 3765 * A racing thread's fault, or swapoff, may have already 3766 * updated the pte, and even removed page from swap cache: in 3767 * those cases unuse_pte()'s pte_same() test will fail; but 3768 * there's also a KSM case which does need to charge the page. 3769 */ 3770 if (!PageSwapCache(page)) { 3771 struct mem_cgroup *memcg; 3772 3773 memcg = mem_cgroup_try_charge_mm(mm, gfp_mask, 1, true); 3774 if (!memcg) 3775 return -ENOMEM; 3776 *memcgp = memcg; 3777 return 0; 3778 } 3779 return __mem_cgroup_try_charge_swapin(mm, page, gfp_mask, memcgp); 3780 } 3781 3782 void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *memcg) 3783 { 3784 if (mem_cgroup_disabled()) 3785 return; 3786 if (!memcg) 3787 return; 3788 __mem_cgroup_cancel_charge(memcg, 1); 3789 } 3790 3791 static void 3792 __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *memcg, 3793 enum charge_type ctype) 3794 { 3795 if (mem_cgroup_disabled()) 3796 return; 3797 if (!memcg) 3798 return; 3799 3800 __mem_cgroup_commit_charge(memcg, page, 1, ctype, true); 3801 /* 3802 * Now swap is on-memory. This means this page may be 3803 * counted both as mem and swap....double count. 3804 * Fix it by uncharging from memsw. Basically, this SwapCache is stable 3805 * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page() 3806 * may call delete_from_swap_cache() before reach here. 3807 */ 3808 if (do_swap_account && PageSwapCache(page)) { 3809 swp_entry_t ent = {.val = page_private(page)}; 3810 mem_cgroup_uncharge_swap(ent); 3811 } 3812 } 3813 3814 void mem_cgroup_commit_charge_swapin(struct page *page, 3815 struct mem_cgroup *memcg) 3816 { 3817 __mem_cgroup_commit_charge_swapin(page, memcg, 3818 MEM_CGROUP_CHARGE_TYPE_ANON); 3819 } 3820 3821 int mem_cgroup_charge_file(struct page *page, struct mm_struct *mm, 3822 gfp_t gfp_mask) 3823 { 3824 enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE; 3825 struct mem_cgroup *memcg; 3826 int ret; 3827 3828 if (mem_cgroup_disabled()) 3829 return 0; 3830 if (PageCompound(page)) 3831 return 0; 3832 3833 if (PageSwapCache(page)) { /* shmem */ 3834 ret = __mem_cgroup_try_charge_swapin(mm, page, 3835 gfp_mask, &memcg); 3836 if (ret) 3837 return ret; 3838 __mem_cgroup_commit_charge_swapin(page, memcg, type); 3839 return 0; 3840 } 3841 3842 memcg = mem_cgroup_try_charge_mm(mm, gfp_mask, 1, true); 3843 if (!memcg) 3844 return -ENOMEM; 3845 __mem_cgroup_commit_charge(memcg, page, 1, type, false); 3846 return 0; 3847 } 3848 3849 static void mem_cgroup_do_uncharge(struct mem_cgroup *memcg, 3850 unsigned int nr_pages, 3851 const enum charge_type ctype) 3852 { 3853 struct memcg_batch_info *batch = NULL; 3854 bool uncharge_memsw = true; 3855 3856 /* If swapout, usage of swap doesn't decrease */ 3857 if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) 3858 uncharge_memsw = false; 3859 3860 batch = ¤t->memcg_batch; 3861 /* 3862 * In usual, we do css_get() when we remember memcg pointer. 3863 * But in this case, we keep res->usage until end of a series of 3864 * uncharges. Then, it's ok to ignore memcg's refcnt. 3865 */ 3866 if (!batch->memcg) 3867 batch->memcg = memcg; 3868 /* 3869 * do_batch > 0 when unmapping pages or inode invalidate/truncate. 3870 * In those cases, all pages freed continuously can be expected to be in 3871 * the same cgroup and we have chance to coalesce uncharges. 3872 * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE) 3873 * because we want to do uncharge as soon as possible. 3874 */ 3875 3876 if (!batch->do_batch || test_thread_flag(TIF_MEMDIE)) 3877 goto direct_uncharge; 3878 3879 if (nr_pages > 1) 3880 goto direct_uncharge; 3881 3882 /* 3883 * In typical case, batch->memcg == mem. This means we can 3884 * merge a series of uncharges to an uncharge of res_counter. 3885 * If not, we uncharge res_counter ony by one. 3886 */ 3887 if (batch->memcg != memcg) 3888 goto direct_uncharge; 3889 /* remember freed charge and uncharge it later */ 3890 batch->nr_pages++; 3891 if (uncharge_memsw) 3892 batch->memsw_nr_pages++; 3893 return; 3894 direct_uncharge: 3895 res_counter_uncharge(&memcg->res, nr_pages * PAGE_SIZE); 3896 if (uncharge_memsw) 3897 res_counter_uncharge(&memcg->memsw, nr_pages * PAGE_SIZE); 3898 if (unlikely(batch->memcg != memcg)) 3899 memcg_oom_recover(memcg); 3900 } 3901 3902 /* 3903 * uncharge if !page_mapped(page) 3904 */ 3905 static struct mem_cgroup * 3906 __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype, 3907 bool end_migration) 3908 { 3909 struct mem_cgroup *memcg = NULL; 3910 unsigned int nr_pages = 1; 3911 struct page_cgroup *pc; 3912 bool anon; 3913 3914 if (mem_cgroup_disabled()) 3915 return NULL; 3916 3917 if (PageTransHuge(page)) { 3918 nr_pages <<= compound_order(page); 3919 VM_BUG_ON_PAGE(!PageTransHuge(page), page); 3920 } 3921 /* 3922 * Check if our page_cgroup is valid 3923 */ 3924 pc = lookup_page_cgroup(page); 3925 if (unlikely(!PageCgroupUsed(pc))) 3926 return NULL; 3927 3928 lock_page_cgroup(pc); 3929 3930 memcg = pc->mem_cgroup; 3931 3932 if (!PageCgroupUsed(pc)) 3933 goto unlock_out; 3934 3935 anon = PageAnon(page); 3936 3937 switch (ctype) { 3938 case MEM_CGROUP_CHARGE_TYPE_ANON: 3939 /* 3940 * Generally PageAnon tells if it's the anon statistics to be 3941 * updated; but sometimes e.g. mem_cgroup_uncharge_page() is 3942 * used before page reached the stage of being marked PageAnon. 3943 */ 3944 anon = true; 3945 /* fallthrough */ 3946 case MEM_CGROUP_CHARGE_TYPE_DROP: 3947 /* See mem_cgroup_prepare_migration() */ 3948 if (page_mapped(page)) 3949 goto unlock_out; 3950 /* 3951 * Pages under migration may not be uncharged. But 3952 * end_migration() /must/ be the one uncharging the 3953 * unused post-migration page and so it has to call 3954 * here with the migration bit still set. See the 3955 * res_counter handling below. 3956 */ 3957 if (!end_migration && PageCgroupMigration(pc)) 3958 goto unlock_out; 3959 break; 3960 case MEM_CGROUP_CHARGE_TYPE_SWAPOUT: 3961 if (!PageAnon(page)) { /* Shared memory */ 3962 if (page->mapping && !page_is_file_cache(page)) 3963 goto unlock_out; 3964 } else if (page_mapped(page)) /* Anon */ 3965 goto unlock_out; 3966 break; 3967 default: 3968 break; 3969 } 3970 3971 mem_cgroup_charge_statistics(memcg, page, anon, -nr_pages); 3972 3973 ClearPageCgroupUsed(pc); 3974 /* 3975 * pc->mem_cgroup is not cleared here. It will be accessed when it's 3976 * freed from LRU. This is safe because uncharged page is expected not 3977 * to be reused (freed soon). Exception is SwapCache, it's handled by 3978 * special functions. 3979 */ 3980 3981 unlock_page_cgroup(pc); 3982 /* 3983 * even after unlock, we have memcg->res.usage here and this memcg 3984 * will never be freed, so it's safe to call css_get(). 3985 */ 3986 memcg_check_events(memcg, page); 3987 if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) { 3988 mem_cgroup_swap_statistics(memcg, true); 3989 css_get(&memcg->css); 3990 } 3991 /* 3992 * Migration does not charge the res_counter for the 3993 * replacement page, so leave it alone when phasing out the 3994 * page that is unused after the migration. 3995 */ 3996 if (!end_migration && !mem_cgroup_is_root(memcg)) 3997 mem_cgroup_do_uncharge(memcg, nr_pages, ctype); 3998 3999 return memcg; 4000 4001 unlock_out: 4002 unlock_page_cgroup(pc); 4003 return NULL; 4004 } 4005 4006 void mem_cgroup_uncharge_page(struct page *page) 4007 { 4008 /* early check. */ 4009 if (page_mapped(page)) 4010 return; 4011 VM_BUG_ON_PAGE(page->mapping && !PageAnon(page), page); 4012 /* 4013 * If the page is in swap cache, uncharge should be deferred 4014 * to the swap path, which also properly accounts swap usage 4015 * and handles memcg lifetime. 4016 * 4017 * Note that this check is not stable and reclaim may add the 4018 * page to swap cache at any time after this. However, if the 4019 * page is not in swap cache by the time page->mapcount hits 4020 * 0, there won't be any page table references to the swap 4021 * slot, and reclaim will free it and not actually write the 4022 * page to disk. 4023 */ 4024 if (PageSwapCache(page)) 4025 return; 4026 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_ANON, false); 4027 } 4028 4029 void mem_cgroup_uncharge_cache_page(struct page *page) 4030 { 4031 VM_BUG_ON_PAGE(page_mapped(page), page); 4032 VM_BUG_ON_PAGE(page->mapping, page); 4033 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE, false); 4034 } 4035 4036 /* 4037 * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate. 4038 * In that cases, pages are freed continuously and we can expect pages 4039 * are in the same memcg. All these calls itself limits the number of 4040 * pages freed at once, then uncharge_start/end() is called properly. 4041 * This may be called prural(2) times in a context, 4042 */ 4043 4044 void mem_cgroup_uncharge_start(void) 4045 { 4046 current->memcg_batch.do_batch++; 4047 /* We can do nest. */ 4048 if (current->memcg_batch.do_batch == 1) { 4049 current->memcg_batch.memcg = NULL; 4050 current->memcg_batch.nr_pages = 0; 4051 current->memcg_batch.memsw_nr_pages = 0; 4052 } 4053 } 4054 4055 void mem_cgroup_uncharge_end(void) 4056 { 4057 struct memcg_batch_info *batch = ¤t->memcg_batch; 4058 4059 if (!batch->do_batch) 4060 return; 4061 4062 batch->do_batch--; 4063 if (batch->do_batch) /* If stacked, do nothing. */ 4064 return; 4065 4066 if (!batch->memcg) 4067 return; 4068 /* 4069 * This "batch->memcg" is valid without any css_get/put etc... 4070 * bacause we hide charges behind us. 4071 */ 4072 if (batch->nr_pages) 4073 res_counter_uncharge(&batch->memcg->res, 4074 batch->nr_pages * PAGE_SIZE); 4075 if (batch->memsw_nr_pages) 4076 res_counter_uncharge(&batch->memcg->memsw, 4077 batch->memsw_nr_pages * PAGE_SIZE); 4078 memcg_oom_recover(batch->memcg); 4079 /* forget this pointer (for sanity check) */ 4080 batch->memcg = NULL; 4081 } 4082 4083 #ifdef CONFIG_SWAP 4084 /* 4085 * called after __delete_from_swap_cache() and drop "page" account. 4086 * memcg information is recorded to swap_cgroup of "ent" 4087 */ 4088 void 4089 mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout) 4090 { 4091 struct mem_cgroup *memcg; 4092 int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT; 4093 4094 if (!swapout) /* this was a swap cache but the swap is unused ! */ 4095 ctype = MEM_CGROUP_CHARGE_TYPE_DROP; 4096 4097 memcg = __mem_cgroup_uncharge_common(page, ctype, false); 4098 4099 /* 4100 * record memcg information, if swapout && memcg != NULL, 4101 * css_get() was called in uncharge(). 4102 */ 4103 if (do_swap_account && swapout && memcg) 4104 swap_cgroup_record(ent, mem_cgroup_id(memcg)); 4105 } 4106 #endif 4107 4108 #ifdef CONFIG_MEMCG_SWAP 4109 /* 4110 * called from swap_entry_free(). remove record in swap_cgroup and 4111 * uncharge "memsw" account. 4112 */ 4113 void mem_cgroup_uncharge_swap(swp_entry_t ent) 4114 { 4115 struct mem_cgroup *memcg; 4116 unsigned short id; 4117 4118 if (!do_swap_account) 4119 return; 4120 4121 id = swap_cgroup_record(ent, 0); 4122 rcu_read_lock(); 4123 memcg = mem_cgroup_lookup(id); 4124 if (memcg) { 4125 /* 4126 * We uncharge this because swap is freed. This memcg can 4127 * be obsolete one. We avoid calling css_tryget_online(). 4128 */ 4129 if (!mem_cgroup_is_root(memcg)) 4130 res_counter_uncharge(&memcg->memsw, PAGE_SIZE); 4131 mem_cgroup_swap_statistics(memcg, false); 4132 css_put(&memcg->css); 4133 } 4134 rcu_read_unlock(); 4135 } 4136 4137 /** 4138 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record. 4139 * @entry: swap entry to be moved 4140 * @from: mem_cgroup which the entry is moved from 4141 * @to: mem_cgroup which the entry is moved to 4142 * 4143 * It succeeds only when the swap_cgroup's record for this entry is the same 4144 * as the mem_cgroup's id of @from. 4145 * 4146 * Returns 0 on success, -EINVAL on failure. 4147 * 4148 * The caller must have charged to @to, IOW, called res_counter_charge() about 4149 * both res and memsw, and called css_get(). 4150 */ 4151 static int mem_cgroup_move_swap_account(swp_entry_t entry, 4152 struct mem_cgroup *from, struct mem_cgroup *to) 4153 { 4154 unsigned short old_id, new_id; 4155 4156 old_id = mem_cgroup_id(from); 4157 new_id = mem_cgroup_id(to); 4158 4159 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) { 4160 mem_cgroup_swap_statistics(from, false); 4161 mem_cgroup_swap_statistics(to, true); 4162 /* 4163 * This function is only called from task migration context now. 4164 * It postpones res_counter and refcount handling till the end 4165 * of task migration(mem_cgroup_clear_mc()) for performance 4166 * improvement. But we cannot postpone css_get(to) because if 4167 * the process that has been moved to @to does swap-in, the 4168 * refcount of @to might be decreased to 0. 4169 * 4170 * We are in attach() phase, so the cgroup is guaranteed to be 4171 * alive, so we can just call css_get(). 4172 */ 4173 css_get(&to->css); 4174 return 0; 4175 } 4176 return -EINVAL; 4177 } 4178 #else 4179 static inline int mem_cgroup_move_swap_account(swp_entry_t entry, 4180 struct mem_cgroup *from, struct mem_cgroup *to) 4181 { 4182 return -EINVAL; 4183 } 4184 #endif 4185 4186 /* 4187 * Before starting migration, account PAGE_SIZE to mem_cgroup that the old 4188 * page belongs to. 4189 */ 4190 void mem_cgroup_prepare_migration(struct page *page, struct page *newpage, 4191 struct mem_cgroup **memcgp) 4192 { 4193 struct mem_cgroup *memcg = NULL; 4194 unsigned int nr_pages = 1; 4195 struct page_cgroup *pc; 4196 enum charge_type ctype; 4197 4198 *memcgp = NULL; 4199 4200 if (mem_cgroup_disabled()) 4201 return; 4202 4203 if (PageTransHuge(page)) 4204 nr_pages <<= compound_order(page); 4205 4206 pc = lookup_page_cgroup(page); 4207 lock_page_cgroup(pc); 4208 if (PageCgroupUsed(pc)) { 4209 memcg = pc->mem_cgroup; 4210 css_get(&memcg->css); 4211 /* 4212 * At migrating an anonymous page, its mapcount goes down 4213 * to 0 and uncharge() will be called. But, even if it's fully 4214 * unmapped, migration may fail and this page has to be 4215 * charged again. We set MIGRATION flag here and delay uncharge 4216 * until end_migration() is called 4217 * 4218 * Corner Case Thinking 4219 * A) 4220 * When the old page was mapped as Anon and it's unmap-and-freed 4221 * while migration was ongoing. 4222 * If unmap finds the old page, uncharge() of it will be delayed 4223 * until end_migration(). If unmap finds a new page, it's 4224 * uncharged when it make mapcount to be 1->0. If unmap code 4225 * finds swap_migration_entry, the new page will not be mapped 4226 * and end_migration() will find it(mapcount==0). 4227 * 4228 * B) 4229 * When the old page was mapped but migraion fails, the kernel 4230 * remaps it. A charge for it is kept by MIGRATION flag even 4231 * if mapcount goes down to 0. We can do remap successfully 4232 * without charging it again. 4233 * 4234 * C) 4235 * The "old" page is under lock_page() until the end of 4236 * migration, so, the old page itself will not be swapped-out. 4237 * If the new page is swapped out before end_migraton, our 4238 * hook to usual swap-out path will catch the event. 4239 */ 4240 if (PageAnon(page)) 4241 SetPageCgroupMigration(pc); 4242 } 4243 unlock_page_cgroup(pc); 4244 /* 4245 * If the page is not charged at this point, 4246 * we return here. 4247 */ 4248 if (!memcg) 4249 return; 4250 4251 *memcgp = memcg; 4252 /* 4253 * We charge new page before it's used/mapped. So, even if unlock_page() 4254 * is called before end_migration, we can catch all events on this new 4255 * page. In the case new page is migrated but not remapped, new page's 4256 * mapcount will be finally 0 and we call uncharge in end_migration(). 4257 */ 4258 if (PageAnon(page)) 4259 ctype = MEM_CGROUP_CHARGE_TYPE_ANON; 4260 else 4261 ctype = MEM_CGROUP_CHARGE_TYPE_CACHE; 4262 /* 4263 * The page is committed to the memcg, but it's not actually 4264 * charged to the res_counter since we plan on replacing the 4265 * old one and only one page is going to be left afterwards. 4266 */ 4267 __mem_cgroup_commit_charge(memcg, newpage, nr_pages, ctype, false); 4268 } 4269 4270 /* remove redundant charge if migration failed*/ 4271 void mem_cgroup_end_migration(struct mem_cgroup *memcg, 4272 struct page *oldpage, struct page *newpage, bool migration_ok) 4273 { 4274 struct page *used, *unused; 4275 struct page_cgroup *pc; 4276 bool anon; 4277 4278 if (!memcg) 4279 return; 4280 4281 if (!migration_ok) { 4282 used = oldpage; 4283 unused = newpage; 4284 } else { 4285 used = newpage; 4286 unused = oldpage; 4287 } 4288 anon = PageAnon(used); 4289 __mem_cgroup_uncharge_common(unused, 4290 anon ? MEM_CGROUP_CHARGE_TYPE_ANON 4291 : MEM_CGROUP_CHARGE_TYPE_CACHE, 4292 true); 4293 css_put(&memcg->css); 4294 /* 4295 * We disallowed uncharge of pages under migration because mapcount 4296 * of the page goes down to zero, temporarly. 4297 * Clear the flag and check the page should be charged. 4298 */ 4299 pc = lookup_page_cgroup(oldpage); 4300 lock_page_cgroup(pc); 4301 ClearPageCgroupMigration(pc); 4302 unlock_page_cgroup(pc); 4303 4304 /* 4305 * If a page is a file cache, radix-tree replacement is very atomic 4306 * and we can skip this check. When it was an Anon page, its mapcount 4307 * goes down to 0. But because we added MIGRATION flage, it's not 4308 * uncharged yet. There are several case but page->mapcount check 4309 * and USED bit check in mem_cgroup_uncharge_page() will do enough 4310 * check. (see prepare_charge() also) 4311 */ 4312 if (anon) 4313 mem_cgroup_uncharge_page(used); 4314 } 4315 4316 /* 4317 * At replace page cache, newpage is not under any memcg but it's on 4318 * LRU. So, this function doesn't touch res_counter but handles LRU 4319 * in correct way. Both pages are locked so we cannot race with uncharge. 4320 */ 4321 void mem_cgroup_replace_page_cache(struct page *oldpage, 4322 struct page *newpage) 4323 { 4324 struct mem_cgroup *memcg = NULL; 4325 struct page_cgroup *pc; 4326 enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE; 4327 4328 if (mem_cgroup_disabled()) 4329 return; 4330 4331 pc = lookup_page_cgroup(oldpage); 4332 /* fix accounting on old pages */ 4333 lock_page_cgroup(pc); 4334 if (PageCgroupUsed(pc)) { 4335 memcg = pc->mem_cgroup; 4336 mem_cgroup_charge_statistics(memcg, oldpage, false, -1); 4337 ClearPageCgroupUsed(pc); 4338 } 4339 unlock_page_cgroup(pc); 4340 4341 /* 4342 * When called from shmem_replace_page(), in some cases the 4343 * oldpage has already been charged, and in some cases not. 4344 */ 4345 if (!memcg) 4346 return; 4347 /* 4348 * Even if newpage->mapping was NULL before starting replacement, 4349 * the newpage may be on LRU(or pagevec for LRU) already. We lock 4350 * LRU while we overwrite pc->mem_cgroup. 4351 */ 4352 __mem_cgroup_commit_charge(memcg, newpage, 1, type, true); 4353 } 4354 4355 #ifdef CONFIG_DEBUG_VM 4356 static struct page_cgroup *lookup_page_cgroup_used(struct page *page) 4357 { 4358 struct page_cgroup *pc; 4359 4360 pc = lookup_page_cgroup(page); 4361 /* 4362 * Can be NULL while feeding pages into the page allocator for 4363 * the first time, i.e. during boot or memory hotplug; 4364 * or when mem_cgroup_disabled(). 4365 */ 4366 if (likely(pc) && PageCgroupUsed(pc)) 4367 return pc; 4368 return NULL; 4369 } 4370 4371 bool mem_cgroup_bad_page_check(struct page *page) 4372 { 4373 if (mem_cgroup_disabled()) 4374 return false; 4375 4376 return lookup_page_cgroup_used(page) != NULL; 4377 } 4378 4379 void mem_cgroup_print_bad_page(struct page *page) 4380 { 4381 struct page_cgroup *pc; 4382 4383 pc = lookup_page_cgroup_used(page); 4384 if (pc) { 4385 pr_alert("pc:%p pc->flags:%lx pc->mem_cgroup:%p\n", 4386 pc, pc->flags, pc->mem_cgroup); 4387 } 4388 } 4389 #endif 4390 4391 static int mem_cgroup_resize_limit(struct mem_cgroup *memcg, 4392 unsigned long long val) 4393 { 4394 int retry_count; 4395 u64 memswlimit, memlimit; 4396 int ret = 0; 4397 int children = mem_cgroup_count_children(memcg); 4398 u64 curusage, oldusage; 4399 int enlarge; 4400 4401 /* 4402 * For keeping hierarchical_reclaim simple, how long we should retry 4403 * is depends on callers. We set our retry-count to be function 4404 * of # of children which we should visit in this loop. 4405 */ 4406 retry_count = MEM_CGROUP_RECLAIM_RETRIES * children; 4407 4408 oldusage = res_counter_read_u64(&memcg->res, RES_USAGE); 4409 4410 enlarge = 0; 4411 while (retry_count) { 4412 if (signal_pending(current)) { 4413 ret = -EINTR; 4414 break; 4415 } 4416 /* 4417 * Rather than hide all in some function, I do this in 4418 * open coded manner. You see what this really does. 4419 * We have to guarantee memcg->res.limit <= memcg->memsw.limit. 4420 */ 4421 mutex_lock(&set_limit_mutex); 4422 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 4423 if (memswlimit < val) { 4424 ret = -EINVAL; 4425 mutex_unlock(&set_limit_mutex); 4426 break; 4427 } 4428 4429 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT); 4430 if (memlimit < val) 4431 enlarge = 1; 4432 4433 ret = res_counter_set_limit(&memcg->res, val); 4434 if (!ret) { 4435 if (memswlimit == val) 4436 memcg->memsw_is_minimum = true; 4437 else 4438 memcg->memsw_is_minimum = false; 4439 } 4440 mutex_unlock(&set_limit_mutex); 4441 4442 if (!ret) 4443 break; 4444 4445 mem_cgroup_reclaim(memcg, GFP_KERNEL, 4446 MEM_CGROUP_RECLAIM_SHRINK); 4447 curusage = res_counter_read_u64(&memcg->res, RES_USAGE); 4448 /* Usage is reduced ? */ 4449 if (curusage >= oldusage) 4450 retry_count--; 4451 else 4452 oldusage = curusage; 4453 } 4454 if (!ret && enlarge) 4455 memcg_oom_recover(memcg); 4456 4457 return ret; 4458 } 4459 4460 static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg, 4461 unsigned long long val) 4462 { 4463 int retry_count; 4464 u64 memlimit, memswlimit, oldusage, curusage; 4465 int children = mem_cgroup_count_children(memcg); 4466 int ret = -EBUSY; 4467 int enlarge = 0; 4468 4469 /* see mem_cgroup_resize_res_limit */ 4470 retry_count = children * MEM_CGROUP_RECLAIM_RETRIES; 4471 oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE); 4472 while (retry_count) { 4473 if (signal_pending(current)) { 4474 ret = -EINTR; 4475 break; 4476 } 4477 /* 4478 * Rather than hide all in some function, I do this in 4479 * open coded manner. You see what this really does. 4480 * We have to guarantee memcg->res.limit <= memcg->memsw.limit. 4481 */ 4482 mutex_lock(&set_limit_mutex); 4483 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT); 4484 if (memlimit > val) { 4485 ret = -EINVAL; 4486 mutex_unlock(&set_limit_mutex); 4487 break; 4488 } 4489 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 4490 if (memswlimit < val) 4491 enlarge = 1; 4492 ret = res_counter_set_limit(&memcg->memsw, val); 4493 if (!ret) { 4494 if (memlimit == val) 4495 memcg->memsw_is_minimum = true; 4496 else 4497 memcg->memsw_is_minimum = false; 4498 } 4499 mutex_unlock(&set_limit_mutex); 4500 4501 if (!ret) 4502 break; 4503 4504 mem_cgroup_reclaim(memcg, GFP_KERNEL, 4505 MEM_CGROUP_RECLAIM_NOSWAP | 4506 MEM_CGROUP_RECLAIM_SHRINK); 4507 curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE); 4508 /* Usage is reduced ? */ 4509 if (curusage >= oldusage) 4510 retry_count--; 4511 else 4512 oldusage = curusage; 4513 } 4514 if (!ret && enlarge) 4515 memcg_oom_recover(memcg); 4516 return ret; 4517 } 4518 4519 unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order, 4520 gfp_t gfp_mask, 4521 unsigned long *total_scanned) 4522 { 4523 unsigned long nr_reclaimed = 0; 4524 struct mem_cgroup_per_zone *mz, *next_mz = NULL; 4525 unsigned long reclaimed; 4526 int loop = 0; 4527 struct mem_cgroup_tree_per_zone *mctz; 4528 unsigned long long excess; 4529 unsigned long nr_scanned; 4530 4531 if (order > 0) 4532 return 0; 4533 4534 mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone)); 4535 /* 4536 * This loop can run a while, specially if mem_cgroup's continuously 4537 * keep exceeding their soft limit and putting the system under 4538 * pressure 4539 */ 4540 do { 4541 if (next_mz) 4542 mz = next_mz; 4543 else 4544 mz = mem_cgroup_largest_soft_limit_node(mctz); 4545 if (!mz) 4546 break; 4547 4548 nr_scanned = 0; 4549 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, zone, 4550 gfp_mask, &nr_scanned); 4551 nr_reclaimed += reclaimed; 4552 *total_scanned += nr_scanned; 4553 spin_lock(&mctz->lock); 4554 4555 /* 4556 * If we failed to reclaim anything from this memory cgroup 4557 * it is time to move on to the next cgroup 4558 */ 4559 next_mz = NULL; 4560 if (!reclaimed) { 4561 do { 4562 /* 4563 * Loop until we find yet another one. 4564 * 4565 * By the time we get the soft_limit lock 4566 * again, someone might have aded the 4567 * group back on the RB tree. Iterate to 4568 * make sure we get a different mem. 4569 * mem_cgroup_largest_soft_limit_node returns 4570 * NULL if no other cgroup is present on 4571 * the tree 4572 */ 4573 next_mz = 4574 __mem_cgroup_largest_soft_limit_node(mctz); 4575 if (next_mz == mz) 4576 css_put(&next_mz->memcg->css); 4577 else /* next_mz == NULL or other memcg */ 4578 break; 4579 } while (1); 4580 } 4581 __mem_cgroup_remove_exceeded(mz, mctz); 4582 excess = res_counter_soft_limit_excess(&mz->memcg->res); 4583 /* 4584 * One school of thought says that we should not add 4585 * back the node to the tree if reclaim returns 0. 4586 * But our reclaim could return 0, simply because due 4587 * to priority we are exposing a smaller subset of 4588 * memory to reclaim from. Consider this as a longer 4589 * term TODO. 4590 */ 4591 /* If excess == 0, no tree ops */ 4592 __mem_cgroup_insert_exceeded(mz, mctz, excess); 4593 spin_unlock(&mctz->lock); 4594 css_put(&mz->memcg->css); 4595 loop++; 4596 /* 4597 * Could not reclaim anything and there are no more 4598 * mem cgroups to try or we seem to be looping without 4599 * reclaiming anything. 4600 */ 4601 if (!nr_reclaimed && 4602 (next_mz == NULL || 4603 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS)) 4604 break; 4605 } while (!nr_reclaimed); 4606 if (next_mz) 4607 css_put(&next_mz->memcg->css); 4608 return nr_reclaimed; 4609 } 4610 4611 /** 4612 * mem_cgroup_force_empty_list - clears LRU of a group 4613 * @memcg: group to clear 4614 * @node: NUMA node 4615 * @zid: zone id 4616 * @lru: lru to to clear 4617 * 4618 * Traverse a specified page_cgroup list and try to drop them all. This doesn't 4619 * reclaim the pages page themselves - pages are moved to the parent (or root) 4620 * group. 4621 */ 4622 static void mem_cgroup_force_empty_list(struct mem_cgroup *memcg, 4623 int node, int zid, enum lru_list lru) 4624 { 4625 struct lruvec *lruvec; 4626 unsigned long flags; 4627 struct list_head *list; 4628 struct page *busy; 4629 struct zone *zone; 4630 4631 zone = &NODE_DATA(node)->node_zones[zid]; 4632 lruvec = mem_cgroup_zone_lruvec(zone, memcg); 4633 list = &lruvec->lists[lru]; 4634 4635 busy = NULL; 4636 do { 4637 struct page_cgroup *pc; 4638 struct page *page; 4639 4640 spin_lock_irqsave(&zone->lru_lock, flags); 4641 if (list_empty(list)) { 4642 spin_unlock_irqrestore(&zone->lru_lock, flags); 4643 break; 4644 } 4645 page = list_entry(list->prev, struct page, lru); 4646 if (busy == page) { 4647 list_move(&page->lru, list); 4648 busy = NULL; 4649 spin_unlock_irqrestore(&zone->lru_lock, flags); 4650 continue; 4651 } 4652 spin_unlock_irqrestore(&zone->lru_lock, flags); 4653 4654 pc = lookup_page_cgroup(page); 4655 4656 if (mem_cgroup_move_parent(page, pc, memcg)) { 4657 /* found lock contention or "pc" is obsolete. */ 4658 busy = page; 4659 } else 4660 busy = NULL; 4661 cond_resched(); 4662 } while (!list_empty(list)); 4663 } 4664 4665 /* 4666 * make mem_cgroup's charge to be 0 if there is no task by moving 4667 * all the charges and pages to the parent. 4668 * This enables deleting this mem_cgroup. 4669 * 4670 * Caller is responsible for holding css reference on the memcg. 4671 */ 4672 static void mem_cgroup_reparent_charges(struct mem_cgroup *memcg) 4673 { 4674 int node, zid; 4675 u64 usage; 4676 4677 do { 4678 /* This is for making all *used* pages to be on LRU. */ 4679 lru_add_drain_all(); 4680 drain_all_stock_sync(memcg); 4681 mem_cgroup_start_move(memcg); 4682 for_each_node_state(node, N_MEMORY) { 4683 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 4684 enum lru_list lru; 4685 for_each_lru(lru) { 4686 mem_cgroup_force_empty_list(memcg, 4687 node, zid, lru); 4688 } 4689 } 4690 } 4691 mem_cgroup_end_move(memcg); 4692 memcg_oom_recover(memcg); 4693 cond_resched(); 4694 4695 /* 4696 * Kernel memory may not necessarily be trackable to a specific 4697 * process. So they are not migrated, and therefore we can't 4698 * expect their value to drop to 0 here. 4699 * Having res filled up with kmem only is enough. 4700 * 4701 * This is a safety check because mem_cgroup_force_empty_list 4702 * could have raced with mem_cgroup_replace_page_cache callers 4703 * so the lru seemed empty but the page could have been added 4704 * right after the check. RES_USAGE should be safe as we always 4705 * charge before adding to the LRU. 4706 */ 4707 usage = res_counter_read_u64(&memcg->res, RES_USAGE) - 4708 res_counter_read_u64(&memcg->kmem, RES_USAGE); 4709 } while (usage > 0); 4710 } 4711 4712 /* 4713 * Test whether @memcg has children, dead or alive. Note that this 4714 * function doesn't care whether @memcg has use_hierarchy enabled and 4715 * returns %true if there are child csses according to the cgroup 4716 * hierarchy. Testing use_hierarchy is the caller's responsiblity. 4717 */ 4718 static inline bool memcg_has_children(struct mem_cgroup *memcg) 4719 { 4720 bool ret; 4721 4722 /* 4723 * The lock does not prevent addition or deletion of children, but 4724 * it prevents a new child from being initialized based on this 4725 * parent in css_online(), so it's enough to decide whether 4726 * hierarchically inherited attributes can still be changed or not. 4727 */ 4728 lockdep_assert_held(&memcg_create_mutex); 4729 4730 rcu_read_lock(); 4731 ret = css_next_child(NULL, &memcg->css); 4732 rcu_read_unlock(); 4733 return ret; 4734 } 4735 4736 /* 4737 * Reclaims as many pages from the given memcg as possible and moves 4738 * the rest to the parent. 4739 * 4740 * Caller is responsible for holding css reference for memcg. 4741 */ 4742 static int mem_cgroup_force_empty(struct mem_cgroup *memcg) 4743 { 4744 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES; 4745 4746 /* we call try-to-free pages for make this cgroup empty */ 4747 lru_add_drain_all(); 4748 /* try to free all pages in this cgroup */ 4749 while (nr_retries && res_counter_read_u64(&memcg->res, RES_USAGE) > 0) { 4750 int progress; 4751 4752 if (signal_pending(current)) 4753 return -EINTR; 4754 4755 progress = try_to_free_mem_cgroup_pages(memcg, GFP_KERNEL, 4756 false); 4757 if (!progress) { 4758 nr_retries--; 4759 /* maybe some writeback is necessary */ 4760 congestion_wait(BLK_RW_ASYNC, HZ/10); 4761 } 4762 4763 } 4764 4765 return 0; 4766 } 4767 4768 static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of, 4769 char *buf, size_t nbytes, 4770 loff_t off) 4771 { 4772 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 4773 4774 if (mem_cgroup_is_root(memcg)) 4775 return -EINVAL; 4776 return mem_cgroup_force_empty(memcg) ?: nbytes; 4777 } 4778 4779 static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css, 4780 struct cftype *cft) 4781 { 4782 return mem_cgroup_from_css(css)->use_hierarchy; 4783 } 4784 4785 static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css, 4786 struct cftype *cft, u64 val) 4787 { 4788 int retval = 0; 4789 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4790 struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent); 4791 4792 mutex_lock(&memcg_create_mutex); 4793 4794 if (memcg->use_hierarchy == val) 4795 goto out; 4796 4797 /* 4798 * If parent's use_hierarchy is set, we can't make any modifications 4799 * in the child subtrees. If it is unset, then the change can 4800 * occur, provided the current cgroup has no children. 4801 * 4802 * For the root cgroup, parent_mem is NULL, we allow value to be 4803 * set if there are no children. 4804 */ 4805 if ((!parent_memcg || !parent_memcg->use_hierarchy) && 4806 (val == 1 || val == 0)) { 4807 if (!memcg_has_children(memcg)) 4808 memcg->use_hierarchy = val; 4809 else 4810 retval = -EBUSY; 4811 } else 4812 retval = -EINVAL; 4813 4814 out: 4815 mutex_unlock(&memcg_create_mutex); 4816 4817 return retval; 4818 } 4819 4820 4821 static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *memcg, 4822 enum mem_cgroup_stat_index idx) 4823 { 4824 struct mem_cgroup *iter; 4825 long val = 0; 4826 4827 /* Per-cpu values can be negative, use a signed accumulator */ 4828 for_each_mem_cgroup_tree(iter, memcg) 4829 val += mem_cgroup_read_stat(iter, idx); 4830 4831 if (val < 0) /* race ? */ 4832 val = 0; 4833 return val; 4834 } 4835 4836 static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap) 4837 { 4838 u64 val; 4839 4840 if (!mem_cgroup_is_root(memcg)) { 4841 if (!swap) 4842 return res_counter_read_u64(&memcg->res, RES_USAGE); 4843 else 4844 return res_counter_read_u64(&memcg->memsw, RES_USAGE); 4845 } 4846 4847 /* 4848 * Transparent hugepages are still accounted for in MEM_CGROUP_STAT_RSS 4849 * as well as in MEM_CGROUP_STAT_RSS_HUGE. 4850 */ 4851 val = mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_CACHE); 4852 val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_RSS); 4853 4854 if (swap) 4855 val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_SWAP); 4856 4857 return val << PAGE_SHIFT; 4858 } 4859 4860 static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css, 4861 struct cftype *cft) 4862 { 4863 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4864 u64 val; 4865 int name; 4866 enum res_type type; 4867 4868 type = MEMFILE_TYPE(cft->private); 4869 name = MEMFILE_ATTR(cft->private); 4870 4871 switch (type) { 4872 case _MEM: 4873 if (name == RES_USAGE) 4874 val = mem_cgroup_usage(memcg, false); 4875 else 4876 val = res_counter_read_u64(&memcg->res, name); 4877 break; 4878 case _MEMSWAP: 4879 if (name == RES_USAGE) 4880 val = mem_cgroup_usage(memcg, true); 4881 else 4882 val = res_counter_read_u64(&memcg->memsw, name); 4883 break; 4884 case _KMEM: 4885 val = res_counter_read_u64(&memcg->kmem, name); 4886 break; 4887 default: 4888 BUG(); 4889 } 4890 4891 return val; 4892 } 4893 4894 #ifdef CONFIG_MEMCG_KMEM 4895 /* should be called with activate_kmem_mutex held */ 4896 static int __memcg_activate_kmem(struct mem_cgroup *memcg, 4897 unsigned long long limit) 4898 { 4899 int err = 0; 4900 int memcg_id; 4901 4902 if (memcg_kmem_is_active(memcg)) 4903 return 0; 4904 4905 /* 4906 * We are going to allocate memory for data shared by all memory 4907 * cgroups so let's stop accounting here. 4908 */ 4909 memcg_stop_kmem_account(); 4910 4911 /* 4912 * For simplicity, we won't allow this to be disabled. It also can't 4913 * be changed if the cgroup has children already, or if tasks had 4914 * already joined. 4915 * 4916 * If tasks join before we set the limit, a person looking at 4917 * kmem.usage_in_bytes will have no way to determine when it took 4918 * place, which makes the value quite meaningless. 4919 * 4920 * After it first became limited, changes in the value of the limit are 4921 * of course permitted. 4922 */ 4923 mutex_lock(&memcg_create_mutex); 4924 if (cgroup_has_tasks(memcg->css.cgroup) || 4925 (memcg->use_hierarchy && memcg_has_children(memcg))) 4926 err = -EBUSY; 4927 mutex_unlock(&memcg_create_mutex); 4928 if (err) 4929 goto out; 4930 4931 memcg_id = ida_simple_get(&kmem_limited_groups, 4932 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL); 4933 if (memcg_id < 0) { 4934 err = memcg_id; 4935 goto out; 4936 } 4937 4938 /* 4939 * Make sure we have enough space for this cgroup in each root cache's 4940 * memcg_params. 4941 */ 4942 mutex_lock(&memcg_slab_mutex); 4943 err = memcg_update_all_caches(memcg_id + 1); 4944 mutex_unlock(&memcg_slab_mutex); 4945 if (err) 4946 goto out_rmid; 4947 4948 memcg->kmemcg_id = memcg_id; 4949 INIT_LIST_HEAD(&memcg->memcg_slab_caches); 4950 4951 /* 4952 * We couldn't have accounted to this cgroup, because it hasn't got the 4953 * active bit set yet, so this should succeed. 4954 */ 4955 err = res_counter_set_limit(&memcg->kmem, limit); 4956 VM_BUG_ON(err); 4957 4958 static_key_slow_inc(&memcg_kmem_enabled_key); 4959 /* 4960 * Setting the active bit after enabling static branching will 4961 * guarantee no one starts accounting before all call sites are 4962 * patched. 4963 */ 4964 memcg_kmem_set_active(memcg); 4965 out: 4966 memcg_resume_kmem_account(); 4967 return err; 4968 4969 out_rmid: 4970 ida_simple_remove(&kmem_limited_groups, memcg_id); 4971 goto out; 4972 } 4973 4974 static int memcg_activate_kmem(struct mem_cgroup *memcg, 4975 unsigned long long limit) 4976 { 4977 int ret; 4978 4979 mutex_lock(&activate_kmem_mutex); 4980 ret = __memcg_activate_kmem(memcg, limit); 4981 mutex_unlock(&activate_kmem_mutex); 4982 return ret; 4983 } 4984 4985 static int memcg_update_kmem_limit(struct mem_cgroup *memcg, 4986 unsigned long long val) 4987 { 4988 int ret; 4989 4990 if (!memcg_kmem_is_active(memcg)) 4991 ret = memcg_activate_kmem(memcg, val); 4992 else 4993 ret = res_counter_set_limit(&memcg->kmem, val); 4994 return ret; 4995 } 4996 4997 static int memcg_propagate_kmem(struct mem_cgroup *memcg) 4998 { 4999 int ret = 0; 5000 struct mem_cgroup *parent = parent_mem_cgroup(memcg); 5001 5002 if (!parent) 5003 return 0; 5004 5005 mutex_lock(&activate_kmem_mutex); 5006 /* 5007 * If the parent cgroup is not kmem-active now, it cannot be activated 5008 * after this point, because it has at least one child already. 5009 */ 5010 if (memcg_kmem_is_active(parent)) 5011 ret = __memcg_activate_kmem(memcg, RES_COUNTER_MAX); 5012 mutex_unlock(&activate_kmem_mutex); 5013 return ret; 5014 } 5015 #else 5016 static int memcg_update_kmem_limit(struct mem_cgroup *memcg, 5017 unsigned long long val) 5018 { 5019 return -EINVAL; 5020 } 5021 #endif /* CONFIG_MEMCG_KMEM */ 5022 5023 /* 5024 * The user of this function is... 5025 * RES_LIMIT. 5026 */ 5027 static ssize_t mem_cgroup_write(struct kernfs_open_file *of, 5028 char *buf, size_t nbytes, loff_t off) 5029 { 5030 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 5031 enum res_type type; 5032 int name; 5033 unsigned long long val; 5034 int ret; 5035 5036 buf = strstrip(buf); 5037 type = MEMFILE_TYPE(of_cft(of)->private); 5038 name = MEMFILE_ATTR(of_cft(of)->private); 5039 5040 switch (name) { 5041 case RES_LIMIT: 5042 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */ 5043 ret = -EINVAL; 5044 break; 5045 } 5046 /* This function does all necessary parse...reuse it */ 5047 ret = res_counter_memparse_write_strategy(buf, &val); 5048 if (ret) 5049 break; 5050 if (type == _MEM) 5051 ret = mem_cgroup_resize_limit(memcg, val); 5052 else if (type == _MEMSWAP) 5053 ret = mem_cgroup_resize_memsw_limit(memcg, val); 5054 else if (type == _KMEM) 5055 ret = memcg_update_kmem_limit(memcg, val); 5056 else 5057 return -EINVAL; 5058 break; 5059 case RES_SOFT_LIMIT: 5060 ret = res_counter_memparse_write_strategy(buf, &val); 5061 if (ret) 5062 break; 5063 /* 5064 * For memsw, soft limits are hard to implement in terms 5065 * of semantics, for now, we support soft limits for 5066 * control without swap 5067 */ 5068 if (type == _MEM) 5069 ret = res_counter_set_soft_limit(&memcg->res, val); 5070 else 5071 ret = -EINVAL; 5072 break; 5073 default: 5074 ret = -EINVAL; /* should be BUG() ? */ 5075 break; 5076 } 5077 return ret ?: nbytes; 5078 } 5079 5080 static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg, 5081 unsigned long long *mem_limit, unsigned long long *memsw_limit) 5082 { 5083 unsigned long long min_limit, min_memsw_limit, tmp; 5084 5085 min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT); 5086 min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 5087 if (!memcg->use_hierarchy) 5088 goto out; 5089 5090 while (memcg->css.parent) { 5091 memcg = mem_cgroup_from_css(memcg->css.parent); 5092 if (!memcg->use_hierarchy) 5093 break; 5094 tmp = res_counter_read_u64(&memcg->res, RES_LIMIT); 5095 min_limit = min(min_limit, tmp); 5096 tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 5097 min_memsw_limit = min(min_memsw_limit, tmp); 5098 } 5099 out: 5100 *mem_limit = min_limit; 5101 *memsw_limit = min_memsw_limit; 5102 } 5103 5104 static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf, 5105 size_t nbytes, loff_t off) 5106 { 5107 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 5108 int name; 5109 enum res_type type; 5110 5111 type = MEMFILE_TYPE(of_cft(of)->private); 5112 name = MEMFILE_ATTR(of_cft(of)->private); 5113 5114 switch (name) { 5115 case RES_MAX_USAGE: 5116 if (type == _MEM) 5117 res_counter_reset_max(&memcg->res); 5118 else if (type == _MEMSWAP) 5119 res_counter_reset_max(&memcg->memsw); 5120 else if (type == _KMEM) 5121 res_counter_reset_max(&memcg->kmem); 5122 else 5123 return -EINVAL; 5124 break; 5125 case RES_FAILCNT: 5126 if (type == _MEM) 5127 res_counter_reset_failcnt(&memcg->res); 5128 else if (type == _MEMSWAP) 5129 res_counter_reset_failcnt(&memcg->memsw); 5130 else if (type == _KMEM) 5131 res_counter_reset_failcnt(&memcg->kmem); 5132 else 5133 return -EINVAL; 5134 break; 5135 } 5136 5137 return nbytes; 5138 } 5139 5140 static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css, 5141 struct cftype *cft) 5142 { 5143 return mem_cgroup_from_css(css)->move_charge_at_immigrate; 5144 } 5145 5146 #ifdef CONFIG_MMU 5147 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css, 5148 struct cftype *cft, u64 val) 5149 { 5150 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5151 5152 if (val >= (1 << NR_MOVE_TYPE)) 5153 return -EINVAL; 5154 5155 /* 5156 * No kind of locking is needed in here, because ->can_attach() will 5157 * check this value once in the beginning of the process, and then carry 5158 * on with stale data. This means that changes to this value will only 5159 * affect task migrations starting after the change. 5160 */ 5161 memcg->move_charge_at_immigrate = val; 5162 return 0; 5163 } 5164 #else 5165 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css, 5166 struct cftype *cft, u64 val) 5167 { 5168 return -ENOSYS; 5169 } 5170 #endif 5171 5172 #ifdef CONFIG_NUMA 5173 static int memcg_numa_stat_show(struct seq_file *m, void *v) 5174 { 5175 struct numa_stat { 5176 const char *name; 5177 unsigned int lru_mask; 5178 }; 5179 5180 static const struct numa_stat stats[] = { 5181 { "total", LRU_ALL }, 5182 { "file", LRU_ALL_FILE }, 5183 { "anon", LRU_ALL_ANON }, 5184 { "unevictable", BIT(LRU_UNEVICTABLE) }, 5185 }; 5186 const struct numa_stat *stat; 5187 int nid; 5188 unsigned long nr; 5189 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 5190 5191 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) { 5192 nr = mem_cgroup_nr_lru_pages(memcg, stat->lru_mask); 5193 seq_printf(m, "%s=%lu", stat->name, nr); 5194 for_each_node_state(nid, N_MEMORY) { 5195 nr = mem_cgroup_node_nr_lru_pages(memcg, nid, 5196 stat->lru_mask); 5197 seq_printf(m, " N%d=%lu", nid, nr); 5198 } 5199 seq_putc(m, '\n'); 5200 } 5201 5202 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) { 5203 struct mem_cgroup *iter; 5204 5205 nr = 0; 5206 for_each_mem_cgroup_tree(iter, memcg) 5207 nr += mem_cgroup_nr_lru_pages(iter, stat->lru_mask); 5208 seq_printf(m, "hierarchical_%s=%lu", stat->name, nr); 5209 for_each_node_state(nid, N_MEMORY) { 5210 nr = 0; 5211 for_each_mem_cgroup_tree(iter, memcg) 5212 nr += mem_cgroup_node_nr_lru_pages( 5213 iter, nid, stat->lru_mask); 5214 seq_printf(m, " N%d=%lu", nid, nr); 5215 } 5216 seq_putc(m, '\n'); 5217 } 5218 5219 return 0; 5220 } 5221 #endif /* CONFIG_NUMA */ 5222 5223 static inline void mem_cgroup_lru_names_not_uptodate(void) 5224 { 5225 BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS); 5226 } 5227 5228 static int memcg_stat_show(struct seq_file *m, void *v) 5229 { 5230 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 5231 struct mem_cgroup *mi; 5232 unsigned int i; 5233 5234 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) { 5235 if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account) 5236 continue; 5237 seq_printf(m, "%s %ld\n", mem_cgroup_stat_names[i], 5238 mem_cgroup_read_stat(memcg, i) * PAGE_SIZE); 5239 } 5240 5241 for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) 5242 seq_printf(m, "%s %lu\n", mem_cgroup_events_names[i], 5243 mem_cgroup_read_events(memcg, i)); 5244 5245 for (i = 0; i < NR_LRU_LISTS; i++) 5246 seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i], 5247 mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE); 5248 5249 /* Hierarchical information */ 5250 { 5251 unsigned long long limit, memsw_limit; 5252 memcg_get_hierarchical_limit(memcg, &limit, &memsw_limit); 5253 seq_printf(m, "hierarchical_memory_limit %llu\n", limit); 5254 if (do_swap_account) 5255 seq_printf(m, "hierarchical_memsw_limit %llu\n", 5256 memsw_limit); 5257 } 5258 5259 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) { 5260 long long val = 0; 5261 5262 if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account) 5263 continue; 5264 for_each_mem_cgroup_tree(mi, memcg) 5265 val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE; 5266 seq_printf(m, "total_%s %lld\n", mem_cgroup_stat_names[i], val); 5267 } 5268 5269 for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) { 5270 unsigned long long val = 0; 5271 5272 for_each_mem_cgroup_tree(mi, memcg) 5273 val += mem_cgroup_read_events(mi, i); 5274 seq_printf(m, "total_%s %llu\n", 5275 mem_cgroup_events_names[i], val); 5276 } 5277 5278 for (i = 0; i < NR_LRU_LISTS; i++) { 5279 unsigned long long val = 0; 5280 5281 for_each_mem_cgroup_tree(mi, memcg) 5282 val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE; 5283 seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val); 5284 } 5285 5286 #ifdef CONFIG_DEBUG_VM 5287 { 5288 int nid, zid; 5289 struct mem_cgroup_per_zone *mz; 5290 struct zone_reclaim_stat *rstat; 5291 unsigned long recent_rotated[2] = {0, 0}; 5292 unsigned long recent_scanned[2] = {0, 0}; 5293 5294 for_each_online_node(nid) 5295 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 5296 mz = &memcg->nodeinfo[nid]->zoneinfo[zid]; 5297 rstat = &mz->lruvec.reclaim_stat; 5298 5299 recent_rotated[0] += rstat->recent_rotated[0]; 5300 recent_rotated[1] += rstat->recent_rotated[1]; 5301 recent_scanned[0] += rstat->recent_scanned[0]; 5302 recent_scanned[1] += rstat->recent_scanned[1]; 5303 } 5304 seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]); 5305 seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]); 5306 seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]); 5307 seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]); 5308 } 5309 #endif 5310 5311 return 0; 5312 } 5313 5314 static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css, 5315 struct cftype *cft) 5316 { 5317 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5318 5319 return mem_cgroup_swappiness(memcg); 5320 } 5321 5322 static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css, 5323 struct cftype *cft, u64 val) 5324 { 5325 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5326 5327 if (val > 100) 5328 return -EINVAL; 5329 5330 if (css->parent) 5331 memcg->swappiness = val; 5332 else 5333 vm_swappiness = val; 5334 5335 return 0; 5336 } 5337 5338 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap) 5339 { 5340 struct mem_cgroup_threshold_ary *t; 5341 u64 usage; 5342 int i; 5343 5344 rcu_read_lock(); 5345 if (!swap) 5346 t = rcu_dereference(memcg->thresholds.primary); 5347 else 5348 t = rcu_dereference(memcg->memsw_thresholds.primary); 5349 5350 if (!t) 5351 goto unlock; 5352 5353 usage = mem_cgroup_usage(memcg, swap); 5354 5355 /* 5356 * current_threshold points to threshold just below or equal to usage. 5357 * If it's not true, a threshold was crossed after last 5358 * call of __mem_cgroup_threshold(). 5359 */ 5360 i = t->current_threshold; 5361 5362 /* 5363 * Iterate backward over array of thresholds starting from 5364 * current_threshold and check if a threshold is crossed. 5365 * If none of thresholds below usage is crossed, we read 5366 * only one element of the array here. 5367 */ 5368 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--) 5369 eventfd_signal(t->entries[i].eventfd, 1); 5370 5371 /* i = current_threshold + 1 */ 5372 i++; 5373 5374 /* 5375 * Iterate forward over array of thresholds starting from 5376 * current_threshold+1 and check if a threshold is crossed. 5377 * If none of thresholds above usage is crossed, we read 5378 * only one element of the array here. 5379 */ 5380 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++) 5381 eventfd_signal(t->entries[i].eventfd, 1); 5382 5383 /* Update current_threshold */ 5384 t->current_threshold = i - 1; 5385 unlock: 5386 rcu_read_unlock(); 5387 } 5388 5389 static void mem_cgroup_threshold(struct mem_cgroup *memcg) 5390 { 5391 while (memcg) { 5392 __mem_cgroup_threshold(memcg, false); 5393 if (do_swap_account) 5394 __mem_cgroup_threshold(memcg, true); 5395 5396 memcg = parent_mem_cgroup(memcg); 5397 } 5398 } 5399 5400 static int compare_thresholds(const void *a, const void *b) 5401 { 5402 const struct mem_cgroup_threshold *_a = a; 5403 const struct mem_cgroup_threshold *_b = b; 5404 5405 if (_a->threshold > _b->threshold) 5406 return 1; 5407 5408 if (_a->threshold < _b->threshold) 5409 return -1; 5410 5411 return 0; 5412 } 5413 5414 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg) 5415 { 5416 struct mem_cgroup_eventfd_list *ev; 5417 5418 list_for_each_entry(ev, &memcg->oom_notify, list) 5419 eventfd_signal(ev->eventfd, 1); 5420 return 0; 5421 } 5422 5423 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg) 5424 { 5425 struct mem_cgroup *iter; 5426 5427 for_each_mem_cgroup_tree(iter, memcg) 5428 mem_cgroup_oom_notify_cb(iter); 5429 } 5430 5431 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg, 5432 struct eventfd_ctx *eventfd, const char *args, enum res_type type) 5433 { 5434 struct mem_cgroup_thresholds *thresholds; 5435 struct mem_cgroup_threshold_ary *new; 5436 u64 threshold, usage; 5437 int i, size, ret; 5438 5439 ret = res_counter_memparse_write_strategy(args, &threshold); 5440 if (ret) 5441 return ret; 5442 5443 mutex_lock(&memcg->thresholds_lock); 5444 5445 if (type == _MEM) 5446 thresholds = &memcg->thresholds; 5447 else if (type == _MEMSWAP) 5448 thresholds = &memcg->memsw_thresholds; 5449 else 5450 BUG(); 5451 5452 usage = mem_cgroup_usage(memcg, type == _MEMSWAP); 5453 5454 /* Check if a threshold crossed before adding a new one */ 5455 if (thresholds->primary) 5456 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 5457 5458 size = thresholds->primary ? thresholds->primary->size + 1 : 1; 5459 5460 /* Allocate memory for new array of thresholds */ 5461 new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold), 5462 GFP_KERNEL); 5463 if (!new) { 5464 ret = -ENOMEM; 5465 goto unlock; 5466 } 5467 new->size = size; 5468 5469 /* Copy thresholds (if any) to new array */ 5470 if (thresholds->primary) { 5471 memcpy(new->entries, thresholds->primary->entries, (size - 1) * 5472 sizeof(struct mem_cgroup_threshold)); 5473 } 5474 5475 /* Add new threshold */ 5476 new->entries[size - 1].eventfd = eventfd; 5477 new->entries[size - 1].threshold = threshold; 5478 5479 /* Sort thresholds. Registering of new threshold isn't time-critical */ 5480 sort(new->entries, size, sizeof(struct mem_cgroup_threshold), 5481 compare_thresholds, NULL); 5482 5483 /* Find current threshold */ 5484 new->current_threshold = -1; 5485 for (i = 0; i < size; i++) { 5486 if (new->entries[i].threshold <= usage) { 5487 /* 5488 * new->current_threshold will not be used until 5489 * rcu_assign_pointer(), so it's safe to increment 5490 * it here. 5491 */ 5492 ++new->current_threshold; 5493 } else 5494 break; 5495 } 5496 5497 /* Free old spare buffer and save old primary buffer as spare */ 5498 kfree(thresholds->spare); 5499 thresholds->spare = thresholds->primary; 5500 5501 rcu_assign_pointer(thresholds->primary, new); 5502 5503 /* To be sure that nobody uses thresholds */ 5504 synchronize_rcu(); 5505 5506 unlock: 5507 mutex_unlock(&memcg->thresholds_lock); 5508 5509 return ret; 5510 } 5511 5512 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg, 5513 struct eventfd_ctx *eventfd, const char *args) 5514 { 5515 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM); 5516 } 5517 5518 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg, 5519 struct eventfd_ctx *eventfd, const char *args) 5520 { 5521 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP); 5522 } 5523 5524 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, 5525 struct eventfd_ctx *eventfd, enum res_type type) 5526 { 5527 struct mem_cgroup_thresholds *thresholds; 5528 struct mem_cgroup_threshold_ary *new; 5529 u64 usage; 5530 int i, j, size; 5531 5532 mutex_lock(&memcg->thresholds_lock); 5533 if (type == _MEM) 5534 thresholds = &memcg->thresholds; 5535 else if (type == _MEMSWAP) 5536 thresholds = &memcg->memsw_thresholds; 5537 else 5538 BUG(); 5539 5540 if (!thresholds->primary) 5541 goto unlock; 5542 5543 usage = mem_cgroup_usage(memcg, type == _MEMSWAP); 5544 5545 /* Check if a threshold crossed before removing */ 5546 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 5547 5548 /* Calculate new number of threshold */ 5549 size = 0; 5550 for (i = 0; i < thresholds->primary->size; i++) { 5551 if (thresholds->primary->entries[i].eventfd != eventfd) 5552 size++; 5553 } 5554 5555 new = thresholds->spare; 5556 5557 /* Set thresholds array to NULL if we don't have thresholds */ 5558 if (!size) { 5559 kfree(new); 5560 new = NULL; 5561 goto swap_buffers; 5562 } 5563 5564 new->size = size; 5565 5566 /* Copy thresholds and find current threshold */ 5567 new->current_threshold = -1; 5568 for (i = 0, j = 0; i < thresholds->primary->size; i++) { 5569 if (thresholds->primary->entries[i].eventfd == eventfd) 5570 continue; 5571 5572 new->entries[j] = thresholds->primary->entries[i]; 5573 if (new->entries[j].threshold <= usage) { 5574 /* 5575 * new->current_threshold will not be used 5576 * until rcu_assign_pointer(), so it's safe to increment 5577 * it here. 5578 */ 5579 ++new->current_threshold; 5580 } 5581 j++; 5582 } 5583 5584 swap_buffers: 5585 /* Swap primary and spare array */ 5586 thresholds->spare = thresholds->primary; 5587 /* If all events are unregistered, free the spare array */ 5588 if (!new) { 5589 kfree(thresholds->spare); 5590 thresholds->spare = NULL; 5591 } 5592 5593 rcu_assign_pointer(thresholds->primary, new); 5594 5595 /* To be sure that nobody uses thresholds */ 5596 synchronize_rcu(); 5597 unlock: 5598 mutex_unlock(&memcg->thresholds_lock); 5599 } 5600 5601 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, 5602 struct eventfd_ctx *eventfd) 5603 { 5604 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM); 5605 } 5606 5607 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg, 5608 struct eventfd_ctx *eventfd) 5609 { 5610 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP); 5611 } 5612 5613 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg, 5614 struct eventfd_ctx *eventfd, const char *args) 5615 { 5616 struct mem_cgroup_eventfd_list *event; 5617 5618 event = kmalloc(sizeof(*event), GFP_KERNEL); 5619 if (!event) 5620 return -ENOMEM; 5621 5622 spin_lock(&memcg_oom_lock); 5623 5624 event->eventfd = eventfd; 5625 list_add(&event->list, &memcg->oom_notify); 5626 5627 /* already in OOM ? */ 5628 if (atomic_read(&memcg->under_oom)) 5629 eventfd_signal(eventfd, 1); 5630 spin_unlock(&memcg_oom_lock); 5631 5632 return 0; 5633 } 5634 5635 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg, 5636 struct eventfd_ctx *eventfd) 5637 { 5638 struct mem_cgroup_eventfd_list *ev, *tmp; 5639 5640 spin_lock(&memcg_oom_lock); 5641 5642 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) { 5643 if (ev->eventfd == eventfd) { 5644 list_del(&ev->list); 5645 kfree(ev); 5646 } 5647 } 5648 5649 spin_unlock(&memcg_oom_lock); 5650 } 5651 5652 static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v) 5653 { 5654 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf)); 5655 5656 seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable); 5657 seq_printf(sf, "under_oom %d\n", (bool)atomic_read(&memcg->under_oom)); 5658 return 0; 5659 } 5660 5661 static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css, 5662 struct cftype *cft, u64 val) 5663 { 5664 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5665 5666 /* cannot set to root cgroup and only 0 and 1 are allowed */ 5667 if (!css->parent || !((val == 0) || (val == 1))) 5668 return -EINVAL; 5669 5670 memcg->oom_kill_disable = val; 5671 if (!val) 5672 memcg_oom_recover(memcg); 5673 5674 return 0; 5675 } 5676 5677 #ifdef CONFIG_MEMCG_KMEM 5678 static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss) 5679 { 5680 int ret; 5681 5682 memcg->kmemcg_id = -1; 5683 ret = memcg_propagate_kmem(memcg); 5684 if (ret) 5685 return ret; 5686 5687 return mem_cgroup_sockets_init(memcg, ss); 5688 } 5689 5690 static void memcg_destroy_kmem(struct mem_cgroup *memcg) 5691 { 5692 mem_cgroup_sockets_destroy(memcg); 5693 } 5694 5695 static void kmem_cgroup_css_offline(struct mem_cgroup *memcg) 5696 { 5697 if (!memcg_kmem_is_active(memcg)) 5698 return; 5699 5700 /* 5701 * kmem charges can outlive the cgroup. In the case of slab 5702 * pages, for instance, a page contain objects from various 5703 * processes. As we prevent from taking a reference for every 5704 * such allocation we have to be careful when doing uncharge 5705 * (see memcg_uncharge_kmem) and here during offlining. 5706 * 5707 * The idea is that that only the _last_ uncharge which sees 5708 * the dead memcg will drop the last reference. An additional 5709 * reference is taken here before the group is marked dead 5710 * which is then paired with css_put during uncharge resp. here. 5711 * 5712 * Although this might sound strange as this path is called from 5713 * css_offline() when the referencemight have dropped down to 0 and 5714 * shouldn't be incremented anymore (css_tryget_online() would 5715 * fail) we do not have other options because of the kmem 5716 * allocations lifetime. 5717 */ 5718 css_get(&memcg->css); 5719 5720 memcg_kmem_mark_dead(memcg); 5721 5722 if (res_counter_read_u64(&memcg->kmem, RES_USAGE) != 0) 5723 return; 5724 5725 if (memcg_kmem_test_and_clear_dead(memcg)) 5726 css_put(&memcg->css); 5727 } 5728 #else 5729 static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss) 5730 { 5731 return 0; 5732 } 5733 5734 static void memcg_destroy_kmem(struct mem_cgroup *memcg) 5735 { 5736 } 5737 5738 static void kmem_cgroup_css_offline(struct mem_cgroup *memcg) 5739 { 5740 } 5741 #endif 5742 5743 /* 5744 * DO NOT USE IN NEW FILES. 5745 * 5746 * "cgroup.event_control" implementation. 5747 * 5748 * This is way over-engineered. It tries to support fully configurable 5749 * events for each user. Such level of flexibility is completely 5750 * unnecessary especially in the light of the planned unified hierarchy. 5751 * 5752 * Please deprecate this and replace with something simpler if at all 5753 * possible. 5754 */ 5755 5756 /* 5757 * Unregister event and free resources. 5758 * 5759 * Gets called from workqueue. 5760 */ 5761 static void memcg_event_remove(struct work_struct *work) 5762 { 5763 struct mem_cgroup_event *event = 5764 container_of(work, struct mem_cgroup_event, remove); 5765 struct mem_cgroup *memcg = event->memcg; 5766 5767 remove_wait_queue(event->wqh, &event->wait); 5768 5769 event->unregister_event(memcg, event->eventfd); 5770 5771 /* Notify userspace the event is going away. */ 5772 eventfd_signal(event->eventfd, 1); 5773 5774 eventfd_ctx_put(event->eventfd); 5775 kfree(event); 5776 css_put(&memcg->css); 5777 } 5778 5779 /* 5780 * Gets called on POLLHUP on eventfd when user closes it. 5781 * 5782 * Called with wqh->lock held and interrupts disabled. 5783 */ 5784 static int memcg_event_wake(wait_queue_t *wait, unsigned mode, 5785 int sync, void *key) 5786 { 5787 struct mem_cgroup_event *event = 5788 container_of(wait, struct mem_cgroup_event, wait); 5789 struct mem_cgroup *memcg = event->memcg; 5790 unsigned long flags = (unsigned long)key; 5791 5792 if (flags & POLLHUP) { 5793 /* 5794 * If the event has been detached at cgroup removal, we 5795 * can simply return knowing the other side will cleanup 5796 * for us. 5797 * 5798 * We can't race against event freeing since the other 5799 * side will require wqh->lock via remove_wait_queue(), 5800 * which we hold. 5801 */ 5802 spin_lock(&memcg->event_list_lock); 5803 if (!list_empty(&event->list)) { 5804 list_del_init(&event->list); 5805 /* 5806 * We are in atomic context, but cgroup_event_remove() 5807 * may sleep, so we have to call it in workqueue. 5808 */ 5809 schedule_work(&event->remove); 5810 } 5811 spin_unlock(&memcg->event_list_lock); 5812 } 5813 5814 return 0; 5815 } 5816 5817 static void memcg_event_ptable_queue_proc(struct file *file, 5818 wait_queue_head_t *wqh, poll_table *pt) 5819 { 5820 struct mem_cgroup_event *event = 5821 container_of(pt, struct mem_cgroup_event, pt); 5822 5823 event->wqh = wqh; 5824 add_wait_queue(wqh, &event->wait); 5825 } 5826 5827 /* 5828 * DO NOT USE IN NEW FILES. 5829 * 5830 * Parse input and register new cgroup event handler. 5831 * 5832 * Input must be in format '<event_fd> <control_fd> <args>'. 5833 * Interpretation of args is defined by control file implementation. 5834 */ 5835 static ssize_t memcg_write_event_control(struct kernfs_open_file *of, 5836 char *buf, size_t nbytes, loff_t off) 5837 { 5838 struct cgroup_subsys_state *css = of_css(of); 5839 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5840 struct mem_cgroup_event *event; 5841 struct cgroup_subsys_state *cfile_css; 5842 unsigned int efd, cfd; 5843 struct fd efile; 5844 struct fd cfile; 5845 const char *name; 5846 char *endp; 5847 int ret; 5848 5849 buf = strstrip(buf); 5850 5851 efd = simple_strtoul(buf, &endp, 10); 5852 if (*endp != ' ') 5853 return -EINVAL; 5854 buf = endp + 1; 5855 5856 cfd = simple_strtoul(buf, &endp, 10); 5857 if ((*endp != ' ') && (*endp != '\0')) 5858 return -EINVAL; 5859 buf = endp + 1; 5860 5861 event = kzalloc(sizeof(*event), GFP_KERNEL); 5862 if (!event) 5863 return -ENOMEM; 5864 5865 event->memcg = memcg; 5866 INIT_LIST_HEAD(&event->list); 5867 init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc); 5868 init_waitqueue_func_entry(&event->wait, memcg_event_wake); 5869 INIT_WORK(&event->remove, memcg_event_remove); 5870 5871 efile = fdget(efd); 5872 if (!efile.file) { 5873 ret = -EBADF; 5874 goto out_kfree; 5875 } 5876 5877 event->eventfd = eventfd_ctx_fileget(efile.file); 5878 if (IS_ERR(event->eventfd)) { 5879 ret = PTR_ERR(event->eventfd); 5880 goto out_put_efile; 5881 } 5882 5883 cfile = fdget(cfd); 5884 if (!cfile.file) { 5885 ret = -EBADF; 5886 goto out_put_eventfd; 5887 } 5888 5889 /* the process need read permission on control file */ 5890 /* AV: shouldn't we check that it's been opened for read instead? */ 5891 ret = inode_permission(file_inode(cfile.file), MAY_READ); 5892 if (ret < 0) 5893 goto out_put_cfile; 5894 5895 /* 5896 * Determine the event callbacks and set them in @event. This used 5897 * to be done via struct cftype but cgroup core no longer knows 5898 * about these events. The following is crude but the whole thing 5899 * is for compatibility anyway. 5900 * 5901 * DO NOT ADD NEW FILES. 5902 */ 5903 name = cfile.file->f_dentry->d_name.name; 5904 5905 if (!strcmp(name, "memory.usage_in_bytes")) { 5906 event->register_event = mem_cgroup_usage_register_event; 5907 event->unregister_event = mem_cgroup_usage_unregister_event; 5908 } else if (!strcmp(name, "memory.oom_control")) { 5909 event->register_event = mem_cgroup_oom_register_event; 5910 event->unregister_event = mem_cgroup_oom_unregister_event; 5911 } else if (!strcmp(name, "memory.pressure_level")) { 5912 event->register_event = vmpressure_register_event; 5913 event->unregister_event = vmpressure_unregister_event; 5914 } else if (!strcmp(name, "memory.memsw.usage_in_bytes")) { 5915 event->register_event = memsw_cgroup_usage_register_event; 5916 event->unregister_event = memsw_cgroup_usage_unregister_event; 5917 } else { 5918 ret = -EINVAL; 5919 goto out_put_cfile; 5920 } 5921 5922 /* 5923 * Verify @cfile should belong to @css. Also, remaining events are 5924 * automatically removed on cgroup destruction but the removal is 5925 * asynchronous, so take an extra ref on @css. 5926 */ 5927 cfile_css = css_tryget_online_from_dir(cfile.file->f_dentry->d_parent, 5928 &memory_cgrp_subsys); 5929 ret = -EINVAL; 5930 if (IS_ERR(cfile_css)) 5931 goto out_put_cfile; 5932 if (cfile_css != css) { 5933 css_put(cfile_css); 5934 goto out_put_cfile; 5935 } 5936 5937 ret = event->register_event(memcg, event->eventfd, buf); 5938 if (ret) 5939 goto out_put_css; 5940 5941 efile.file->f_op->poll(efile.file, &event->pt); 5942 5943 spin_lock(&memcg->event_list_lock); 5944 list_add(&event->list, &memcg->event_list); 5945 spin_unlock(&memcg->event_list_lock); 5946 5947 fdput(cfile); 5948 fdput(efile); 5949 5950 return nbytes; 5951 5952 out_put_css: 5953 css_put(css); 5954 out_put_cfile: 5955 fdput(cfile); 5956 out_put_eventfd: 5957 eventfd_ctx_put(event->eventfd); 5958 out_put_efile: 5959 fdput(efile); 5960 out_kfree: 5961 kfree(event); 5962 5963 return ret; 5964 } 5965 5966 static struct cftype mem_cgroup_files[] = { 5967 { 5968 .name = "usage_in_bytes", 5969 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE), 5970 .read_u64 = mem_cgroup_read_u64, 5971 }, 5972 { 5973 .name = "max_usage_in_bytes", 5974 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE), 5975 .write = mem_cgroup_reset, 5976 .read_u64 = mem_cgroup_read_u64, 5977 }, 5978 { 5979 .name = "limit_in_bytes", 5980 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT), 5981 .write = mem_cgroup_write, 5982 .read_u64 = mem_cgroup_read_u64, 5983 }, 5984 { 5985 .name = "soft_limit_in_bytes", 5986 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT), 5987 .write = mem_cgroup_write, 5988 .read_u64 = mem_cgroup_read_u64, 5989 }, 5990 { 5991 .name = "failcnt", 5992 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT), 5993 .write = mem_cgroup_reset, 5994 .read_u64 = mem_cgroup_read_u64, 5995 }, 5996 { 5997 .name = "stat", 5998 .seq_show = memcg_stat_show, 5999 }, 6000 { 6001 .name = "force_empty", 6002 .write = mem_cgroup_force_empty_write, 6003 }, 6004 { 6005 .name = "use_hierarchy", 6006 .flags = CFTYPE_INSANE, 6007 .write_u64 = mem_cgroup_hierarchy_write, 6008 .read_u64 = mem_cgroup_hierarchy_read, 6009 }, 6010 { 6011 .name = "cgroup.event_control", /* XXX: for compat */ 6012 .write = memcg_write_event_control, 6013 .flags = CFTYPE_NO_PREFIX, 6014 .mode = S_IWUGO, 6015 }, 6016 { 6017 .name = "swappiness", 6018 .read_u64 = mem_cgroup_swappiness_read, 6019 .write_u64 = mem_cgroup_swappiness_write, 6020 }, 6021 { 6022 .name = "move_charge_at_immigrate", 6023 .read_u64 = mem_cgroup_move_charge_read, 6024 .write_u64 = mem_cgroup_move_charge_write, 6025 }, 6026 { 6027 .name = "oom_control", 6028 .seq_show = mem_cgroup_oom_control_read, 6029 .write_u64 = mem_cgroup_oom_control_write, 6030 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL), 6031 }, 6032 { 6033 .name = "pressure_level", 6034 }, 6035 #ifdef CONFIG_NUMA 6036 { 6037 .name = "numa_stat", 6038 .seq_show = memcg_numa_stat_show, 6039 }, 6040 #endif 6041 #ifdef CONFIG_MEMCG_KMEM 6042 { 6043 .name = "kmem.limit_in_bytes", 6044 .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT), 6045 .write = mem_cgroup_write, 6046 .read_u64 = mem_cgroup_read_u64, 6047 }, 6048 { 6049 .name = "kmem.usage_in_bytes", 6050 .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE), 6051 .read_u64 = mem_cgroup_read_u64, 6052 }, 6053 { 6054 .name = "kmem.failcnt", 6055 .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT), 6056 .write = mem_cgroup_reset, 6057 .read_u64 = mem_cgroup_read_u64, 6058 }, 6059 { 6060 .name = "kmem.max_usage_in_bytes", 6061 .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE), 6062 .write = mem_cgroup_reset, 6063 .read_u64 = mem_cgroup_read_u64, 6064 }, 6065 #ifdef CONFIG_SLABINFO 6066 { 6067 .name = "kmem.slabinfo", 6068 .seq_show = mem_cgroup_slabinfo_read, 6069 }, 6070 #endif 6071 #endif 6072 { }, /* terminate */ 6073 }; 6074 6075 #ifdef CONFIG_MEMCG_SWAP 6076 static struct cftype memsw_cgroup_files[] = { 6077 { 6078 .name = "memsw.usage_in_bytes", 6079 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE), 6080 .read_u64 = mem_cgroup_read_u64, 6081 }, 6082 { 6083 .name = "memsw.max_usage_in_bytes", 6084 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE), 6085 .write = mem_cgroup_reset, 6086 .read_u64 = mem_cgroup_read_u64, 6087 }, 6088 { 6089 .name = "memsw.limit_in_bytes", 6090 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT), 6091 .write = mem_cgroup_write, 6092 .read_u64 = mem_cgroup_read_u64, 6093 }, 6094 { 6095 .name = "memsw.failcnt", 6096 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT), 6097 .write = mem_cgroup_reset, 6098 .read_u64 = mem_cgroup_read_u64, 6099 }, 6100 { }, /* terminate */ 6101 }; 6102 #endif 6103 static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node) 6104 { 6105 struct mem_cgroup_per_node *pn; 6106 struct mem_cgroup_per_zone *mz; 6107 int zone, tmp = node; 6108 /* 6109 * This routine is called against possible nodes. 6110 * But it's BUG to call kmalloc() against offline node. 6111 * 6112 * TODO: this routine can waste much memory for nodes which will 6113 * never be onlined. It's better to use memory hotplug callback 6114 * function. 6115 */ 6116 if (!node_state(node, N_NORMAL_MEMORY)) 6117 tmp = -1; 6118 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp); 6119 if (!pn) 6120 return 1; 6121 6122 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 6123 mz = &pn->zoneinfo[zone]; 6124 lruvec_init(&mz->lruvec); 6125 mz->usage_in_excess = 0; 6126 mz->on_tree = false; 6127 mz->memcg = memcg; 6128 } 6129 memcg->nodeinfo[node] = pn; 6130 return 0; 6131 } 6132 6133 static void free_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node) 6134 { 6135 kfree(memcg->nodeinfo[node]); 6136 } 6137 6138 static struct mem_cgroup *mem_cgroup_alloc(void) 6139 { 6140 struct mem_cgroup *memcg; 6141 size_t size; 6142 6143 size = sizeof(struct mem_cgroup); 6144 size += nr_node_ids * sizeof(struct mem_cgroup_per_node *); 6145 6146 memcg = kzalloc(size, GFP_KERNEL); 6147 if (!memcg) 6148 return NULL; 6149 6150 memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu); 6151 if (!memcg->stat) 6152 goto out_free; 6153 spin_lock_init(&memcg->pcp_counter_lock); 6154 return memcg; 6155 6156 out_free: 6157 kfree(memcg); 6158 return NULL; 6159 } 6160 6161 /* 6162 * At destroying mem_cgroup, references from swap_cgroup can remain. 6163 * (scanning all at force_empty is too costly...) 6164 * 6165 * Instead of clearing all references at force_empty, we remember 6166 * the number of reference from swap_cgroup and free mem_cgroup when 6167 * it goes down to 0. 6168 * 6169 * Removal of cgroup itself succeeds regardless of refs from swap. 6170 */ 6171 6172 static void __mem_cgroup_free(struct mem_cgroup *memcg) 6173 { 6174 int node; 6175 6176 mem_cgroup_remove_from_trees(memcg); 6177 6178 for_each_node(node) 6179 free_mem_cgroup_per_zone_info(memcg, node); 6180 6181 free_percpu(memcg->stat); 6182 6183 /* 6184 * We need to make sure that (at least for now), the jump label 6185 * destruction code runs outside of the cgroup lock. This is because 6186 * get_online_cpus(), which is called from the static_branch update, 6187 * can't be called inside the cgroup_lock. cpusets are the ones 6188 * enforcing this dependency, so if they ever change, we might as well. 6189 * 6190 * schedule_work() will guarantee this happens. Be careful if you need 6191 * to move this code around, and make sure it is outside 6192 * the cgroup_lock. 6193 */ 6194 disarm_static_keys(memcg); 6195 kfree(memcg); 6196 } 6197 6198 /* 6199 * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled. 6200 */ 6201 struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg) 6202 { 6203 if (!memcg->res.parent) 6204 return NULL; 6205 return mem_cgroup_from_res_counter(memcg->res.parent, res); 6206 } 6207 EXPORT_SYMBOL(parent_mem_cgroup); 6208 6209 static void __init mem_cgroup_soft_limit_tree_init(void) 6210 { 6211 struct mem_cgroup_tree_per_node *rtpn; 6212 struct mem_cgroup_tree_per_zone *rtpz; 6213 int tmp, node, zone; 6214 6215 for_each_node(node) { 6216 tmp = node; 6217 if (!node_state(node, N_NORMAL_MEMORY)) 6218 tmp = -1; 6219 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp); 6220 BUG_ON(!rtpn); 6221 6222 soft_limit_tree.rb_tree_per_node[node] = rtpn; 6223 6224 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 6225 rtpz = &rtpn->rb_tree_per_zone[zone]; 6226 rtpz->rb_root = RB_ROOT; 6227 spin_lock_init(&rtpz->lock); 6228 } 6229 } 6230 } 6231 6232 static struct cgroup_subsys_state * __ref 6233 mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 6234 { 6235 struct mem_cgroup *memcg; 6236 long error = -ENOMEM; 6237 int node; 6238 6239 memcg = mem_cgroup_alloc(); 6240 if (!memcg) 6241 return ERR_PTR(error); 6242 6243 for_each_node(node) 6244 if (alloc_mem_cgroup_per_zone_info(memcg, node)) 6245 goto free_out; 6246 6247 /* root ? */ 6248 if (parent_css == NULL) { 6249 root_mem_cgroup = memcg; 6250 res_counter_init(&memcg->res, NULL); 6251 res_counter_init(&memcg->memsw, NULL); 6252 res_counter_init(&memcg->kmem, NULL); 6253 } 6254 6255 memcg->last_scanned_node = MAX_NUMNODES; 6256 INIT_LIST_HEAD(&memcg->oom_notify); 6257 memcg->move_charge_at_immigrate = 0; 6258 mutex_init(&memcg->thresholds_lock); 6259 spin_lock_init(&memcg->move_lock); 6260 vmpressure_init(&memcg->vmpressure); 6261 INIT_LIST_HEAD(&memcg->event_list); 6262 spin_lock_init(&memcg->event_list_lock); 6263 6264 return &memcg->css; 6265 6266 free_out: 6267 __mem_cgroup_free(memcg); 6268 return ERR_PTR(error); 6269 } 6270 6271 static int 6272 mem_cgroup_css_online(struct cgroup_subsys_state *css) 6273 { 6274 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 6275 struct mem_cgroup *parent = mem_cgroup_from_css(css->parent); 6276 6277 if (css->id > MEM_CGROUP_ID_MAX) 6278 return -ENOSPC; 6279 6280 if (!parent) 6281 return 0; 6282 6283 mutex_lock(&memcg_create_mutex); 6284 6285 memcg->use_hierarchy = parent->use_hierarchy; 6286 memcg->oom_kill_disable = parent->oom_kill_disable; 6287 memcg->swappiness = mem_cgroup_swappiness(parent); 6288 6289 if (parent->use_hierarchy) { 6290 res_counter_init(&memcg->res, &parent->res); 6291 res_counter_init(&memcg->memsw, &parent->memsw); 6292 res_counter_init(&memcg->kmem, &parent->kmem); 6293 6294 /* 6295 * No need to take a reference to the parent because cgroup 6296 * core guarantees its existence. 6297 */ 6298 } else { 6299 res_counter_init(&memcg->res, NULL); 6300 res_counter_init(&memcg->memsw, NULL); 6301 res_counter_init(&memcg->kmem, NULL); 6302 /* 6303 * Deeper hierachy with use_hierarchy == false doesn't make 6304 * much sense so let cgroup subsystem know about this 6305 * unfortunate state in our controller. 6306 */ 6307 if (parent != root_mem_cgroup) 6308 memory_cgrp_subsys.broken_hierarchy = true; 6309 } 6310 mutex_unlock(&memcg_create_mutex); 6311 6312 return memcg_init_kmem(memcg, &memory_cgrp_subsys); 6313 } 6314 6315 /* 6316 * Announce all parents that a group from their hierarchy is gone. 6317 */ 6318 static void mem_cgroup_invalidate_reclaim_iterators(struct mem_cgroup *memcg) 6319 { 6320 struct mem_cgroup *parent = memcg; 6321 6322 while ((parent = parent_mem_cgroup(parent))) 6323 mem_cgroup_iter_invalidate(parent); 6324 6325 /* 6326 * if the root memcg is not hierarchical we have to check it 6327 * explicitely. 6328 */ 6329 if (!root_mem_cgroup->use_hierarchy) 6330 mem_cgroup_iter_invalidate(root_mem_cgroup); 6331 } 6332 6333 static void mem_cgroup_css_offline(struct cgroup_subsys_state *css) 6334 { 6335 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 6336 struct mem_cgroup_event *event, *tmp; 6337 struct cgroup_subsys_state *iter; 6338 6339 /* 6340 * Unregister events and notify userspace. 6341 * Notify userspace about cgroup removing only after rmdir of cgroup 6342 * directory to avoid race between userspace and kernelspace. 6343 */ 6344 spin_lock(&memcg->event_list_lock); 6345 list_for_each_entry_safe(event, tmp, &memcg->event_list, list) { 6346 list_del_init(&event->list); 6347 schedule_work(&event->remove); 6348 } 6349 spin_unlock(&memcg->event_list_lock); 6350 6351 kmem_cgroup_css_offline(memcg); 6352 6353 mem_cgroup_invalidate_reclaim_iterators(memcg); 6354 6355 /* 6356 * This requires that offlining is serialized. Right now that is 6357 * guaranteed because css_killed_work_fn() holds the cgroup_mutex. 6358 */ 6359 css_for_each_descendant_post(iter, css) 6360 mem_cgroup_reparent_charges(mem_cgroup_from_css(iter)); 6361 6362 memcg_unregister_all_caches(memcg); 6363 vmpressure_cleanup(&memcg->vmpressure); 6364 } 6365 6366 static void mem_cgroup_css_free(struct cgroup_subsys_state *css) 6367 { 6368 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 6369 /* 6370 * XXX: css_offline() would be where we should reparent all 6371 * memory to prepare the cgroup for destruction. However, 6372 * memcg does not do css_tryget_online() and res_counter charging 6373 * under the same RCU lock region, which means that charging 6374 * could race with offlining. Offlining only happens to 6375 * cgroups with no tasks in them but charges can show up 6376 * without any tasks from the swapin path when the target 6377 * memcg is looked up from the swapout record and not from the 6378 * current task as it usually is. A race like this can leak 6379 * charges and put pages with stale cgroup pointers into 6380 * circulation: 6381 * 6382 * #0 #1 6383 * lookup_swap_cgroup_id() 6384 * rcu_read_lock() 6385 * mem_cgroup_lookup() 6386 * css_tryget_online() 6387 * rcu_read_unlock() 6388 * disable css_tryget_online() 6389 * call_rcu() 6390 * offline_css() 6391 * reparent_charges() 6392 * res_counter_charge() 6393 * css_put() 6394 * css_free() 6395 * pc->mem_cgroup = dead memcg 6396 * add page to lru 6397 * 6398 * The bulk of the charges are still moved in offline_css() to 6399 * avoid pinning a lot of pages in case a long-term reference 6400 * like a swapout record is deferring the css_free() to long 6401 * after offlining. But this makes sure we catch any charges 6402 * made after offlining: 6403 */ 6404 mem_cgroup_reparent_charges(memcg); 6405 6406 memcg_destroy_kmem(memcg); 6407 __mem_cgroup_free(memcg); 6408 } 6409 6410 #ifdef CONFIG_MMU 6411 /* Handlers for move charge at task migration. */ 6412 #define PRECHARGE_COUNT_AT_ONCE 256 6413 static int mem_cgroup_do_precharge(unsigned long count) 6414 { 6415 int ret = 0; 6416 int batch_count = PRECHARGE_COUNT_AT_ONCE; 6417 struct mem_cgroup *memcg = mc.to; 6418 6419 if (mem_cgroup_is_root(memcg)) { 6420 mc.precharge += count; 6421 /* we don't need css_get for root */ 6422 return ret; 6423 } 6424 /* try to charge at once */ 6425 if (count > 1) { 6426 struct res_counter *dummy; 6427 /* 6428 * "memcg" cannot be under rmdir() because we've already checked 6429 * by cgroup_lock_live_cgroup() that it is not removed and we 6430 * are still under the same cgroup_mutex. So we can postpone 6431 * css_get(). 6432 */ 6433 if (res_counter_charge(&memcg->res, PAGE_SIZE * count, &dummy)) 6434 goto one_by_one; 6435 if (do_swap_account && res_counter_charge(&memcg->memsw, 6436 PAGE_SIZE * count, &dummy)) { 6437 res_counter_uncharge(&memcg->res, PAGE_SIZE * count); 6438 goto one_by_one; 6439 } 6440 mc.precharge += count; 6441 return ret; 6442 } 6443 one_by_one: 6444 /* fall back to one by one charge */ 6445 while (count--) { 6446 if (signal_pending(current)) { 6447 ret = -EINTR; 6448 break; 6449 } 6450 if (!batch_count--) { 6451 batch_count = PRECHARGE_COUNT_AT_ONCE; 6452 cond_resched(); 6453 } 6454 ret = mem_cgroup_try_charge(memcg, GFP_KERNEL, 1, false); 6455 if (ret) 6456 /* mem_cgroup_clear_mc() will do uncharge later */ 6457 return ret; 6458 mc.precharge++; 6459 } 6460 return ret; 6461 } 6462 6463 /** 6464 * get_mctgt_type - get target type of moving charge 6465 * @vma: the vma the pte to be checked belongs 6466 * @addr: the address corresponding to the pte to be checked 6467 * @ptent: the pte to be checked 6468 * @target: the pointer the target page or swap ent will be stored(can be NULL) 6469 * 6470 * Returns 6471 * 0(MC_TARGET_NONE): if the pte is not a target for move charge. 6472 * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for 6473 * move charge. if @target is not NULL, the page is stored in target->page 6474 * with extra refcnt got(Callers should handle it). 6475 * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a 6476 * target for charge migration. if @target is not NULL, the entry is stored 6477 * in target->ent. 6478 * 6479 * Called with pte lock held. 6480 */ 6481 union mc_target { 6482 struct page *page; 6483 swp_entry_t ent; 6484 }; 6485 6486 enum mc_target_type { 6487 MC_TARGET_NONE = 0, 6488 MC_TARGET_PAGE, 6489 MC_TARGET_SWAP, 6490 }; 6491 6492 static struct page *mc_handle_present_pte(struct vm_area_struct *vma, 6493 unsigned long addr, pte_t ptent) 6494 { 6495 struct page *page = vm_normal_page(vma, addr, ptent); 6496 6497 if (!page || !page_mapped(page)) 6498 return NULL; 6499 if (PageAnon(page)) { 6500 /* we don't move shared anon */ 6501 if (!move_anon()) 6502 return NULL; 6503 } else if (!move_file()) 6504 /* we ignore mapcount for file pages */ 6505 return NULL; 6506 if (!get_page_unless_zero(page)) 6507 return NULL; 6508 6509 return page; 6510 } 6511 6512 #ifdef CONFIG_SWAP 6513 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma, 6514 unsigned long addr, pte_t ptent, swp_entry_t *entry) 6515 { 6516 struct page *page = NULL; 6517 swp_entry_t ent = pte_to_swp_entry(ptent); 6518 6519 if (!move_anon() || non_swap_entry(ent)) 6520 return NULL; 6521 /* 6522 * Because lookup_swap_cache() updates some statistics counter, 6523 * we call find_get_page() with swapper_space directly. 6524 */ 6525 page = find_get_page(swap_address_space(ent), ent.val); 6526 if (do_swap_account) 6527 entry->val = ent.val; 6528 6529 return page; 6530 } 6531 #else 6532 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma, 6533 unsigned long addr, pte_t ptent, swp_entry_t *entry) 6534 { 6535 return NULL; 6536 } 6537 #endif 6538 6539 static struct page *mc_handle_file_pte(struct vm_area_struct *vma, 6540 unsigned long addr, pte_t ptent, swp_entry_t *entry) 6541 { 6542 struct page *page = NULL; 6543 struct address_space *mapping; 6544 pgoff_t pgoff; 6545 6546 if (!vma->vm_file) /* anonymous vma */ 6547 return NULL; 6548 if (!move_file()) 6549 return NULL; 6550 6551 mapping = vma->vm_file->f_mapping; 6552 if (pte_none(ptent)) 6553 pgoff = linear_page_index(vma, addr); 6554 else /* pte_file(ptent) is true */ 6555 pgoff = pte_to_pgoff(ptent); 6556 6557 /* page is moved even if it's not RSS of this task(page-faulted). */ 6558 #ifdef CONFIG_SWAP 6559 /* shmem/tmpfs may report page out on swap: account for that too. */ 6560 if (shmem_mapping(mapping)) { 6561 page = find_get_entry(mapping, pgoff); 6562 if (radix_tree_exceptional_entry(page)) { 6563 swp_entry_t swp = radix_to_swp_entry(page); 6564 if (do_swap_account) 6565 *entry = swp; 6566 page = find_get_page(swap_address_space(swp), swp.val); 6567 } 6568 } else 6569 page = find_get_page(mapping, pgoff); 6570 #else 6571 page = find_get_page(mapping, pgoff); 6572 #endif 6573 return page; 6574 } 6575 6576 static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma, 6577 unsigned long addr, pte_t ptent, union mc_target *target) 6578 { 6579 struct page *page = NULL; 6580 struct page_cgroup *pc; 6581 enum mc_target_type ret = MC_TARGET_NONE; 6582 swp_entry_t ent = { .val = 0 }; 6583 6584 if (pte_present(ptent)) 6585 page = mc_handle_present_pte(vma, addr, ptent); 6586 else if (is_swap_pte(ptent)) 6587 page = mc_handle_swap_pte(vma, addr, ptent, &ent); 6588 else if (pte_none(ptent) || pte_file(ptent)) 6589 page = mc_handle_file_pte(vma, addr, ptent, &ent); 6590 6591 if (!page && !ent.val) 6592 return ret; 6593 if (page) { 6594 pc = lookup_page_cgroup(page); 6595 /* 6596 * Do only loose check w/o page_cgroup lock. 6597 * mem_cgroup_move_account() checks the pc is valid or not under 6598 * the lock. 6599 */ 6600 if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) { 6601 ret = MC_TARGET_PAGE; 6602 if (target) 6603 target->page = page; 6604 } 6605 if (!ret || !target) 6606 put_page(page); 6607 } 6608 /* There is a swap entry and a page doesn't exist or isn't charged */ 6609 if (ent.val && !ret && 6610 mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) { 6611 ret = MC_TARGET_SWAP; 6612 if (target) 6613 target->ent = ent; 6614 } 6615 return ret; 6616 } 6617 6618 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 6619 /* 6620 * We don't consider swapping or file mapped pages because THP does not 6621 * support them for now. 6622 * Caller should make sure that pmd_trans_huge(pmd) is true. 6623 */ 6624 static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma, 6625 unsigned long addr, pmd_t pmd, union mc_target *target) 6626 { 6627 struct page *page = NULL; 6628 struct page_cgroup *pc; 6629 enum mc_target_type ret = MC_TARGET_NONE; 6630 6631 page = pmd_page(pmd); 6632 VM_BUG_ON_PAGE(!page || !PageHead(page), page); 6633 if (!move_anon()) 6634 return ret; 6635 pc = lookup_page_cgroup(page); 6636 if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) { 6637 ret = MC_TARGET_PAGE; 6638 if (target) { 6639 get_page(page); 6640 target->page = page; 6641 } 6642 } 6643 return ret; 6644 } 6645 #else 6646 static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma, 6647 unsigned long addr, pmd_t pmd, union mc_target *target) 6648 { 6649 return MC_TARGET_NONE; 6650 } 6651 #endif 6652 6653 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd, 6654 unsigned long addr, unsigned long end, 6655 struct mm_walk *walk) 6656 { 6657 struct vm_area_struct *vma = walk->private; 6658 pte_t *pte; 6659 spinlock_t *ptl; 6660 6661 if (pmd_trans_huge_lock(pmd, vma, &ptl) == 1) { 6662 if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE) 6663 mc.precharge += HPAGE_PMD_NR; 6664 spin_unlock(ptl); 6665 return 0; 6666 } 6667 6668 if (pmd_trans_unstable(pmd)) 6669 return 0; 6670 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 6671 for (; addr != end; pte++, addr += PAGE_SIZE) 6672 if (get_mctgt_type(vma, addr, *pte, NULL)) 6673 mc.precharge++; /* increment precharge temporarily */ 6674 pte_unmap_unlock(pte - 1, ptl); 6675 cond_resched(); 6676 6677 return 0; 6678 } 6679 6680 static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm) 6681 { 6682 unsigned long precharge; 6683 struct vm_area_struct *vma; 6684 6685 down_read(&mm->mmap_sem); 6686 for (vma = mm->mmap; vma; vma = vma->vm_next) { 6687 struct mm_walk mem_cgroup_count_precharge_walk = { 6688 .pmd_entry = mem_cgroup_count_precharge_pte_range, 6689 .mm = mm, 6690 .private = vma, 6691 }; 6692 if (is_vm_hugetlb_page(vma)) 6693 continue; 6694 walk_page_range(vma->vm_start, vma->vm_end, 6695 &mem_cgroup_count_precharge_walk); 6696 } 6697 up_read(&mm->mmap_sem); 6698 6699 precharge = mc.precharge; 6700 mc.precharge = 0; 6701 6702 return precharge; 6703 } 6704 6705 static int mem_cgroup_precharge_mc(struct mm_struct *mm) 6706 { 6707 unsigned long precharge = mem_cgroup_count_precharge(mm); 6708 6709 VM_BUG_ON(mc.moving_task); 6710 mc.moving_task = current; 6711 return mem_cgroup_do_precharge(precharge); 6712 } 6713 6714 /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */ 6715 static void __mem_cgroup_clear_mc(void) 6716 { 6717 struct mem_cgroup *from = mc.from; 6718 struct mem_cgroup *to = mc.to; 6719 int i; 6720 6721 /* we must uncharge all the leftover precharges from mc.to */ 6722 if (mc.precharge) { 6723 __mem_cgroup_cancel_charge(mc.to, mc.precharge); 6724 mc.precharge = 0; 6725 } 6726 /* 6727 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so 6728 * we must uncharge here. 6729 */ 6730 if (mc.moved_charge) { 6731 __mem_cgroup_cancel_charge(mc.from, mc.moved_charge); 6732 mc.moved_charge = 0; 6733 } 6734 /* we must fixup refcnts and charges */ 6735 if (mc.moved_swap) { 6736 /* uncharge swap account from the old cgroup */ 6737 if (!mem_cgroup_is_root(mc.from)) 6738 res_counter_uncharge(&mc.from->memsw, 6739 PAGE_SIZE * mc.moved_swap); 6740 6741 for (i = 0; i < mc.moved_swap; i++) 6742 css_put(&mc.from->css); 6743 6744 if (!mem_cgroup_is_root(mc.to)) { 6745 /* 6746 * we charged both to->res and to->memsw, so we should 6747 * uncharge to->res. 6748 */ 6749 res_counter_uncharge(&mc.to->res, 6750 PAGE_SIZE * mc.moved_swap); 6751 } 6752 /* we've already done css_get(mc.to) */ 6753 mc.moved_swap = 0; 6754 } 6755 memcg_oom_recover(from); 6756 memcg_oom_recover(to); 6757 wake_up_all(&mc.waitq); 6758 } 6759 6760 static void mem_cgroup_clear_mc(void) 6761 { 6762 struct mem_cgroup *from = mc.from; 6763 6764 /* 6765 * we must clear moving_task before waking up waiters at the end of 6766 * task migration. 6767 */ 6768 mc.moving_task = NULL; 6769 __mem_cgroup_clear_mc(); 6770 spin_lock(&mc.lock); 6771 mc.from = NULL; 6772 mc.to = NULL; 6773 spin_unlock(&mc.lock); 6774 mem_cgroup_end_move(from); 6775 } 6776 6777 static int mem_cgroup_can_attach(struct cgroup_subsys_state *css, 6778 struct cgroup_taskset *tset) 6779 { 6780 struct task_struct *p = cgroup_taskset_first(tset); 6781 int ret = 0; 6782 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 6783 unsigned long move_charge_at_immigrate; 6784 6785 /* 6786 * We are now commited to this value whatever it is. Changes in this 6787 * tunable will only affect upcoming migrations, not the current one. 6788 * So we need to save it, and keep it going. 6789 */ 6790 move_charge_at_immigrate = memcg->move_charge_at_immigrate; 6791 if (move_charge_at_immigrate) { 6792 struct mm_struct *mm; 6793 struct mem_cgroup *from = mem_cgroup_from_task(p); 6794 6795 VM_BUG_ON(from == memcg); 6796 6797 mm = get_task_mm(p); 6798 if (!mm) 6799 return 0; 6800 /* We move charges only when we move a owner of the mm */ 6801 if (mm->owner == p) { 6802 VM_BUG_ON(mc.from); 6803 VM_BUG_ON(mc.to); 6804 VM_BUG_ON(mc.precharge); 6805 VM_BUG_ON(mc.moved_charge); 6806 VM_BUG_ON(mc.moved_swap); 6807 mem_cgroup_start_move(from); 6808 spin_lock(&mc.lock); 6809 mc.from = from; 6810 mc.to = memcg; 6811 mc.immigrate_flags = move_charge_at_immigrate; 6812 spin_unlock(&mc.lock); 6813 /* We set mc.moving_task later */ 6814 6815 ret = mem_cgroup_precharge_mc(mm); 6816 if (ret) 6817 mem_cgroup_clear_mc(); 6818 } 6819 mmput(mm); 6820 } 6821 return ret; 6822 } 6823 6824 static void mem_cgroup_cancel_attach(struct cgroup_subsys_state *css, 6825 struct cgroup_taskset *tset) 6826 { 6827 mem_cgroup_clear_mc(); 6828 } 6829 6830 static int mem_cgroup_move_charge_pte_range(pmd_t *pmd, 6831 unsigned long addr, unsigned long end, 6832 struct mm_walk *walk) 6833 { 6834 int ret = 0; 6835 struct vm_area_struct *vma = walk->private; 6836 pte_t *pte; 6837 spinlock_t *ptl; 6838 enum mc_target_type target_type; 6839 union mc_target target; 6840 struct page *page; 6841 struct page_cgroup *pc; 6842 6843 /* 6844 * We don't take compound_lock() here but no race with splitting thp 6845 * happens because: 6846 * - if pmd_trans_huge_lock() returns 1, the relevant thp is not 6847 * under splitting, which means there's no concurrent thp split, 6848 * - if another thread runs into split_huge_page() just after we 6849 * entered this if-block, the thread must wait for page table lock 6850 * to be unlocked in __split_huge_page_splitting(), where the main 6851 * part of thp split is not executed yet. 6852 */ 6853 if (pmd_trans_huge_lock(pmd, vma, &ptl) == 1) { 6854 if (mc.precharge < HPAGE_PMD_NR) { 6855 spin_unlock(ptl); 6856 return 0; 6857 } 6858 target_type = get_mctgt_type_thp(vma, addr, *pmd, &target); 6859 if (target_type == MC_TARGET_PAGE) { 6860 page = target.page; 6861 if (!isolate_lru_page(page)) { 6862 pc = lookup_page_cgroup(page); 6863 if (!mem_cgroup_move_account(page, HPAGE_PMD_NR, 6864 pc, mc.from, mc.to)) { 6865 mc.precharge -= HPAGE_PMD_NR; 6866 mc.moved_charge += HPAGE_PMD_NR; 6867 } 6868 putback_lru_page(page); 6869 } 6870 put_page(page); 6871 } 6872 spin_unlock(ptl); 6873 return 0; 6874 } 6875 6876 if (pmd_trans_unstable(pmd)) 6877 return 0; 6878 retry: 6879 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 6880 for (; addr != end; addr += PAGE_SIZE) { 6881 pte_t ptent = *(pte++); 6882 swp_entry_t ent; 6883 6884 if (!mc.precharge) 6885 break; 6886 6887 switch (get_mctgt_type(vma, addr, ptent, &target)) { 6888 case MC_TARGET_PAGE: 6889 page = target.page; 6890 if (isolate_lru_page(page)) 6891 goto put; 6892 pc = lookup_page_cgroup(page); 6893 if (!mem_cgroup_move_account(page, 1, pc, 6894 mc.from, mc.to)) { 6895 mc.precharge--; 6896 /* we uncharge from mc.from later. */ 6897 mc.moved_charge++; 6898 } 6899 putback_lru_page(page); 6900 put: /* get_mctgt_type() gets the page */ 6901 put_page(page); 6902 break; 6903 case MC_TARGET_SWAP: 6904 ent = target.ent; 6905 if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) { 6906 mc.precharge--; 6907 /* we fixup refcnts and charges later. */ 6908 mc.moved_swap++; 6909 } 6910 break; 6911 default: 6912 break; 6913 } 6914 } 6915 pte_unmap_unlock(pte - 1, ptl); 6916 cond_resched(); 6917 6918 if (addr != end) { 6919 /* 6920 * We have consumed all precharges we got in can_attach(). 6921 * We try charge one by one, but don't do any additional 6922 * charges to mc.to if we have failed in charge once in attach() 6923 * phase. 6924 */ 6925 ret = mem_cgroup_do_precharge(1); 6926 if (!ret) 6927 goto retry; 6928 } 6929 6930 return ret; 6931 } 6932 6933 static void mem_cgroup_move_charge(struct mm_struct *mm) 6934 { 6935 struct vm_area_struct *vma; 6936 6937 lru_add_drain_all(); 6938 retry: 6939 if (unlikely(!down_read_trylock(&mm->mmap_sem))) { 6940 /* 6941 * Someone who are holding the mmap_sem might be waiting in 6942 * waitq. So we cancel all extra charges, wake up all waiters, 6943 * and retry. Because we cancel precharges, we might not be able 6944 * to move enough charges, but moving charge is a best-effort 6945 * feature anyway, so it wouldn't be a big problem. 6946 */ 6947 __mem_cgroup_clear_mc(); 6948 cond_resched(); 6949 goto retry; 6950 } 6951 for (vma = mm->mmap; vma; vma = vma->vm_next) { 6952 int ret; 6953 struct mm_walk mem_cgroup_move_charge_walk = { 6954 .pmd_entry = mem_cgroup_move_charge_pte_range, 6955 .mm = mm, 6956 .private = vma, 6957 }; 6958 if (is_vm_hugetlb_page(vma)) 6959 continue; 6960 ret = walk_page_range(vma->vm_start, vma->vm_end, 6961 &mem_cgroup_move_charge_walk); 6962 if (ret) 6963 /* 6964 * means we have consumed all precharges and failed in 6965 * doing additional charge. Just abandon here. 6966 */ 6967 break; 6968 } 6969 up_read(&mm->mmap_sem); 6970 } 6971 6972 static void mem_cgroup_move_task(struct cgroup_subsys_state *css, 6973 struct cgroup_taskset *tset) 6974 { 6975 struct task_struct *p = cgroup_taskset_first(tset); 6976 struct mm_struct *mm = get_task_mm(p); 6977 6978 if (mm) { 6979 if (mc.to) 6980 mem_cgroup_move_charge(mm); 6981 mmput(mm); 6982 } 6983 if (mc.to) 6984 mem_cgroup_clear_mc(); 6985 } 6986 #else /* !CONFIG_MMU */ 6987 static int mem_cgroup_can_attach(struct cgroup_subsys_state *css, 6988 struct cgroup_taskset *tset) 6989 { 6990 return 0; 6991 } 6992 static void mem_cgroup_cancel_attach(struct cgroup_subsys_state *css, 6993 struct cgroup_taskset *tset) 6994 { 6995 } 6996 static void mem_cgroup_move_task(struct cgroup_subsys_state *css, 6997 struct cgroup_taskset *tset) 6998 { 6999 } 7000 #endif 7001 7002 /* 7003 * Cgroup retains root cgroups across [un]mount cycles making it necessary 7004 * to verify sane_behavior flag on each mount attempt. 7005 */ 7006 static void mem_cgroup_bind(struct cgroup_subsys_state *root_css) 7007 { 7008 /* 7009 * use_hierarchy is forced with sane_behavior. cgroup core 7010 * guarantees that @root doesn't have any children, so turning it 7011 * on for the root memcg is enough. 7012 */ 7013 if (cgroup_sane_behavior(root_css->cgroup)) 7014 mem_cgroup_from_css(root_css)->use_hierarchy = true; 7015 } 7016 7017 struct cgroup_subsys memory_cgrp_subsys = { 7018 .css_alloc = mem_cgroup_css_alloc, 7019 .css_online = mem_cgroup_css_online, 7020 .css_offline = mem_cgroup_css_offline, 7021 .css_free = mem_cgroup_css_free, 7022 .can_attach = mem_cgroup_can_attach, 7023 .cancel_attach = mem_cgroup_cancel_attach, 7024 .attach = mem_cgroup_move_task, 7025 .bind = mem_cgroup_bind, 7026 .base_cftypes = mem_cgroup_files, 7027 .early_init = 0, 7028 }; 7029 7030 #ifdef CONFIG_MEMCG_SWAP 7031 static int __init enable_swap_account(char *s) 7032 { 7033 if (!strcmp(s, "1")) 7034 really_do_swap_account = 1; 7035 else if (!strcmp(s, "0")) 7036 really_do_swap_account = 0; 7037 return 1; 7038 } 7039 __setup("swapaccount=", enable_swap_account); 7040 7041 static void __init memsw_file_init(void) 7042 { 7043 WARN_ON(cgroup_add_cftypes(&memory_cgrp_subsys, memsw_cgroup_files)); 7044 } 7045 7046 static void __init enable_swap_cgroup(void) 7047 { 7048 if (!mem_cgroup_disabled() && really_do_swap_account) { 7049 do_swap_account = 1; 7050 memsw_file_init(); 7051 } 7052 } 7053 7054 #else 7055 static void __init enable_swap_cgroup(void) 7056 { 7057 } 7058 #endif 7059 7060 /* 7061 * subsys_initcall() for memory controller. 7062 * 7063 * Some parts like hotcpu_notifier() have to be initialized from this context 7064 * because of lock dependencies (cgroup_lock -> cpu hotplug) but basically 7065 * everything that doesn't depend on a specific mem_cgroup structure should 7066 * be initialized from here. 7067 */ 7068 static int __init mem_cgroup_init(void) 7069 { 7070 hotcpu_notifier(memcg_cpu_hotplug_callback, 0); 7071 enable_swap_cgroup(); 7072 mem_cgroup_soft_limit_tree_init(); 7073 memcg_stock_init(); 7074 return 0; 7075 } 7076 subsys_initcall(mem_cgroup_init); 7077