1c942fddfSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later
28cdea7c0SBalbir Singh /* memcontrol.c - Memory Controller
38cdea7c0SBalbir Singh *
48cdea7c0SBalbir Singh * Copyright IBM Corporation, 2007
58cdea7c0SBalbir Singh * Author Balbir Singh <balbir@linux.vnet.ibm.com>
68cdea7c0SBalbir Singh *
778fb7466SPavel Emelianov * Copyright 2007 OpenVZ SWsoft Inc
878fb7466SPavel Emelianov * Author: Pavel Emelianov <xemul@openvz.org>
978fb7466SPavel Emelianov *
102e72b634SKirill A. Shutemov * Memory thresholds
112e72b634SKirill A. Shutemov * Copyright (C) 2009 Nokia Corporation
122e72b634SKirill A. Shutemov * Author: Kirill A. Shutemov
132e72b634SKirill A. Shutemov *
147ae1e1d0SGlauber Costa * Kernel Memory Controller
157ae1e1d0SGlauber Costa * Copyright (C) 2012 Parallels Inc. and Google Inc.
167ae1e1d0SGlauber Costa * Authors: Glauber Costa and Suleiman Souhlal
177ae1e1d0SGlauber Costa *
181575e68bSJohannes Weiner * Native page reclaim
191575e68bSJohannes Weiner * Charge lifetime sanitation
201575e68bSJohannes Weiner * Lockless page tracking & accounting
211575e68bSJohannes Weiner * Unified hierarchy configuration model
221575e68bSJohannes Weiner * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
236168d0daSAlex Shi *
246168d0daSAlex Shi * Per memcg lru locking
256168d0daSAlex Shi * Copyright (C) 2020 Alibaba, Inc, Alex Shi
268cdea7c0SBalbir Singh */
278cdea7c0SBalbir Singh
283e32cb2eSJohannes Weiner #include <linux/page_counter.h>
298cdea7c0SBalbir Singh #include <linux/memcontrol.h>
308cdea7c0SBalbir Singh #include <linux/cgroup.h>
31a520110eSChristoph Hellwig #include <linux/pagewalk.h>
326e84f315SIngo Molnar #include <linux/sched/mm.h>
333a4f8a0bSHugh Dickins #include <linux/shmem_fs.h>
344ffef5feSDaisuke Nishimura #include <linux/hugetlb.h>
35d13d1443SKAMEZAWA Hiroyuki #include <linux/pagemap.h>
361ff9e6e1SChris Down #include <linux/vm_event_item.h>
37d52aa412SKAMEZAWA Hiroyuki #include <linux/smp.h>
388a9f3ccdSBalbir Singh #include <linux/page-flags.h>
3966e1707bSBalbir Singh #include <linux/backing-dev.h>
408a9f3ccdSBalbir Singh #include <linux/bit_spinlock.h>
418a9f3ccdSBalbir Singh #include <linux/rcupdate.h>
42e222432bSBalbir Singh #include <linux/limits.h>
43b9e15bafSPaul Gortmaker #include <linux/export.h>
448c7c6e34SKAMEZAWA Hiroyuki #include <linux/mutex.h>
45bb4cc1a8SAndrew Morton #include <linux/rbtree.h>
46b6ac57d5SBalbir Singh #include <linux/slab.h>
4766e1707bSBalbir Singh #include <linux/swap.h>
4802491447SDaisuke Nishimura #include <linux/swapops.h>
4966e1707bSBalbir Singh #include <linux/spinlock.h>
502e72b634SKirill A. Shutemov #include <linux/eventfd.h>
5179bd9814STejun Heo #include <linux/poll.h>
522e72b634SKirill A. Shutemov #include <linux/sort.h>
5366e1707bSBalbir Singh #include <linux/fs.h>
54d2ceb9b7SKAMEZAWA Hiroyuki #include <linux/seq_file.h>
5570ddf637SAnton Vorontsov #include <linux/vmpressure.h>
56dc90f084SChristoph Hellwig #include <linux/memremap.h>
57b69408e8SChristoph Lameter #include <linux/mm_inline.h>
585d1ea48bSJohannes Weiner #include <linux/swap_cgroup.h>
59cdec2e42SKAMEZAWA Hiroyuki #include <linux/cpu.h>
60158e0a2dSKAMEZAWA Hiroyuki #include <linux/oom.h>
610056f4e6SJohannes Weiner #include <linux/lockdep.h>
6279bd9814STejun Heo #include <linux/file.h>
6303248addSEric W. Biederman #include <linux/resume_user_mode.h>
640e4b01dfSChris Down #include <linux/psi.h>
65c8713d0bSJohannes Weiner #include <linux/seq_buf.h>
666a792697SMichal Hocko #include <linux/sched/isolation.h>
6708e552c6SKAMEZAWA Hiroyuki #include "internal.h"
68d1a4c0b3SGlauber Costa #include <net/sock.h>
694bd2c1eeSMichal Hocko #include <net/ip.h>
70f35c3a8eSQiang Huang #include "slab.h"
71014bb1deSNeilBrown #include "swap.h"
728cdea7c0SBalbir Singh
737c0f6ba6SLinus Torvalds #include <linux/uaccess.h>
748697d331SBalbir Singh
75cc8e970cSKOSAKI Motohiro #include <trace/events/vmscan.h>
76cc8e970cSKOSAKI Motohiro
77073219e9STejun Heo struct cgroup_subsys memory_cgrp_subsys __read_mostly;
78073219e9STejun Heo EXPORT_SYMBOL(memory_cgrp_subsys);
7968ae564bSDavid Rientjes
807d828602SJohannes Weiner struct mem_cgroup *root_mem_cgroup __read_mostly;
817d828602SJohannes Weiner
8237d5985cSRoman Gushchin /* Active memory cgroup to use from an interrupt context */
8337d5985cSRoman Gushchin DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg);
84c74d40e8SDan Schatzberg EXPORT_PER_CPU_SYMBOL_GPL(int_active_memcg);
8537d5985cSRoman Gushchin
86f7e1cb6eSJohannes Weiner /* Socket memory accounting disabled? */
870f0cace3SRoman Gushchin static bool cgroup_memory_nosocket __ro_after_init;
88f7e1cb6eSJohannes Weiner
8904823c83SVladimir Davydov /* Kernel memory accounting disabled? */
9017c17367SMuchun Song static bool cgroup_memory_nokmem __ro_after_init;
9104823c83SVladimir Davydov
92b6c1a8afSYafang Shao /* BPF memory accounting disabled? */
93b6c1a8afSYafang Shao static bool cgroup_memory_nobpf __ro_after_init;
94b6c1a8afSYafang Shao
9597b27821STejun Heo #ifdef CONFIG_CGROUP_WRITEBACK
9697b27821STejun Heo static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq);
9797b27821STejun Heo #endif
9897b27821STejun Heo
997941d214SJohannes Weiner /* Whether legacy memory+swap accounting is active */
do_memsw_account(void)1007941d214SJohannes Weiner static bool do_memsw_account(void)
1017941d214SJohannes Weiner {
102b25806dcSJohannes Weiner return !cgroup_subsys_on_dfl(memory_cgrp_subsys);
1037941d214SJohannes Weiner }
1047941d214SJohannes Weiner
105a0db00fcSKirill A. Shutemov #define THRESHOLDS_EVENTS_TARGET 128
106a0db00fcSKirill A. Shutemov #define SOFTLIMIT_EVENTS_TARGET 1024
107e9f8974fSJohannes Weiner
108bb4cc1a8SAndrew Morton /*
109bb4cc1a8SAndrew Morton * Cgroups above their limits are maintained in a RB-Tree, independent of
110bb4cc1a8SAndrew Morton * their hierarchy representation
111bb4cc1a8SAndrew Morton */
112bb4cc1a8SAndrew Morton
113ef8f2327SMel Gorman struct mem_cgroup_tree_per_node {
114bb4cc1a8SAndrew Morton struct rb_root rb_root;
115fa90b2fdSDavidlohr Bueso struct rb_node *rb_rightmost;
116bb4cc1a8SAndrew Morton spinlock_t lock;
117bb4cc1a8SAndrew Morton };
118bb4cc1a8SAndrew Morton
119bb4cc1a8SAndrew Morton struct mem_cgroup_tree {
120bb4cc1a8SAndrew Morton struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
121bb4cc1a8SAndrew Morton };
122bb4cc1a8SAndrew Morton
123bb4cc1a8SAndrew Morton static struct mem_cgroup_tree soft_limit_tree __read_mostly;
124bb4cc1a8SAndrew Morton
1259490ff27SKAMEZAWA Hiroyuki /* for OOM */
1269490ff27SKAMEZAWA Hiroyuki struct mem_cgroup_eventfd_list {
1279490ff27SKAMEZAWA Hiroyuki struct list_head list;
1289490ff27SKAMEZAWA Hiroyuki struct eventfd_ctx *eventfd;
1299490ff27SKAMEZAWA Hiroyuki };
1302e72b634SKirill A. Shutemov
13179bd9814STejun Heo /*
13279bd9814STejun Heo * cgroup_event represents events which userspace want to receive.
13379bd9814STejun Heo */
1343bc942f3STejun Heo struct mem_cgroup_event {
13579bd9814STejun Heo /*
13659b6f873STejun Heo * memcg which the event belongs to.
13779bd9814STejun Heo */
13859b6f873STejun Heo struct mem_cgroup *memcg;
13979bd9814STejun Heo /*
14079bd9814STejun Heo * eventfd to signal userspace about the event.
14179bd9814STejun Heo */
14279bd9814STejun Heo struct eventfd_ctx *eventfd;
14379bd9814STejun Heo /*
14479bd9814STejun Heo * Each of these stored in a list by the cgroup.
14579bd9814STejun Heo */
14679bd9814STejun Heo struct list_head list;
14779bd9814STejun Heo /*
148fba94807STejun Heo * register_event() callback will be used to add new userspace
149fba94807STejun Heo * waiter for changes related to this event. Use eventfd_signal()
150fba94807STejun Heo * on eventfd to send notification to userspace.
151fba94807STejun Heo */
15259b6f873STejun Heo int (*register_event)(struct mem_cgroup *memcg,
153347c4a87STejun Heo struct eventfd_ctx *eventfd, const char *args);
154fba94807STejun Heo /*
155fba94807STejun Heo * unregister_event() callback will be called when userspace closes
156fba94807STejun Heo * the eventfd or on cgroup removing. This callback must be set,
157fba94807STejun Heo * if you want provide notification functionality.
158fba94807STejun Heo */
15959b6f873STejun Heo void (*unregister_event)(struct mem_cgroup *memcg,
160fba94807STejun Heo struct eventfd_ctx *eventfd);
161fba94807STejun Heo /*
16279bd9814STejun Heo * All fields below needed to unregister event when
16379bd9814STejun Heo * userspace closes eventfd.
16479bd9814STejun Heo */
16579bd9814STejun Heo poll_table pt;
16679bd9814STejun Heo wait_queue_head_t *wqh;
167ac6424b9SIngo Molnar wait_queue_entry_t wait;
16879bd9814STejun Heo struct work_struct remove;
16979bd9814STejun Heo };
17079bd9814STejun Heo
171c0ff4b85SRaghavendra K T static void mem_cgroup_threshold(struct mem_cgroup *memcg);
172c0ff4b85SRaghavendra K T static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
1732e72b634SKirill A. Shutemov
1747dc74be0SDaisuke Nishimura /* Stuffs for move charges at task migration. */
1757dc74be0SDaisuke Nishimura /*
1761dfab5abSJohannes Weiner * Types of charges to be moved.
1777dc74be0SDaisuke Nishimura */
1781dfab5abSJohannes Weiner #define MOVE_ANON 0x1U
1791dfab5abSJohannes Weiner #define MOVE_FILE 0x2U
1801dfab5abSJohannes Weiner #define MOVE_MASK (MOVE_ANON | MOVE_FILE)
1817dc74be0SDaisuke Nishimura
1824ffef5feSDaisuke Nishimura /* "mc" and its members are protected by cgroup_mutex */
1834ffef5feSDaisuke Nishimura static struct move_charge_struct {
184b1dd693eSDaisuke Nishimura spinlock_t lock; /* for from, to */
185264a0ae1STejun Heo struct mm_struct *mm;
1864ffef5feSDaisuke Nishimura struct mem_cgroup *from;
1874ffef5feSDaisuke Nishimura struct mem_cgroup *to;
1881dfab5abSJohannes Weiner unsigned long flags;
1894ffef5feSDaisuke Nishimura unsigned long precharge;
190854ffa8dSDaisuke Nishimura unsigned long moved_charge;
191483c30b5SDaisuke Nishimura unsigned long moved_swap;
1928033b97cSDaisuke Nishimura struct task_struct *moving_task; /* a task moving charges */
1938033b97cSDaisuke Nishimura wait_queue_head_t waitq; /* a waitq for other context */
1948033b97cSDaisuke Nishimura } mc = {
1952bd9bb20SKAMEZAWA Hiroyuki .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
1968033b97cSDaisuke Nishimura .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
1978033b97cSDaisuke Nishimura };
1984ffef5feSDaisuke Nishimura
1994e416953SBalbir Singh /*
200f4d005afSMiaohe Lin * Maximum loops in mem_cgroup_soft_reclaim(), used for soft
2014e416953SBalbir Singh * limit reclaim to prevent infinite loops, if they ever occur.
2024e416953SBalbir Singh */
203a0db00fcSKirill A. Shutemov #define MEM_CGROUP_MAX_RECLAIM_LOOPS 100
204bb4cc1a8SAndrew Morton #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
2054e416953SBalbir Singh
2068c7c6e34SKAMEZAWA Hiroyuki /* for encoding cft->private value on file */
20786ae53e1SGlauber Costa enum res_type {
20886ae53e1SGlauber Costa _MEM,
20986ae53e1SGlauber Costa _MEMSWAP,
210510fc4e1SGlauber Costa _KMEM,
211d55f90bfSVladimir Davydov _TCP,
21286ae53e1SGlauber Costa };
21386ae53e1SGlauber Costa
214a0db00fcSKirill A. Shutemov #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
215a0db00fcSKirill A. Shutemov #define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)
2168c7c6e34SKAMEZAWA Hiroyuki #define MEMFILE_ATTR(val) ((val) & 0xffff)
2178c7c6e34SKAMEZAWA Hiroyuki
218b05706f1SKirill Tkhai /*
219b05706f1SKirill Tkhai * Iteration constructs for visiting all cgroups (under a tree). If
220b05706f1SKirill Tkhai * loops are exited prematurely (break), mem_cgroup_iter_break() must
221b05706f1SKirill Tkhai * be used for reference counting.
222b05706f1SKirill Tkhai */
223b05706f1SKirill Tkhai #define for_each_mem_cgroup_tree(iter, root) \
224b05706f1SKirill Tkhai for (iter = mem_cgroup_iter(root, NULL, NULL); \
225b05706f1SKirill Tkhai iter != NULL; \
226b05706f1SKirill Tkhai iter = mem_cgroup_iter(root, iter, NULL))
227b05706f1SKirill Tkhai
228b05706f1SKirill Tkhai #define for_each_mem_cgroup(iter) \
229b05706f1SKirill Tkhai for (iter = mem_cgroup_iter(NULL, NULL, NULL); \
230b05706f1SKirill Tkhai iter != NULL; \
231b05706f1SKirill Tkhai iter = mem_cgroup_iter(NULL, iter, NULL))
232b05706f1SKirill Tkhai
task_is_dying(void)233a4ebf1b6SVasily Averin static inline bool task_is_dying(void)
2347775faceSTetsuo Handa {
2357775faceSTetsuo Handa return tsk_is_oom_victim(current) || fatal_signal_pending(current) ||
2367775faceSTetsuo Handa (current->flags & PF_EXITING);
2377775faceSTetsuo Handa }
2387775faceSTetsuo Handa
23970ddf637SAnton Vorontsov /* Some nice accessors for the vmpressure. */
memcg_to_vmpressure(struct mem_cgroup * memcg)24070ddf637SAnton Vorontsov struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
24170ddf637SAnton Vorontsov {
24270ddf637SAnton Vorontsov if (!memcg)
24370ddf637SAnton Vorontsov memcg = root_mem_cgroup;
24470ddf637SAnton Vorontsov return &memcg->vmpressure;
24570ddf637SAnton Vorontsov }
24670ddf637SAnton Vorontsov
vmpressure_to_memcg(struct vmpressure * vmpr)2479647875bSHui Su struct mem_cgroup *vmpressure_to_memcg(struct vmpressure *vmpr)
24870ddf637SAnton Vorontsov {
2499647875bSHui Su return container_of(vmpr, struct mem_cgroup, vmpressure);
25070ddf637SAnton Vorontsov }
25170ddf637SAnton Vorontsov
25284c07d11SKirill Tkhai #ifdef CONFIG_MEMCG_KMEM
2530764db9bSRoman Gushchin static DEFINE_SPINLOCK(objcg_lock);
254bf4f0599SRoman Gushchin
mem_cgroup_kmem_disabled(void)2554d5c8aedSRoman Gushchin bool mem_cgroup_kmem_disabled(void)
2564d5c8aedSRoman Gushchin {
2574d5c8aedSRoman Gushchin return cgroup_memory_nokmem;
2584d5c8aedSRoman Gushchin }
2594d5c8aedSRoman Gushchin
260f1286faeSMuchun Song static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg,
261c1a660deSRoman Gushchin unsigned int nr_pages);
262c1a660deSRoman Gushchin
obj_cgroup_release(struct percpu_ref * ref)263bf4f0599SRoman Gushchin static void obj_cgroup_release(struct percpu_ref *ref)
264bf4f0599SRoman Gushchin {
265bf4f0599SRoman Gushchin struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt);
266bf4f0599SRoman Gushchin unsigned int nr_bytes;
267bf4f0599SRoman Gushchin unsigned int nr_pages;
268bf4f0599SRoman Gushchin unsigned long flags;
269bf4f0599SRoman Gushchin
270bf4f0599SRoman Gushchin /*
271bf4f0599SRoman Gushchin * At this point all allocated objects are freed, and
272bf4f0599SRoman Gushchin * objcg->nr_charged_bytes can't have an arbitrary byte value.
273bf4f0599SRoman Gushchin * However, it can be PAGE_SIZE or (x * PAGE_SIZE).
274bf4f0599SRoman Gushchin *
275bf4f0599SRoman Gushchin * The following sequence can lead to it:
276bf4f0599SRoman Gushchin * 1) CPU0: objcg == stock->cached_objcg
277bf4f0599SRoman Gushchin * 2) CPU1: we do a small allocation (e.g. 92 bytes),
278bf4f0599SRoman Gushchin * PAGE_SIZE bytes are charged
279bf4f0599SRoman Gushchin * 3) CPU1: a process from another memcg is allocating something,
280bf4f0599SRoman Gushchin * the stock if flushed,
281bf4f0599SRoman Gushchin * objcg->nr_charged_bytes = PAGE_SIZE - 92
282bf4f0599SRoman Gushchin * 5) CPU0: we do release this object,
283bf4f0599SRoman Gushchin * 92 bytes are added to stock->nr_bytes
284bf4f0599SRoman Gushchin * 6) CPU0: stock is flushed,
285bf4f0599SRoman Gushchin * 92 bytes are added to objcg->nr_charged_bytes
286bf4f0599SRoman Gushchin *
287bf4f0599SRoman Gushchin * In the result, nr_charged_bytes == PAGE_SIZE.
288bf4f0599SRoman Gushchin * This page will be uncharged in obj_cgroup_release().
289bf4f0599SRoman Gushchin */
290bf4f0599SRoman Gushchin nr_bytes = atomic_read(&objcg->nr_charged_bytes);
291bf4f0599SRoman Gushchin WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1));
292bf4f0599SRoman Gushchin nr_pages = nr_bytes >> PAGE_SHIFT;
293bf4f0599SRoman Gushchin
294bf4f0599SRoman Gushchin if (nr_pages)
295f1286faeSMuchun Song obj_cgroup_uncharge_pages(objcg, nr_pages);
296271dd6b1SMuchun Song
2970764db9bSRoman Gushchin spin_lock_irqsave(&objcg_lock, flags);
298bf4f0599SRoman Gushchin list_del(&objcg->list);
2990764db9bSRoman Gushchin spin_unlock_irqrestore(&objcg_lock, flags);
300bf4f0599SRoman Gushchin
301bf4f0599SRoman Gushchin percpu_ref_exit(ref);
302bf4f0599SRoman Gushchin kfree_rcu(objcg, rcu);
303bf4f0599SRoman Gushchin }
304bf4f0599SRoman Gushchin
obj_cgroup_alloc(void)305bf4f0599SRoman Gushchin static struct obj_cgroup *obj_cgroup_alloc(void)
306bf4f0599SRoman Gushchin {
307bf4f0599SRoman Gushchin struct obj_cgroup *objcg;
308bf4f0599SRoman Gushchin int ret;
309bf4f0599SRoman Gushchin
310bf4f0599SRoman Gushchin objcg = kzalloc(sizeof(struct obj_cgroup), GFP_KERNEL);
311bf4f0599SRoman Gushchin if (!objcg)
312bf4f0599SRoman Gushchin return NULL;
313bf4f0599SRoman Gushchin
314bf4f0599SRoman Gushchin ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0,
315bf4f0599SRoman Gushchin GFP_KERNEL);
316bf4f0599SRoman Gushchin if (ret) {
317bf4f0599SRoman Gushchin kfree(objcg);
318bf4f0599SRoman Gushchin return NULL;
319bf4f0599SRoman Gushchin }
320bf4f0599SRoman Gushchin INIT_LIST_HEAD(&objcg->list);
321bf4f0599SRoman Gushchin return objcg;
322bf4f0599SRoman Gushchin }
323bf4f0599SRoman Gushchin
memcg_reparent_objcgs(struct mem_cgroup * memcg,struct mem_cgroup * parent)324bf4f0599SRoman Gushchin static void memcg_reparent_objcgs(struct mem_cgroup *memcg,
325bf4f0599SRoman Gushchin struct mem_cgroup *parent)
326bf4f0599SRoman Gushchin {
327bf4f0599SRoman Gushchin struct obj_cgroup *objcg, *iter;
328bf4f0599SRoman Gushchin
329bf4f0599SRoman Gushchin objcg = rcu_replace_pointer(memcg->objcg, NULL, true);
330bf4f0599SRoman Gushchin
3310764db9bSRoman Gushchin spin_lock_irq(&objcg_lock);
332bf4f0599SRoman Gushchin
3339838354eSMuchun Song /* 1) Ready to reparent active objcg. */
3349838354eSMuchun Song list_add(&objcg->list, &memcg->objcg_list);
3359838354eSMuchun Song /* 2) Reparent active objcg and already reparented objcgs to parent. */
3369838354eSMuchun Song list_for_each_entry(iter, &memcg->objcg_list, list)
3379838354eSMuchun Song WRITE_ONCE(iter->memcg, parent);
3389838354eSMuchun Song /* 3) Move already reparented objcgs to the parent's list */
339bf4f0599SRoman Gushchin list_splice(&memcg->objcg_list, &parent->objcg_list);
340bf4f0599SRoman Gushchin
3410764db9bSRoman Gushchin spin_unlock_irq(&objcg_lock);
342bf4f0599SRoman Gushchin
343bf4f0599SRoman Gushchin percpu_ref_kill(&objcg->refcnt);
344bf4f0599SRoman Gushchin }
345bf4f0599SRoman Gushchin
34655007d84SGlauber Costa /*
347d7f25f8aSGlauber Costa * A lot of the calls to the cache allocation functions are expected to be
348272911a4SRoman Gushchin * inlined by the compiler. Since the calls to memcg_slab_pre_alloc_hook() are
349d7f25f8aSGlauber Costa * conditional to this static branch, we'll have to allow modules that does
350d7f25f8aSGlauber Costa * kmem_cache_alloc and the such to see this symbol as well
351d7f25f8aSGlauber Costa */
352f7a449f7SRoman Gushchin DEFINE_STATIC_KEY_FALSE(memcg_kmem_online_key);
353f7a449f7SRoman Gushchin EXPORT_SYMBOL(memcg_kmem_online_key);
354b6c1a8afSYafang Shao
355b6c1a8afSYafang Shao DEFINE_STATIC_KEY_FALSE(memcg_bpf_enabled_key);
356b6c1a8afSYafang Shao EXPORT_SYMBOL(memcg_bpf_enabled_key);
3570a432dcbSYang Shi #endif
35817cc4dfeSTejun Heo
359ad7fa852STejun Heo /**
36075376c6fSMatthew Wilcox (Oracle) * mem_cgroup_css_from_folio - css of the memcg associated with a folio
36175376c6fSMatthew Wilcox (Oracle) * @folio: folio of interest
362ad7fa852STejun Heo *
363ad7fa852STejun Heo * If memcg is bound to the default hierarchy, css of the memcg associated
36475376c6fSMatthew Wilcox (Oracle) * with @folio is returned. The returned css remains associated with @folio
365ad7fa852STejun Heo * until it is released.
366ad7fa852STejun Heo *
367ad7fa852STejun Heo * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
368ad7fa852STejun Heo * is returned.
369ad7fa852STejun Heo */
mem_cgroup_css_from_folio(struct folio * folio)37075376c6fSMatthew Wilcox (Oracle) struct cgroup_subsys_state *mem_cgroup_css_from_folio(struct folio *folio)
371ad7fa852STejun Heo {
37275376c6fSMatthew Wilcox (Oracle) struct mem_cgroup *memcg = folio_memcg(folio);
373ad7fa852STejun Heo
3749e10a130STejun Heo if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
375ad7fa852STejun Heo memcg = root_mem_cgroup;
376ad7fa852STejun Heo
377ad7fa852STejun Heo return &memcg->css;
378ad7fa852STejun Heo }
379ad7fa852STejun Heo
3802fc04524SVladimir Davydov /**
3812fc04524SVladimir Davydov * page_cgroup_ino - return inode number of the memcg a page is charged to
3822fc04524SVladimir Davydov * @page: the page
3832fc04524SVladimir Davydov *
3842fc04524SVladimir Davydov * Look up the closest online ancestor of the memory cgroup @page is charged to
3852fc04524SVladimir Davydov * and return its inode number or 0 if @page is not charged to any cgroup. It
3862fc04524SVladimir Davydov * is safe to call this function without holding a reference to @page.
3872fc04524SVladimir Davydov *
3882fc04524SVladimir Davydov * Note, this function is inherently racy, because there is nothing to prevent
3892fc04524SVladimir Davydov * the cgroup inode from getting torn down and potentially reallocated a moment
3902fc04524SVladimir Davydov * after page_cgroup_ino() returns, so it only should be used by callers that
3912fc04524SVladimir Davydov * do not care (such as procfs interfaces).
3922fc04524SVladimir Davydov */
page_cgroup_ino(struct page * page)3932fc04524SVladimir Davydov ino_t page_cgroup_ino(struct page *page)
3942fc04524SVladimir Davydov {
3952fc04524SVladimir Davydov struct mem_cgroup *memcg;
3962fc04524SVladimir Davydov unsigned long ino = 0;
3972fc04524SVladimir Davydov
3982fc04524SVladimir Davydov rcu_read_lock();
399ec342603SYosry Ahmed /* page_folio() is racy here, but the entire function is racy anyway */
400ec342603SYosry Ahmed memcg = folio_memcg_check(page_folio(page));
401286e04b8SRoman Gushchin
4022fc04524SVladimir Davydov while (memcg && !(memcg->css.flags & CSS_ONLINE))
4032fc04524SVladimir Davydov memcg = parent_mem_cgroup(memcg);
4042fc04524SVladimir Davydov if (memcg)
4052fc04524SVladimir Davydov ino = cgroup_ino(memcg->css.cgroup);
4062fc04524SVladimir Davydov rcu_read_unlock();
4072fc04524SVladimir Davydov return ino;
4082fc04524SVladimir Davydov }
4092fc04524SVladimir Davydov
__mem_cgroup_insert_exceeded(struct mem_cgroup_per_node * mz,struct mem_cgroup_tree_per_node * mctz,unsigned long new_usage_in_excess)410ef8f2327SMel Gorman static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
411ef8f2327SMel Gorman struct mem_cgroup_tree_per_node *mctz,
4123e32cb2eSJohannes Weiner unsigned long new_usage_in_excess)
413bb4cc1a8SAndrew Morton {
414bb4cc1a8SAndrew Morton struct rb_node **p = &mctz->rb_root.rb_node;
415bb4cc1a8SAndrew Morton struct rb_node *parent = NULL;
416ef8f2327SMel Gorman struct mem_cgroup_per_node *mz_node;
417fa90b2fdSDavidlohr Bueso bool rightmost = true;
418bb4cc1a8SAndrew Morton
419bb4cc1a8SAndrew Morton if (mz->on_tree)
420bb4cc1a8SAndrew Morton return;
421bb4cc1a8SAndrew Morton
422bb4cc1a8SAndrew Morton mz->usage_in_excess = new_usage_in_excess;
423bb4cc1a8SAndrew Morton if (!mz->usage_in_excess)
424bb4cc1a8SAndrew Morton return;
425bb4cc1a8SAndrew Morton while (*p) {
426bb4cc1a8SAndrew Morton parent = *p;
427ef8f2327SMel Gorman mz_node = rb_entry(parent, struct mem_cgroup_per_node,
428bb4cc1a8SAndrew Morton tree_node);
429fa90b2fdSDavidlohr Bueso if (mz->usage_in_excess < mz_node->usage_in_excess) {
430bb4cc1a8SAndrew Morton p = &(*p)->rb_left;
431fa90b2fdSDavidlohr Bueso rightmost = false;
432378876b0SMiaohe Lin } else {
433bb4cc1a8SAndrew Morton p = &(*p)->rb_right;
434bb4cc1a8SAndrew Morton }
435378876b0SMiaohe Lin }
436fa90b2fdSDavidlohr Bueso
437fa90b2fdSDavidlohr Bueso if (rightmost)
438fa90b2fdSDavidlohr Bueso mctz->rb_rightmost = &mz->tree_node;
439fa90b2fdSDavidlohr Bueso
440bb4cc1a8SAndrew Morton rb_link_node(&mz->tree_node, parent, p);
441bb4cc1a8SAndrew Morton rb_insert_color(&mz->tree_node, &mctz->rb_root);
442bb4cc1a8SAndrew Morton mz->on_tree = true;
443bb4cc1a8SAndrew Morton }
444bb4cc1a8SAndrew Morton
__mem_cgroup_remove_exceeded(struct mem_cgroup_per_node * mz,struct mem_cgroup_tree_per_node * mctz)445ef8f2327SMel Gorman static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
446ef8f2327SMel Gorman struct mem_cgroup_tree_per_node *mctz)
447bb4cc1a8SAndrew Morton {
448bb4cc1a8SAndrew Morton if (!mz->on_tree)
449bb4cc1a8SAndrew Morton return;
450fa90b2fdSDavidlohr Bueso
451fa90b2fdSDavidlohr Bueso if (&mz->tree_node == mctz->rb_rightmost)
452fa90b2fdSDavidlohr Bueso mctz->rb_rightmost = rb_prev(&mz->tree_node);
453fa90b2fdSDavidlohr Bueso
454bb4cc1a8SAndrew Morton rb_erase(&mz->tree_node, &mctz->rb_root);
455bb4cc1a8SAndrew Morton mz->on_tree = false;
456bb4cc1a8SAndrew Morton }
457bb4cc1a8SAndrew Morton
mem_cgroup_remove_exceeded(struct mem_cgroup_per_node * mz,struct mem_cgroup_tree_per_node * mctz)458ef8f2327SMel Gorman static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
459ef8f2327SMel Gorman struct mem_cgroup_tree_per_node *mctz)
460bb4cc1a8SAndrew Morton {
4610a31bc97SJohannes Weiner unsigned long flags;
4620a31bc97SJohannes Weiner
4630a31bc97SJohannes Weiner spin_lock_irqsave(&mctz->lock, flags);
464cf2c8127SJohannes Weiner __mem_cgroup_remove_exceeded(mz, mctz);
4650a31bc97SJohannes Weiner spin_unlock_irqrestore(&mctz->lock, flags);
466bb4cc1a8SAndrew Morton }
467bb4cc1a8SAndrew Morton
soft_limit_excess(struct mem_cgroup * memcg)4683e32cb2eSJohannes Weiner static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
4693e32cb2eSJohannes Weiner {
4703e32cb2eSJohannes Weiner unsigned long nr_pages = page_counter_read(&memcg->memory);
4714db0c3c2SJason Low unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
4723e32cb2eSJohannes Weiner unsigned long excess = 0;
4733e32cb2eSJohannes Weiner
4743e32cb2eSJohannes Weiner if (nr_pages > soft_limit)
4753e32cb2eSJohannes Weiner excess = nr_pages - soft_limit;
4763e32cb2eSJohannes Weiner
4773e32cb2eSJohannes Weiner return excess;
4783e32cb2eSJohannes Weiner }
479bb4cc1a8SAndrew Morton
mem_cgroup_update_tree(struct mem_cgroup * memcg,int nid)480658b69c9SMatthew Wilcox (Oracle) static void mem_cgroup_update_tree(struct mem_cgroup *memcg, int nid)
481bb4cc1a8SAndrew Morton {
4823e32cb2eSJohannes Weiner unsigned long excess;
483ef8f2327SMel Gorman struct mem_cgroup_per_node *mz;
484ef8f2327SMel Gorman struct mem_cgroup_tree_per_node *mctz;
485bb4cc1a8SAndrew Morton
486e4dde56cSYu Zhao if (lru_gen_enabled()) {
48736c7b4dbST.J. Alumbaugh if (soft_limit_excess(memcg))
4885c7e7a0dST.J. Alumbaugh lru_gen_soft_reclaim(memcg, nid);
489e4dde56cSYu Zhao return;
490e4dde56cSYu Zhao }
491e4dde56cSYu Zhao
4922ab082baSMatthew Wilcox (Oracle) mctz = soft_limit_tree.rb_tree_per_node[nid];
493bfc7228bSLaurent Dufour if (!mctz)
494bfc7228bSLaurent Dufour return;
495bb4cc1a8SAndrew Morton /*
496bb4cc1a8SAndrew Morton * Necessary to update all ancestors when hierarchy is used.
497bb4cc1a8SAndrew Morton * because their event counter is not touched.
498bb4cc1a8SAndrew Morton */
499bb4cc1a8SAndrew Morton for (; memcg; memcg = parent_mem_cgroup(memcg)) {
500658b69c9SMatthew Wilcox (Oracle) mz = memcg->nodeinfo[nid];
5013e32cb2eSJohannes Weiner excess = soft_limit_excess(memcg);
502bb4cc1a8SAndrew Morton /*
503bb4cc1a8SAndrew Morton * We have to update the tree if mz is on RB-tree or
504bb4cc1a8SAndrew Morton * mem is over its softlimit.
505bb4cc1a8SAndrew Morton */
506bb4cc1a8SAndrew Morton if (excess || mz->on_tree) {
5070a31bc97SJohannes Weiner unsigned long flags;
5080a31bc97SJohannes Weiner
5090a31bc97SJohannes Weiner spin_lock_irqsave(&mctz->lock, flags);
510bb4cc1a8SAndrew Morton /* if on-tree, remove it */
511bb4cc1a8SAndrew Morton if (mz->on_tree)
512cf2c8127SJohannes Weiner __mem_cgroup_remove_exceeded(mz, mctz);
513bb4cc1a8SAndrew Morton /*
514bb4cc1a8SAndrew Morton * Insert again. mz->usage_in_excess will be updated.
515bb4cc1a8SAndrew Morton * If excess is 0, no tree ops.
516bb4cc1a8SAndrew Morton */
517cf2c8127SJohannes Weiner __mem_cgroup_insert_exceeded(mz, mctz, excess);
5180a31bc97SJohannes Weiner spin_unlock_irqrestore(&mctz->lock, flags);
519bb4cc1a8SAndrew Morton }
520bb4cc1a8SAndrew Morton }
521bb4cc1a8SAndrew Morton }
522bb4cc1a8SAndrew Morton
mem_cgroup_remove_from_trees(struct mem_cgroup * memcg)523bb4cc1a8SAndrew Morton static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
524bb4cc1a8SAndrew Morton {
525ef8f2327SMel Gorman struct mem_cgroup_tree_per_node *mctz;
526ef8f2327SMel Gorman struct mem_cgroup_per_node *mz;
527ef8f2327SMel Gorman int nid;
528bb4cc1a8SAndrew Morton
529e231875bSJianyu Zhan for_each_node(nid) {
530a3747b53SJohannes Weiner mz = memcg->nodeinfo[nid];
5312ab082baSMatthew Wilcox (Oracle) mctz = soft_limit_tree.rb_tree_per_node[nid];
532bfc7228bSLaurent Dufour if (mctz)
533cf2c8127SJohannes Weiner mem_cgroup_remove_exceeded(mz, mctz);
534bb4cc1a8SAndrew Morton }
535bb4cc1a8SAndrew Morton }
536bb4cc1a8SAndrew Morton
537ef8f2327SMel Gorman static struct mem_cgroup_per_node *
__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node * mctz)538ef8f2327SMel Gorman __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
539bb4cc1a8SAndrew Morton {
540ef8f2327SMel Gorman struct mem_cgroup_per_node *mz;
541bb4cc1a8SAndrew Morton
542bb4cc1a8SAndrew Morton retry:
543bb4cc1a8SAndrew Morton mz = NULL;
544fa90b2fdSDavidlohr Bueso if (!mctz->rb_rightmost)
545bb4cc1a8SAndrew Morton goto done; /* Nothing to reclaim from */
546bb4cc1a8SAndrew Morton
547fa90b2fdSDavidlohr Bueso mz = rb_entry(mctz->rb_rightmost,
548fa90b2fdSDavidlohr Bueso struct mem_cgroup_per_node, tree_node);
549bb4cc1a8SAndrew Morton /*
550bb4cc1a8SAndrew Morton * Remove the node now but someone else can add it back,
551bb4cc1a8SAndrew Morton * we will to add it back at the end of reclaim to its correct
552bb4cc1a8SAndrew Morton * position in the tree.
553bb4cc1a8SAndrew Morton */
554cf2c8127SJohannes Weiner __mem_cgroup_remove_exceeded(mz, mctz);
5553e32cb2eSJohannes Weiner if (!soft_limit_excess(mz->memcg) ||
5568965aa28SShakeel Butt !css_tryget(&mz->memcg->css))
557bb4cc1a8SAndrew Morton goto retry;
558bb4cc1a8SAndrew Morton done:
559bb4cc1a8SAndrew Morton return mz;
560bb4cc1a8SAndrew Morton }
561bb4cc1a8SAndrew Morton
562ef8f2327SMel Gorman static struct mem_cgroup_per_node *
mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node * mctz)563ef8f2327SMel Gorman mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
564bb4cc1a8SAndrew Morton {
565ef8f2327SMel Gorman struct mem_cgroup_per_node *mz;
566bb4cc1a8SAndrew Morton
5670a31bc97SJohannes Weiner spin_lock_irq(&mctz->lock);
568bb4cc1a8SAndrew Morton mz = __mem_cgroup_largest_soft_limit_node(mctz);
5690a31bc97SJohannes Weiner spin_unlock_irq(&mctz->lock);
570bb4cc1a8SAndrew Morton return mz;
571bb4cc1a8SAndrew Morton }
572bb4cc1a8SAndrew Morton
57311192d9cSShakeel Butt /*
57411192d9cSShakeel Butt * memcg and lruvec stats flushing
57511192d9cSShakeel Butt *
57611192d9cSShakeel Butt * Many codepaths leading to stats update or read are performance sensitive and
57711192d9cSShakeel Butt * adding stats flushing in such codepaths is not desirable. So, to optimize the
57811192d9cSShakeel Butt * flushing the kernel does:
57911192d9cSShakeel Butt *
58011192d9cSShakeel Butt * 1) Periodically and asynchronously flush the stats every 2 seconds to not let
58111192d9cSShakeel Butt * rstat update tree grow unbounded.
58211192d9cSShakeel Butt *
58311192d9cSShakeel Butt * 2) Flush the stats synchronously on reader side only when there are more than
58411192d9cSShakeel Butt * (MEMCG_CHARGE_BATCH * nr_cpus) update events. Though this optimization
58511192d9cSShakeel Butt * will let stats be out of sync by atmost (MEMCG_CHARGE_BATCH * nr_cpus) but
58611192d9cSShakeel Butt * only for 2 seconds due to (1).
58711192d9cSShakeel Butt */
58811192d9cSShakeel Butt static void flush_memcg_stats_dwork(struct work_struct *w);
58911192d9cSShakeel Butt static DECLARE_DEFERRABLE_WORK(stats_flush_dwork, flush_memcg_stats_dwork);
59011192d9cSShakeel Butt static DEFINE_PER_CPU(unsigned int, stats_updates);
5913cd9992bSYosry Ahmed static atomic_t stats_flush_ongoing = ATOMIC_INIT(0);
59211192d9cSShakeel Butt static atomic_t stats_flush_threshold = ATOMIC_INIT(0);
5939b301615SShakeel Butt static u64 flush_next_time;
5949b301615SShakeel Butt
5959b301615SShakeel Butt #define FLUSH_TIME (2UL*HZ)
59611192d9cSShakeel Butt
597be3e67b5SSebastian Andrzej Siewior /*
598be3e67b5SSebastian Andrzej Siewior * Accessors to ensure that preemption is disabled on PREEMPT_RT because it can
599be3e67b5SSebastian Andrzej Siewior * not rely on this as part of an acquired spinlock_t lock. These functions are
600be3e67b5SSebastian Andrzej Siewior * never used in hardirq context on PREEMPT_RT and therefore disabling preemtion
601be3e67b5SSebastian Andrzej Siewior * is sufficient.
602be3e67b5SSebastian Andrzej Siewior */
memcg_stats_lock(void)603be3e67b5SSebastian Andrzej Siewior static void memcg_stats_lock(void)
604be3e67b5SSebastian Andrzej Siewior {
605e575d401SThomas Gleixner preempt_disable_nested();
606e575d401SThomas Gleixner VM_WARN_ON_IRQS_ENABLED();
607be3e67b5SSebastian Andrzej Siewior }
608be3e67b5SSebastian Andrzej Siewior
__memcg_stats_lock(void)609be3e67b5SSebastian Andrzej Siewior static void __memcg_stats_lock(void)
610be3e67b5SSebastian Andrzej Siewior {
611e575d401SThomas Gleixner preempt_disable_nested();
612be3e67b5SSebastian Andrzej Siewior }
613be3e67b5SSebastian Andrzej Siewior
memcg_stats_unlock(void)614be3e67b5SSebastian Andrzej Siewior static void memcg_stats_unlock(void)
615be3e67b5SSebastian Andrzej Siewior {
616e575d401SThomas Gleixner preempt_enable_nested();
617be3e67b5SSebastian Andrzej Siewior }
618be3e67b5SSebastian Andrzej Siewior
memcg_rstat_updated(struct mem_cgroup * memcg,int val)6195b3be698SShakeel Butt static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val)
62011192d9cSShakeel Butt {
6215b3be698SShakeel Butt unsigned int x;
6225b3be698SShakeel Butt
623f9d911caSYosry Ahmed if (!val)
624f9d911caSYosry Ahmed return;
625f9d911caSYosry Ahmed
62611192d9cSShakeel Butt cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id());
6275b3be698SShakeel Butt
6285b3be698SShakeel Butt x = __this_cpu_add_return(stats_updates, abs(val));
6295b3be698SShakeel Butt if (x > MEMCG_CHARGE_BATCH) {
630873f64b7SJiebin Sun /*
631873f64b7SJiebin Sun * If stats_flush_threshold exceeds the threshold
632873f64b7SJiebin Sun * (>num_online_cpus()), cgroup stats update will be triggered
633873f64b7SJiebin Sun * in __mem_cgroup_flush_stats(). Increasing this var further
634873f64b7SJiebin Sun * is redundant and simply adds overhead in atomic update.
635873f64b7SJiebin Sun */
636873f64b7SJiebin Sun if (atomic_read(&stats_flush_threshold) <= num_online_cpus())
6375b3be698SShakeel Butt atomic_add(x / MEMCG_CHARGE_BATCH, &stats_flush_threshold);
6385b3be698SShakeel Butt __this_cpu_write(stats_updates, 0);
6395b3be698SShakeel Butt }
64011192d9cSShakeel Butt }
64111192d9cSShakeel Butt
do_flush_stats(void)64235822fdaSYosry Ahmed static void do_flush_stats(void)
64311192d9cSShakeel Butt {
6443cd9992bSYosry Ahmed /*
6453cd9992bSYosry Ahmed * We always flush the entire tree, so concurrent flushers can just
6463cd9992bSYosry Ahmed * skip. This avoids a thundering herd problem on the rstat global lock
6473cd9992bSYosry Ahmed * from memcg flushers (e.g. reclaim, refault, etc).
6483cd9992bSYosry Ahmed */
6493cd9992bSYosry Ahmed if (atomic_read(&stats_flush_ongoing) ||
6503cd9992bSYosry Ahmed atomic_xchg(&stats_flush_ongoing, 1))
65111192d9cSShakeel Butt return;
65211192d9cSShakeel Butt
6533cd9992bSYosry Ahmed WRITE_ONCE(flush_next_time, jiffies_64 + 2*FLUSH_TIME);
6549fad9aeeSYosry Ahmed
6559fad9aeeSYosry Ahmed cgroup_rstat_flush(root_mem_cgroup->css.cgroup);
6569fad9aeeSYosry Ahmed
65711192d9cSShakeel Butt atomic_set(&stats_flush_threshold, 0);
6583cd9992bSYosry Ahmed atomic_set(&stats_flush_ongoing, 0);
65911192d9cSShakeel Butt }
66011192d9cSShakeel Butt
mem_cgroup_flush_stats(void)6619fad9aeeSYosry Ahmed void mem_cgroup_flush_stats(void)
6629fad9aeeSYosry Ahmed {
66335822fdaSYosry Ahmed if (atomic_read(&stats_flush_threshold) > num_online_cpus())
66435822fdaSYosry Ahmed do_flush_stats();
6659fad9aeeSYosry Ahmed }
6669fad9aeeSYosry Ahmed
mem_cgroup_flush_stats_ratelimited(void)6674009b2f1SYosry Ahmed void mem_cgroup_flush_stats_ratelimited(void)
6689b301615SShakeel Butt {
6693cd9992bSYosry Ahmed if (time_after64(jiffies_64, READ_ONCE(flush_next_time)))
6704009b2f1SYosry Ahmed mem_cgroup_flush_stats();
6719b301615SShakeel Butt }
6729b301615SShakeel Butt
flush_memcg_stats_dwork(struct work_struct * w)67311192d9cSShakeel Butt static void flush_memcg_stats_dwork(struct work_struct *w)
67411192d9cSShakeel Butt {
6759fad9aeeSYosry Ahmed /*
6769fad9aeeSYosry Ahmed * Always flush here so that flushing in latency-sensitive paths is
6779fad9aeeSYosry Ahmed * as cheap as possible.
6789fad9aeeSYosry Ahmed */
67935822fdaSYosry Ahmed do_flush_stats();
6809b301615SShakeel Butt queue_delayed_work(system_unbound_wq, &stats_flush_dwork, FLUSH_TIME);
68111192d9cSShakeel Butt }
68211192d9cSShakeel Butt
683d396def5SShakeel Butt /* Subset of vm_event_item to report for memcg event stats */
684d396def5SShakeel Butt static const unsigned int memcg_vm_event_stat[] = {
6858278f1c7SShakeel Butt PGPGIN,
6868278f1c7SShakeel Butt PGPGOUT,
687d396def5SShakeel Butt PGSCAN_KSWAPD,
688d396def5SShakeel Butt PGSCAN_DIRECT,
68957e9cc50SJohannes Weiner PGSCAN_KHUGEPAGED,
690d396def5SShakeel Butt PGSTEAL_KSWAPD,
691d396def5SShakeel Butt PGSTEAL_DIRECT,
69257e9cc50SJohannes Weiner PGSTEAL_KHUGEPAGED,
693d396def5SShakeel Butt PGFAULT,
694d396def5SShakeel Butt PGMAJFAULT,
695d396def5SShakeel Butt PGREFILL,
696d396def5SShakeel Butt PGACTIVATE,
697d396def5SShakeel Butt PGDEACTIVATE,
698d396def5SShakeel Butt PGLAZYFREE,
699d396def5SShakeel Butt PGLAZYFREED,
700d396def5SShakeel Butt #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP)
701d396def5SShakeel Butt ZSWPIN,
702d396def5SShakeel Butt ZSWPOUT,
703d396def5SShakeel Butt #endif
704d396def5SShakeel Butt #ifdef CONFIG_TRANSPARENT_HUGEPAGE
705d396def5SShakeel Butt THP_FAULT_ALLOC,
706d396def5SShakeel Butt THP_COLLAPSE_ALLOC,
707d396def5SShakeel Butt #endif
708d396def5SShakeel Butt };
709d396def5SShakeel Butt
7108278f1c7SShakeel Butt #define NR_MEMCG_EVENTS ARRAY_SIZE(memcg_vm_event_stat)
7118278f1c7SShakeel Butt static int mem_cgroup_events_index[NR_VM_EVENT_ITEMS] __read_mostly;
7128278f1c7SShakeel Butt
init_memcg_events(void)7138278f1c7SShakeel Butt static void init_memcg_events(void)
7148278f1c7SShakeel Butt {
7158278f1c7SShakeel Butt int i;
7168278f1c7SShakeel Butt
7178278f1c7SShakeel Butt for (i = 0; i < NR_MEMCG_EVENTS; ++i)
7188278f1c7SShakeel Butt mem_cgroup_events_index[memcg_vm_event_stat[i]] = i + 1;
7198278f1c7SShakeel Butt }
7208278f1c7SShakeel Butt
memcg_events_index(enum vm_event_item idx)7218278f1c7SShakeel Butt static inline int memcg_events_index(enum vm_event_item idx)
7228278f1c7SShakeel Butt {
7238278f1c7SShakeel Butt return mem_cgroup_events_index[idx] - 1;
7248278f1c7SShakeel Butt }
7258278f1c7SShakeel Butt
726410f8e82SShakeel Butt struct memcg_vmstats_percpu {
727410f8e82SShakeel Butt /* Local (CPU and cgroup) page state & events */
728410f8e82SShakeel Butt long state[MEMCG_NR_STAT];
7298278f1c7SShakeel Butt unsigned long events[NR_MEMCG_EVENTS];
730410f8e82SShakeel Butt
731410f8e82SShakeel Butt /* Delta calculation for lockless upward propagation */
732410f8e82SShakeel Butt long state_prev[MEMCG_NR_STAT];
7338278f1c7SShakeel Butt unsigned long events_prev[NR_MEMCG_EVENTS];
734410f8e82SShakeel Butt
735410f8e82SShakeel Butt /* Cgroup1: threshold notifications & softlimit tree updates */
736410f8e82SShakeel Butt unsigned long nr_page_events;
737410f8e82SShakeel Butt unsigned long targets[MEM_CGROUP_NTARGETS];
738410f8e82SShakeel Butt };
739410f8e82SShakeel Butt
740410f8e82SShakeel Butt struct memcg_vmstats {
741410f8e82SShakeel Butt /* Aggregated (CPU and subtree) page state & events */
742410f8e82SShakeel Butt long state[MEMCG_NR_STAT];
7438278f1c7SShakeel Butt unsigned long events[NR_MEMCG_EVENTS];
744410f8e82SShakeel Butt
745f82e6bf9SYosry Ahmed /* Non-hierarchical (CPU aggregated) page state & events */
746f82e6bf9SYosry Ahmed long state_local[MEMCG_NR_STAT];
747f82e6bf9SYosry Ahmed unsigned long events_local[NR_MEMCG_EVENTS];
748f82e6bf9SYosry Ahmed
749410f8e82SShakeel Butt /* Pending child counts during tree propagation */
750410f8e82SShakeel Butt long state_pending[MEMCG_NR_STAT];
7518278f1c7SShakeel Butt unsigned long events_pending[NR_MEMCG_EVENTS];
752410f8e82SShakeel Butt };
753410f8e82SShakeel Butt
memcg_page_state(struct mem_cgroup * memcg,int idx)754410f8e82SShakeel Butt unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx)
755410f8e82SShakeel Butt {
756410f8e82SShakeel Butt long x = READ_ONCE(memcg->vmstats->state[idx]);
757410f8e82SShakeel Butt #ifdef CONFIG_SMP
758410f8e82SShakeel Butt if (x < 0)
759410f8e82SShakeel Butt x = 0;
760410f8e82SShakeel Butt #endif
761410f8e82SShakeel Butt return x;
762410f8e82SShakeel Butt }
763410f8e82SShakeel Butt
764db9adbcbSJohannes Weiner /**
765db9adbcbSJohannes Weiner * __mod_memcg_state - update cgroup memory statistics
766db9adbcbSJohannes Weiner * @memcg: the memory cgroup
767db9adbcbSJohannes Weiner * @idx: the stat item - can be enum memcg_stat_item or enum node_stat_item
768db9adbcbSJohannes Weiner * @val: delta to add to the counter, can be negative
769db9adbcbSJohannes Weiner */
__mod_memcg_state(struct mem_cgroup * memcg,int idx,int val)770db9adbcbSJohannes Weiner void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val)
771db9adbcbSJohannes Weiner {
772db9adbcbSJohannes Weiner if (mem_cgroup_disabled())
773db9adbcbSJohannes Weiner return;
774db9adbcbSJohannes Weiner
7752d146aa3SJohannes Weiner __this_cpu_add(memcg->vmstats_percpu->state[idx], val);
7765b3be698SShakeel Butt memcg_rstat_updated(memcg, val);
777db9adbcbSJohannes Weiner }
778db9adbcbSJohannes Weiner
7792d146aa3SJohannes Weiner /* idx can be of type enum memcg_stat_item or node_stat_item. */
memcg_page_state_local(struct mem_cgroup * memcg,int idx)780a18e6e6eSJohannes Weiner static unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx)
781a18e6e6eSJohannes Weiner {
782f82e6bf9SYosry Ahmed long x = READ_ONCE(memcg->vmstats->state_local[idx]);
783a18e6e6eSJohannes Weiner
784a18e6e6eSJohannes Weiner #ifdef CONFIG_SMP
785a18e6e6eSJohannes Weiner if (x < 0)
786a18e6e6eSJohannes Weiner x = 0;
787a18e6e6eSJohannes Weiner #endif
788a18e6e6eSJohannes Weiner return x;
789a18e6e6eSJohannes Weiner }
790a18e6e6eSJohannes Weiner
__mod_memcg_lruvec_state(struct lruvec * lruvec,enum node_stat_item idx,int val)791eedc4e5aSRoman Gushchin void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
792eedc4e5aSRoman Gushchin int val)
793eedc4e5aSRoman Gushchin {
794eedc4e5aSRoman Gushchin struct mem_cgroup_per_node *pn;
795eedc4e5aSRoman Gushchin struct mem_cgroup *memcg;
796eedc4e5aSRoman Gushchin
797eedc4e5aSRoman Gushchin pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
798eedc4e5aSRoman Gushchin memcg = pn->memcg;
799eedc4e5aSRoman Gushchin
800be3e67b5SSebastian Andrzej Siewior /*
801be3e67b5SSebastian Andrzej Siewior * The caller from rmap relay on disabled preemption becase they never
802be3e67b5SSebastian Andrzej Siewior * update their counter from in-interrupt context. For these two
803be3e67b5SSebastian Andrzej Siewior * counters we check that the update is never performed from an
804be3e67b5SSebastian Andrzej Siewior * interrupt context while other caller need to have disabled interrupt.
805be3e67b5SSebastian Andrzej Siewior */
806be3e67b5SSebastian Andrzej Siewior __memcg_stats_lock();
807e575d401SThomas Gleixner if (IS_ENABLED(CONFIG_DEBUG_VM)) {
808be3e67b5SSebastian Andrzej Siewior switch (idx) {
809be3e67b5SSebastian Andrzej Siewior case NR_ANON_MAPPED:
810be3e67b5SSebastian Andrzej Siewior case NR_FILE_MAPPED:
811be3e67b5SSebastian Andrzej Siewior case NR_ANON_THPS:
812be3e67b5SSebastian Andrzej Siewior case NR_SHMEM_PMDMAPPED:
813be3e67b5SSebastian Andrzej Siewior case NR_FILE_PMDMAPPED:
814be3e67b5SSebastian Andrzej Siewior WARN_ON_ONCE(!in_task());
815be3e67b5SSebastian Andrzej Siewior break;
816be3e67b5SSebastian Andrzej Siewior default:
817e575d401SThomas Gleixner VM_WARN_ON_IRQS_ENABLED();
818be3e67b5SSebastian Andrzej Siewior }
819be3e67b5SSebastian Andrzej Siewior }
820be3e67b5SSebastian Andrzej Siewior
821eedc4e5aSRoman Gushchin /* Update memcg */
82211192d9cSShakeel Butt __this_cpu_add(memcg->vmstats_percpu->state[idx], val);
823eedc4e5aSRoman Gushchin
824eedc4e5aSRoman Gushchin /* Update lruvec */
8257e1c0d6fSShakeel Butt __this_cpu_add(pn->lruvec_stats_percpu->state[idx], val);
82611192d9cSShakeel Butt
8275b3be698SShakeel Butt memcg_rstat_updated(memcg, val);
828be3e67b5SSebastian Andrzej Siewior memcg_stats_unlock();
829eedc4e5aSRoman Gushchin }
830eedc4e5aSRoman Gushchin
831db9adbcbSJohannes Weiner /**
832db9adbcbSJohannes Weiner * __mod_lruvec_state - update lruvec memory statistics
833db9adbcbSJohannes Weiner * @lruvec: the lruvec
834db9adbcbSJohannes Weiner * @idx: the stat item
835db9adbcbSJohannes Weiner * @val: delta to add to the counter, can be negative
836db9adbcbSJohannes Weiner *
837db9adbcbSJohannes Weiner * The lruvec is the intersection of the NUMA node and a cgroup. This
838db9adbcbSJohannes Weiner * function updates the all three counters that are affected by a
839db9adbcbSJohannes Weiner * change of state at this level: per-node, per-cgroup, per-lruvec.
840db9adbcbSJohannes Weiner */
__mod_lruvec_state(struct lruvec * lruvec,enum node_stat_item idx,int val)841db9adbcbSJohannes Weiner void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
842db9adbcbSJohannes Weiner int val)
843db9adbcbSJohannes Weiner {
844db9adbcbSJohannes Weiner /* Update node */
845eedc4e5aSRoman Gushchin __mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
846db9adbcbSJohannes Weiner
847eedc4e5aSRoman Gushchin /* Update memcg and lruvec */
848eedc4e5aSRoman Gushchin if (!mem_cgroup_disabled())
849eedc4e5aSRoman Gushchin __mod_memcg_lruvec_state(lruvec, idx, val);
850db9adbcbSJohannes Weiner }
851db9adbcbSJohannes Weiner
__mod_lruvec_page_state(struct page * page,enum node_stat_item idx,int val)852c47d5032SShakeel Butt void __mod_lruvec_page_state(struct page *page, enum node_stat_item idx,
853c47d5032SShakeel Butt int val)
854c47d5032SShakeel Butt {
855c47d5032SShakeel Butt struct page *head = compound_head(page); /* rmap on tail pages */
856b4e0b68fSMuchun Song struct mem_cgroup *memcg;
857c47d5032SShakeel Butt pg_data_t *pgdat = page_pgdat(page);
858c47d5032SShakeel Butt struct lruvec *lruvec;
859c47d5032SShakeel Butt
860b4e0b68fSMuchun Song rcu_read_lock();
861b4e0b68fSMuchun Song memcg = page_memcg(head);
862c47d5032SShakeel Butt /* Untracked pages have no memcg, no lruvec. Update only the node */
863d635a69dSLinus Torvalds if (!memcg) {
864b4e0b68fSMuchun Song rcu_read_unlock();
865c47d5032SShakeel Butt __mod_node_page_state(pgdat, idx, val);
866c47d5032SShakeel Butt return;
867c47d5032SShakeel Butt }
868c47d5032SShakeel Butt
869d635a69dSLinus Torvalds lruvec = mem_cgroup_lruvec(memcg, pgdat);
870c47d5032SShakeel Butt __mod_lruvec_state(lruvec, idx, val);
871b4e0b68fSMuchun Song rcu_read_unlock();
872c47d5032SShakeel Butt }
873f0c0c115SShakeel Butt EXPORT_SYMBOL(__mod_lruvec_page_state);
874c47d5032SShakeel Butt
__mod_lruvec_kmem_state(void * p,enum node_stat_item idx,int val)875da3ceeffSMuchun Song void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val)
876ec9f0238SRoman Gushchin {
8774f103c63SRoman Gushchin pg_data_t *pgdat = page_pgdat(virt_to_page(p));
878ec9f0238SRoman Gushchin struct mem_cgroup *memcg;
879ec9f0238SRoman Gushchin struct lruvec *lruvec;
880ec9f0238SRoman Gushchin
881ec9f0238SRoman Gushchin rcu_read_lock();
882fc4db90fSRoman Gushchin memcg = mem_cgroup_from_slab_obj(p);
883ec9f0238SRoman Gushchin
8848faeb1ffSMuchun Song /*
8858faeb1ffSMuchun Song * Untracked pages have no memcg, no lruvec. Update only the
8868faeb1ffSMuchun Song * node. If we reparent the slab objects to the root memcg,
8878faeb1ffSMuchun Song * when we free the slab object, we need to update the per-memcg
8888faeb1ffSMuchun Song * vmstats to keep it correct for the root memcg.
8898faeb1ffSMuchun Song */
8908faeb1ffSMuchun Song if (!memcg) {
891ec9f0238SRoman Gushchin __mod_node_page_state(pgdat, idx, val);
892ec9f0238SRoman Gushchin } else {
893867e5e1dSJohannes Weiner lruvec = mem_cgroup_lruvec(memcg, pgdat);
894ec9f0238SRoman Gushchin __mod_lruvec_state(lruvec, idx, val);
895ec9f0238SRoman Gushchin }
896ec9f0238SRoman Gushchin rcu_read_unlock();
897ec9f0238SRoman Gushchin }
898ec9f0238SRoman Gushchin
899db9adbcbSJohannes Weiner /**
900db9adbcbSJohannes Weiner * __count_memcg_events - account VM events in a cgroup
901db9adbcbSJohannes Weiner * @memcg: the memory cgroup
902db9adbcbSJohannes Weiner * @idx: the event item
903f0953a1bSIngo Molnar * @count: the number of events that occurred
904db9adbcbSJohannes Weiner */
__count_memcg_events(struct mem_cgroup * memcg,enum vm_event_item idx,unsigned long count)905db9adbcbSJohannes Weiner void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
906db9adbcbSJohannes Weiner unsigned long count)
907db9adbcbSJohannes Weiner {
9088278f1c7SShakeel Butt int index = memcg_events_index(idx);
9098278f1c7SShakeel Butt
9108278f1c7SShakeel Butt if (mem_cgroup_disabled() || index < 0)
911db9adbcbSJohannes Weiner return;
912db9adbcbSJohannes Weiner
913be3e67b5SSebastian Andrzej Siewior memcg_stats_lock();
9148278f1c7SShakeel Butt __this_cpu_add(memcg->vmstats_percpu->events[index], count);
9155b3be698SShakeel Butt memcg_rstat_updated(memcg, count);
916be3e67b5SSebastian Andrzej Siewior memcg_stats_unlock();
917db9adbcbSJohannes Weiner }
918db9adbcbSJohannes Weiner
memcg_events(struct mem_cgroup * memcg,int event)91942a30035SJohannes Weiner static unsigned long memcg_events(struct mem_cgroup *memcg, int event)
920e9f8974fSJohannes Weiner {
9218278f1c7SShakeel Butt int index = memcg_events_index(event);
9228278f1c7SShakeel Butt
9238278f1c7SShakeel Butt if (index < 0)
9248278f1c7SShakeel Butt return 0;
9258278f1c7SShakeel Butt return READ_ONCE(memcg->vmstats->events[index]);
926e9f8974fSJohannes Weiner }
927e9f8974fSJohannes Weiner
memcg_events_local(struct mem_cgroup * memcg,int event)92842a30035SJohannes Weiner static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event)
92942a30035SJohannes Weiner {
9308278f1c7SShakeel Butt int index = memcg_events_index(event);
9318278f1c7SShakeel Butt
9328278f1c7SShakeel Butt if (index < 0)
9338278f1c7SShakeel Butt return 0;
934815744d7SJohannes Weiner
935f82e6bf9SYosry Ahmed return READ_ONCE(memcg->vmstats->events_local[index]);
93642a30035SJohannes Weiner }
93742a30035SJohannes Weiner
mem_cgroup_charge_statistics(struct mem_cgroup * memcg,int nr_pages)938c0ff4b85SRaghavendra K T static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
9393fba69a5SJohannes Weiner int nr_pages)
940d52aa412SKAMEZAWA Hiroyuki {
941e401f176SKAMEZAWA Hiroyuki /* pagein of a big page is an event. So, ignore page size */
942e401f176SKAMEZAWA Hiroyuki if (nr_pages > 0)
943c9019e9bSJohannes Weiner __count_memcg_events(memcg, PGPGIN, 1);
9443751d604SKAMEZAWA Hiroyuki else {
945c9019e9bSJohannes Weiner __count_memcg_events(memcg, PGPGOUT, 1);
9463751d604SKAMEZAWA Hiroyuki nr_pages = -nr_pages; /* for event */
9473751d604SKAMEZAWA Hiroyuki }
948e401f176SKAMEZAWA Hiroyuki
949871789d4SChris Down __this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages);
9506d12e2d8SKAMEZAWA Hiroyuki }
951d52aa412SKAMEZAWA Hiroyuki
mem_cgroup_event_ratelimit(struct mem_cgroup * memcg,enum mem_cgroup_events_target target)952f53d7ce3SJohannes Weiner static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
953f53d7ce3SJohannes Weiner enum mem_cgroup_events_target target)
954d2265e6fSKAMEZAWA Hiroyuki {
9557a159cc9SJohannes Weiner unsigned long val, next;
9567a159cc9SJohannes Weiner
957871789d4SChris Down val = __this_cpu_read(memcg->vmstats_percpu->nr_page_events);
958871789d4SChris Down next = __this_cpu_read(memcg->vmstats_percpu->targets[target]);
9597a159cc9SJohannes Weiner /* from time_after() in jiffies.h */
9606a1a8b80SMichal Hocko if ((long)(next - val) < 0) {
9617a159cc9SJohannes Weiner switch (target) {
9627a159cc9SJohannes Weiner case MEM_CGROUP_TARGET_THRESH:
9637a159cc9SJohannes Weiner next = val + THRESHOLDS_EVENTS_TARGET;
9647a159cc9SJohannes Weiner break;
965bb4cc1a8SAndrew Morton case MEM_CGROUP_TARGET_SOFTLIMIT:
966bb4cc1a8SAndrew Morton next = val + SOFTLIMIT_EVENTS_TARGET;
967bb4cc1a8SAndrew Morton break;
9687a159cc9SJohannes Weiner default:
969f53d7ce3SJohannes Weiner break;
9707a159cc9SJohannes Weiner }
971871789d4SChris Down __this_cpu_write(memcg->vmstats_percpu->targets[target], next);
972f53d7ce3SJohannes Weiner return true;
973f53d7ce3SJohannes Weiner }
974f53d7ce3SJohannes Weiner return false;
975d2265e6fSKAMEZAWA Hiroyuki }
976d2265e6fSKAMEZAWA Hiroyuki
977d2265e6fSKAMEZAWA Hiroyuki /*
978d2265e6fSKAMEZAWA Hiroyuki * Check events in order.
979d2265e6fSKAMEZAWA Hiroyuki *
980d2265e6fSKAMEZAWA Hiroyuki */
memcg_check_events(struct mem_cgroup * memcg,int nid)9818e88bd2dSMatthew Wilcox (Oracle) static void memcg_check_events(struct mem_cgroup *memcg, int nid)
982d2265e6fSKAMEZAWA Hiroyuki {
9832343e88dSSebastian Andrzej Siewior if (IS_ENABLED(CONFIG_PREEMPT_RT))
9842343e88dSSebastian Andrzej Siewior return;
9852343e88dSSebastian Andrzej Siewior
986d2265e6fSKAMEZAWA Hiroyuki /* threshold event is triggered in finer grain than soft limit */
987f53d7ce3SJohannes Weiner if (unlikely(mem_cgroup_event_ratelimit(memcg,
988f53d7ce3SJohannes Weiner MEM_CGROUP_TARGET_THRESH))) {
989bb4cc1a8SAndrew Morton bool do_softlimit;
990f53d7ce3SJohannes Weiner
991bb4cc1a8SAndrew Morton do_softlimit = mem_cgroup_event_ratelimit(memcg,
992bb4cc1a8SAndrew Morton MEM_CGROUP_TARGET_SOFTLIMIT);
993f53d7ce3SJohannes Weiner mem_cgroup_threshold(memcg);
994bb4cc1a8SAndrew Morton if (unlikely(do_softlimit))
9958e88bd2dSMatthew Wilcox (Oracle) mem_cgroup_update_tree(memcg, nid);
9960a31bc97SJohannes Weiner }
997d2265e6fSKAMEZAWA Hiroyuki }
998d2265e6fSKAMEZAWA Hiroyuki
mem_cgroup_from_task(struct task_struct * p)999cf475ad2SBalbir Singh struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
100078fb7466SPavel Emelianov {
100131a78f23SBalbir Singh /*
100231a78f23SBalbir Singh * mm_update_next_owner() may clear mm->owner to NULL
100331a78f23SBalbir Singh * if it races with swapoff, page migration, etc.
100431a78f23SBalbir Singh * So this can be called with p == NULL.
100531a78f23SBalbir Singh */
100631a78f23SBalbir Singh if (unlikely(!p))
100731a78f23SBalbir Singh return NULL;
100831a78f23SBalbir Singh
1009073219e9STejun Heo return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
101078fb7466SPavel Emelianov }
101133398cf2SMichal Hocko EXPORT_SYMBOL(mem_cgroup_from_task);
101278fb7466SPavel Emelianov
active_memcg(void)101304f94e3fSDan Schatzberg static __always_inline struct mem_cgroup *active_memcg(void)
101404f94e3fSDan Schatzberg {
101555a68c82SVasily Averin if (!in_task())
101604f94e3fSDan Schatzberg return this_cpu_read(int_active_memcg);
101704f94e3fSDan Schatzberg else
101804f94e3fSDan Schatzberg return current->active_memcg;
101904f94e3fSDan Schatzberg }
102004f94e3fSDan Schatzberg
1021d46eb14bSShakeel Butt /**
1022d46eb14bSShakeel Butt * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg.
1023d46eb14bSShakeel Butt * @mm: mm from which memcg should be extracted. It can be NULL.
1024d46eb14bSShakeel Butt *
102504f94e3fSDan Schatzberg * Obtain a reference on mm->memcg and returns it if successful. If mm
102604f94e3fSDan Schatzberg * is NULL, then the memcg is chosen as follows:
102704f94e3fSDan Schatzberg * 1) The active memcg, if set.
102804f94e3fSDan Schatzberg * 2) current->mm->memcg, if available
102904f94e3fSDan Schatzberg * 3) root memcg
103004f94e3fSDan Schatzberg * If mem_cgroup is disabled, NULL is returned.
1031d46eb14bSShakeel Butt */
get_mem_cgroup_from_mm(struct mm_struct * mm)1032d46eb14bSShakeel Butt struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
103354595fe2SKAMEZAWA Hiroyuki {
1034d46eb14bSShakeel Butt struct mem_cgroup *memcg;
1035d46eb14bSShakeel Butt
1036d46eb14bSShakeel Butt if (mem_cgroup_disabled())
1037d46eb14bSShakeel Butt return NULL;
10380b7f569eSKAMEZAWA Hiroyuki
10396f6acb00SMichal Hocko /*
1040f0953a1bSIngo Molnar * Page cache insertions can happen without an
10416f6acb00SMichal Hocko * actual mm context, e.g. during disk probing
10426f6acb00SMichal Hocko * on boot, loopback IO, acct() writes etc.
10432884b6b7SMuchun Song *
10442884b6b7SMuchun Song * No need to css_get on root memcg as the reference
10452884b6b7SMuchun Song * counting is disabled on the root level in the
10462884b6b7SMuchun Song * cgroup core. See CSS_NO_REF.
10476f6acb00SMichal Hocko */
104804f94e3fSDan Schatzberg if (unlikely(!mm)) {
104904f94e3fSDan Schatzberg memcg = active_memcg();
105004f94e3fSDan Schatzberg if (unlikely(memcg)) {
105104f94e3fSDan Schatzberg /* remote memcg must hold a ref */
105204f94e3fSDan Schatzberg css_get(&memcg->css);
105304f94e3fSDan Schatzberg return memcg;
105404f94e3fSDan Schatzberg }
105504f94e3fSDan Schatzberg mm = current->mm;
10566f6acb00SMichal Hocko if (unlikely(!mm))
10572884b6b7SMuchun Song return root_mem_cgroup;
105804f94e3fSDan Schatzberg }
10592884b6b7SMuchun Song
10602884b6b7SMuchun Song rcu_read_lock();
10612884b6b7SMuchun Song do {
1062c0ff4b85SRaghavendra K T memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1063c0ff4b85SRaghavendra K T if (unlikely(!memcg))
1064df381975SJohannes Weiner memcg = root_mem_cgroup;
106500d484f3SRoman Gushchin } while (!css_tryget(&memcg->css));
106654595fe2SKAMEZAWA Hiroyuki rcu_read_unlock();
1067c0ff4b85SRaghavendra K T return memcg;
106854595fe2SKAMEZAWA Hiroyuki }
1069d46eb14bSShakeel Butt EXPORT_SYMBOL(get_mem_cgroup_from_mm);
1070d46eb14bSShakeel Butt
memcg_kmem_bypass(void)10714127c650SRoman Gushchin static __always_inline bool memcg_kmem_bypass(void)
10724127c650SRoman Gushchin {
10734127c650SRoman Gushchin /* Allow remote memcg charging from any context. */
10744127c650SRoman Gushchin if (unlikely(active_memcg()))
10754127c650SRoman Gushchin return false;
10764127c650SRoman Gushchin
10774127c650SRoman Gushchin /* Memcg to charge can't be determined. */
10786126891cSVasily Averin if (!in_task() || !current->mm || (current->flags & PF_KTHREAD))
10794127c650SRoman Gushchin return true;
10804127c650SRoman Gushchin
10814127c650SRoman Gushchin return false;
10824127c650SRoman Gushchin }
10834127c650SRoman Gushchin
1084f745c6f5SShakeel Butt /**
10855660048cSJohannes Weiner * mem_cgroup_iter - iterate over memory cgroup hierarchy
10865660048cSJohannes Weiner * @root: hierarchy root
10875660048cSJohannes Weiner * @prev: previously returned memcg, NULL on first invocation
10885660048cSJohannes Weiner * @reclaim: cookie for shared reclaim walks, NULL for full walks
10895660048cSJohannes Weiner *
10905660048cSJohannes Weiner * Returns references to children of the hierarchy below @root, or
10915660048cSJohannes Weiner * @root itself, or %NULL after a full round-trip.
10925660048cSJohannes Weiner *
10935660048cSJohannes Weiner * Caller must pass the return value in @prev on subsequent
10945660048cSJohannes Weiner * invocations for reference counting, or use mem_cgroup_iter_break()
10955660048cSJohannes Weiner * to cancel a hierarchy walk before the round-trip is complete.
10965660048cSJohannes Weiner *
109705bdc520SMiaohe Lin * Reclaimers can specify a node in @reclaim to divide up the memcgs
109805bdc520SMiaohe Lin * in the hierarchy among all concurrent reclaimers operating on the
109905bdc520SMiaohe Lin * same node.
11005660048cSJohannes Weiner */
mem_cgroup_iter(struct mem_cgroup * root,struct mem_cgroup * prev,struct mem_cgroup_reclaim_cookie * reclaim)1101694fbc0fSAndrew Morton struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
11029f3a0d09SJohannes Weiner struct mem_cgroup *prev,
1103694fbc0fSAndrew Morton struct mem_cgroup_reclaim_cookie *reclaim)
110414067bb3SKAMEZAWA Hiroyuki {
11053f649ab7SKees Cook struct mem_cgroup_reclaim_iter *iter;
11065ac8fb31SJohannes Weiner struct cgroup_subsys_state *css = NULL;
11079f3a0d09SJohannes Weiner struct mem_cgroup *memcg = NULL;
11085ac8fb31SJohannes Weiner struct mem_cgroup *pos = NULL;
11097d74b06fSKAMEZAWA Hiroyuki
1110694fbc0fSAndrew Morton if (mem_cgroup_disabled())
1111694fbc0fSAndrew Morton return NULL;
11125660048cSJohannes Weiner
1113711d3d2cSKAMEZAWA Hiroyuki if (!root)
1114711d3d2cSKAMEZAWA Hiroyuki root = root_mem_cgroup;
1115711d3d2cSKAMEZAWA Hiroyuki
1116542f85f9SMichal Hocko rcu_read_lock();
11179f3a0d09SJohannes Weiner
1118527a5ec9SJohannes Weiner if (reclaim) {
1119ef8f2327SMel Gorman struct mem_cgroup_per_node *mz;
1120527a5ec9SJohannes Weiner
1121a3747b53SJohannes Weiner mz = root->nodeinfo[reclaim->pgdat->node_id];
11229da83f3fSYafang Shao iter = &mz->iter;
11235ac8fb31SJohannes Weiner
1124a9320aaeSWei Yang /*
1125a9320aaeSWei Yang * On start, join the current reclaim iteration cycle.
1126a9320aaeSWei Yang * Exit when a concurrent walker completes it.
1127a9320aaeSWei Yang */
1128a9320aaeSWei Yang if (!prev)
1129a9320aaeSWei Yang reclaim->generation = iter->generation;
1130a9320aaeSWei Yang else if (reclaim->generation != iter->generation)
1131542f85f9SMichal Hocko goto out_unlock;
11325ac8fb31SJohannes Weiner
11336df38689SVladimir Davydov while (1) {
11344db0c3c2SJason Low pos = READ_ONCE(iter->position);
11356df38689SVladimir Davydov if (!pos || css_tryget(&pos->css))
11366df38689SVladimir Davydov break;
11375ac8fb31SJohannes Weiner /*
11386df38689SVladimir Davydov * css reference reached zero, so iter->position will
11396df38689SVladimir Davydov * be cleared by ->css_released. However, we should not
11406df38689SVladimir Davydov * rely on this happening soon, because ->css_released
11416df38689SVladimir Davydov * is called from a work queue, and by busy-waiting we
11426df38689SVladimir Davydov * might block it. So we clear iter->position right
11436df38689SVladimir Davydov * away.
11445ac8fb31SJohannes Weiner */
11456df38689SVladimir Davydov (void)cmpxchg(&iter->position, pos, NULL);
11466df38689SVladimir Davydov }
114789d8330cSWei Yang } else if (prev) {
114889d8330cSWei Yang pos = prev;
1149542f85f9SMichal Hocko }
11505f578161SMichal Hocko
11515ac8fb31SJohannes Weiner if (pos)
11525ac8fb31SJohannes Weiner css = &pos->css;
11535ac8fb31SJohannes Weiner
11545ac8fb31SJohannes Weiner for (;;) {
11555ac8fb31SJohannes Weiner css = css_next_descendant_pre(css, &root->css);
11565ac8fb31SJohannes Weiner if (!css) {
11575ac8fb31SJohannes Weiner /*
11585ac8fb31SJohannes Weiner * Reclaimers share the hierarchy walk, and a
11595ac8fb31SJohannes Weiner * new one might jump in right at the end of
11605ac8fb31SJohannes Weiner * the hierarchy - make sure they see at least
11615ac8fb31SJohannes Weiner * one group and restart from the beginning.
11625ac8fb31SJohannes Weiner */
11635ac8fb31SJohannes Weiner if (!prev)
11645ac8fb31SJohannes Weiner continue;
11655ac8fb31SJohannes Weiner break;
11665f578161SMichal Hocko }
11679f3a0d09SJohannes Weiner
11685ac8fb31SJohannes Weiner /*
11695ac8fb31SJohannes Weiner * Verify the css and acquire a reference. The root
11705ac8fb31SJohannes Weiner * is provided by the caller, so we know it's alive
11715ac8fb31SJohannes Weiner * and kicking, and don't take an extra reference.
11725ac8fb31SJohannes Weiner */
117341555dadSWei Yang if (css == &root->css || css_tryget(css)) {
11745ac8fb31SJohannes Weiner memcg = mem_cgroup_from_css(css);
11755ac8fb31SJohannes Weiner break;
117641555dadSWei Yang }
11775ac8fb31SJohannes Weiner }
11789f3a0d09SJohannes Weiner
1179527a5ec9SJohannes Weiner if (reclaim) {
11805ac8fb31SJohannes Weiner /*
11816df38689SVladimir Davydov * The position could have already been updated by a competing
11826df38689SVladimir Davydov * thread, so check that the value hasn't changed since we read
11836df38689SVladimir Davydov * it to avoid reclaiming from the same cgroup twice.
11845ac8fb31SJohannes Weiner */
11856df38689SVladimir Davydov (void)cmpxchg(&iter->position, pos, memcg);
11866df38689SVladimir Davydov
11875ac8fb31SJohannes Weiner if (pos)
11885ac8fb31SJohannes Weiner css_put(&pos->css);
1189542f85f9SMichal Hocko
119019f39402SMichal Hocko if (!memcg)
1191527a5ec9SJohannes Weiner iter->generation++;
1192527a5ec9SJohannes Weiner }
11939f3a0d09SJohannes Weiner
1194542f85f9SMichal Hocko out_unlock:
1195542f85f9SMichal Hocko rcu_read_unlock();
1196c40046f3SMichal Hocko if (prev && prev != root)
1197c40046f3SMichal Hocko css_put(&prev->css);
1198c40046f3SMichal Hocko
11999f3a0d09SJohannes Weiner return memcg;
12009f3a0d09SJohannes Weiner }
12019f3a0d09SJohannes Weiner
12025660048cSJohannes Weiner /**
12035660048cSJohannes Weiner * mem_cgroup_iter_break - abort a hierarchy walk prematurely
12045660048cSJohannes Weiner * @root: hierarchy root
12055660048cSJohannes Weiner * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
12065660048cSJohannes Weiner */
mem_cgroup_iter_break(struct mem_cgroup * root,struct mem_cgroup * prev)12075660048cSJohannes Weiner void mem_cgroup_iter_break(struct mem_cgroup *root,
12089f3a0d09SJohannes Weiner struct mem_cgroup *prev)
12099f3a0d09SJohannes Weiner {
12109f3a0d09SJohannes Weiner if (!root)
12119f3a0d09SJohannes Weiner root = root_mem_cgroup;
12129f3a0d09SJohannes Weiner if (prev && prev != root)
12139f3a0d09SJohannes Weiner css_put(&prev->css);
12149f3a0d09SJohannes Weiner }
12159f3a0d09SJohannes Weiner
__invalidate_reclaim_iterators(struct mem_cgroup * from,struct mem_cgroup * dead_memcg)121654a83d6bSMiles Chen static void __invalidate_reclaim_iterators(struct mem_cgroup *from,
121754a83d6bSMiles Chen struct mem_cgroup *dead_memcg)
12186df38689SVladimir Davydov {
12196df38689SVladimir Davydov struct mem_cgroup_reclaim_iter *iter;
1220ef8f2327SMel Gorman struct mem_cgroup_per_node *mz;
1221ef8f2327SMel Gorman int nid;
12226df38689SVladimir Davydov
12236df38689SVladimir Davydov for_each_node(nid) {
1224a3747b53SJohannes Weiner mz = from->nodeinfo[nid];
12259da83f3fSYafang Shao iter = &mz->iter;
12269da83f3fSYafang Shao cmpxchg(&iter->position, dead_memcg, NULL);
12276df38689SVladimir Davydov }
12286df38689SVladimir Davydov }
122954a83d6bSMiles Chen
invalidate_reclaim_iterators(struct mem_cgroup * dead_memcg)123054a83d6bSMiles Chen static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
123154a83d6bSMiles Chen {
123254a83d6bSMiles Chen struct mem_cgroup *memcg = dead_memcg;
123354a83d6bSMiles Chen struct mem_cgroup *last;
123454a83d6bSMiles Chen
123554a83d6bSMiles Chen do {
123654a83d6bSMiles Chen __invalidate_reclaim_iterators(memcg, dead_memcg);
123754a83d6bSMiles Chen last = memcg;
123854a83d6bSMiles Chen } while ((memcg = parent_mem_cgroup(memcg)));
123954a83d6bSMiles Chen
124054a83d6bSMiles Chen /*
1241b8dd3ee9Sxupanda * When cgroup1 non-hierarchy mode is used,
124254a83d6bSMiles Chen * parent_mem_cgroup() does not walk all the way up to the
124354a83d6bSMiles Chen * cgroup root (root_mem_cgroup). So we have to handle
124454a83d6bSMiles Chen * dead_memcg from cgroup root separately.
124554a83d6bSMiles Chen */
12467848ed62SKamalesh Babulal if (!mem_cgroup_is_root(last))
124754a83d6bSMiles Chen __invalidate_reclaim_iterators(root_mem_cgroup,
124854a83d6bSMiles Chen dead_memcg);
12496df38689SVladimir Davydov }
12506df38689SVladimir Davydov
1251925b7673SJohannes Weiner /**
12527c5f64f8SVladimir Davydov * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
12537c5f64f8SVladimir Davydov * @memcg: hierarchy root
12547c5f64f8SVladimir Davydov * @fn: function to call for each task
12557c5f64f8SVladimir Davydov * @arg: argument passed to @fn
12567c5f64f8SVladimir Davydov *
12577c5f64f8SVladimir Davydov * This function iterates over tasks attached to @memcg or to any of its
12587c5f64f8SVladimir Davydov * descendants and calls @fn for each task. If @fn returns a non-zero
1259025b7799SZhangPeng * value, the function breaks the iteration loop. Otherwise, it will iterate
1260025b7799SZhangPeng * over all tasks and return 0.
12617c5f64f8SVladimir Davydov *
12627c5f64f8SVladimir Davydov * This function must not be called for the root memory cgroup.
12637c5f64f8SVladimir Davydov */
mem_cgroup_scan_tasks(struct mem_cgroup * memcg,int (* fn)(struct task_struct *,void *),void * arg)1264025b7799SZhangPeng void mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
12657c5f64f8SVladimir Davydov int (*fn)(struct task_struct *, void *), void *arg)
12667c5f64f8SVladimir Davydov {
12677c5f64f8SVladimir Davydov struct mem_cgroup *iter;
12687c5f64f8SVladimir Davydov int ret = 0;
12697c5f64f8SVladimir Davydov
12707848ed62SKamalesh Babulal BUG_ON(mem_cgroup_is_root(memcg));
12717c5f64f8SVladimir Davydov
12727c5f64f8SVladimir Davydov for_each_mem_cgroup_tree(iter, memcg) {
12737c5f64f8SVladimir Davydov struct css_task_iter it;
12747c5f64f8SVladimir Davydov struct task_struct *task;
12757c5f64f8SVladimir Davydov
1276f168a9a5STetsuo Handa css_task_iter_start(&iter->css, CSS_TASK_ITER_PROCS, &it);
12777c5f64f8SVladimir Davydov while (!ret && (task = css_task_iter_next(&it)))
12787c5f64f8SVladimir Davydov ret = fn(task, arg);
12797c5f64f8SVladimir Davydov css_task_iter_end(&it);
12807c5f64f8SVladimir Davydov if (ret) {
12817c5f64f8SVladimir Davydov mem_cgroup_iter_break(memcg, iter);
12827c5f64f8SVladimir Davydov break;
12837c5f64f8SVladimir Davydov }
12847c5f64f8SVladimir Davydov }
12857c5f64f8SVladimir Davydov }
12867c5f64f8SVladimir Davydov
12876168d0daSAlex Shi #ifdef CONFIG_DEBUG_VM
lruvec_memcg_debug(struct lruvec * lruvec,struct folio * folio)1288e809c3feSMatthew Wilcox (Oracle) void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio)
12896168d0daSAlex Shi {
12906168d0daSAlex Shi struct mem_cgroup *memcg;
12916168d0daSAlex Shi
12926168d0daSAlex Shi if (mem_cgroup_disabled())
12936168d0daSAlex Shi return;
12946168d0daSAlex Shi
1295e809c3feSMatthew Wilcox (Oracle) memcg = folio_memcg(folio);
12966168d0daSAlex Shi
12976168d0daSAlex Shi if (!memcg)
12987848ed62SKamalesh Babulal VM_BUG_ON_FOLIO(!mem_cgroup_is_root(lruvec_memcg(lruvec)), folio);
12996168d0daSAlex Shi else
1300e809c3feSMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(lruvec_memcg(lruvec) != memcg, folio);
13016168d0daSAlex Shi }
13026168d0daSAlex Shi #endif
13036168d0daSAlex Shi
13047c5f64f8SVladimir Davydov /**
1305e809c3feSMatthew Wilcox (Oracle) * folio_lruvec_lock - Lock the lruvec for a folio.
1306e809c3feSMatthew Wilcox (Oracle) * @folio: Pointer to the folio.
13076168d0daSAlex Shi *
1308d7e3aba5SAlex Shi * These functions are safe to use under any of the following conditions:
1309e809c3feSMatthew Wilcox (Oracle) * - folio locked
1310e809c3feSMatthew Wilcox (Oracle) * - folio_test_lru false
1311e809c3feSMatthew Wilcox (Oracle) * - folio_memcg_lock()
1312e809c3feSMatthew Wilcox (Oracle) * - folio frozen (refcount of 0)
1313e809c3feSMatthew Wilcox (Oracle) *
1314e809c3feSMatthew Wilcox (Oracle) * Return: The lruvec this folio is on with its lock held.
13156168d0daSAlex Shi */
folio_lruvec_lock(struct folio * folio)1316e809c3feSMatthew Wilcox (Oracle) struct lruvec *folio_lruvec_lock(struct folio *folio)
13176168d0daSAlex Shi {
1318e809c3feSMatthew Wilcox (Oracle) struct lruvec *lruvec = folio_lruvec(folio);
13196168d0daSAlex Shi
13206168d0daSAlex Shi spin_lock(&lruvec->lru_lock);
1321e809c3feSMatthew Wilcox (Oracle) lruvec_memcg_debug(lruvec, folio);
13226168d0daSAlex Shi
13236168d0daSAlex Shi return lruvec;
13246168d0daSAlex Shi }
13256168d0daSAlex Shi
1326e809c3feSMatthew Wilcox (Oracle) /**
1327e809c3feSMatthew Wilcox (Oracle) * folio_lruvec_lock_irq - Lock the lruvec for a folio.
1328e809c3feSMatthew Wilcox (Oracle) * @folio: Pointer to the folio.
1329e809c3feSMatthew Wilcox (Oracle) *
1330e809c3feSMatthew Wilcox (Oracle) * These functions are safe to use under any of the following conditions:
1331e809c3feSMatthew Wilcox (Oracle) * - folio locked
1332e809c3feSMatthew Wilcox (Oracle) * - folio_test_lru false
1333e809c3feSMatthew Wilcox (Oracle) * - folio_memcg_lock()
1334e809c3feSMatthew Wilcox (Oracle) * - folio frozen (refcount of 0)
1335e809c3feSMatthew Wilcox (Oracle) *
1336e809c3feSMatthew Wilcox (Oracle) * Return: The lruvec this folio is on with its lock held and interrupts
1337e809c3feSMatthew Wilcox (Oracle) * disabled.
1338e809c3feSMatthew Wilcox (Oracle) */
folio_lruvec_lock_irq(struct folio * folio)1339e809c3feSMatthew Wilcox (Oracle) struct lruvec *folio_lruvec_lock_irq(struct folio *folio)
13406168d0daSAlex Shi {
1341e809c3feSMatthew Wilcox (Oracle) struct lruvec *lruvec = folio_lruvec(folio);
13426168d0daSAlex Shi
13436168d0daSAlex Shi spin_lock_irq(&lruvec->lru_lock);
1344e809c3feSMatthew Wilcox (Oracle) lruvec_memcg_debug(lruvec, folio);
13456168d0daSAlex Shi
13466168d0daSAlex Shi return lruvec;
13476168d0daSAlex Shi }
13486168d0daSAlex Shi
1349e809c3feSMatthew Wilcox (Oracle) /**
1350e809c3feSMatthew Wilcox (Oracle) * folio_lruvec_lock_irqsave - Lock the lruvec for a folio.
1351e809c3feSMatthew Wilcox (Oracle) * @folio: Pointer to the folio.
1352e809c3feSMatthew Wilcox (Oracle) * @flags: Pointer to irqsave flags.
1353e809c3feSMatthew Wilcox (Oracle) *
1354e809c3feSMatthew Wilcox (Oracle) * These functions are safe to use under any of the following conditions:
1355e809c3feSMatthew Wilcox (Oracle) * - folio locked
1356e809c3feSMatthew Wilcox (Oracle) * - folio_test_lru false
1357e809c3feSMatthew Wilcox (Oracle) * - folio_memcg_lock()
1358e809c3feSMatthew Wilcox (Oracle) * - folio frozen (refcount of 0)
1359e809c3feSMatthew Wilcox (Oracle) *
1360e809c3feSMatthew Wilcox (Oracle) * Return: The lruvec this folio is on with its lock held and interrupts
1361e809c3feSMatthew Wilcox (Oracle) * disabled.
1362e809c3feSMatthew Wilcox (Oracle) */
folio_lruvec_lock_irqsave(struct folio * folio,unsigned long * flags)1363e809c3feSMatthew Wilcox (Oracle) struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
1364e809c3feSMatthew Wilcox (Oracle) unsigned long *flags)
13656168d0daSAlex Shi {
1366e809c3feSMatthew Wilcox (Oracle) struct lruvec *lruvec = folio_lruvec(folio);
13676168d0daSAlex Shi
13686168d0daSAlex Shi spin_lock_irqsave(&lruvec->lru_lock, *flags);
1369e809c3feSMatthew Wilcox (Oracle) lruvec_memcg_debug(lruvec, folio);
13706168d0daSAlex Shi
13716168d0daSAlex Shi return lruvec;
13726168d0daSAlex Shi }
13736168d0daSAlex Shi
13746168d0daSAlex Shi /**
1375fa9add64SHugh Dickins * mem_cgroup_update_lru_size - account for adding or removing an lru page
1376fa9add64SHugh Dickins * @lruvec: mem_cgroup per zone lru vector
1377fa9add64SHugh Dickins * @lru: index of lru list the page is sitting on
1378b4536f0cSMichal Hocko * @zid: zone id of the accounted pages
1379fa9add64SHugh Dickins * @nr_pages: positive when adding or negative when removing
1380925b7673SJohannes Weiner *
1381ca707239SHugh Dickins * This function must be called under lru_lock, just before a page is added
138207ca7606SHugh Dickins * to or just after a page is removed from an lru list.
1383925b7673SJohannes Weiner */
mem_cgroup_update_lru_size(struct lruvec * lruvec,enum lru_list lru,int zid,int nr_pages)1384fa9add64SHugh Dickins void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
1385b4536f0cSMichal Hocko int zid, int nr_pages)
1386925b7673SJohannes Weiner {
1387ef8f2327SMel Gorman struct mem_cgroup_per_node *mz;
1388fa9add64SHugh Dickins unsigned long *lru_size;
1389ca707239SHugh Dickins long size;
1390925b7673SJohannes Weiner
1391925b7673SJohannes Weiner if (mem_cgroup_disabled())
1392925b7673SJohannes Weiner return;
1393925b7673SJohannes Weiner
1394ef8f2327SMel Gorman mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
1395b4536f0cSMichal Hocko lru_size = &mz->lru_zone_size[zid][lru];
1396ca707239SHugh Dickins
1397ca707239SHugh Dickins if (nr_pages < 0)
1398fa9add64SHugh Dickins *lru_size += nr_pages;
1399ca707239SHugh Dickins
1400ca707239SHugh Dickins size = *lru_size;
1401b4536f0cSMichal Hocko if (WARN_ONCE(size < 0,
1402b4536f0cSMichal Hocko "%s(%p, %d, %d): lru_size %ld\n",
1403b4536f0cSMichal Hocko __func__, lruvec, lru, nr_pages, size)) {
1404ca707239SHugh Dickins VM_BUG_ON(1);
1405ca707239SHugh Dickins *lru_size = 0;
1406ca707239SHugh Dickins }
1407ca707239SHugh Dickins
1408ca707239SHugh Dickins if (nr_pages > 0)
1409ca707239SHugh Dickins *lru_size += nr_pages;
141008e552c6SKAMEZAWA Hiroyuki }
1411544122e5SKAMEZAWA Hiroyuki
141219942822SJohannes Weiner /**
14139d11ea9fSJohannes Weiner * mem_cgroup_margin - calculate chargeable space of a memory cgroup
1414dad7557eSWanpeng Li * @memcg: the memory cgroup
141519942822SJohannes Weiner *
14169d11ea9fSJohannes Weiner * Returns the maximum amount of memory @mem can be charged with, in
14177ec99d62SJohannes Weiner * pages.
141819942822SJohannes Weiner */
mem_cgroup_margin(struct mem_cgroup * memcg)1419c0ff4b85SRaghavendra K T static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
142019942822SJohannes Weiner {
14213e32cb2eSJohannes Weiner unsigned long margin = 0;
14223e32cb2eSJohannes Weiner unsigned long count;
14233e32cb2eSJohannes Weiner unsigned long limit;
14249d11ea9fSJohannes Weiner
14253e32cb2eSJohannes Weiner count = page_counter_read(&memcg->memory);
1426bbec2e15SRoman Gushchin limit = READ_ONCE(memcg->memory.max);
14273e32cb2eSJohannes Weiner if (count < limit)
14283e32cb2eSJohannes Weiner margin = limit - count;
14293e32cb2eSJohannes Weiner
14307941d214SJohannes Weiner if (do_memsw_account()) {
14313e32cb2eSJohannes Weiner count = page_counter_read(&memcg->memsw);
1432bbec2e15SRoman Gushchin limit = READ_ONCE(memcg->memsw.max);
14331c4448edSKaixu Xia if (count < limit)
14343e32cb2eSJohannes Weiner margin = min(margin, limit - count);
1435cbedbac3SLi RongQing else
1436cbedbac3SLi RongQing margin = 0;
14373e32cb2eSJohannes Weiner }
14383e32cb2eSJohannes Weiner
14393e32cb2eSJohannes Weiner return margin;
144019942822SJohannes Weiner }
144119942822SJohannes Weiner
1442619d094bSKAMEZAWA Hiroyuki /*
1443bdcbb659SQiang Huang * A routine for checking "mem" is under move_account() or not.
144432047e2aSKAMEZAWA Hiroyuki *
1445bdcbb659SQiang Huang * Checking a cgroup is mc.from or mc.to or under hierarchy of
1446bdcbb659SQiang Huang * moving cgroups. This is for waiting at high-memory pressure
1447bdcbb659SQiang Huang * caused by "move".
144832047e2aSKAMEZAWA Hiroyuki */
mem_cgroup_under_move(struct mem_cgroup * memcg)1449c0ff4b85SRaghavendra K T static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
14504b534334SKAMEZAWA Hiroyuki {
14512bd9bb20SKAMEZAWA Hiroyuki struct mem_cgroup *from;
14522bd9bb20SKAMEZAWA Hiroyuki struct mem_cgroup *to;
14534b534334SKAMEZAWA Hiroyuki bool ret = false;
14542bd9bb20SKAMEZAWA Hiroyuki /*
14552bd9bb20SKAMEZAWA Hiroyuki * Unlike task_move routines, we access mc.to, mc.from not under
14562bd9bb20SKAMEZAWA Hiroyuki * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
14572bd9bb20SKAMEZAWA Hiroyuki */
14582bd9bb20SKAMEZAWA Hiroyuki spin_lock(&mc.lock);
14592bd9bb20SKAMEZAWA Hiroyuki from = mc.from;
14602bd9bb20SKAMEZAWA Hiroyuki to = mc.to;
14612bd9bb20SKAMEZAWA Hiroyuki if (!from)
14622bd9bb20SKAMEZAWA Hiroyuki goto unlock;
14633e92041dSMichal Hocko
14642314b42dSJohannes Weiner ret = mem_cgroup_is_descendant(from, memcg) ||
14652314b42dSJohannes Weiner mem_cgroup_is_descendant(to, memcg);
14662bd9bb20SKAMEZAWA Hiroyuki unlock:
14672bd9bb20SKAMEZAWA Hiroyuki spin_unlock(&mc.lock);
14684b534334SKAMEZAWA Hiroyuki return ret;
14694b534334SKAMEZAWA Hiroyuki }
14704b534334SKAMEZAWA Hiroyuki
mem_cgroup_wait_acct_move(struct mem_cgroup * memcg)1471c0ff4b85SRaghavendra K T static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
14724b534334SKAMEZAWA Hiroyuki {
14734b534334SKAMEZAWA Hiroyuki if (mc.moving_task && current != mc.moving_task) {
1474c0ff4b85SRaghavendra K T if (mem_cgroup_under_move(memcg)) {
14754b534334SKAMEZAWA Hiroyuki DEFINE_WAIT(wait);
14764b534334SKAMEZAWA Hiroyuki prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
14774b534334SKAMEZAWA Hiroyuki /* moving charge context might have finished. */
14784b534334SKAMEZAWA Hiroyuki if (mc.moving_task)
14794b534334SKAMEZAWA Hiroyuki schedule();
14804b534334SKAMEZAWA Hiroyuki finish_wait(&mc.waitq, &wait);
14814b534334SKAMEZAWA Hiroyuki return true;
14824b534334SKAMEZAWA Hiroyuki }
14834b534334SKAMEZAWA Hiroyuki }
14844b534334SKAMEZAWA Hiroyuki return false;
14854b534334SKAMEZAWA Hiroyuki }
14864b534334SKAMEZAWA Hiroyuki
14875f9a4f4aSMuchun Song struct memory_stat {
14885f9a4f4aSMuchun Song const char *name;
14895f9a4f4aSMuchun Song unsigned int idx;
14905f9a4f4aSMuchun Song };
14915f9a4f4aSMuchun Song
149257b2847dSMuchun Song static const struct memory_stat memory_stats[] = {
1493fff66b79SMuchun Song { "anon", NR_ANON_MAPPED },
1494fff66b79SMuchun Song { "file", NR_FILE_PAGES },
1495a8c49af3SYosry Ahmed { "kernel", MEMCG_KMEM },
1496fff66b79SMuchun Song { "kernel_stack", NR_KERNEL_STACK_KB },
1497fff66b79SMuchun Song { "pagetables", NR_PAGETABLE },
1498ebc97a52SYosry Ahmed { "sec_pagetables", NR_SECONDARY_PAGETABLE },
1499fff66b79SMuchun Song { "percpu", MEMCG_PERCPU_B },
1500fff66b79SMuchun Song { "sock", MEMCG_SOCK },
15014e5aa1f4SShakeel Butt { "vmalloc", MEMCG_VMALLOC },
1502fff66b79SMuchun Song { "shmem", NR_SHMEM },
1503f4840ccfSJohannes Weiner #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP)
1504f4840ccfSJohannes Weiner { "zswap", MEMCG_ZSWAP_B },
1505f4840ccfSJohannes Weiner { "zswapped", MEMCG_ZSWAPPED },
1506f4840ccfSJohannes Weiner #endif
1507fff66b79SMuchun Song { "file_mapped", NR_FILE_MAPPED },
1508fff66b79SMuchun Song { "file_dirty", NR_FILE_DIRTY },
1509fff66b79SMuchun Song { "file_writeback", NR_WRITEBACK },
1510b6038942SShakeel Butt #ifdef CONFIG_SWAP
1511b6038942SShakeel Butt { "swapcached", NR_SWAPCACHE },
1512b6038942SShakeel Butt #endif
15135f9a4f4aSMuchun Song #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1514fff66b79SMuchun Song { "anon_thp", NR_ANON_THPS },
1515fff66b79SMuchun Song { "file_thp", NR_FILE_THPS },
1516fff66b79SMuchun Song { "shmem_thp", NR_SHMEM_THPS },
15175f9a4f4aSMuchun Song #endif
1518fff66b79SMuchun Song { "inactive_anon", NR_INACTIVE_ANON },
1519fff66b79SMuchun Song { "active_anon", NR_ACTIVE_ANON },
1520fff66b79SMuchun Song { "inactive_file", NR_INACTIVE_FILE },
1521fff66b79SMuchun Song { "active_file", NR_ACTIVE_FILE },
1522fff66b79SMuchun Song { "unevictable", NR_UNEVICTABLE },
1523fff66b79SMuchun Song { "slab_reclaimable", NR_SLAB_RECLAIMABLE_B },
1524fff66b79SMuchun Song { "slab_unreclaimable", NR_SLAB_UNRECLAIMABLE_B },
15255f9a4f4aSMuchun Song
15265f9a4f4aSMuchun Song /* The memory events */
1527fff66b79SMuchun Song { "workingset_refault_anon", WORKINGSET_REFAULT_ANON },
1528fff66b79SMuchun Song { "workingset_refault_file", WORKINGSET_REFAULT_FILE },
1529fff66b79SMuchun Song { "workingset_activate_anon", WORKINGSET_ACTIVATE_ANON },
1530fff66b79SMuchun Song { "workingset_activate_file", WORKINGSET_ACTIVATE_FILE },
1531fff66b79SMuchun Song { "workingset_restore_anon", WORKINGSET_RESTORE_ANON },
1532fff66b79SMuchun Song { "workingset_restore_file", WORKINGSET_RESTORE_FILE },
1533fff66b79SMuchun Song { "workingset_nodereclaim", WORKINGSET_NODERECLAIM },
15345f9a4f4aSMuchun Song };
15355f9a4f4aSMuchun Song
1536fff66b79SMuchun Song /* Translate stat items to the correct unit for memory.stat output */
memcg_page_state_unit(int item)1537fff66b79SMuchun Song static int memcg_page_state_unit(int item)
1538fff66b79SMuchun Song {
1539fff66b79SMuchun Song switch (item) {
1540fff66b79SMuchun Song case MEMCG_PERCPU_B:
1541f4840ccfSJohannes Weiner case MEMCG_ZSWAP_B:
1542fff66b79SMuchun Song case NR_SLAB_RECLAIMABLE_B:
1543fff66b79SMuchun Song case NR_SLAB_UNRECLAIMABLE_B:
1544fff66b79SMuchun Song case WORKINGSET_REFAULT_ANON:
1545fff66b79SMuchun Song case WORKINGSET_REFAULT_FILE:
1546fff66b79SMuchun Song case WORKINGSET_ACTIVATE_ANON:
1547fff66b79SMuchun Song case WORKINGSET_ACTIVATE_FILE:
1548fff66b79SMuchun Song case WORKINGSET_RESTORE_ANON:
1549fff66b79SMuchun Song case WORKINGSET_RESTORE_FILE:
1550fff66b79SMuchun Song case WORKINGSET_NODERECLAIM:
1551fff66b79SMuchun Song return 1;
1552fff66b79SMuchun Song case NR_KERNEL_STACK_KB:
1553fff66b79SMuchun Song return SZ_1K;
1554fff66b79SMuchun Song default:
1555fff66b79SMuchun Song return PAGE_SIZE;
1556fff66b79SMuchun Song }
1557fff66b79SMuchun Song }
1558fff66b79SMuchun Song
memcg_page_state_output(struct mem_cgroup * memcg,int item)1559fff66b79SMuchun Song static inline unsigned long memcg_page_state_output(struct mem_cgroup *memcg,
1560fff66b79SMuchun Song int item)
1561fff66b79SMuchun Song {
1562fff66b79SMuchun Song return memcg_page_state(memcg, item) * memcg_page_state_unit(item);
1563fff66b79SMuchun Song }
1564fff66b79SMuchun Song
memcg_stat_format(struct mem_cgroup * memcg,struct seq_buf * s)1565dddb44ffSYosry Ahmed static void memcg_stat_format(struct mem_cgroup *memcg, struct seq_buf *s)
1566c8713d0bSJohannes Weiner {
1567c8713d0bSJohannes Weiner int i;
156871cd3113SJohannes Weiner
1569c8713d0bSJohannes Weiner /*
1570c8713d0bSJohannes Weiner * Provide statistics on the state of the memory subsystem as
1571c8713d0bSJohannes Weiner * well as cumulative event counters that show past behavior.
1572c8713d0bSJohannes Weiner *
1573c8713d0bSJohannes Weiner * This list is ordered following a combination of these gradients:
1574c8713d0bSJohannes Weiner * 1) generic big picture -> specifics and details
1575c8713d0bSJohannes Weiner * 2) reflecting userspace activity -> reflecting kernel heuristics
1576c8713d0bSJohannes Weiner *
1577c8713d0bSJohannes Weiner * Current memory state:
1578c8713d0bSJohannes Weiner */
1579fd25a9e0SShakeel Butt mem_cgroup_flush_stats();
1580c8713d0bSJohannes Weiner
15815f9a4f4aSMuchun Song for (i = 0; i < ARRAY_SIZE(memory_stats); i++) {
15825f9a4f4aSMuchun Song u64 size;
1583c8713d0bSJohannes Weiner
1584fff66b79SMuchun Song size = memcg_page_state_output(memcg, memory_stats[i].idx);
15855b42360cSYosry Ahmed seq_buf_printf(s, "%s %llu\n", memory_stats[i].name, size);
1586c8713d0bSJohannes Weiner
15875f9a4f4aSMuchun Song if (unlikely(memory_stats[i].idx == NR_SLAB_UNRECLAIMABLE_B)) {
1588fff66b79SMuchun Song size += memcg_page_state_output(memcg,
1589fff66b79SMuchun Song NR_SLAB_RECLAIMABLE_B);
15905b42360cSYosry Ahmed seq_buf_printf(s, "slab %llu\n", size);
15915f9a4f4aSMuchun Song }
15925f9a4f4aSMuchun Song }
1593c8713d0bSJohannes Weiner
1594c8713d0bSJohannes Weiner /* Accumulated memory events */
15955b42360cSYosry Ahmed seq_buf_printf(s, "pgscan %lu\n",
1596c8713d0bSJohannes Weiner memcg_events(memcg, PGSCAN_KSWAPD) +
159757e9cc50SJohannes Weiner memcg_events(memcg, PGSCAN_DIRECT) +
159857e9cc50SJohannes Weiner memcg_events(memcg, PGSCAN_KHUGEPAGED));
15995b42360cSYosry Ahmed seq_buf_printf(s, "pgsteal %lu\n",
1600c8713d0bSJohannes Weiner memcg_events(memcg, PGSTEAL_KSWAPD) +
160157e9cc50SJohannes Weiner memcg_events(memcg, PGSTEAL_DIRECT) +
160257e9cc50SJohannes Weiner memcg_events(memcg, PGSTEAL_KHUGEPAGED));
1603c8713d0bSJohannes Weiner
16048278f1c7SShakeel Butt for (i = 0; i < ARRAY_SIZE(memcg_vm_event_stat); i++) {
16058278f1c7SShakeel Butt if (memcg_vm_event_stat[i] == PGPGIN ||
16068278f1c7SShakeel Butt memcg_vm_event_stat[i] == PGPGOUT)
16078278f1c7SShakeel Butt continue;
16088278f1c7SShakeel Butt
16095b42360cSYosry Ahmed seq_buf_printf(s, "%s %lu\n",
1610673520f8SQi Zheng vm_event_name(memcg_vm_event_stat[i]),
1611673520f8SQi Zheng memcg_events(memcg, memcg_vm_event_stat[i]));
16128278f1c7SShakeel Butt }
1613c8713d0bSJohannes Weiner
1614c8713d0bSJohannes Weiner /* The above should easily fit into one page */
16155b42360cSYosry Ahmed WARN_ON_ONCE(seq_buf_has_overflowed(s));
1616c8713d0bSJohannes Weiner }
161771cd3113SJohannes Weiner
1618dddb44ffSYosry Ahmed static void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s);
1619dddb44ffSYosry Ahmed
memory_stat_format(struct mem_cgroup * memcg,struct seq_buf * s)1620dddb44ffSYosry Ahmed static void memory_stat_format(struct mem_cgroup *memcg, struct seq_buf *s)
1621dddb44ffSYosry Ahmed {
1622dddb44ffSYosry Ahmed if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
1623dddb44ffSYosry Ahmed memcg_stat_format(memcg, s);
1624dddb44ffSYosry Ahmed else
1625dddb44ffSYosry Ahmed memcg1_stat_format(memcg, s);
1626dddb44ffSYosry Ahmed WARN_ON_ONCE(seq_buf_has_overflowed(s));
1627dddb44ffSYosry Ahmed }
1628dddb44ffSYosry Ahmed
1629e222432bSBalbir Singh /**
1630f0c867d9Syuzhoujian * mem_cgroup_print_oom_context: Print OOM information relevant to
1631f0c867d9Syuzhoujian * memory controller.
1632e222432bSBalbir Singh * @memcg: The memory cgroup that went over limit
1633e222432bSBalbir Singh * @p: Task that is going to be killed
1634e222432bSBalbir Singh *
1635e222432bSBalbir Singh * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1636e222432bSBalbir Singh * enabled
1637e222432bSBalbir Singh */
mem_cgroup_print_oom_context(struct mem_cgroup * memcg,struct task_struct * p)1638f0c867d9Syuzhoujian void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
1639f0c867d9Syuzhoujian {
1640f0c867d9Syuzhoujian rcu_read_lock();
1641f0c867d9Syuzhoujian
1642f0c867d9Syuzhoujian if (memcg) {
1643f0c867d9Syuzhoujian pr_cont(",oom_memcg=");
1644f0c867d9Syuzhoujian pr_cont_cgroup_path(memcg->css.cgroup);
1645f0c867d9Syuzhoujian } else
1646f0c867d9Syuzhoujian pr_cont(",global_oom");
1647f0c867d9Syuzhoujian if (p) {
1648f0c867d9Syuzhoujian pr_cont(",task_memcg=");
1649f0c867d9Syuzhoujian pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
1650f0c867d9Syuzhoujian }
1651f0c867d9Syuzhoujian rcu_read_unlock();
1652f0c867d9Syuzhoujian }
1653f0c867d9Syuzhoujian
1654f0c867d9Syuzhoujian /**
1655f0c867d9Syuzhoujian * mem_cgroup_print_oom_meminfo: Print OOM memory information relevant to
1656f0c867d9Syuzhoujian * memory controller.
1657f0c867d9Syuzhoujian * @memcg: The memory cgroup that went over limit
1658f0c867d9Syuzhoujian */
mem_cgroup_print_oom_meminfo(struct mem_cgroup * memcg)1659f0c867d9Syuzhoujian void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
1660e222432bSBalbir Singh {
166168aaee14STetsuo Handa /* Use static buffer, for the caller is holding oom_lock. */
166268aaee14STetsuo Handa static char buf[PAGE_SIZE];
16635b42360cSYosry Ahmed struct seq_buf s;
166468aaee14STetsuo Handa
166568aaee14STetsuo Handa lockdep_assert_held(&oom_lock);
1666e222432bSBalbir Singh
16673e32cb2eSJohannes Weiner pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
16683e32cb2eSJohannes Weiner K((u64)page_counter_read(&memcg->memory)),
166915b42562SChris Down K((u64)READ_ONCE(memcg->memory.max)), memcg->memory.failcnt);
1670c8713d0bSJohannes Weiner if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
1671c8713d0bSJohannes Weiner pr_info("swap: usage %llukB, limit %llukB, failcnt %lu\n",
1672c8713d0bSJohannes Weiner K((u64)page_counter_read(&memcg->swap)),
167332d087cdSChris Down K((u64)READ_ONCE(memcg->swap.max)), memcg->swap.failcnt);
1674c8713d0bSJohannes Weiner else {
16753e32cb2eSJohannes Weiner pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
16763e32cb2eSJohannes Weiner K((u64)page_counter_read(&memcg->memsw)),
1677bbec2e15SRoman Gushchin K((u64)memcg->memsw.max), memcg->memsw.failcnt);
16783e32cb2eSJohannes Weiner pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
16793e32cb2eSJohannes Weiner K((u64)page_counter_read(&memcg->kmem)),
1680bbec2e15SRoman Gushchin K((u64)memcg->kmem.max), memcg->kmem.failcnt);
1681c8713d0bSJohannes Weiner }
168258cf188eSSha Zhengju
1683e61734c5STejun Heo pr_info("Memory cgroup stats for ");
1684c8713d0bSJohannes Weiner pr_cont_cgroup_path(memcg->css.cgroup);
168558cf188eSSha Zhengju pr_cont(":");
16865b42360cSYosry Ahmed seq_buf_init(&s, buf, sizeof(buf));
16875b42360cSYosry Ahmed memory_stat_format(memcg, &s);
16885b42360cSYosry Ahmed seq_buf_do_printk(&s, KERN_INFO);
1689e222432bSBalbir Singh }
1690e222432bSBalbir Singh
169181d39c20SKAMEZAWA Hiroyuki /*
1692a63d83f4SDavid Rientjes * Return the memory (and swap, if configured) limit for a memcg.
1693a63d83f4SDavid Rientjes */
mem_cgroup_get_max(struct mem_cgroup * memcg)1694bbec2e15SRoman Gushchin unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
1695a63d83f4SDavid Rientjes {
16968d387a5fSWaiman Long unsigned long max = READ_ONCE(memcg->memory.max);
1697a63d83f4SDavid Rientjes
1698b94c4e94SJohannes Weiner if (do_memsw_account()) {
16999a5a8f19SMichal Hocko if (mem_cgroup_swappiness(memcg)) {
17008d387a5fSWaiman Long /* Calculate swap excess capacity from memsw limit */
17018d387a5fSWaiman Long unsigned long swap = READ_ONCE(memcg->memsw.max) - max;
17029a5a8f19SMichal Hocko
17038d387a5fSWaiman Long max += min(swap, (unsigned long)total_swap_pages);
17048d387a5fSWaiman Long }
1705b94c4e94SJohannes Weiner } else {
1706b94c4e94SJohannes Weiner if (mem_cgroup_swappiness(memcg))
1707b94c4e94SJohannes Weiner max += min(READ_ONCE(memcg->swap.max),
1708b94c4e94SJohannes Weiner (unsigned long)total_swap_pages);
17099a5a8f19SMichal Hocko }
1710bbec2e15SRoman Gushchin return max;
1711a63d83f4SDavid Rientjes }
1712a63d83f4SDavid Rientjes
mem_cgroup_size(struct mem_cgroup * memcg)17139783aa99SChris Down unsigned long mem_cgroup_size(struct mem_cgroup *memcg)
17149783aa99SChris Down {
17159783aa99SChris Down return page_counter_read(&memcg->memory);
17169783aa99SChris Down }
17179783aa99SChris Down
mem_cgroup_out_of_memory(struct mem_cgroup * memcg,gfp_t gfp_mask,int order)1718b6e6edcfSJohannes Weiner static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
17199cbb78bbSDavid Rientjes int order)
17209cbb78bbSDavid Rientjes {
17216e0fc46dSDavid Rientjes struct oom_control oc = {
17226e0fc46dSDavid Rientjes .zonelist = NULL,
17236e0fc46dSDavid Rientjes .nodemask = NULL,
17242a966b77SVladimir Davydov .memcg = memcg,
17256e0fc46dSDavid Rientjes .gfp_mask = gfp_mask,
17266e0fc46dSDavid Rientjes .order = order,
17276e0fc46dSDavid Rientjes };
17281378b37dSYafang Shao bool ret = true;
17299cbb78bbSDavid Rientjes
17307775faceSTetsuo Handa if (mutex_lock_killable(&oom_lock))
17317775faceSTetsuo Handa return true;
17321378b37dSYafang Shao
17331378b37dSYafang Shao if (mem_cgroup_margin(memcg) >= (1 << order))
17341378b37dSYafang Shao goto unlock;
17351378b37dSYafang Shao
17367775faceSTetsuo Handa /*
17377775faceSTetsuo Handa * A few threads which were not waiting at mutex_lock_killable() can
17387775faceSTetsuo Handa * fail to bail out. Therefore, check again after holding oom_lock.
17397775faceSTetsuo Handa */
1740a4ebf1b6SVasily Averin ret = task_is_dying() || out_of_memory(&oc);
17411378b37dSYafang Shao
17421378b37dSYafang Shao unlock:
1743dc56401fSJohannes Weiner mutex_unlock(&oom_lock);
17447c5f64f8SVladimir Davydov return ret;
17459cbb78bbSDavid Rientjes }
17469cbb78bbSDavid Rientjes
mem_cgroup_soft_reclaim(struct mem_cgroup * root_memcg,pg_data_t * pgdat,gfp_t gfp_mask,unsigned long * total_scanned)17470608f43dSAndrew Morton static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1748ef8f2327SMel Gorman pg_data_t *pgdat,
17490608f43dSAndrew Morton gfp_t gfp_mask,
17500608f43dSAndrew Morton unsigned long *total_scanned)
17516d61ef40SBalbir Singh {
17520608f43dSAndrew Morton struct mem_cgroup *victim = NULL;
17530608f43dSAndrew Morton int total = 0;
17540608f43dSAndrew Morton int loop = 0;
17550608f43dSAndrew Morton unsigned long excess;
17560608f43dSAndrew Morton unsigned long nr_scanned;
17570608f43dSAndrew Morton struct mem_cgroup_reclaim_cookie reclaim = {
1758ef8f2327SMel Gorman .pgdat = pgdat,
17590608f43dSAndrew Morton };
17609d11ea9fSJohannes Weiner
17613e32cb2eSJohannes Weiner excess = soft_limit_excess(root_memcg);
17626d61ef40SBalbir Singh
17630608f43dSAndrew Morton while (1) {
17640608f43dSAndrew Morton victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
17650608f43dSAndrew Morton if (!victim) {
17660608f43dSAndrew Morton loop++;
17670608f43dSAndrew Morton if (loop >= 2) {
17684e416953SBalbir Singh /*
17690608f43dSAndrew Morton * If we have not been able to reclaim
17700608f43dSAndrew Morton * anything, it might because there are
17710608f43dSAndrew Morton * no reclaimable pages under this hierarchy
17724e416953SBalbir Singh */
17730608f43dSAndrew Morton if (!total)
17740608f43dSAndrew Morton break;
17750608f43dSAndrew Morton /*
17760608f43dSAndrew Morton * We want to do more targeted reclaim.
17770608f43dSAndrew Morton * excess >> 2 is not to excessive so as to
17780608f43dSAndrew Morton * reclaim too much, nor too less that we keep
17790608f43dSAndrew Morton * coming back to reclaim from this cgroup
17800608f43dSAndrew Morton */
17810608f43dSAndrew Morton if (total >= (excess >> 2) ||
17820608f43dSAndrew Morton (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
17830608f43dSAndrew Morton break;
17844e416953SBalbir Singh }
17850608f43dSAndrew Morton continue;
17860608f43dSAndrew Morton }
1787a9dd0a83SMel Gorman total += mem_cgroup_shrink_node(victim, gfp_mask, false,
1788ef8f2327SMel Gorman pgdat, &nr_scanned);
17890608f43dSAndrew Morton *total_scanned += nr_scanned;
17903e32cb2eSJohannes Weiner if (!soft_limit_excess(root_memcg))
17910608f43dSAndrew Morton break;
17920608f43dSAndrew Morton }
17930608f43dSAndrew Morton mem_cgroup_iter_break(root_memcg, victim);
17940608f43dSAndrew Morton return total;
17956d61ef40SBalbir Singh }
17966d61ef40SBalbir Singh
17970056f4e6SJohannes Weiner #ifdef CONFIG_LOCKDEP
17980056f4e6SJohannes Weiner static struct lockdep_map memcg_oom_lock_dep_map = {
17990056f4e6SJohannes Weiner .name = "memcg_oom_lock",
18000056f4e6SJohannes Weiner };
18010056f4e6SJohannes Weiner #endif
18020056f4e6SJohannes Weiner
1803fb2a6fc5SJohannes Weiner static DEFINE_SPINLOCK(memcg_oom_lock);
1804fb2a6fc5SJohannes Weiner
1805867578cbSKAMEZAWA Hiroyuki /*
1806867578cbSKAMEZAWA Hiroyuki * Check OOM-Killer is already running under our hierarchy.
1807867578cbSKAMEZAWA Hiroyuki * If someone is running, return false.
1808867578cbSKAMEZAWA Hiroyuki */
mem_cgroup_oom_trylock(struct mem_cgroup * memcg)1809fb2a6fc5SJohannes Weiner static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
1810867578cbSKAMEZAWA Hiroyuki {
181179dfdaccSMichal Hocko struct mem_cgroup *iter, *failed = NULL;
181279dfdaccSMichal Hocko
1813fb2a6fc5SJohannes Weiner spin_lock(&memcg_oom_lock);
1814fb2a6fc5SJohannes Weiner
18159f3a0d09SJohannes Weiner for_each_mem_cgroup_tree(iter, memcg) {
181623751be0SJohannes Weiner if (iter->oom_lock) {
181779dfdaccSMichal Hocko /*
181879dfdaccSMichal Hocko * this subtree of our hierarchy is already locked
181979dfdaccSMichal Hocko * so we cannot give a lock.
182079dfdaccSMichal Hocko */
182179dfdaccSMichal Hocko failed = iter;
18229f3a0d09SJohannes Weiner mem_cgroup_iter_break(memcg, iter);
18239f3a0d09SJohannes Weiner break;
182423751be0SJohannes Weiner } else
182523751be0SJohannes Weiner iter->oom_lock = true;
182679dfdaccSMichal Hocko }
182779dfdaccSMichal Hocko
1828fb2a6fc5SJohannes Weiner if (failed) {
182979dfdaccSMichal Hocko /*
1830fb2a6fc5SJohannes Weiner * OK, we failed to lock the whole subtree so we have
1831fb2a6fc5SJohannes Weiner * to clean up what we set up to the failing subtree
183279dfdaccSMichal Hocko */
18339f3a0d09SJohannes Weiner for_each_mem_cgroup_tree(iter, memcg) {
183479dfdaccSMichal Hocko if (iter == failed) {
18359f3a0d09SJohannes Weiner mem_cgroup_iter_break(memcg, iter);
18369f3a0d09SJohannes Weiner break;
183779dfdaccSMichal Hocko }
183879dfdaccSMichal Hocko iter->oom_lock = false;
183979dfdaccSMichal Hocko }
18400056f4e6SJohannes Weiner } else
18410056f4e6SJohannes Weiner mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
184279dfdaccSMichal Hocko
1843fb2a6fc5SJohannes Weiner spin_unlock(&memcg_oom_lock);
1844fb2a6fc5SJohannes Weiner
1845fb2a6fc5SJohannes Weiner return !failed;
1846fb2a6fc5SJohannes Weiner }
1847fb2a6fc5SJohannes Weiner
mem_cgroup_oom_unlock(struct mem_cgroup * memcg)1848fb2a6fc5SJohannes Weiner static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
184979dfdaccSMichal Hocko {
18507d74b06fSKAMEZAWA Hiroyuki struct mem_cgroup *iter;
1851a636b327SKAMEZAWA Hiroyuki
1852fb2a6fc5SJohannes Weiner spin_lock(&memcg_oom_lock);
18535facae4fSQian Cai mutex_release(&memcg_oom_lock_dep_map, _RET_IP_);
1854c0ff4b85SRaghavendra K T for_each_mem_cgroup_tree(iter, memcg)
185579dfdaccSMichal Hocko iter->oom_lock = false;
1856fb2a6fc5SJohannes Weiner spin_unlock(&memcg_oom_lock);
18577d74b06fSKAMEZAWA Hiroyuki }
1858867578cbSKAMEZAWA Hiroyuki
mem_cgroup_mark_under_oom(struct mem_cgroup * memcg)1859c0ff4b85SRaghavendra K T static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
186079dfdaccSMichal Hocko {
186179dfdaccSMichal Hocko struct mem_cgroup *iter;
186279dfdaccSMichal Hocko
1863c2b42d3cSTejun Heo spin_lock(&memcg_oom_lock);
1864c0ff4b85SRaghavendra K T for_each_mem_cgroup_tree(iter, memcg)
1865c2b42d3cSTejun Heo iter->under_oom++;
1866c2b42d3cSTejun Heo spin_unlock(&memcg_oom_lock);
1867a636b327SKAMEZAWA Hiroyuki }
18680b7f569eSKAMEZAWA Hiroyuki
mem_cgroup_unmark_under_oom(struct mem_cgroup * memcg)1869c0ff4b85SRaghavendra K T static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
18700b7f569eSKAMEZAWA Hiroyuki {
18717d74b06fSKAMEZAWA Hiroyuki struct mem_cgroup *iter;
18727d74b06fSKAMEZAWA Hiroyuki
1873867578cbSKAMEZAWA Hiroyuki /*
1874f0953a1bSIngo Molnar * Be careful about under_oom underflows because a child memcg
18757a52d4d8SMiaohe Lin * could have been added after mem_cgroup_mark_under_oom.
1876867578cbSKAMEZAWA Hiroyuki */
1877c2b42d3cSTejun Heo spin_lock(&memcg_oom_lock);
1878c0ff4b85SRaghavendra K T for_each_mem_cgroup_tree(iter, memcg)
1879c2b42d3cSTejun Heo if (iter->under_oom > 0)
1880c2b42d3cSTejun Heo iter->under_oom--;
1881c2b42d3cSTejun Heo spin_unlock(&memcg_oom_lock);
18820b7f569eSKAMEZAWA Hiroyuki }
18830b7f569eSKAMEZAWA Hiroyuki
1884867578cbSKAMEZAWA Hiroyuki static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1885867578cbSKAMEZAWA Hiroyuki
1886dc98df5aSKAMEZAWA Hiroyuki struct oom_wait_info {
1887d79154bbSHugh Dickins struct mem_cgroup *memcg;
1888ac6424b9SIngo Molnar wait_queue_entry_t wait;
1889dc98df5aSKAMEZAWA Hiroyuki };
1890dc98df5aSKAMEZAWA Hiroyuki
memcg_oom_wake_function(wait_queue_entry_t * wait,unsigned mode,int sync,void * arg)1891ac6424b9SIngo Molnar static int memcg_oom_wake_function(wait_queue_entry_t *wait,
1892dc98df5aSKAMEZAWA Hiroyuki unsigned mode, int sync, void *arg)
1893dc98df5aSKAMEZAWA Hiroyuki {
1894d79154bbSHugh Dickins struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
1895d79154bbSHugh Dickins struct mem_cgroup *oom_wait_memcg;
1896dc98df5aSKAMEZAWA Hiroyuki struct oom_wait_info *oom_wait_info;
1897dc98df5aSKAMEZAWA Hiroyuki
1898dc98df5aSKAMEZAWA Hiroyuki oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1899d79154bbSHugh Dickins oom_wait_memcg = oom_wait_info->memcg;
1900dc98df5aSKAMEZAWA Hiroyuki
19012314b42dSJohannes Weiner if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
19022314b42dSJohannes Weiner !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
1903dc98df5aSKAMEZAWA Hiroyuki return 0;
1904dc98df5aSKAMEZAWA Hiroyuki return autoremove_wake_function(wait, mode, sync, arg);
1905dc98df5aSKAMEZAWA Hiroyuki }
1906dc98df5aSKAMEZAWA Hiroyuki
memcg_oom_recover(struct mem_cgroup * memcg)1907c0ff4b85SRaghavendra K T static void memcg_oom_recover(struct mem_cgroup *memcg)
19083c11ecf4SKAMEZAWA Hiroyuki {
1909c2b42d3cSTejun Heo /*
1910c2b42d3cSTejun Heo * For the following lockless ->under_oom test, the only required
1911c2b42d3cSTejun Heo * guarantee is that it must see the state asserted by an OOM when
1912c2b42d3cSTejun Heo * this function is called as a result of userland actions
1913c2b42d3cSTejun Heo * triggered by the notification of the OOM. This is trivially
1914c2b42d3cSTejun Heo * achieved by invoking mem_cgroup_mark_under_oom() before
1915c2b42d3cSTejun Heo * triggering notification.
1916c2b42d3cSTejun Heo */
1917c2b42d3cSTejun Heo if (memcg && memcg->under_oom)
1918f4b90b70STejun Heo __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
19193c11ecf4SKAMEZAWA Hiroyuki }
19203c11ecf4SKAMEZAWA Hiroyuki
1921becdf89dSShakeel Butt /*
1922becdf89dSShakeel Butt * Returns true if successfully killed one or more processes. Though in some
1923becdf89dSShakeel Butt * corner cases it can return true even without killing any process.
1924becdf89dSShakeel Butt */
mem_cgroup_oom(struct mem_cgroup * memcg,gfp_t mask,int order)1925becdf89dSShakeel Butt static bool mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
1926867578cbSKAMEZAWA Hiroyuki {
1927becdf89dSShakeel Butt bool locked, ret;
19287056d3a3SMichal Hocko
192929ef680aSMichal Hocko if (order > PAGE_ALLOC_COSTLY_ORDER)
1930becdf89dSShakeel Butt return false;
193129ef680aSMichal Hocko
19327a1adfddSRoman Gushchin memcg_memory_event(memcg, MEMCG_OOM);
19337a1adfddSRoman Gushchin
1934867578cbSKAMEZAWA Hiroyuki /*
193549426420SJohannes Weiner * We are in the middle of the charge context here, so we
193649426420SJohannes Weiner * don't want to block when potentially sitting on a callstack
193749426420SJohannes Weiner * that holds all kinds of filesystem and mm locks.
193849426420SJohannes Weiner *
193929ef680aSMichal Hocko * cgroup1 allows disabling the OOM killer and waiting for outside
194029ef680aSMichal Hocko * handling until the charge can succeed; remember the context and put
194129ef680aSMichal Hocko * the task to sleep at the end of the page fault when all locks are
194229ef680aSMichal Hocko * released.
194349426420SJohannes Weiner *
194429ef680aSMichal Hocko * On the other hand, in-kernel OOM killer allows for an async victim
194529ef680aSMichal Hocko * memory reclaim (oom_reaper) and that means that we are not solely
194629ef680aSMichal Hocko * relying on the oom victim to make a forward progress and we can
194729ef680aSMichal Hocko * invoke the oom killer here.
194829ef680aSMichal Hocko *
194929ef680aSMichal Hocko * Please note that mem_cgroup_out_of_memory might fail to find a
195029ef680aSMichal Hocko * victim and then we have to bail out from the charge path.
1951867578cbSKAMEZAWA Hiroyuki */
195217c56de6SYue Zhao if (READ_ONCE(memcg->oom_kill_disable)) {
1953becdf89dSShakeel Butt if (current->in_user_fault) {
195449426420SJohannes Weiner css_get(&memcg->css);
1955626ebc41STejun Heo current->memcg_in_oom = memcg;
1956626ebc41STejun Heo current->memcg_oom_gfp_mask = mask;
1957626ebc41STejun Heo current->memcg_oom_order = order;
1958becdf89dSShakeel Butt }
1959becdf89dSShakeel Butt return false;
196029ef680aSMichal Hocko }
196129ef680aSMichal Hocko
19627056d3a3SMichal Hocko mem_cgroup_mark_under_oom(memcg);
196329ef680aSMichal Hocko
19647056d3a3SMichal Hocko locked = mem_cgroup_oom_trylock(memcg);
19657056d3a3SMichal Hocko
19667056d3a3SMichal Hocko if (locked)
19677056d3a3SMichal Hocko mem_cgroup_oom_notify(memcg);
19687056d3a3SMichal Hocko
19697056d3a3SMichal Hocko mem_cgroup_unmark_under_oom(memcg);
1970becdf89dSShakeel Butt ret = mem_cgroup_out_of_memory(memcg, mask, order);
19717056d3a3SMichal Hocko
19727056d3a3SMichal Hocko if (locked)
19737056d3a3SMichal Hocko mem_cgroup_oom_unlock(memcg);
19747056d3a3SMichal Hocko
19757056d3a3SMichal Hocko return ret;
197649426420SJohannes Weiner }
197749426420SJohannes Weiner
197849426420SJohannes Weiner /**
197949426420SJohannes Weiner * mem_cgroup_oom_synchronize - complete memcg OOM handling
198049426420SJohannes Weiner * @handle: actually kill/wait or just clean up the OOM state
198149426420SJohannes Weiner *
198249426420SJohannes Weiner * This has to be called at the end of a page fault if the memcg OOM
198349426420SJohannes Weiner * handler was enabled.
198449426420SJohannes Weiner *
198549426420SJohannes Weiner * Memcg supports userspace OOM handling where failed allocations must
198649426420SJohannes Weiner * sleep on a waitqueue until the userspace task resolves the
198749426420SJohannes Weiner * situation. Sleeping directly in the charge context with all kinds
198849426420SJohannes Weiner * of locks held is not a good idea, instead we remember an OOM state
198949426420SJohannes Weiner * in the task and mem_cgroup_oom_synchronize() has to be called at
199049426420SJohannes Weiner * the end of the page fault to complete the OOM handling.
199149426420SJohannes Weiner *
199249426420SJohannes Weiner * Returns %true if an ongoing memcg OOM situation was detected and
199349426420SJohannes Weiner * completed, %false otherwise.
199449426420SJohannes Weiner */
mem_cgroup_oom_synchronize(bool handle)199549426420SJohannes Weiner bool mem_cgroup_oom_synchronize(bool handle)
199649426420SJohannes Weiner {
1997626ebc41STejun Heo struct mem_cgroup *memcg = current->memcg_in_oom;
199849426420SJohannes Weiner struct oom_wait_info owait;
199949426420SJohannes Weiner bool locked;
200049426420SJohannes Weiner
200149426420SJohannes Weiner /* OOM is global, do not handle */
200249426420SJohannes Weiner if (!memcg)
200349426420SJohannes Weiner return false;
200449426420SJohannes Weiner
20057c5f64f8SVladimir Davydov if (!handle)
200649426420SJohannes Weiner goto cleanup;
200749426420SJohannes Weiner
200849426420SJohannes Weiner owait.memcg = memcg;
200949426420SJohannes Weiner owait.wait.flags = 0;
201049426420SJohannes Weiner owait.wait.func = memcg_oom_wake_function;
201149426420SJohannes Weiner owait.wait.private = current;
20122055da97SIngo Molnar INIT_LIST_HEAD(&owait.wait.entry);
201349426420SJohannes Weiner
201449426420SJohannes Weiner prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
2015fb2a6fc5SJohannes Weiner mem_cgroup_mark_under_oom(memcg);
2016fb2a6fc5SJohannes Weiner
2017fb2a6fc5SJohannes Weiner locked = mem_cgroup_oom_trylock(memcg);
2018fb2a6fc5SJohannes Weiner
20193c11ecf4SKAMEZAWA Hiroyuki if (locked)
2020c0ff4b85SRaghavendra K T mem_cgroup_oom_notify(memcg);
2021867578cbSKAMEZAWA Hiroyuki
202249426420SJohannes Weiner schedule();
20233812c8c8SJohannes Weiner mem_cgroup_unmark_under_oom(memcg);
202449426420SJohannes Weiner finish_wait(&memcg_oom_waitq, &owait.wait);
202549426420SJohannes Weiner
202618b1d18bSHaifeng Xu if (locked)
2027fb2a6fc5SJohannes Weiner mem_cgroup_oom_unlock(memcg);
202849426420SJohannes Weiner cleanup:
2029626ebc41STejun Heo current->memcg_in_oom = NULL;
20303812c8c8SJohannes Weiner css_put(&memcg->css);
2031867578cbSKAMEZAWA Hiroyuki return true;
20320b7f569eSKAMEZAWA Hiroyuki }
20330b7f569eSKAMEZAWA Hiroyuki
2034d7365e78SJohannes Weiner /**
20353d8b38ebSRoman Gushchin * mem_cgroup_get_oom_group - get a memory cgroup to clean up after OOM
20363d8b38ebSRoman Gushchin * @victim: task to be killed by the OOM killer
20373d8b38ebSRoman Gushchin * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM
20383d8b38ebSRoman Gushchin *
20393d8b38ebSRoman Gushchin * Returns a pointer to a memory cgroup, which has to be cleaned up
20403d8b38ebSRoman Gushchin * by killing all belonging OOM-killable tasks.
20413d8b38ebSRoman Gushchin *
20423d8b38ebSRoman Gushchin * Caller has to call mem_cgroup_put() on the returned non-NULL memcg.
20433d8b38ebSRoman Gushchin */
mem_cgroup_get_oom_group(struct task_struct * victim,struct mem_cgroup * oom_domain)20443d8b38ebSRoman Gushchin struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
20453d8b38ebSRoman Gushchin struct mem_cgroup *oom_domain)
20463d8b38ebSRoman Gushchin {
20473d8b38ebSRoman Gushchin struct mem_cgroup *oom_group = NULL;
20483d8b38ebSRoman Gushchin struct mem_cgroup *memcg;
20493d8b38ebSRoman Gushchin
20503d8b38ebSRoman Gushchin if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
20513d8b38ebSRoman Gushchin return NULL;
20523d8b38ebSRoman Gushchin
20533d8b38ebSRoman Gushchin if (!oom_domain)
20543d8b38ebSRoman Gushchin oom_domain = root_mem_cgroup;
20553d8b38ebSRoman Gushchin
20563d8b38ebSRoman Gushchin rcu_read_lock();
20573d8b38ebSRoman Gushchin
20583d8b38ebSRoman Gushchin memcg = mem_cgroup_from_task(victim);
20597848ed62SKamalesh Babulal if (mem_cgroup_is_root(memcg))
20603d8b38ebSRoman Gushchin goto out;
20613d8b38ebSRoman Gushchin
20623d8b38ebSRoman Gushchin /*
206348fe267cSRoman Gushchin * If the victim task has been asynchronously moved to a different
206448fe267cSRoman Gushchin * memory cgroup, we might end up killing tasks outside oom_domain.
206548fe267cSRoman Gushchin * In this case it's better to ignore memory.group.oom.
206648fe267cSRoman Gushchin */
206748fe267cSRoman Gushchin if (unlikely(!mem_cgroup_is_descendant(memcg, oom_domain)))
206848fe267cSRoman Gushchin goto out;
206948fe267cSRoman Gushchin
207048fe267cSRoman Gushchin /*
20713d8b38ebSRoman Gushchin * Traverse the memory cgroup hierarchy from the victim task's
20723d8b38ebSRoman Gushchin * cgroup up to the OOMing cgroup (or root) to find the
20733d8b38ebSRoman Gushchin * highest-level memory cgroup with oom.group set.
20743d8b38ebSRoman Gushchin */
20753d8b38ebSRoman Gushchin for (; memcg; memcg = parent_mem_cgroup(memcg)) {
2076eaf7b66bSYue Zhao if (READ_ONCE(memcg->oom_group))
20773d8b38ebSRoman Gushchin oom_group = memcg;
20783d8b38ebSRoman Gushchin
20793d8b38ebSRoman Gushchin if (memcg == oom_domain)
20803d8b38ebSRoman Gushchin break;
20813d8b38ebSRoman Gushchin }
20823d8b38ebSRoman Gushchin
20833d8b38ebSRoman Gushchin if (oom_group)
20843d8b38ebSRoman Gushchin css_get(&oom_group->css);
20853d8b38ebSRoman Gushchin out:
20863d8b38ebSRoman Gushchin rcu_read_unlock();
20873d8b38ebSRoman Gushchin
20883d8b38ebSRoman Gushchin return oom_group;
20893d8b38ebSRoman Gushchin }
20903d8b38ebSRoman Gushchin
mem_cgroup_print_oom_group(struct mem_cgroup * memcg)20913d8b38ebSRoman Gushchin void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
20923d8b38ebSRoman Gushchin {
20933d8b38ebSRoman Gushchin pr_info("Tasks in ");
20943d8b38ebSRoman Gushchin pr_cont_cgroup_path(memcg->css.cgroup);
20953d8b38ebSRoman Gushchin pr_cont(" are going to be killed due to memory.oom.group set\n");
20963d8b38ebSRoman Gushchin }
20973d8b38ebSRoman Gushchin
20983d8b38ebSRoman Gushchin /**
2099f70ad448SMatthew Wilcox (Oracle) * folio_memcg_lock - Bind a folio to its memcg.
2100f70ad448SMatthew Wilcox (Oracle) * @folio: The folio.
210132047e2aSKAMEZAWA Hiroyuki *
2102f70ad448SMatthew Wilcox (Oracle) * This function prevents unlocked LRU folios from being moved to
2103739f79fcSJohannes Weiner * another cgroup.
2104739f79fcSJohannes Weiner *
2105f70ad448SMatthew Wilcox (Oracle) * It ensures lifetime of the bound memcg. The caller is responsible
2106f70ad448SMatthew Wilcox (Oracle) * for the lifetime of the folio.
2107d69b042fSBalbir Singh */
folio_memcg_lock(struct folio * folio)2108f70ad448SMatthew Wilcox (Oracle) void folio_memcg_lock(struct folio *folio)
210989c06bd5SKAMEZAWA Hiroyuki {
211089c06bd5SKAMEZAWA Hiroyuki struct mem_cgroup *memcg;
21116de22619SJohannes Weiner unsigned long flags;
211289c06bd5SKAMEZAWA Hiroyuki
21136de22619SJohannes Weiner /*
21146de22619SJohannes Weiner * The RCU lock is held throughout the transaction. The fast
21156de22619SJohannes Weiner * path can get away without acquiring the memcg->move_lock
21166de22619SJohannes Weiner * because page moving starts with an RCU grace period.
21176de22619SJohannes Weiner */
2118d7365e78SJohannes Weiner rcu_read_lock();
2119d7365e78SJohannes Weiner
2120d7365e78SJohannes Weiner if (mem_cgroup_disabled())
21211c824a68SJohannes Weiner return;
212289c06bd5SKAMEZAWA Hiroyuki again:
2123f70ad448SMatthew Wilcox (Oracle) memcg = folio_memcg(folio);
212429833315SJohannes Weiner if (unlikely(!memcg))
21251c824a68SJohannes Weiner return;
2126d7365e78SJohannes Weiner
212720ad50d6SAlex Shi #ifdef CONFIG_PROVE_LOCKING
212820ad50d6SAlex Shi local_irq_save(flags);
212920ad50d6SAlex Shi might_lock(&memcg->move_lock);
213020ad50d6SAlex Shi local_irq_restore(flags);
213120ad50d6SAlex Shi #endif
213220ad50d6SAlex Shi
2133bdcbb659SQiang Huang if (atomic_read(&memcg->moving_account) <= 0)
21341c824a68SJohannes Weiner return;
213589c06bd5SKAMEZAWA Hiroyuki
21366de22619SJohannes Weiner spin_lock_irqsave(&memcg->move_lock, flags);
2137f70ad448SMatthew Wilcox (Oracle) if (memcg != folio_memcg(folio)) {
21386de22619SJohannes Weiner spin_unlock_irqrestore(&memcg->move_lock, flags);
213989c06bd5SKAMEZAWA Hiroyuki goto again;
214089c06bd5SKAMEZAWA Hiroyuki }
21416de22619SJohannes Weiner
21426de22619SJohannes Weiner /*
21431c824a68SJohannes Weiner * When charge migration first begins, we can have multiple
21441c824a68SJohannes Weiner * critical sections holding the fast-path RCU lock and one
21451c824a68SJohannes Weiner * holding the slowpath move_lock. Track the task who has the
21466c77b607SKefeng Wang * move_lock for folio_memcg_unlock().
21476de22619SJohannes Weiner */
21486de22619SJohannes Weiner memcg->move_lock_task = current;
21496de22619SJohannes Weiner memcg->move_lock_flags = flags;
215089c06bd5SKAMEZAWA Hiroyuki }
2151f70ad448SMatthew Wilcox (Oracle)
__folio_memcg_unlock(struct mem_cgroup * memcg)2152f70ad448SMatthew Wilcox (Oracle) static void __folio_memcg_unlock(struct mem_cgroup *memcg)
2153d7365e78SJohannes Weiner {
21546de22619SJohannes Weiner if (memcg && memcg->move_lock_task == current) {
21556de22619SJohannes Weiner unsigned long flags = memcg->move_lock_flags;
21566de22619SJohannes Weiner
21576de22619SJohannes Weiner memcg->move_lock_task = NULL;
21586de22619SJohannes Weiner memcg->move_lock_flags = 0;
21596de22619SJohannes Weiner
21606de22619SJohannes Weiner spin_unlock_irqrestore(&memcg->move_lock, flags);
21616de22619SJohannes Weiner }
2162d7365e78SJohannes Weiner
2163d7365e78SJohannes Weiner rcu_read_unlock();
216489c06bd5SKAMEZAWA Hiroyuki }
2165739f79fcSJohannes Weiner
2166739f79fcSJohannes Weiner /**
2167f70ad448SMatthew Wilcox (Oracle) * folio_memcg_unlock - Release the binding between a folio and its memcg.
2168f70ad448SMatthew Wilcox (Oracle) * @folio: The folio.
2169f70ad448SMatthew Wilcox (Oracle) *
2170f70ad448SMatthew Wilcox (Oracle) * This releases the binding created by folio_memcg_lock(). This does
2171f70ad448SMatthew Wilcox (Oracle) * not change the accounting of this folio to its memcg, but it does
2172f70ad448SMatthew Wilcox (Oracle) * permit others to change it.
2173739f79fcSJohannes Weiner */
folio_memcg_unlock(struct folio * folio)2174f70ad448SMatthew Wilcox (Oracle) void folio_memcg_unlock(struct folio *folio)
2175f70ad448SMatthew Wilcox (Oracle) {
2176f70ad448SMatthew Wilcox (Oracle) __folio_memcg_unlock(folio_memcg(folio));
2177f70ad448SMatthew Wilcox (Oracle) }
2178f70ad448SMatthew Wilcox (Oracle)
2179fead2b86SMichal Hocko struct memcg_stock_pcp {
218056751146SSebastian Andrzej Siewior local_lock_t stock_lock;
2181fead2b86SMichal Hocko struct mem_cgroup *cached; /* this never be root cgroup */
2182fead2b86SMichal Hocko unsigned int nr_pages;
2183fead2b86SMichal Hocko
2184bf4f0599SRoman Gushchin #ifdef CONFIG_MEMCG_KMEM
2185bf4f0599SRoman Gushchin struct obj_cgroup *cached_objcg;
218668ac5b3cSWaiman Long struct pglist_data *cached_pgdat;
2187bf4f0599SRoman Gushchin unsigned int nr_bytes;
218868ac5b3cSWaiman Long int nr_slab_reclaimable_b;
218968ac5b3cSWaiman Long int nr_slab_unreclaimable_b;
2190bf4f0599SRoman Gushchin #endif
2191bf4f0599SRoman Gushchin
2192cdec2e42SKAMEZAWA Hiroyuki struct work_struct work;
219326fe6168SKAMEZAWA Hiroyuki unsigned long flags;
2194a0db00fcSKirill A. Shutemov #define FLUSHING_CACHED_CHARGE 0
2195cdec2e42SKAMEZAWA Hiroyuki };
219656751146SSebastian Andrzej Siewior static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock) = {
219756751146SSebastian Andrzej Siewior .stock_lock = INIT_LOCAL_LOCK(stock_lock),
219856751146SSebastian Andrzej Siewior };
21999f50fad6SMichal Hocko static DEFINE_MUTEX(percpu_charge_mutex);
2200cdec2e42SKAMEZAWA Hiroyuki
2201bf4f0599SRoman Gushchin #ifdef CONFIG_MEMCG_KMEM
220256751146SSebastian Andrzej Siewior static struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock);
2203bf4f0599SRoman Gushchin static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2204bf4f0599SRoman Gushchin struct mem_cgroup *root_memcg);
2205a8c49af3SYosry Ahmed static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages);
2206bf4f0599SRoman Gushchin
2207bf4f0599SRoman Gushchin #else
drain_obj_stock(struct memcg_stock_pcp * stock)220856751146SSebastian Andrzej Siewior static inline struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock)
2209bf4f0599SRoman Gushchin {
221056751146SSebastian Andrzej Siewior return NULL;
2211bf4f0599SRoman Gushchin }
obj_stock_flush_required(struct memcg_stock_pcp * stock,struct mem_cgroup * root_memcg)2212bf4f0599SRoman Gushchin static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2213bf4f0599SRoman Gushchin struct mem_cgroup *root_memcg)
2214bf4f0599SRoman Gushchin {
2215bf4f0599SRoman Gushchin return false;
2216bf4f0599SRoman Gushchin }
memcg_account_kmem(struct mem_cgroup * memcg,int nr_pages)2217a8c49af3SYosry Ahmed static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages)
2218a8c49af3SYosry Ahmed {
2219a8c49af3SYosry Ahmed }
2220bf4f0599SRoman Gushchin #endif
2221bf4f0599SRoman Gushchin
2222a0956d54SSuleiman Souhlal /**
2223a0956d54SSuleiman Souhlal * consume_stock: Try to consume stocked charge on this cpu.
2224a0956d54SSuleiman Souhlal * @memcg: memcg to consume from.
2225a0956d54SSuleiman Souhlal * @nr_pages: how many pages to charge.
2226a0956d54SSuleiman Souhlal *
2227a0956d54SSuleiman Souhlal * The charges will only happen if @memcg matches the current cpu's memcg
2228a0956d54SSuleiman Souhlal * stock, and at least @nr_pages are available in that stock. Failure to
2229a0956d54SSuleiman Souhlal * service an allocation will refill the stock.
2230a0956d54SSuleiman Souhlal *
2231a0956d54SSuleiman Souhlal * returns true if successful, false otherwise.
2232cdec2e42SKAMEZAWA Hiroyuki */
consume_stock(struct mem_cgroup * memcg,unsigned int nr_pages)2233a0956d54SSuleiman Souhlal static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2234cdec2e42SKAMEZAWA Hiroyuki {
2235cdec2e42SKAMEZAWA Hiroyuki struct memcg_stock_pcp *stock;
2236db2ba40cSJohannes Weiner unsigned long flags;
22373e32cb2eSJohannes Weiner bool ret = false;
2238cdec2e42SKAMEZAWA Hiroyuki
2239a983b5ebSJohannes Weiner if (nr_pages > MEMCG_CHARGE_BATCH)
22403e32cb2eSJohannes Weiner return ret;
2241a0956d54SSuleiman Souhlal
224256751146SSebastian Andrzej Siewior local_lock_irqsave(&memcg_stock.stock_lock, flags);
2243db2ba40cSJohannes Weiner
2244db2ba40cSJohannes Weiner stock = this_cpu_ptr(&memcg_stock);
2245f785a8f2SRoman Gushchin if (memcg == READ_ONCE(stock->cached) && stock->nr_pages >= nr_pages) {
2246a0956d54SSuleiman Souhlal stock->nr_pages -= nr_pages;
22473e32cb2eSJohannes Weiner ret = true;
22483e32cb2eSJohannes Weiner }
2249db2ba40cSJohannes Weiner
225056751146SSebastian Andrzej Siewior local_unlock_irqrestore(&memcg_stock.stock_lock, flags);
2251db2ba40cSJohannes Weiner
2252cdec2e42SKAMEZAWA Hiroyuki return ret;
2253cdec2e42SKAMEZAWA Hiroyuki }
2254cdec2e42SKAMEZAWA Hiroyuki
2255cdec2e42SKAMEZAWA Hiroyuki /*
22563e32cb2eSJohannes Weiner * Returns stocks cached in percpu and reset cached information.
2257cdec2e42SKAMEZAWA Hiroyuki */
drain_stock(struct memcg_stock_pcp * stock)2258cdec2e42SKAMEZAWA Hiroyuki static void drain_stock(struct memcg_stock_pcp *stock)
2259cdec2e42SKAMEZAWA Hiroyuki {
2260f785a8f2SRoman Gushchin struct mem_cgroup *old = READ_ONCE(stock->cached);
2261cdec2e42SKAMEZAWA Hiroyuki
22621a3e1f40SJohannes Weiner if (!old)
22631a3e1f40SJohannes Weiner return;
22641a3e1f40SJohannes Weiner
226511c9ea4eSJohannes Weiner if (stock->nr_pages) {
22663e32cb2eSJohannes Weiner page_counter_uncharge(&old->memory, stock->nr_pages);
22677941d214SJohannes Weiner if (do_memsw_account())
22683e32cb2eSJohannes Weiner page_counter_uncharge(&old->memsw, stock->nr_pages);
226911c9ea4eSJohannes Weiner stock->nr_pages = 0;
2270cdec2e42SKAMEZAWA Hiroyuki }
22711a3e1f40SJohannes Weiner
22721a3e1f40SJohannes Weiner css_put(&old->css);
2273f785a8f2SRoman Gushchin WRITE_ONCE(stock->cached, NULL);
2274cdec2e42SKAMEZAWA Hiroyuki }
2275cdec2e42SKAMEZAWA Hiroyuki
drain_local_stock(struct work_struct * dummy)2276cdec2e42SKAMEZAWA Hiroyuki static void drain_local_stock(struct work_struct *dummy)
2277cdec2e42SKAMEZAWA Hiroyuki {
2278db2ba40cSJohannes Weiner struct memcg_stock_pcp *stock;
227956751146SSebastian Andrzej Siewior struct obj_cgroup *old = NULL;
2280db2ba40cSJohannes Weiner unsigned long flags;
2281db2ba40cSJohannes Weiner
228272f0184cSMichal Hocko /*
22835c49cf9aSMichal Hocko * The only protection from cpu hotplug (memcg_hotplug_cpu_dead) vs.
22845c49cf9aSMichal Hocko * drain_stock races is that we always operate on local CPU stock
22855c49cf9aSMichal Hocko * here with IRQ disabled
228672f0184cSMichal Hocko */
228756751146SSebastian Andrzej Siewior local_lock_irqsave(&memcg_stock.stock_lock, flags);
2288db2ba40cSJohannes Weiner
2289db2ba40cSJohannes Weiner stock = this_cpu_ptr(&memcg_stock);
229056751146SSebastian Andrzej Siewior old = drain_obj_stock(stock);
2291cdec2e42SKAMEZAWA Hiroyuki drain_stock(stock);
229226fe6168SKAMEZAWA Hiroyuki clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
2293db2ba40cSJohannes Weiner
229456751146SSebastian Andrzej Siewior local_unlock_irqrestore(&memcg_stock.stock_lock, flags);
229556751146SSebastian Andrzej Siewior if (old)
229656751146SSebastian Andrzej Siewior obj_cgroup_put(old);
2297cdec2e42SKAMEZAWA Hiroyuki }
2298cdec2e42SKAMEZAWA Hiroyuki
2299cdec2e42SKAMEZAWA Hiroyuki /*
23003e32cb2eSJohannes Weiner * Cache charges(val) to local per_cpu area.
2301320cc51dSGreg Thelen * This will be consumed by consume_stock() function, later.
2302cdec2e42SKAMEZAWA Hiroyuki */
__refill_stock(struct mem_cgroup * memcg,unsigned int nr_pages)2303af9a3b69SJohannes Weiner static void __refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2304cdec2e42SKAMEZAWA Hiroyuki {
2305db2ba40cSJohannes Weiner struct memcg_stock_pcp *stock;
2306db2ba40cSJohannes Weiner
2307db2ba40cSJohannes Weiner stock = this_cpu_ptr(&memcg_stock);
2308f785a8f2SRoman Gushchin if (READ_ONCE(stock->cached) != memcg) { /* reset if necessary */
2309cdec2e42SKAMEZAWA Hiroyuki drain_stock(stock);
23101a3e1f40SJohannes Weiner css_get(&memcg->css);
2311f785a8f2SRoman Gushchin WRITE_ONCE(stock->cached, memcg);
2312cdec2e42SKAMEZAWA Hiroyuki }
231311c9ea4eSJohannes Weiner stock->nr_pages += nr_pages;
2314db2ba40cSJohannes Weiner
2315a983b5ebSJohannes Weiner if (stock->nr_pages > MEMCG_CHARGE_BATCH)
2316475d0487SRoman Gushchin drain_stock(stock);
2317af9a3b69SJohannes Weiner }
2318475d0487SRoman Gushchin
refill_stock(struct mem_cgroup * memcg,unsigned int nr_pages)2319af9a3b69SJohannes Weiner static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2320af9a3b69SJohannes Weiner {
2321af9a3b69SJohannes Weiner unsigned long flags;
2322af9a3b69SJohannes Weiner
232356751146SSebastian Andrzej Siewior local_lock_irqsave(&memcg_stock.stock_lock, flags);
2324af9a3b69SJohannes Weiner __refill_stock(memcg, nr_pages);
232556751146SSebastian Andrzej Siewior local_unlock_irqrestore(&memcg_stock.stock_lock, flags);
2326cdec2e42SKAMEZAWA Hiroyuki }
2327cdec2e42SKAMEZAWA Hiroyuki
2328cdec2e42SKAMEZAWA Hiroyuki /*
2329c0ff4b85SRaghavendra K T * Drains all per-CPU charge caches for given root_memcg resp. subtree
23306d3d6aa2SJohannes Weiner * of the hierarchy under it.
2331cdec2e42SKAMEZAWA Hiroyuki */
drain_all_stock(struct mem_cgroup * root_memcg)23326d3d6aa2SJohannes Weiner static void drain_all_stock(struct mem_cgroup *root_memcg)
2333cdec2e42SKAMEZAWA Hiroyuki {
233426fe6168SKAMEZAWA Hiroyuki int cpu, curcpu;
2335d38144b7SMichal Hocko
23366d3d6aa2SJohannes Weiner /* If someone's already draining, avoid adding running more workers. */
23376d3d6aa2SJohannes Weiner if (!mutex_trylock(&percpu_charge_mutex))
23386d3d6aa2SJohannes Weiner return;
233972f0184cSMichal Hocko /*
234072f0184cSMichal Hocko * Notify other cpus that system-wide "drain" is running
234172f0184cSMichal Hocko * We do not care about races with the cpu hotplug because cpu down
234272f0184cSMichal Hocko * as well as workers from this path always operate on the local
234372f0184cSMichal Hocko * per-cpu data. CPU up doesn't touch memcg_stock at all.
234472f0184cSMichal Hocko */
23450790ed62SSebastian Andrzej Siewior migrate_disable();
23460790ed62SSebastian Andrzej Siewior curcpu = smp_processor_id();
2347cdec2e42SKAMEZAWA Hiroyuki for_each_online_cpu(cpu) {
2348cdec2e42SKAMEZAWA Hiroyuki struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2349c0ff4b85SRaghavendra K T struct mem_cgroup *memcg;
2350e1a366beSRoman Gushchin bool flush = false;
235126fe6168SKAMEZAWA Hiroyuki
2352e1a366beSRoman Gushchin rcu_read_lock();
2353f785a8f2SRoman Gushchin memcg = READ_ONCE(stock->cached);
2354e1a366beSRoman Gushchin if (memcg && stock->nr_pages &&
2355e1a366beSRoman Gushchin mem_cgroup_is_descendant(memcg, root_memcg))
2356e1a366beSRoman Gushchin flush = true;
235727fb0956SMiaohe Lin else if (obj_stock_flush_required(stock, root_memcg))
2358bf4f0599SRoman Gushchin flush = true;
2359e1a366beSRoman Gushchin rcu_read_unlock();
2360e1a366beSRoman Gushchin
2361e1a366beSRoman Gushchin if (flush &&
2362e1a366beSRoman Gushchin !test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
2363d1a05b69SMichal Hocko if (cpu == curcpu)
2364d1a05b69SMichal Hocko drain_local_stock(&stock->work);
23656a792697SMichal Hocko else if (!cpu_is_isolated(cpu))
2366cdec2e42SKAMEZAWA Hiroyuki schedule_work_on(cpu, &stock->work);
2367cdec2e42SKAMEZAWA Hiroyuki }
2368d1a05b69SMichal Hocko }
23690790ed62SSebastian Andrzej Siewior migrate_enable();
23709f50fad6SMichal Hocko mutex_unlock(&percpu_charge_mutex);
2371cdec2e42SKAMEZAWA Hiroyuki }
2372cdec2e42SKAMEZAWA Hiroyuki
memcg_hotplug_cpu_dead(unsigned int cpu)2373308167fcSSebastian Andrzej Siewior static int memcg_hotplug_cpu_dead(unsigned int cpu)
2374cdec2e42SKAMEZAWA Hiroyuki {
2375cdec2e42SKAMEZAWA Hiroyuki struct memcg_stock_pcp *stock;
2376cdec2e42SKAMEZAWA Hiroyuki
2377cdec2e42SKAMEZAWA Hiroyuki stock = &per_cpu(memcg_stock, cpu);
2378cdec2e42SKAMEZAWA Hiroyuki drain_stock(stock);
2379a983b5ebSJohannes Weiner
2380308167fcSSebastian Andrzej Siewior return 0;
2381cdec2e42SKAMEZAWA Hiroyuki }
2382cdec2e42SKAMEZAWA Hiroyuki
reclaim_high(struct mem_cgroup * memcg,unsigned int nr_pages,gfp_t gfp_mask)2383b3ff9291SChris Down static unsigned long reclaim_high(struct mem_cgroup *memcg,
2384f7e1cb6eSJohannes Weiner unsigned int nr_pages,
2385f7e1cb6eSJohannes Weiner gfp_t gfp_mask)
2386f7e1cb6eSJohannes Weiner {
2387b3ff9291SChris Down unsigned long nr_reclaimed = 0;
2388b3ff9291SChris Down
2389f7e1cb6eSJohannes Weiner do {
2390e22c6ed9SJohannes Weiner unsigned long pflags;
2391e22c6ed9SJohannes Weiner
2392d1663a90SJakub Kicinski if (page_counter_read(&memcg->memory) <=
2393d1663a90SJakub Kicinski READ_ONCE(memcg->memory.high))
2394f7e1cb6eSJohannes Weiner continue;
2395e22c6ed9SJohannes Weiner
2396e27be240SJohannes Weiner memcg_memory_event(memcg, MEMCG_HIGH);
2397e22c6ed9SJohannes Weiner
2398e22c6ed9SJohannes Weiner psi_memstall_enter(&pflags);
2399b3ff9291SChris Down nr_reclaimed += try_to_free_mem_cgroup_pages(memcg, nr_pages,
240073b73bacSYosry Ahmed gfp_mask,
240155ab834aSMichal Hocko MEMCG_RECLAIM_MAY_SWAP);
2402e22c6ed9SJohannes Weiner psi_memstall_leave(&pflags);
24034bf17307SChris Down } while ((memcg = parent_mem_cgroup(memcg)) &&
24044bf17307SChris Down !mem_cgroup_is_root(memcg));
2405b3ff9291SChris Down
2406b3ff9291SChris Down return nr_reclaimed;
2407f7e1cb6eSJohannes Weiner }
2408f7e1cb6eSJohannes Weiner
high_work_func(struct work_struct * work)2409f7e1cb6eSJohannes Weiner static void high_work_func(struct work_struct *work)
2410f7e1cb6eSJohannes Weiner {
2411f7e1cb6eSJohannes Weiner struct mem_cgroup *memcg;
2412f7e1cb6eSJohannes Weiner
2413f7e1cb6eSJohannes Weiner memcg = container_of(work, struct mem_cgroup, high_work);
2414a983b5ebSJohannes Weiner reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);
2415f7e1cb6eSJohannes Weiner }
2416f7e1cb6eSJohannes Weiner
2417b23afb93STejun Heo /*
24180e4b01dfSChris Down * Clamp the maximum sleep time per allocation batch to 2 seconds. This is
24190e4b01dfSChris Down * enough to still cause a significant slowdown in most cases, while still
24200e4b01dfSChris Down * allowing diagnostics and tracing to proceed without becoming stuck.
24210e4b01dfSChris Down */
24220e4b01dfSChris Down #define MEMCG_MAX_HIGH_DELAY_JIFFIES (2UL*HZ)
24230e4b01dfSChris Down
24240e4b01dfSChris Down /*
24250e4b01dfSChris Down * When calculating the delay, we use these either side of the exponentiation to
24260e4b01dfSChris Down * maintain precision and scale to a reasonable number of jiffies (see the table
24270e4b01dfSChris Down * below.
24280e4b01dfSChris Down *
24290e4b01dfSChris Down * - MEMCG_DELAY_PRECISION_SHIFT: Extra precision bits while translating the
24300e4b01dfSChris Down * overage ratio to a delay.
2431ac5ddd0fSRandy Dunlap * - MEMCG_DELAY_SCALING_SHIFT: The number of bits to scale down the
24320e4b01dfSChris Down * proposed penalty in order to reduce to a reasonable number of jiffies, and
24330e4b01dfSChris Down * to produce a reasonable delay curve.
24340e4b01dfSChris Down *
24350e4b01dfSChris Down * MEMCG_DELAY_SCALING_SHIFT just happens to be a number that produces a
24360e4b01dfSChris Down * reasonable delay curve compared to precision-adjusted overage, not
24370e4b01dfSChris Down * penalising heavily at first, but still making sure that growth beyond the
24380e4b01dfSChris Down * limit penalises misbehaviour cgroups by slowing them down exponentially. For
24390e4b01dfSChris Down * example, with a high of 100 megabytes:
24400e4b01dfSChris Down *
24410e4b01dfSChris Down * +-------+------------------------+
24420e4b01dfSChris Down * | usage | time to allocate in ms |
24430e4b01dfSChris Down * +-------+------------------------+
24440e4b01dfSChris Down * | 100M | 0 |
24450e4b01dfSChris Down * | 101M | 6 |
24460e4b01dfSChris Down * | 102M | 25 |
24470e4b01dfSChris Down * | 103M | 57 |
24480e4b01dfSChris Down * | 104M | 102 |
24490e4b01dfSChris Down * | 105M | 159 |
24500e4b01dfSChris Down * | 106M | 230 |
24510e4b01dfSChris Down * | 107M | 313 |
24520e4b01dfSChris Down * | 108M | 409 |
24530e4b01dfSChris Down * | 109M | 518 |
24540e4b01dfSChris Down * | 110M | 639 |
24550e4b01dfSChris Down * | 111M | 774 |
24560e4b01dfSChris Down * | 112M | 921 |
24570e4b01dfSChris Down * | 113M | 1081 |
24580e4b01dfSChris Down * | 114M | 1254 |
24590e4b01dfSChris Down * | 115M | 1439 |
24600e4b01dfSChris Down * | 116M | 1638 |
24610e4b01dfSChris Down * | 117M | 1849 |
24620e4b01dfSChris Down * | 118M | 2000 |
24630e4b01dfSChris Down * | 119M | 2000 |
24640e4b01dfSChris Down * | 120M | 2000 |
24650e4b01dfSChris Down * +-------+------------------------+
24660e4b01dfSChris Down */
24670e4b01dfSChris Down #define MEMCG_DELAY_PRECISION_SHIFT 20
24680e4b01dfSChris Down #define MEMCG_DELAY_SCALING_SHIFT 14
24690e4b01dfSChris Down
calculate_overage(unsigned long usage,unsigned long high)24708a5dbc65SJakub Kicinski static u64 calculate_overage(unsigned long usage, unsigned long high)
2471b23afb93STejun Heo {
2472e26733e0SChris Down u64 overage;
2473b23afb93STejun Heo
24749b8b1754SJakub Kicinski if (usage <= high)
24758a5dbc65SJakub Kicinski return 0;
24769b8b1754SJakub Kicinski
24770e4b01dfSChris Down /*
2478e26733e0SChris Down * Prevent division by 0 in overage calculation by acting as if
2479e26733e0SChris Down * it was a threshold of 1 page
2480e26733e0SChris Down */
2481e26733e0SChris Down high = max(high, 1UL);
2482e26733e0SChris Down
2483e26733e0SChris Down overage = usage - high;
2484e26733e0SChris Down overage <<= MEMCG_DELAY_PRECISION_SHIFT;
24858a5dbc65SJakub Kicinski return div64_u64(overage, high);
24868a5dbc65SJakub Kicinski }
2487e26733e0SChris Down
mem_find_max_overage(struct mem_cgroup * memcg)24888a5dbc65SJakub Kicinski static u64 mem_find_max_overage(struct mem_cgroup *memcg)
24898a5dbc65SJakub Kicinski {
24908a5dbc65SJakub Kicinski u64 overage, max_overage = 0;
24918a5dbc65SJakub Kicinski
24928a5dbc65SJakub Kicinski do {
24938a5dbc65SJakub Kicinski overage = calculate_overage(page_counter_read(&memcg->memory),
2494d1663a90SJakub Kicinski READ_ONCE(memcg->memory.high));
24958a5dbc65SJakub Kicinski max_overage = max(overage, max_overage);
2496e26733e0SChris Down } while ((memcg = parent_mem_cgroup(memcg)) &&
2497e26733e0SChris Down !mem_cgroup_is_root(memcg));
2498e26733e0SChris Down
24998a5dbc65SJakub Kicinski return max_overage;
25008a5dbc65SJakub Kicinski }
25018a5dbc65SJakub Kicinski
swap_find_max_overage(struct mem_cgroup * memcg)25024b82ab4fSJakub Kicinski static u64 swap_find_max_overage(struct mem_cgroup *memcg)
25034b82ab4fSJakub Kicinski {
25044b82ab4fSJakub Kicinski u64 overage, max_overage = 0;
25054b82ab4fSJakub Kicinski
25064b82ab4fSJakub Kicinski do {
25074b82ab4fSJakub Kicinski overage = calculate_overage(page_counter_read(&memcg->swap),
25084b82ab4fSJakub Kicinski READ_ONCE(memcg->swap.high));
25094b82ab4fSJakub Kicinski if (overage)
25104b82ab4fSJakub Kicinski memcg_memory_event(memcg, MEMCG_SWAP_HIGH);
25114b82ab4fSJakub Kicinski max_overage = max(overage, max_overage);
25124b82ab4fSJakub Kicinski } while ((memcg = parent_mem_cgroup(memcg)) &&
25134b82ab4fSJakub Kicinski !mem_cgroup_is_root(memcg));
25144b82ab4fSJakub Kicinski
25154b82ab4fSJakub Kicinski return max_overage;
25164b82ab4fSJakub Kicinski }
25174b82ab4fSJakub Kicinski
25188a5dbc65SJakub Kicinski /*
25198a5dbc65SJakub Kicinski * Get the number of jiffies that we should penalise a mischievous cgroup which
25208a5dbc65SJakub Kicinski * is exceeding its memory.high by checking both it and its ancestors.
25218a5dbc65SJakub Kicinski */
calculate_high_delay(struct mem_cgroup * memcg,unsigned int nr_pages,u64 max_overage)25228a5dbc65SJakub Kicinski static unsigned long calculate_high_delay(struct mem_cgroup *memcg,
25238a5dbc65SJakub Kicinski unsigned int nr_pages,
25248a5dbc65SJakub Kicinski u64 max_overage)
25258a5dbc65SJakub Kicinski {
25268a5dbc65SJakub Kicinski unsigned long penalty_jiffies;
25278a5dbc65SJakub Kicinski
2528e26733e0SChris Down if (!max_overage)
2529e26733e0SChris Down return 0;
2530e26733e0SChris Down
2531e26733e0SChris Down /*
25320e4b01dfSChris Down * We use overage compared to memory.high to calculate the number of
25330e4b01dfSChris Down * jiffies to sleep (penalty_jiffies). Ideally this value should be
25340e4b01dfSChris Down * fairly lenient on small overages, and increasingly harsh when the
25350e4b01dfSChris Down * memcg in question makes it clear that it has no intention of stopping
25360e4b01dfSChris Down * its crazy behaviour, so we exponentially increase the delay based on
25370e4b01dfSChris Down * overage amount.
25380e4b01dfSChris Down */
2539e26733e0SChris Down penalty_jiffies = max_overage * max_overage * HZ;
2540e26733e0SChris Down penalty_jiffies >>= MEMCG_DELAY_PRECISION_SHIFT;
2541e26733e0SChris Down penalty_jiffies >>= MEMCG_DELAY_SCALING_SHIFT;
25420e4b01dfSChris Down
25430e4b01dfSChris Down /*
25440e4b01dfSChris Down * Factor in the task's own contribution to the overage, such that four
25450e4b01dfSChris Down * N-sized allocations are throttled approximately the same as one
25460e4b01dfSChris Down * 4N-sized allocation.
25470e4b01dfSChris Down *
25480e4b01dfSChris Down * MEMCG_CHARGE_BATCH pages is nominal, so work out how much smaller or
25490e4b01dfSChris Down * larger the current charge patch is than that.
25500e4b01dfSChris Down */
2551ff144e69SJakub Kicinski return penalty_jiffies * nr_pages / MEMCG_CHARGE_BATCH;
2552e26733e0SChris Down }
2553e26733e0SChris Down
2554e26733e0SChris Down /*
2555e26733e0SChris Down * Scheduled by try_charge() to be executed from the userland return path
2556e26733e0SChris Down * and reclaims memory over the high limit.
2557e26733e0SChris Down */
mem_cgroup_handle_over_high(gfp_t gfp_mask)25589ea9cb00SJohannes Weiner void mem_cgroup_handle_over_high(gfp_t gfp_mask)
2559e26733e0SChris Down {
2560e26733e0SChris Down unsigned long penalty_jiffies;
2561e26733e0SChris Down unsigned long pflags;
2562b3ff9291SChris Down unsigned long nr_reclaimed;
2563e26733e0SChris Down unsigned int nr_pages = current->memcg_nr_pages_over_high;
2564d977aa93SChris Down int nr_retries = MAX_RECLAIM_RETRIES;
2565e26733e0SChris Down struct mem_cgroup *memcg;
2566b3ff9291SChris Down bool in_retry = false;
2567e26733e0SChris Down
2568e26733e0SChris Down if (likely(!nr_pages))
2569e26733e0SChris Down return;
2570e26733e0SChris Down
2571e26733e0SChris Down memcg = get_mem_cgroup_from_mm(current->mm);
2572e26733e0SChris Down current->memcg_nr_pages_over_high = 0;
2573e26733e0SChris Down
2574b3ff9291SChris Down retry_reclaim:
2575b3ff9291SChris Down /*
2576b3ff9291SChris Down * The allocating task should reclaim at least the batch size, but for
2577b3ff9291SChris Down * subsequent retries we only want to do what's necessary to prevent oom
2578b3ff9291SChris Down * or breaching resource isolation.
2579b3ff9291SChris Down *
2580b3ff9291SChris Down * This is distinct from memory.max or page allocator behaviour because
2581b3ff9291SChris Down * memory.high is currently batched, whereas memory.max and the page
2582b3ff9291SChris Down * allocator run every time an allocation is made.
2583b3ff9291SChris Down */
2584b3ff9291SChris Down nr_reclaimed = reclaim_high(memcg,
2585b3ff9291SChris Down in_retry ? SWAP_CLUSTER_MAX : nr_pages,
25869ea9cb00SJohannes Weiner gfp_mask);
2587b3ff9291SChris Down
2588e26733e0SChris Down /*
2589e26733e0SChris Down * memory.high is breached and reclaim is unable to keep up. Throttle
2590e26733e0SChris Down * allocators proactively to slow down excessive growth.
2591e26733e0SChris Down */
25928a5dbc65SJakub Kicinski penalty_jiffies = calculate_high_delay(memcg, nr_pages,
25938a5dbc65SJakub Kicinski mem_find_max_overage(memcg));
25940e4b01dfSChris Down
25954b82ab4fSJakub Kicinski penalty_jiffies += calculate_high_delay(memcg, nr_pages,
25964b82ab4fSJakub Kicinski swap_find_max_overage(memcg));
25974b82ab4fSJakub Kicinski
25980e4b01dfSChris Down /*
2599ff144e69SJakub Kicinski * Clamp the max delay per usermode return so as to still keep the
2600ff144e69SJakub Kicinski * application moving forwards and also permit diagnostics, albeit
2601ff144e69SJakub Kicinski * extremely slowly.
2602ff144e69SJakub Kicinski */
2603ff144e69SJakub Kicinski penalty_jiffies = min(penalty_jiffies, MEMCG_MAX_HIGH_DELAY_JIFFIES);
2604ff144e69SJakub Kicinski
2605ff144e69SJakub Kicinski /*
26060e4b01dfSChris Down * Don't sleep if the amount of jiffies this memcg owes us is so low
26070e4b01dfSChris Down * that it's not even worth doing, in an attempt to be nice to those who
26080e4b01dfSChris Down * go only a small amount over their memory.high value and maybe haven't
26090e4b01dfSChris Down * been aggressively reclaimed enough yet.
26100e4b01dfSChris Down */
26110e4b01dfSChris Down if (penalty_jiffies <= HZ / 100)
26120e4b01dfSChris Down goto out;
26130e4b01dfSChris Down
26140e4b01dfSChris Down /*
2615b3ff9291SChris Down * If reclaim is making forward progress but we're still over
2616b3ff9291SChris Down * memory.high, we want to encourage that rather than doing allocator
2617b3ff9291SChris Down * throttling.
2618b3ff9291SChris Down */
2619b3ff9291SChris Down if (nr_reclaimed || nr_retries--) {
2620b3ff9291SChris Down in_retry = true;
2621b3ff9291SChris Down goto retry_reclaim;
2622b3ff9291SChris Down }
2623b3ff9291SChris Down
2624b3ff9291SChris Down /*
26250e4b01dfSChris Down * If we exit early, we're guaranteed to die (since
26260e4b01dfSChris Down * schedule_timeout_killable sets TASK_KILLABLE). This means we don't
26270e4b01dfSChris Down * need to account for any ill-begotten jiffies to pay them off later.
26280e4b01dfSChris Down */
26290e4b01dfSChris Down psi_memstall_enter(&pflags);
26300e4b01dfSChris Down schedule_timeout_killable(penalty_jiffies);
26310e4b01dfSChris Down psi_memstall_leave(&pflags);
26320e4b01dfSChris Down
26330e4b01dfSChris Down out:
26340e4b01dfSChris Down css_put(&memcg->css);
2635b23afb93STejun Heo }
2636b23afb93STejun Heo
try_charge_memcg(struct mem_cgroup * memcg,gfp_t gfp_mask,unsigned int nr_pages)2637c5c8b16bSMuchun Song static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask,
26380029e19eSMichal Hocko unsigned int nr_pages)
26398a9f3ccdSBalbir Singh {
2640a983b5ebSJohannes Weiner unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages);
2641d977aa93SChris Down int nr_retries = MAX_RECLAIM_RETRIES;
26426539cc05SJohannes Weiner struct mem_cgroup *mem_over_limit;
26433e32cb2eSJohannes Weiner struct page_counter *counter;
26446539cc05SJohannes Weiner unsigned long nr_reclaimed;
2645a4ebf1b6SVasily Averin bool passed_oom = false;
264673b73bacSYosry Ahmed unsigned int reclaim_options = MEMCG_RECLAIM_MAY_SWAP;
2647b70a2a21SJohannes Weiner bool drained = false;
2648d6e103a7SRoman Gushchin bool raised_max_event = false;
2649e22c6ed9SJohannes Weiner unsigned long pflags;
2650a636b327SKAMEZAWA Hiroyuki
26516539cc05SJohannes Weiner retry:
2652a0956d54SSuleiman Souhlal if (consume_stock(memcg, nr_pages))
265310d53c74STejun Heo return 0;
26548a9f3ccdSBalbir Singh
26557941d214SJohannes Weiner if (!do_memsw_account() ||
26566071ca52SJohannes Weiner page_counter_try_charge(&memcg->memsw, batch, &counter)) {
26576071ca52SJohannes Weiner if (page_counter_try_charge(&memcg->memory, batch, &counter))
26586539cc05SJohannes Weiner goto done_restock;
26597941d214SJohannes Weiner if (do_memsw_account())
26603e32cb2eSJohannes Weiner page_counter_uncharge(&memcg->memsw, batch);
26613e32cb2eSJohannes Weiner mem_over_limit = mem_cgroup_from_counter(counter, memory);
26623fbe7244SJohannes Weiner } else {
26633e32cb2eSJohannes Weiner mem_over_limit = mem_cgroup_from_counter(counter, memsw);
266473b73bacSYosry Ahmed reclaim_options &= ~MEMCG_RECLAIM_MAY_SWAP;
26653fbe7244SJohannes Weiner }
26667a81b88cSKAMEZAWA Hiroyuki
26676539cc05SJohannes Weiner if (batch > nr_pages) {
26686539cc05SJohannes Weiner batch = nr_pages;
26696539cc05SJohannes Weiner goto retry;
26706539cc05SJohannes Weiner }
26716539cc05SJohannes Weiner
267206b078fcSJohannes Weiner /*
267389a28483SJohannes Weiner * Prevent unbounded recursion when reclaim operations need to
267489a28483SJohannes Weiner * allocate memory. This might exceed the limits temporarily,
267589a28483SJohannes Weiner * but we prefer facilitating memory reclaim and getting back
267689a28483SJohannes Weiner * under the limit over triggering OOM kills in these cases.
267789a28483SJohannes Weiner */
267889a28483SJohannes Weiner if (unlikely(current->flags & PF_MEMALLOC))
267989a28483SJohannes Weiner goto force;
268089a28483SJohannes Weiner
268106b078fcSJohannes Weiner if (unlikely(task_in_memcg_oom(current)))
268206b078fcSJohannes Weiner goto nomem;
268306b078fcSJohannes Weiner
2684d0164adcSMel Gorman if (!gfpflags_allow_blocking(gfp_mask))
26856539cc05SJohannes Weiner goto nomem;
26866539cc05SJohannes Weiner
2687e27be240SJohannes Weiner memcg_memory_event(mem_over_limit, MEMCG_MAX);
2688d6e103a7SRoman Gushchin raised_max_event = true;
2689241994edSJohannes Weiner
2690e22c6ed9SJohannes Weiner psi_memstall_enter(&pflags);
2691b70a2a21SJohannes Weiner nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
269255ab834aSMichal Hocko gfp_mask, reclaim_options);
2693e22c6ed9SJohannes Weiner psi_memstall_leave(&pflags);
26946539cc05SJohannes Weiner
269561e02c74SJohannes Weiner if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
26966539cc05SJohannes Weiner goto retry;
269728c34c29SJohannes Weiner
2698b70a2a21SJohannes Weiner if (!drained) {
26996d3d6aa2SJohannes Weiner drain_all_stock(mem_over_limit);
2700b70a2a21SJohannes Weiner drained = true;
2701b70a2a21SJohannes Weiner goto retry;
2702b70a2a21SJohannes Weiner }
2703b70a2a21SJohannes Weiner
270428c34c29SJohannes Weiner if (gfp_mask & __GFP_NORETRY)
270528c34c29SJohannes Weiner goto nomem;
27066539cc05SJohannes Weiner /*
27076539cc05SJohannes Weiner * Even though the limit is exceeded at this point, reclaim
27086539cc05SJohannes Weiner * may have been able to free some pages. Retry the charge
27096539cc05SJohannes Weiner * before killing the task.
27106539cc05SJohannes Weiner *
27116539cc05SJohannes Weiner * Only for regular pages, though: huge pages are rather
27126539cc05SJohannes Weiner * unlikely to succeed so close to the limit, and we fall back
27136539cc05SJohannes Weiner * to regular pages anyway in case of failure.
27146539cc05SJohannes Weiner */
271561e02c74SJohannes Weiner if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
27166539cc05SJohannes Weiner goto retry;
27176539cc05SJohannes Weiner /*
27186539cc05SJohannes Weiner * At task move, charge accounts can be doubly counted. So, it's
27196539cc05SJohannes Weiner * better to wait until the end of task_move if something is going on.
27206539cc05SJohannes Weiner */
27216539cc05SJohannes Weiner if (mem_cgroup_wait_acct_move(mem_over_limit))
27226539cc05SJohannes Weiner goto retry;
27236539cc05SJohannes Weiner
27249b130619SJohannes Weiner if (nr_retries--)
27259b130619SJohannes Weiner goto retry;
27269b130619SJohannes Weiner
272738d38493SShakeel Butt if (gfp_mask & __GFP_RETRY_MAYFAIL)
272829ef680aSMichal Hocko goto nomem;
272929ef680aSMichal Hocko
2730a4ebf1b6SVasily Averin /* Avoid endless loop for tasks bypassed by the oom killer */
2731a4ebf1b6SVasily Averin if (passed_oom && task_is_dying())
2732a4ebf1b6SVasily Averin goto nomem;
2733cdec2e42SKAMEZAWA Hiroyuki
273429ef680aSMichal Hocko /*
273529ef680aSMichal Hocko * keep retrying as long as the memcg oom killer is able to make
273629ef680aSMichal Hocko * a forward progress or bypass the charge if the oom killer
273729ef680aSMichal Hocko * couldn't make any progress.
273829ef680aSMichal Hocko */
2739becdf89dSShakeel Butt if (mem_cgroup_oom(mem_over_limit, gfp_mask,
2740becdf89dSShakeel Butt get_order(nr_pages * PAGE_SIZE))) {
2741a4ebf1b6SVasily Averin passed_oom = true;
2742d977aa93SChris Down nr_retries = MAX_RECLAIM_RETRIES;
274329ef680aSMichal Hocko goto retry;
274429ef680aSMichal Hocko }
27457a81b88cSKAMEZAWA Hiroyuki nomem:
27461461e8c2SShakeel Butt /*
27471461e8c2SShakeel Butt * Memcg doesn't have a dedicated reserve for atomic
27481461e8c2SShakeel Butt * allocations. But like the global atomic pool, we need to
27491461e8c2SShakeel Butt * put the burden of reclaim on regular allocation requests
27501461e8c2SShakeel Butt * and let these go through as privileged allocations.
27511461e8c2SShakeel Butt */
27521461e8c2SShakeel Butt if (!(gfp_mask & (__GFP_NOFAIL | __GFP_HIGH)))
27537a81b88cSKAMEZAWA Hiroyuki return -ENOMEM;
275410d53c74STejun Heo force:
275510d53c74STejun Heo /*
2756d6e103a7SRoman Gushchin * If the allocation has to be enforced, don't forget to raise
2757d6e103a7SRoman Gushchin * a MEMCG_MAX event.
2758d6e103a7SRoman Gushchin */
2759d6e103a7SRoman Gushchin if (!raised_max_event)
2760d6e103a7SRoman Gushchin memcg_memory_event(mem_over_limit, MEMCG_MAX);
2761d6e103a7SRoman Gushchin
2762d6e103a7SRoman Gushchin /*
276310d53c74STejun Heo * The allocation either can't fail or will lead to more memory
276410d53c74STejun Heo * being freed very soon. Allow memory usage go over the limit
276510d53c74STejun Heo * temporarily by force charging it.
276610d53c74STejun Heo */
276710d53c74STejun Heo page_counter_charge(&memcg->memory, nr_pages);
27687941d214SJohannes Weiner if (do_memsw_account())
276910d53c74STejun Heo page_counter_charge(&memcg->memsw, nr_pages);
277010d53c74STejun Heo
277110d53c74STejun Heo return 0;
27726539cc05SJohannes Weiner
27736539cc05SJohannes Weiner done_restock:
27746539cc05SJohannes Weiner if (batch > nr_pages)
27756539cc05SJohannes Weiner refill_stock(memcg, batch - nr_pages);
2776b23afb93STejun Heo
2777241994edSJohannes Weiner /*
2778b23afb93STejun Heo * If the hierarchy is above the normal consumption range, schedule
2779b23afb93STejun Heo * reclaim on returning to userland. We can perform reclaim here
278071baba4bSMel Gorman * if __GFP_RECLAIM but let's always punt for simplicity and so that
2781b23afb93STejun Heo * GFP_KERNEL can consistently be used during reclaim. @memcg is
2782b23afb93STejun Heo * not recorded as it most likely matches current's and won't
2783b23afb93STejun Heo * change in the meantime. As high limit is checked again before
2784b23afb93STejun Heo * reclaim, the cost of mismatch is negligible.
2785241994edSJohannes Weiner */
2786241994edSJohannes Weiner do {
27874b82ab4fSJakub Kicinski bool mem_high, swap_high;
27884b82ab4fSJakub Kicinski
27894b82ab4fSJakub Kicinski mem_high = page_counter_read(&memcg->memory) >
27904b82ab4fSJakub Kicinski READ_ONCE(memcg->memory.high);
27914b82ab4fSJakub Kicinski swap_high = page_counter_read(&memcg->swap) >
27924b82ab4fSJakub Kicinski READ_ONCE(memcg->swap.high);
27934b82ab4fSJakub Kicinski
2794f7e1cb6eSJohannes Weiner /* Don't bother a random interrupted task */
2795086f694aSShakeel Butt if (!in_task()) {
27964b82ab4fSJakub Kicinski if (mem_high) {
2797f7e1cb6eSJohannes Weiner schedule_work(&memcg->high_work);
2798f7e1cb6eSJohannes Weiner break;
2799f7e1cb6eSJohannes Weiner }
28004b82ab4fSJakub Kicinski continue;
28014b82ab4fSJakub Kicinski }
28024b82ab4fSJakub Kicinski
28034b82ab4fSJakub Kicinski if (mem_high || swap_high) {
28044b82ab4fSJakub Kicinski /*
28054b82ab4fSJakub Kicinski * The allocating tasks in this cgroup will need to do
28064b82ab4fSJakub Kicinski * reclaim or be throttled to prevent further growth
28074b82ab4fSJakub Kicinski * of the memory or swap footprints.
28084b82ab4fSJakub Kicinski *
28094b82ab4fSJakub Kicinski * Target some best-effort fairness between the tasks,
28104b82ab4fSJakub Kicinski * and distribute reclaim work and delay penalties
28114b82ab4fSJakub Kicinski * based on how much each task is actually allocating.
28124b82ab4fSJakub Kicinski */
28139516a18aSVladimir Davydov current->memcg_nr_pages_over_high += batch;
2814b23afb93STejun Heo set_notify_resume(current);
2815b23afb93STejun Heo break;
2816b23afb93STejun Heo }
2817241994edSJohannes Weiner } while ((memcg = parent_mem_cgroup(memcg)));
281810d53c74STejun Heo
2819c9afe31eSShakeel Butt if (current->memcg_nr_pages_over_high > MEMCG_CHARGE_BATCH &&
2820c9afe31eSShakeel Butt !(current->flags & PF_MEMALLOC) &&
2821c9afe31eSShakeel Butt gfpflags_allow_blocking(gfp_mask)) {
28229ea9cb00SJohannes Weiner mem_cgroup_handle_over_high(gfp_mask);
2823c9afe31eSShakeel Butt }
282410d53c74STejun Heo return 0;
28257a81b88cSKAMEZAWA Hiroyuki }
28268a9f3ccdSBalbir Singh
try_charge(struct mem_cgroup * memcg,gfp_t gfp_mask,unsigned int nr_pages)2827c5c8b16bSMuchun Song static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2828c5c8b16bSMuchun Song unsigned int nr_pages)
2829c5c8b16bSMuchun Song {
2830c5c8b16bSMuchun Song if (mem_cgroup_is_root(memcg))
2831c5c8b16bSMuchun Song return 0;
2832c5c8b16bSMuchun Song
2833c5c8b16bSMuchun Song return try_charge_memcg(memcg, gfp_mask, nr_pages);
2834c5c8b16bSMuchun Song }
2835c5c8b16bSMuchun Song
cancel_charge(struct mem_cgroup * memcg,unsigned int nr_pages)283658056f77SShakeel Butt static inline void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2837a3032a2cSDaisuke Nishimura {
2838ce00a967SJohannes Weiner if (mem_cgroup_is_root(memcg))
2839ce00a967SJohannes Weiner return;
2840ce00a967SJohannes Weiner
28413e32cb2eSJohannes Weiner page_counter_uncharge(&memcg->memory, nr_pages);
28427941d214SJohannes Weiner if (do_memsw_account())
28433e32cb2eSJohannes Weiner page_counter_uncharge(&memcg->memsw, nr_pages);
2844d01dd17fSKAMEZAWA Hiroyuki }
2845d01dd17fSKAMEZAWA Hiroyuki
commit_charge(struct folio * folio,struct mem_cgroup * memcg)2846118f2875SMatthew Wilcox (Oracle) static void commit_charge(struct folio *folio, struct mem_cgroup *memcg)
28470a31bc97SJohannes Weiner {
2848118f2875SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(folio_memcg(folio), folio);
28490a31bc97SJohannes Weiner /*
2850a5eb011aSAlex Shi * Any of the following ensures page's memcg stability:
28510a31bc97SJohannes Weiner *
2852a0b5b414SJohannes Weiner * - the page lock
2853a0b5b414SJohannes Weiner * - LRU isolation
28546c77b607SKefeng Wang * - folio_memcg_lock()
2855a0b5b414SJohannes Weiner * - exclusive reference
2856018ee47fSYu Zhao * - mem_cgroup_trylock_pages()
28570a31bc97SJohannes Weiner */
2858118f2875SMatthew Wilcox (Oracle) folio->memcg_data = (unsigned long)memcg;
28598a9f3ccdSBalbir Singh }
28608a9f3ccdSBalbir Singh
286184c07d11SKirill Tkhai #ifdef CONFIG_MEMCG_KMEM
286241eb5df1SWaiman Long /*
286341eb5df1SWaiman Long * The allocated objcg pointers array is not accounted directly.
286441eb5df1SWaiman Long * Moreover, it should not come from DMA buffer and is not readily
286541eb5df1SWaiman Long * reclaimable. So those GFP bits should be masked off.
286641eb5df1SWaiman Long */
2867572d3e8bSRoman Gushchin #define OBJCGS_CLEAR_MASK (__GFP_DMA | __GFP_RECLAIMABLE | \
2868572d3e8bSRoman Gushchin __GFP_ACCOUNT | __GFP_NOFAIL)
286941eb5df1SWaiman Long
2870a7ebf564SWaiman Long /*
2871a7ebf564SWaiman Long * mod_objcg_mlstate() may be called with irq enabled, so
2872a7ebf564SWaiman Long * mod_memcg_lruvec_state() should be used.
2873a7ebf564SWaiman Long */
mod_objcg_mlstate(struct obj_cgroup * objcg,struct pglist_data * pgdat,enum node_stat_item idx,int nr)2874a7ebf564SWaiman Long static inline void mod_objcg_mlstate(struct obj_cgroup *objcg,
2875a7ebf564SWaiman Long struct pglist_data *pgdat,
2876a7ebf564SWaiman Long enum node_stat_item idx, int nr)
2877a7ebf564SWaiman Long {
2878a7ebf564SWaiman Long struct mem_cgroup *memcg;
2879a7ebf564SWaiman Long struct lruvec *lruvec;
2880a7ebf564SWaiman Long
2881a7ebf564SWaiman Long rcu_read_lock();
2882a7ebf564SWaiman Long memcg = obj_cgroup_memcg(objcg);
2883a7ebf564SWaiman Long lruvec = mem_cgroup_lruvec(memcg, pgdat);
2884a7ebf564SWaiman Long mod_memcg_lruvec_state(lruvec, idx, nr);
2885a7ebf564SWaiman Long rcu_read_unlock();
2886a7ebf564SWaiman Long }
2887a7ebf564SWaiman Long
memcg_alloc_slab_cgroups(struct slab * slab,struct kmem_cache * s,gfp_t gfp,bool new_slab)28884b5f8d9aSVlastimil Babka int memcg_alloc_slab_cgroups(struct slab *slab, struct kmem_cache *s,
28894b5f8d9aSVlastimil Babka gfp_t gfp, bool new_slab)
289010befea9SRoman Gushchin {
28914b5f8d9aSVlastimil Babka unsigned int objects = objs_per_slab(s, slab);
28922e9bd483SRoman Gushchin unsigned long memcg_data;
289310befea9SRoman Gushchin void *vec;
289410befea9SRoman Gushchin
289541eb5df1SWaiman Long gfp &= ~OBJCGS_CLEAR_MASK;
289610befea9SRoman Gushchin vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp,
28974b5f8d9aSVlastimil Babka slab_nid(slab));
289810befea9SRoman Gushchin if (!vec)
289910befea9SRoman Gushchin return -ENOMEM;
290010befea9SRoman Gushchin
29012e9bd483SRoman Gushchin memcg_data = (unsigned long) vec | MEMCG_DATA_OBJCGS;
29024b5f8d9aSVlastimil Babka if (new_slab) {
29032e9bd483SRoman Gushchin /*
29044b5f8d9aSVlastimil Babka * If the slab is brand new and nobody can yet access its
29054b5f8d9aSVlastimil Babka * memcg_data, no synchronization is required and memcg_data can
29064b5f8d9aSVlastimil Babka * be simply assigned.
29072e9bd483SRoman Gushchin */
29084b5f8d9aSVlastimil Babka slab->memcg_data = memcg_data;
29094b5f8d9aSVlastimil Babka } else if (cmpxchg(&slab->memcg_data, 0, memcg_data)) {
29102e9bd483SRoman Gushchin /*
29114b5f8d9aSVlastimil Babka * If the slab is already in use, somebody can allocate and
29124b5f8d9aSVlastimil Babka * assign obj_cgroups in parallel. In this case the existing
29132e9bd483SRoman Gushchin * objcg vector should be reused.
29142e9bd483SRoman Gushchin */
291510befea9SRoman Gushchin kfree(vec);
29162e9bd483SRoman Gushchin return 0;
29172e9bd483SRoman Gushchin }
291810befea9SRoman Gushchin
29192e9bd483SRoman Gushchin kmemleak_not_leak(vec);
292010befea9SRoman Gushchin return 0;
292110befea9SRoman Gushchin }
292210befea9SRoman Gushchin
2923fc4db90fSRoman Gushchin static __always_inline
mem_cgroup_from_obj_folio(struct folio * folio,void * p)2924fc4db90fSRoman Gushchin struct mem_cgroup *mem_cgroup_from_obj_folio(struct folio *folio, void *p)
29258380ce47SRoman Gushchin {
29268380ce47SRoman Gushchin /*
29279855609bSRoman Gushchin * Slab objects are accounted individually, not per-page.
29289855609bSRoman Gushchin * Memcg membership data for each individual object is saved in
29294b5f8d9aSVlastimil Babka * slab->memcg_data.
29308380ce47SRoman Gushchin */
29314b5f8d9aSVlastimil Babka if (folio_test_slab(folio)) {
29324b5f8d9aSVlastimil Babka struct obj_cgroup **objcgs;
29334b5f8d9aSVlastimil Babka struct slab *slab;
29349855609bSRoman Gushchin unsigned int off;
29359855609bSRoman Gushchin
29364b5f8d9aSVlastimil Babka slab = folio_slab(folio);
29374b5f8d9aSVlastimil Babka objcgs = slab_objcgs(slab);
29384b5f8d9aSVlastimil Babka if (!objcgs)
29394b5f8d9aSVlastimil Babka return NULL;
29404b5f8d9aSVlastimil Babka
29414b5f8d9aSVlastimil Babka off = obj_to_index(slab->slab_cache, slab, p);
29424b5f8d9aSVlastimil Babka if (objcgs[off])
29434b5f8d9aSVlastimil Babka return obj_cgroup_memcg(objcgs[off]);
294410befea9SRoman Gushchin
294510befea9SRoman Gushchin return NULL;
29469855609bSRoman Gushchin }
29478380ce47SRoman Gushchin
2948bcfe06bfSRoman Gushchin /*
2949becacb04SMatthew Wilcox * folio_memcg_check() is used here, because in theory we can encounter
29504b5f8d9aSVlastimil Babka * a folio where the slab flag has been cleared already, but
29514b5f8d9aSVlastimil Babka * slab->memcg_data has not been freed yet
2952becacb04SMatthew Wilcox * folio_memcg_check() will guarantee that a proper memory
2953bcfe06bfSRoman Gushchin * cgroup pointer or NULL will be returned.
2954bcfe06bfSRoman Gushchin */
2955becacb04SMatthew Wilcox return folio_memcg_check(folio);
29568380ce47SRoman Gushchin }
29578380ce47SRoman Gushchin
2958fc4db90fSRoman Gushchin /*
2959fc4db90fSRoman Gushchin * Returns a pointer to the memory cgroup to which the kernel object is charged.
2960fc4db90fSRoman Gushchin *
2961fc4db90fSRoman Gushchin * A passed kernel object can be a slab object, vmalloc object or a generic
2962fc4db90fSRoman Gushchin * kernel page, so different mechanisms for getting the memory cgroup pointer
2963fc4db90fSRoman Gushchin * should be used.
2964fc4db90fSRoman Gushchin *
2965fc4db90fSRoman Gushchin * In certain cases (e.g. kernel stacks or large kmallocs with SLUB) the caller
2966fc4db90fSRoman Gushchin * can not know for sure how the kernel object is implemented.
2967fc4db90fSRoman Gushchin * mem_cgroup_from_obj() can be safely used in such cases.
2968fc4db90fSRoman Gushchin *
2969fc4db90fSRoman Gushchin * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(),
2970fc4db90fSRoman Gushchin * cgroup_mutex, etc.
2971fc4db90fSRoman Gushchin */
mem_cgroup_from_obj(void * p)2972fc4db90fSRoman Gushchin struct mem_cgroup *mem_cgroup_from_obj(void *p)
2973fc4db90fSRoman Gushchin {
2974fc4db90fSRoman Gushchin struct folio *folio;
2975fc4db90fSRoman Gushchin
2976fc4db90fSRoman Gushchin if (mem_cgroup_disabled())
2977fc4db90fSRoman Gushchin return NULL;
2978fc4db90fSRoman Gushchin
2979fc4db90fSRoman Gushchin if (unlikely(is_vmalloc_addr(p)))
2980fc4db90fSRoman Gushchin folio = page_folio(vmalloc_to_page(p));
2981fc4db90fSRoman Gushchin else
2982fc4db90fSRoman Gushchin folio = virt_to_folio(p);
2983fc4db90fSRoman Gushchin
2984fc4db90fSRoman Gushchin return mem_cgroup_from_obj_folio(folio, p);
2985fc4db90fSRoman Gushchin }
2986fc4db90fSRoman Gushchin
2987fc4db90fSRoman Gushchin /*
2988fc4db90fSRoman Gushchin * Returns a pointer to the memory cgroup to which the kernel object is charged.
2989fc4db90fSRoman Gushchin * Similar to mem_cgroup_from_obj(), but faster and not suitable for objects,
2990fc4db90fSRoman Gushchin * allocated using vmalloc().
2991fc4db90fSRoman Gushchin *
2992fc4db90fSRoman Gushchin * A passed kernel object must be a slab object or a generic kernel page.
2993fc4db90fSRoman Gushchin *
2994fc4db90fSRoman Gushchin * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(),
2995fc4db90fSRoman Gushchin * cgroup_mutex, etc.
2996fc4db90fSRoman Gushchin */
mem_cgroup_from_slab_obj(void * p)2997fc4db90fSRoman Gushchin struct mem_cgroup *mem_cgroup_from_slab_obj(void *p)
2998fc4db90fSRoman Gushchin {
2999fc4db90fSRoman Gushchin if (mem_cgroup_disabled())
3000fc4db90fSRoman Gushchin return NULL;
3001fc4db90fSRoman Gushchin
3002fc4db90fSRoman Gushchin return mem_cgroup_from_obj_folio(virt_to_folio(p), p);
3003fc4db90fSRoman Gushchin }
3004fc4db90fSRoman Gushchin
__get_obj_cgroup_from_memcg(struct mem_cgroup * memcg)3005f4840ccfSJohannes Weiner static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg)
3006f4840ccfSJohannes Weiner {
3007f4840ccfSJohannes Weiner struct obj_cgroup *objcg = NULL;
3008f4840ccfSJohannes Weiner
30097848ed62SKamalesh Babulal for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) {
3010f4840ccfSJohannes Weiner objcg = rcu_dereference(memcg->objcg);
3011f4840ccfSJohannes Weiner if (objcg && obj_cgroup_tryget(objcg))
3012f4840ccfSJohannes Weiner break;
3013f4840ccfSJohannes Weiner objcg = NULL;
3014f4840ccfSJohannes Weiner }
3015f4840ccfSJohannes Weiner return objcg;
3016f4840ccfSJohannes Weiner }
3017f4840ccfSJohannes Weiner
get_obj_cgroup_from_current(void)3018bf4f0599SRoman Gushchin __always_inline struct obj_cgroup *get_obj_cgroup_from_current(void)
3019bf4f0599SRoman Gushchin {
3020bf4f0599SRoman Gushchin struct obj_cgroup *objcg = NULL;
3021bf4f0599SRoman Gushchin struct mem_cgroup *memcg;
3022bf4f0599SRoman Gushchin
3023279c3393SRoman Gushchin if (memcg_kmem_bypass())
3024279c3393SRoman Gushchin return NULL;
3025279c3393SRoman Gushchin
3026bf4f0599SRoman Gushchin rcu_read_lock();
302737d5985cSRoman Gushchin if (unlikely(active_memcg()))
302837d5985cSRoman Gushchin memcg = active_memcg();
3029bf4f0599SRoman Gushchin else
3030bf4f0599SRoman Gushchin memcg = mem_cgroup_from_task(current);
3031f4840ccfSJohannes Weiner objcg = __get_obj_cgroup_from_memcg(memcg);
3032bf4f0599SRoman Gushchin rcu_read_unlock();
3033f4840ccfSJohannes Weiner return objcg;
3034f4840ccfSJohannes Weiner }
3035bf4f0599SRoman Gushchin
get_obj_cgroup_from_folio(struct folio * folio)3036074e3e26SMatthew Wilcox (Oracle) struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio)
3037f4840ccfSJohannes Weiner {
3038f4840ccfSJohannes Weiner struct obj_cgroup *objcg;
3039f4840ccfSJohannes Weiner
3040f7a449f7SRoman Gushchin if (!memcg_kmem_online())
3041f4840ccfSJohannes Weiner return NULL;
3042f4840ccfSJohannes Weiner
3043074e3e26SMatthew Wilcox (Oracle) if (folio_memcg_kmem(folio)) {
3044074e3e26SMatthew Wilcox (Oracle) objcg = __folio_objcg(folio);
3045f4840ccfSJohannes Weiner obj_cgroup_get(objcg);
3046f4840ccfSJohannes Weiner } else {
3047f4840ccfSJohannes Weiner struct mem_cgroup *memcg;
3048f4840ccfSJohannes Weiner
3049f4840ccfSJohannes Weiner rcu_read_lock();
3050074e3e26SMatthew Wilcox (Oracle) memcg = __folio_memcg(folio);
3051f4840ccfSJohannes Weiner if (memcg)
3052f4840ccfSJohannes Weiner objcg = __get_obj_cgroup_from_memcg(memcg);
3053f4840ccfSJohannes Weiner else
3054f4840ccfSJohannes Weiner objcg = NULL;
3055f4840ccfSJohannes Weiner rcu_read_unlock();
3056f4840ccfSJohannes Weiner }
3057bf4f0599SRoman Gushchin return objcg;
3058bf4f0599SRoman Gushchin }
3059bf4f0599SRoman Gushchin
memcg_account_kmem(struct mem_cgroup * memcg,int nr_pages)3060a8c49af3SYosry Ahmed static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages)
3061a8c49af3SYosry Ahmed {
3062a8c49af3SYosry Ahmed mod_memcg_state(memcg, MEMCG_KMEM, nr_pages);
3063a8c49af3SYosry Ahmed if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
3064a8c49af3SYosry Ahmed if (nr_pages > 0)
3065a8c49af3SYosry Ahmed page_counter_charge(&memcg->kmem, nr_pages);
3066a8c49af3SYosry Ahmed else
3067a8c49af3SYosry Ahmed page_counter_uncharge(&memcg->kmem, -nr_pages);
3068a8c49af3SYosry Ahmed }
3069a8c49af3SYosry Ahmed }
3070a8c49af3SYosry Ahmed
3071a8c49af3SYosry Ahmed
3072f1286faeSMuchun Song /*
3073f1286faeSMuchun Song * obj_cgroup_uncharge_pages: uncharge a number of kernel pages from a objcg
3074f1286faeSMuchun Song * @objcg: object cgroup to uncharge
3075f1286faeSMuchun Song * @nr_pages: number of pages to uncharge
3076f1286faeSMuchun Song */
obj_cgroup_uncharge_pages(struct obj_cgroup * objcg,unsigned int nr_pages)3077e74d2259SMuchun Song static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg,
3078e74d2259SMuchun Song unsigned int nr_pages)
3079e74d2259SMuchun Song {
3080e74d2259SMuchun Song struct mem_cgroup *memcg;
3081e74d2259SMuchun Song
3082e74d2259SMuchun Song memcg = get_mem_cgroup_from_objcg(objcg);
3083f1286faeSMuchun Song
3084a8c49af3SYosry Ahmed memcg_account_kmem(memcg, -nr_pages);
3085f1286faeSMuchun Song refill_stock(memcg, nr_pages);
3086f1286faeSMuchun Song
3087e74d2259SMuchun Song css_put(&memcg->css);
3088e74d2259SMuchun Song }
3089e74d2259SMuchun Song
3090f1286faeSMuchun Song /*
3091f1286faeSMuchun Song * obj_cgroup_charge_pages: charge a number of kernel pages to a objcg
3092f1286faeSMuchun Song * @objcg: object cgroup to charge
309345264778SVladimir Davydov * @gfp: reclaim mode
309492d0510cSRoman Gushchin * @nr_pages: number of pages to charge
309545264778SVladimir Davydov *
309645264778SVladimir Davydov * Returns 0 on success, an error code on failure.
309745264778SVladimir Davydov */
obj_cgroup_charge_pages(struct obj_cgroup * objcg,gfp_t gfp,unsigned int nr_pages)3098f1286faeSMuchun Song static int obj_cgroup_charge_pages(struct obj_cgroup *objcg, gfp_t gfp,
309992d0510cSRoman Gushchin unsigned int nr_pages)
3100f3ccb2c4SVladimir Davydov {
3101f1286faeSMuchun Song struct mem_cgroup *memcg;
31026071ca52SJohannes Weiner int ret;
3103f3ccb2c4SVladimir Davydov
3104f1286faeSMuchun Song memcg = get_mem_cgroup_from_objcg(objcg);
3105f1286faeSMuchun Song
3106c5c8b16bSMuchun Song ret = try_charge_memcg(memcg, gfp, nr_pages);
310752c29b04SJohannes Weiner if (ret)
3108f1286faeSMuchun Song goto out;
310952c29b04SJohannes Weiner
3110a8c49af3SYosry Ahmed memcg_account_kmem(memcg, nr_pages);
3111f1286faeSMuchun Song out:
3112f1286faeSMuchun Song css_put(&memcg->css);
3113f3ccb2c4SVladimir Davydov
3114f1286faeSMuchun Song return ret;
31154b13f64dSRoman Gushchin }
31164b13f64dSRoman Gushchin
31174b13f64dSRoman Gushchin /**
3118f4b00eabSRoman Gushchin * __memcg_kmem_charge_page: charge a kmem page to the current memory cgroup
311945264778SVladimir Davydov * @page: page to charge
312045264778SVladimir Davydov * @gfp: reclaim mode
312145264778SVladimir Davydov * @order: allocation order
312245264778SVladimir Davydov *
312345264778SVladimir Davydov * Returns 0 on success, an error code on failure.
312445264778SVladimir Davydov */
__memcg_kmem_charge_page(struct page * page,gfp_t gfp,int order)3125f4b00eabSRoman Gushchin int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
31267ae1e1d0SGlauber Costa {
3127b4e0b68fSMuchun Song struct obj_cgroup *objcg;
3128fcff7d7eSVladimir Davydov int ret = 0;
31297ae1e1d0SGlauber Costa
3130b4e0b68fSMuchun Song objcg = get_obj_cgroup_from_current();
3131b4e0b68fSMuchun Song if (objcg) {
3132b4e0b68fSMuchun Song ret = obj_cgroup_charge_pages(objcg, gfp, 1 << order);
31334d96ba35SRoman Gushchin if (!ret) {
3134b4e0b68fSMuchun Song page->memcg_data = (unsigned long)objcg |
313518b2db3bSRoman Gushchin MEMCG_DATA_KMEM;
31361a3e1f40SJohannes Weiner return 0;
3137c4159a75SVladimir Davydov }
3138b4e0b68fSMuchun Song obj_cgroup_put(objcg);
3139279c3393SRoman Gushchin }
3140d05e83a6SVladimir Davydov return ret;
31417ae1e1d0SGlauber Costa }
314249a18eaeSRoman Gushchin
314349a18eaeSRoman Gushchin /**
3144f4b00eabSRoman Gushchin * __memcg_kmem_uncharge_page: uncharge a kmem page
314545264778SVladimir Davydov * @page: page to uncharge
314645264778SVladimir Davydov * @order: allocation order
314745264778SVladimir Davydov */
__memcg_kmem_uncharge_page(struct page * page,int order)3148f4b00eabSRoman Gushchin void __memcg_kmem_uncharge_page(struct page *page, int order)
31497ae1e1d0SGlauber Costa {
31501b7e4464SMatthew Wilcox (Oracle) struct folio *folio = page_folio(page);
3151b4e0b68fSMuchun Song struct obj_cgroup *objcg;
3152f3ccb2c4SVladimir Davydov unsigned int nr_pages = 1 << order;
31537ae1e1d0SGlauber Costa
31541b7e4464SMatthew Wilcox (Oracle) if (!folio_memcg_kmem(folio))
31557ae1e1d0SGlauber Costa return;
31567ae1e1d0SGlauber Costa
31571b7e4464SMatthew Wilcox (Oracle) objcg = __folio_objcg(folio);
3158b4e0b68fSMuchun Song obj_cgroup_uncharge_pages(objcg, nr_pages);
31591b7e4464SMatthew Wilcox (Oracle) folio->memcg_data = 0;
3160b4e0b68fSMuchun Song obj_cgroup_put(objcg);
316160d3fd32SVladimir Davydov }
3162bf4f0599SRoman Gushchin
mod_objcg_state(struct obj_cgroup * objcg,struct pglist_data * pgdat,enum node_stat_item idx,int nr)316368ac5b3cSWaiman Long void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat,
316468ac5b3cSWaiman Long enum node_stat_item idx, int nr)
316568ac5b3cSWaiman Long {
3166fead2b86SMichal Hocko struct memcg_stock_pcp *stock;
316756751146SSebastian Andrzej Siewior struct obj_cgroup *old = NULL;
316868ac5b3cSWaiman Long unsigned long flags;
316968ac5b3cSWaiman Long int *bytes;
317068ac5b3cSWaiman Long
317156751146SSebastian Andrzej Siewior local_lock_irqsave(&memcg_stock.stock_lock, flags);
3172fead2b86SMichal Hocko stock = this_cpu_ptr(&memcg_stock);
3173fead2b86SMichal Hocko
317468ac5b3cSWaiman Long /*
317568ac5b3cSWaiman Long * Save vmstat data in stock and skip vmstat array update unless
317668ac5b3cSWaiman Long * accumulating over a page of vmstat data or when pgdat or idx
317768ac5b3cSWaiman Long * changes.
317868ac5b3cSWaiman Long */
31793b8abb32SRoman Gushchin if (READ_ONCE(stock->cached_objcg) != objcg) {
318056751146SSebastian Andrzej Siewior old = drain_obj_stock(stock);
318168ac5b3cSWaiman Long obj_cgroup_get(objcg);
318268ac5b3cSWaiman Long stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes)
318368ac5b3cSWaiman Long ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0;
31843b8abb32SRoman Gushchin WRITE_ONCE(stock->cached_objcg, objcg);
318568ac5b3cSWaiman Long stock->cached_pgdat = pgdat;
318668ac5b3cSWaiman Long } else if (stock->cached_pgdat != pgdat) {
318768ac5b3cSWaiman Long /* Flush the existing cached vmstat data */
31887fa0dacbSWaiman Long struct pglist_data *oldpg = stock->cached_pgdat;
31897fa0dacbSWaiman Long
319068ac5b3cSWaiman Long if (stock->nr_slab_reclaimable_b) {
31917fa0dacbSWaiman Long mod_objcg_mlstate(objcg, oldpg, NR_SLAB_RECLAIMABLE_B,
319268ac5b3cSWaiman Long stock->nr_slab_reclaimable_b);
319368ac5b3cSWaiman Long stock->nr_slab_reclaimable_b = 0;
319468ac5b3cSWaiman Long }
319568ac5b3cSWaiman Long if (stock->nr_slab_unreclaimable_b) {
31967fa0dacbSWaiman Long mod_objcg_mlstate(objcg, oldpg, NR_SLAB_UNRECLAIMABLE_B,
319768ac5b3cSWaiman Long stock->nr_slab_unreclaimable_b);
319868ac5b3cSWaiman Long stock->nr_slab_unreclaimable_b = 0;
319968ac5b3cSWaiman Long }
320068ac5b3cSWaiman Long stock->cached_pgdat = pgdat;
320168ac5b3cSWaiman Long }
320268ac5b3cSWaiman Long
320368ac5b3cSWaiman Long bytes = (idx == NR_SLAB_RECLAIMABLE_B) ? &stock->nr_slab_reclaimable_b
320468ac5b3cSWaiman Long : &stock->nr_slab_unreclaimable_b;
320568ac5b3cSWaiman Long /*
320668ac5b3cSWaiman Long * Even for large object >= PAGE_SIZE, the vmstat data will still be
320768ac5b3cSWaiman Long * cached locally at least once before pushing it out.
320868ac5b3cSWaiman Long */
320968ac5b3cSWaiman Long if (!*bytes) {
321068ac5b3cSWaiman Long *bytes = nr;
321168ac5b3cSWaiman Long nr = 0;
321268ac5b3cSWaiman Long } else {
321368ac5b3cSWaiman Long *bytes += nr;
321468ac5b3cSWaiman Long if (abs(*bytes) > PAGE_SIZE) {
321568ac5b3cSWaiman Long nr = *bytes;
321668ac5b3cSWaiman Long *bytes = 0;
321768ac5b3cSWaiman Long } else {
321868ac5b3cSWaiman Long nr = 0;
321968ac5b3cSWaiman Long }
322068ac5b3cSWaiman Long }
322168ac5b3cSWaiman Long if (nr)
322268ac5b3cSWaiman Long mod_objcg_mlstate(objcg, pgdat, idx, nr);
322368ac5b3cSWaiman Long
322456751146SSebastian Andrzej Siewior local_unlock_irqrestore(&memcg_stock.stock_lock, flags);
322556751146SSebastian Andrzej Siewior if (old)
322656751146SSebastian Andrzej Siewior obj_cgroup_put(old);
322768ac5b3cSWaiman Long }
322868ac5b3cSWaiman Long
consume_obj_stock(struct obj_cgroup * objcg,unsigned int nr_bytes)3229bf4f0599SRoman Gushchin static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes)
3230bf4f0599SRoman Gushchin {
3231fead2b86SMichal Hocko struct memcg_stock_pcp *stock;
3232bf4f0599SRoman Gushchin unsigned long flags;
3233bf4f0599SRoman Gushchin bool ret = false;
3234bf4f0599SRoman Gushchin
323556751146SSebastian Andrzej Siewior local_lock_irqsave(&memcg_stock.stock_lock, flags);
3236fead2b86SMichal Hocko
3237fead2b86SMichal Hocko stock = this_cpu_ptr(&memcg_stock);
32383b8abb32SRoman Gushchin if (objcg == READ_ONCE(stock->cached_objcg) && stock->nr_bytes >= nr_bytes) {
3239bf4f0599SRoman Gushchin stock->nr_bytes -= nr_bytes;
3240bf4f0599SRoman Gushchin ret = true;
3241bf4f0599SRoman Gushchin }
3242bf4f0599SRoman Gushchin
324356751146SSebastian Andrzej Siewior local_unlock_irqrestore(&memcg_stock.stock_lock, flags);
3244bf4f0599SRoman Gushchin
3245bf4f0599SRoman Gushchin return ret;
3246bf4f0599SRoman Gushchin }
3247bf4f0599SRoman Gushchin
drain_obj_stock(struct memcg_stock_pcp * stock)324856751146SSebastian Andrzej Siewior static struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock)
3249bf4f0599SRoman Gushchin {
32503b8abb32SRoman Gushchin struct obj_cgroup *old = READ_ONCE(stock->cached_objcg);
3251bf4f0599SRoman Gushchin
3252bf4f0599SRoman Gushchin if (!old)
325356751146SSebastian Andrzej Siewior return NULL;
3254bf4f0599SRoman Gushchin
3255bf4f0599SRoman Gushchin if (stock->nr_bytes) {
3256bf4f0599SRoman Gushchin unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT;
3257bf4f0599SRoman Gushchin unsigned int nr_bytes = stock->nr_bytes & (PAGE_SIZE - 1);
3258bf4f0599SRoman Gushchin
3259af9a3b69SJohannes Weiner if (nr_pages) {
3260af9a3b69SJohannes Weiner struct mem_cgroup *memcg;
3261af9a3b69SJohannes Weiner
3262af9a3b69SJohannes Weiner memcg = get_mem_cgroup_from_objcg(old);
3263af9a3b69SJohannes Weiner
3264af9a3b69SJohannes Weiner memcg_account_kmem(memcg, -nr_pages);
3265af9a3b69SJohannes Weiner __refill_stock(memcg, nr_pages);
3266af9a3b69SJohannes Weiner
3267af9a3b69SJohannes Weiner css_put(&memcg->css);
3268af9a3b69SJohannes Weiner }
3269bf4f0599SRoman Gushchin
3270bf4f0599SRoman Gushchin /*
3271bf4f0599SRoman Gushchin * The leftover is flushed to the centralized per-memcg value.
3272bf4f0599SRoman Gushchin * On the next attempt to refill obj stock it will be moved
3273bf4f0599SRoman Gushchin * to a per-cpu stock (probably, on an other CPU), see
3274bf4f0599SRoman Gushchin * refill_obj_stock().
3275bf4f0599SRoman Gushchin *
3276bf4f0599SRoman Gushchin * How often it's flushed is a trade-off between the memory
3277bf4f0599SRoman Gushchin * limit enforcement accuracy and potential CPU contention,
3278bf4f0599SRoman Gushchin * so it might be changed in the future.
3279bf4f0599SRoman Gushchin */
3280bf4f0599SRoman Gushchin atomic_add(nr_bytes, &old->nr_charged_bytes);
3281bf4f0599SRoman Gushchin stock->nr_bytes = 0;
3282bf4f0599SRoman Gushchin }
3283bf4f0599SRoman Gushchin
328468ac5b3cSWaiman Long /*
328568ac5b3cSWaiman Long * Flush the vmstat data in current stock
328668ac5b3cSWaiman Long */
328768ac5b3cSWaiman Long if (stock->nr_slab_reclaimable_b || stock->nr_slab_unreclaimable_b) {
328868ac5b3cSWaiman Long if (stock->nr_slab_reclaimable_b) {
328968ac5b3cSWaiman Long mod_objcg_mlstate(old, stock->cached_pgdat,
329068ac5b3cSWaiman Long NR_SLAB_RECLAIMABLE_B,
329168ac5b3cSWaiman Long stock->nr_slab_reclaimable_b);
329268ac5b3cSWaiman Long stock->nr_slab_reclaimable_b = 0;
329368ac5b3cSWaiman Long }
329468ac5b3cSWaiman Long if (stock->nr_slab_unreclaimable_b) {
329568ac5b3cSWaiman Long mod_objcg_mlstate(old, stock->cached_pgdat,
329668ac5b3cSWaiman Long NR_SLAB_UNRECLAIMABLE_B,
329768ac5b3cSWaiman Long stock->nr_slab_unreclaimable_b);
329868ac5b3cSWaiman Long stock->nr_slab_unreclaimable_b = 0;
329968ac5b3cSWaiman Long }
330068ac5b3cSWaiman Long stock->cached_pgdat = NULL;
330168ac5b3cSWaiman Long }
330268ac5b3cSWaiman Long
33033b8abb32SRoman Gushchin WRITE_ONCE(stock->cached_objcg, NULL);
330456751146SSebastian Andrzej Siewior /*
330556751146SSebastian Andrzej Siewior * The `old' objects needs to be released by the caller via
330656751146SSebastian Andrzej Siewior * obj_cgroup_put() outside of memcg_stock_pcp::stock_lock.
330756751146SSebastian Andrzej Siewior */
330856751146SSebastian Andrzej Siewior return old;
3309bf4f0599SRoman Gushchin }
3310bf4f0599SRoman Gushchin
obj_stock_flush_required(struct memcg_stock_pcp * stock,struct mem_cgroup * root_memcg)3311bf4f0599SRoman Gushchin static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
3312bf4f0599SRoman Gushchin struct mem_cgroup *root_memcg)
3313bf4f0599SRoman Gushchin {
33143b8abb32SRoman Gushchin struct obj_cgroup *objcg = READ_ONCE(stock->cached_objcg);
3315bf4f0599SRoman Gushchin struct mem_cgroup *memcg;
3316bf4f0599SRoman Gushchin
33173b8abb32SRoman Gushchin if (objcg) {
33183b8abb32SRoman Gushchin memcg = obj_cgroup_memcg(objcg);
3319bf4f0599SRoman Gushchin if (memcg && mem_cgroup_is_descendant(memcg, root_memcg))
3320bf4f0599SRoman Gushchin return true;
3321bf4f0599SRoman Gushchin }
3322bf4f0599SRoman Gushchin
3323bf4f0599SRoman Gushchin return false;
3324bf4f0599SRoman Gushchin }
3325bf4f0599SRoman Gushchin
refill_obj_stock(struct obj_cgroup * objcg,unsigned int nr_bytes,bool allow_uncharge)33265387c904SWaiman Long static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes,
33275387c904SWaiman Long bool allow_uncharge)
3328bf4f0599SRoman Gushchin {
3329fead2b86SMichal Hocko struct memcg_stock_pcp *stock;
333056751146SSebastian Andrzej Siewior struct obj_cgroup *old = NULL;
3331bf4f0599SRoman Gushchin unsigned long flags;
33325387c904SWaiman Long unsigned int nr_pages = 0;
3333bf4f0599SRoman Gushchin
333456751146SSebastian Andrzej Siewior local_lock_irqsave(&memcg_stock.stock_lock, flags);
3335fead2b86SMichal Hocko
3336fead2b86SMichal Hocko stock = this_cpu_ptr(&memcg_stock);
33373b8abb32SRoman Gushchin if (READ_ONCE(stock->cached_objcg) != objcg) { /* reset if necessary */
333856751146SSebastian Andrzej Siewior old = drain_obj_stock(stock);
3339bf4f0599SRoman Gushchin obj_cgroup_get(objcg);
33403b8abb32SRoman Gushchin WRITE_ONCE(stock->cached_objcg, objcg);
33415387c904SWaiman Long stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes)
33425387c904SWaiman Long ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0;
33435387c904SWaiman Long allow_uncharge = true; /* Allow uncharge when objcg changes */
3344bf4f0599SRoman Gushchin }
3345bf4f0599SRoman Gushchin stock->nr_bytes += nr_bytes;
3346bf4f0599SRoman Gushchin
33475387c904SWaiman Long if (allow_uncharge && (stock->nr_bytes > PAGE_SIZE)) {
33485387c904SWaiman Long nr_pages = stock->nr_bytes >> PAGE_SHIFT;
33495387c904SWaiman Long stock->nr_bytes &= (PAGE_SIZE - 1);
33505387c904SWaiman Long }
3351bf4f0599SRoman Gushchin
335256751146SSebastian Andrzej Siewior local_unlock_irqrestore(&memcg_stock.stock_lock, flags);
335356751146SSebastian Andrzej Siewior if (old)
335456751146SSebastian Andrzej Siewior obj_cgroup_put(old);
33555387c904SWaiman Long
33565387c904SWaiman Long if (nr_pages)
33575387c904SWaiman Long obj_cgroup_uncharge_pages(objcg, nr_pages);
3358bf4f0599SRoman Gushchin }
3359bf4f0599SRoman Gushchin
obj_cgroup_charge(struct obj_cgroup * objcg,gfp_t gfp,size_t size)3360bf4f0599SRoman Gushchin int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size)
3361bf4f0599SRoman Gushchin {
3362bf4f0599SRoman Gushchin unsigned int nr_pages, nr_bytes;
3363bf4f0599SRoman Gushchin int ret;
3364bf4f0599SRoman Gushchin
3365bf4f0599SRoman Gushchin if (consume_obj_stock(objcg, size))
3366bf4f0599SRoman Gushchin return 0;
3367bf4f0599SRoman Gushchin
3368bf4f0599SRoman Gushchin /*
33695387c904SWaiman Long * In theory, objcg->nr_charged_bytes can have enough
3370bf4f0599SRoman Gushchin * pre-charged bytes to satisfy the allocation. However,
33715387c904SWaiman Long * flushing objcg->nr_charged_bytes requires two atomic
33725387c904SWaiman Long * operations, and objcg->nr_charged_bytes can't be big.
33735387c904SWaiman Long * The shared objcg->nr_charged_bytes can also become a
33745387c904SWaiman Long * performance bottleneck if all tasks of the same memcg are
33755387c904SWaiman Long * trying to update it. So it's better to ignore it and try
33765387c904SWaiman Long * grab some new pages. The stock's nr_bytes will be flushed to
33775387c904SWaiman Long * objcg->nr_charged_bytes later on when objcg changes.
33785387c904SWaiman Long *
33795387c904SWaiman Long * The stock's nr_bytes may contain enough pre-charged bytes
33805387c904SWaiman Long * to allow one less page from being charged, but we can't rely
33815387c904SWaiman Long * on the pre-charged bytes not being changed outside of
33825387c904SWaiman Long * consume_obj_stock() or refill_obj_stock(). So ignore those
33835387c904SWaiman Long * pre-charged bytes as well when charging pages. To avoid a
33845387c904SWaiman Long * page uncharge right after a page charge, we set the
33855387c904SWaiman Long * allow_uncharge flag to false when calling refill_obj_stock()
33865387c904SWaiman Long * to temporarily allow the pre-charged bytes to exceed the page
33875387c904SWaiman Long * size limit. The maximum reachable value of the pre-charged
33885387c904SWaiman Long * bytes is (sizeof(object) + PAGE_SIZE - 2) if there is no data
33895387c904SWaiman Long * race.
3390bf4f0599SRoman Gushchin */
3391bf4f0599SRoman Gushchin nr_pages = size >> PAGE_SHIFT;
3392bf4f0599SRoman Gushchin nr_bytes = size & (PAGE_SIZE - 1);
3393bf4f0599SRoman Gushchin
3394bf4f0599SRoman Gushchin if (nr_bytes)
3395bf4f0599SRoman Gushchin nr_pages += 1;
3396bf4f0599SRoman Gushchin
3397e74d2259SMuchun Song ret = obj_cgroup_charge_pages(objcg, gfp, nr_pages);
3398bf4f0599SRoman Gushchin if (!ret && nr_bytes)
33995387c904SWaiman Long refill_obj_stock(objcg, PAGE_SIZE - nr_bytes, false);
3400bf4f0599SRoman Gushchin
3401bf4f0599SRoman Gushchin return ret;
3402bf4f0599SRoman Gushchin }
3403bf4f0599SRoman Gushchin
obj_cgroup_uncharge(struct obj_cgroup * objcg,size_t size)3404bf4f0599SRoman Gushchin void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size)
3405bf4f0599SRoman Gushchin {
34065387c904SWaiman Long refill_obj_stock(objcg, size, true);
3407bf4f0599SRoman Gushchin }
3408bf4f0599SRoman Gushchin
340984c07d11SKirill Tkhai #endif /* CONFIG_MEMCG_KMEM */
34107ae1e1d0SGlauber Costa
3411ca3e0214SKAMEZAWA Hiroyuki /*
3412be6c8982SZhou Guanghui * Because page_memcg(head) is not set on tails, set it now.
3413ca3e0214SKAMEZAWA Hiroyuki */
split_page_memcg(struct page * head,unsigned int nr)3414be6c8982SZhou Guanghui void split_page_memcg(struct page *head, unsigned int nr)
3415ca3e0214SKAMEZAWA Hiroyuki {
34161b7e4464SMatthew Wilcox (Oracle) struct folio *folio = page_folio(head);
34171b7e4464SMatthew Wilcox (Oracle) struct mem_cgroup *memcg = folio_memcg(folio);
3418e94c8a9cSKAMEZAWA Hiroyuki int i;
3419ca3e0214SKAMEZAWA Hiroyuki
3420be6c8982SZhou Guanghui if (mem_cgroup_disabled() || !memcg)
34213d37c4a9SKAMEZAWA Hiroyuki return;
3422b070e65cSDavid Rientjes
3423be6c8982SZhou Guanghui for (i = 1; i < nr; i++)
34241b7e4464SMatthew Wilcox (Oracle) folio_page(folio, i)->memcg_data = folio->memcg_data;
3425b4e0b68fSMuchun Song
34261b7e4464SMatthew Wilcox (Oracle) if (folio_memcg_kmem(folio))
34271b7e4464SMatthew Wilcox (Oracle) obj_cgroup_get_many(__folio_objcg(folio), nr - 1);
3428b4e0b68fSMuchun Song else
3429be6c8982SZhou Guanghui css_get_many(&memcg->css, nr - 1);
34301a3e1f40SJohannes Weiner }
3431ca3e0214SKAMEZAWA Hiroyuki
3432e55b9f96SJohannes Weiner #ifdef CONFIG_SWAP
343302491447SDaisuke Nishimura /**
343402491447SDaisuke Nishimura * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
343502491447SDaisuke Nishimura * @entry: swap entry to be moved
343602491447SDaisuke Nishimura * @from: mem_cgroup which the entry is moved from
343702491447SDaisuke Nishimura * @to: mem_cgroup which the entry is moved to
343802491447SDaisuke Nishimura *
343902491447SDaisuke Nishimura * It succeeds only when the swap_cgroup's record for this entry is the same
344002491447SDaisuke Nishimura * as the mem_cgroup's id of @from.
344102491447SDaisuke Nishimura *
344202491447SDaisuke Nishimura * Returns 0 on success, -EINVAL on failure.
344302491447SDaisuke Nishimura *
34443e32cb2eSJohannes Weiner * The caller must have charged to @to, IOW, called page_counter_charge() about
344502491447SDaisuke Nishimura * both res and memsw, and called css_get().
344602491447SDaisuke Nishimura */
mem_cgroup_move_swap_account(swp_entry_t entry,struct mem_cgroup * from,struct mem_cgroup * to)344702491447SDaisuke Nishimura static int mem_cgroup_move_swap_account(swp_entry_t entry,
3448e91cbb42SHugh Dickins struct mem_cgroup *from, struct mem_cgroup *to)
344902491447SDaisuke Nishimura {
345002491447SDaisuke Nishimura unsigned short old_id, new_id;
345102491447SDaisuke Nishimura
345234c00c31SLi Zefan old_id = mem_cgroup_id(from);
345334c00c31SLi Zefan new_id = mem_cgroup_id(to);
345402491447SDaisuke Nishimura
345502491447SDaisuke Nishimura if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
3456c9019e9bSJohannes Weiner mod_memcg_state(from, MEMCG_SWAP, -1);
3457c9019e9bSJohannes Weiner mod_memcg_state(to, MEMCG_SWAP, 1);
345802491447SDaisuke Nishimura return 0;
345902491447SDaisuke Nishimura }
346002491447SDaisuke Nishimura return -EINVAL;
346102491447SDaisuke Nishimura }
346202491447SDaisuke Nishimura #else
mem_cgroup_move_swap_account(swp_entry_t entry,struct mem_cgroup * from,struct mem_cgroup * to)346302491447SDaisuke Nishimura static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
3464e91cbb42SHugh Dickins struct mem_cgroup *from, struct mem_cgroup *to)
346502491447SDaisuke Nishimura {
346602491447SDaisuke Nishimura return -EINVAL;
346702491447SDaisuke Nishimura }
34688c7c6e34SKAMEZAWA Hiroyuki #endif
3469d13d1443SKAMEZAWA Hiroyuki
3470bbec2e15SRoman Gushchin static DEFINE_MUTEX(memcg_max_mutex);
3471f212ad7cSDaisuke Nishimura
mem_cgroup_resize_max(struct mem_cgroup * memcg,unsigned long max,bool memsw)3472bbec2e15SRoman Gushchin static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
3473bbec2e15SRoman Gushchin unsigned long max, bool memsw)
3474628f4235SKAMEZAWA Hiroyuki {
34753e32cb2eSJohannes Weiner bool enlarge = false;
3476bb4a7ea2SShakeel Butt bool drained = false;
34773e32cb2eSJohannes Weiner int ret;
3478c054a78cSYu Zhao bool limits_invariant;
3479c054a78cSYu Zhao struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory;
348081d39c20SKAMEZAWA Hiroyuki
34813e32cb2eSJohannes Weiner do {
3482628f4235SKAMEZAWA Hiroyuki if (signal_pending(current)) {
3483628f4235SKAMEZAWA Hiroyuki ret = -EINTR;
3484628f4235SKAMEZAWA Hiroyuki break;
3485628f4235SKAMEZAWA Hiroyuki }
34863e32cb2eSJohannes Weiner
3487bbec2e15SRoman Gushchin mutex_lock(&memcg_max_mutex);
3488c054a78cSYu Zhao /*
3489c054a78cSYu Zhao * Make sure that the new limit (memsw or memory limit) doesn't
3490bbec2e15SRoman Gushchin * break our basic invariant rule memory.max <= memsw.max.
3491c054a78cSYu Zhao */
349215b42562SChris Down limits_invariant = memsw ? max >= READ_ONCE(memcg->memory.max) :
3493bbec2e15SRoman Gushchin max <= memcg->memsw.max;
3494c054a78cSYu Zhao if (!limits_invariant) {
3495bbec2e15SRoman Gushchin mutex_unlock(&memcg_max_mutex);
34968c7c6e34SKAMEZAWA Hiroyuki ret = -EINVAL;
3497628f4235SKAMEZAWA Hiroyuki break;
3498628f4235SKAMEZAWA Hiroyuki }
3499bbec2e15SRoman Gushchin if (max > counter->max)
35003e32cb2eSJohannes Weiner enlarge = true;
3501bbec2e15SRoman Gushchin ret = page_counter_set_max(counter, max);
3502bbec2e15SRoman Gushchin mutex_unlock(&memcg_max_mutex);
35038c7c6e34SKAMEZAWA Hiroyuki
35048c7c6e34SKAMEZAWA Hiroyuki if (!ret)
35058c7c6e34SKAMEZAWA Hiroyuki break;
35068c7c6e34SKAMEZAWA Hiroyuki
3507bb4a7ea2SShakeel Butt if (!drained) {
3508bb4a7ea2SShakeel Butt drain_all_stock(memcg);
3509bb4a7ea2SShakeel Butt drained = true;
3510bb4a7ea2SShakeel Butt continue;
3511bb4a7ea2SShakeel Butt }
3512bb4a7ea2SShakeel Butt
351373b73bacSYosry Ahmed if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL,
351455ab834aSMichal Hocko memsw ? 0 : MEMCG_RECLAIM_MAY_SWAP)) {
35151ab5c056SAndrey Ryabinin ret = -EBUSY;
35161ab5c056SAndrey Ryabinin break;
35171ab5c056SAndrey Ryabinin }
35181ab5c056SAndrey Ryabinin } while (true);
35193e32cb2eSJohannes Weiner
35203c11ecf4SKAMEZAWA Hiroyuki if (!ret && enlarge)
35213c11ecf4SKAMEZAWA Hiroyuki memcg_oom_recover(memcg);
35223e32cb2eSJohannes Weiner
3523628f4235SKAMEZAWA Hiroyuki return ret;
3524628f4235SKAMEZAWA Hiroyuki }
3525628f4235SKAMEZAWA Hiroyuki
mem_cgroup_soft_limit_reclaim(pg_data_t * pgdat,int order,gfp_t gfp_mask,unsigned long * total_scanned)3526ef8f2327SMel Gorman unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
35270608f43dSAndrew Morton gfp_t gfp_mask,
35280608f43dSAndrew Morton unsigned long *total_scanned)
35290608f43dSAndrew Morton {
35300608f43dSAndrew Morton unsigned long nr_reclaimed = 0;
3531ef8f2327SMel Gorman struct mem_cgroup_per_node *mz, *next_mz = NULL;
35320608f43dSAndrew Morton unsigned long reclaimed;
35330608f43dSAndrew Morton int loop = 0;
3534ef8f2327SMel Gorman struct mem_cgroup_tree_per_node *mctz;
35353e32cb2eSJohannes Weiner unsigned long excess;
35360608f43dSAndrew Morton
3537e4dde56cSYu Zhao if (lru_gen_enabled())
3538e4dde56cSYu Zhao return 0;
3539e4dde56cSYu Zhao
35400608f43dSAndrew Morton if (order > 0)
35410608f43dSAndrew Morton return 0;
35420608f43dSAndrew Morton
35432ab082baSMatthew Wilcox (Oracle) mctz = soft_limit_tree.rb_tree_per_node[pgdat->node_id];
3544d6507ff5SMichal Hocko
3545d6507ff5SMichal Hocko /*
3546d6507ff5SMichal Hocko * Do not even bother to check the largest node if the root
3547d6507ff5SMichal Hocko * is empty. Do it lockless to prevent lock bouncing. Races
3548d6507ff5SMichal Hocko * are acceptable as soft limit is best effort anyway.
3549d6507ff5SMichal Hocko */
3550bfc7228bSLaurent Dufour if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
3551d6507ff5SMichal Hocko return 0;
3552d6507ff5SMichal Hocko
35530608f43dSAndrew Morton /*
35540608f43dSAndrew Morton * This loop can run a while, specially if mem_cgroup's continuously
35550608f43dSAndrew Morton * keep exceeding their soft limit and putting the system under
35560608f43dSAndrew Morton * pressure
35570608f43dSAndrew Morton */
35580608f43dSAndrew Morton do {
35590608f43dSAndrew Morton if (next_mz)
35600608f43dSAndrew Morton mz = next_mz;
35610608f43dSAndrew Morton else
35620608f43dSAndrew Morton mz = mem_cgroup_largest_soft_limit_node(mctz);
35630608f43dSAndrew Morton if (!mz)
35640608f43dSAndrew Morton break;
35650608f43dSAndrew Morton
3566ef8f2327SMel Gorman reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
3567d8f65338SMiaohe Lin gfp_mask, total_scanned);
35680608f43dSAndrew Morton nr_reclaimed += reclaimed;
35690a31bc97SJohannes Weiner spin_lock_irq(&mctz->lock);
35700608f43dSAndrew Morton
35710608f43dSAndrew Morton /*
35720608f43dSAndrew Morton * If we failed to reclaim anything from this memory cgroup
35730608f43dSAndrew Morton * it is time to move on to the next cgroup
35740608f43dSAndrew Morton */
35750608f43dSAndrew Morton next_mz = NULL;
3576bc2f2e7fSVladimir Davydov if (!reclaimed)
3577bc2f2e7fSVladimir Davydov next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
3578bc2f2e7fSVladimir Davydov
35793e32cb2eSJohannes Weiner excess = soft_limit_excess(mz->memcg);
35800608f43dSAndrew Morton /*
35810608f43dSAndrew Morton * One school of thought says that we should not add
35820608f43dSAndrew Morton * back the node to the tree if reclaim returns 0.
35830608f43dSAndrew Morton * But our reclaim could return 0, simply because due
35840608f43dSAndrew Morton * to priority we are exposing a smaller subset of
35850608f43dSAndrew Morton * memory to reclaim from. Consider this as a longer
35860608f43dSAndrew Morton * term TODO.
35870608f43dSAndrew Morton */
35880608f43dSAndrew Morton /* If excess == 0, no tree ops */
3589cf2c8127SJohannes Weiner __mem_cgroup_insert_exceeded(mz, mctz, excess);
35900a31bc97SJohannes Weiner spin_unlock_irq(&mctz->lock);
35910608f43dSAndrew Morton css_put(&mz->memcg->css);
35920608f43dSAndrew Morton loop++;
35930608f43dSAndrew Morton /*
35940608f43dSAndrew Morton * Could not reclaim anything and there are no more
35950608f43dSAndrew Morton * mem cgroups to try or we seem to be looping without
35960608f43dSAndrew Morton * reclaiming anything.
35970608f43dSAndrew Morton */
35980608f43dSAndrew Morton if (!nr_reclaimed &&
35990608f43dSAndrew Morton (next_mz == NULL ||
36000608f43dSAndrew Morton loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
36010608f43dSAndrew Morton break;
36020608f43dSAndrew Morton } while (!nr_reclaimed);
36030608f43dSAndrew Morton if (next_mz)
36040608f43dSAndrew Morton css_put(&next_mz->memcg->css);
36050608f43dSAndrew Morton return nr_reclaimed;
36060608f43dSAndrew Morton }
36070608f43dSAndrew Morton
3608ea280e7bSTejun Heo /*
360951038171SGreg Thelen * Reclaims as many pages from the given memcg as possible.
3610c26251f9SMichal Hocko *
3611c26251f9SMichal Hocko * Caller is responsible for holding css reference for memcg.
3612c26251f9SMichal Hocko */
mem_cgroup_force_empty(struct mem_cgroup * memcg)3613c26251f9SMichal Hocko static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
3614c26251f9SMichal Hocko {
3615d977aa93SChris Down int nr_retries = MAX_RECLAIM_RETRIES;
3616c26251f9SMichal Hocko
3617c1e862c1SKAMEZAWA Hiroyuki /* we call try-to-free pages for make this cgroup empty */
3618c1e862c1SKAMEZAWA Hiroyuki lru_add_drain_all();
3619d12c60f6SJunaid Shahid
3620d12c60f6SJunaid Shahid drain_all_stock(memcg);
3621d12c60f6SJunaid Shahid
3622f817ed48SKAMEZAWA Hiroyuki /* try to free all pages in this cgroup */
36233e32cb2eSJohannes Weiner while (nr_retries && page_counter_read(&memcg->memory)) {
3624c26251f9SMichal Hocko if (signal_pending(current))
3625c26251f9SMichal Hocko return -EINTR;
3626c26251f9SMichal Hocko
362773b73bacSYosry Ahmed if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL,
362855ab834aSMichal Hocko MEMCG_RECLAIM_MAY_SWAP))
3629f817ed48SKAMEZAWA Hiroyuki nr_retries--;
3630f817ed48SKAMEZAWA Hiroyuki }
3631ab5196c2SMichal Hocko
3632ab5196c2SMichal Hocko return 0;
3633cc847582SKAMEZAWA Hiroyuki }
3634cc847582SKAMEZAWA Hiroyuki
mem_cgroup_force_empty_write(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)36356770c64eSTejun Heo static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
36366770c64eSTejun Heo char *buf, size_t nbytes,
36376770c64eSTejun Heo loff_t off)
3638c1e862c1SKAMEZAWA Hiroyuki {
36396770c64eSTejun Heo struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3640c26251f9SMichal Hocko
3641d8423011SMichal Hocko if (mem_cgroup_is_root(memcg))
3642d8423011SMichal Hocko return -EINVAL;
36436770c64eSTejun Heo return mem_cgroup_force_empty(memcg) ?: nbytes;
3644c1e862c1SKAMEZAWA Hiroyuki }
3645c1e862c1SKAMEZAWA Hiroyuki
mem_cgroup_hierarchy_read(struct cgroup_subsys_state * css,struct cftype * cft)3646182446d0STejun Heo static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
3647182446d0STejun Heo struct cftype *cft)
364818f59ea7SBalbir Singh {
3649bef8620cSRoman Gushchin return 1;
365018f59ea7SBalbir Singh }
365118f59ea7SBalbir Singh
mem_cgroup_hierarchy_write(struct cgroup_subsys_state * css,struct cftype * cft,u64 val)3652182446d0STejun Heo static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
3653182446d0STejun Heo struct cftype *cft, u64 val)
365418f59ea7SBalbir Singh {
3655bef8620cSRoman Gushchin if (val == 1)
36560b8f73e1SJohannes Weiner return 0;
3657567fb435SGlauber Costa
3658bef8620cSRoman Gushchin pr_warn_once("Non-hierarchical mode is deprecated. "
3659bef8620cSRoman Gushchin "Please report your usecase to linux-mm@kvack.org if you "
3660bef8620cSRoman Gushchin "depend on this functionality.\n");
3661567fb435SGlauber Costa
3662bef8620cSRoman Gushchin return -EINVAL;
366318f59ea7SBalbir Singh }
366418f59ea7SBalbir Singh
mem_cgroup_usage(struct mem_cgroup * memcg,bool swap)36656f646156SAndrew Morton static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
3666ce00a967SJohannes Weiner {
366742a30035SJohannes Weiner unsigned long val;
3668ce00a967SJohannes Weiner
36693e32cb2eSJohannes Weiner if (mem_cgroup_is_root(memcg)) {
3670a2174e95SYosry Ahmed /*
3671f82a7a86SYosry Ahmed * Approximate root's usage from global state. This isn't
3672f82a7a86SYosry Ahmed * perfect, but the root usage was always an approximation.
3673a2174e95SYosry Ahmed */
3674f82a7a86SYosry Ahmed val = global_node_page_state(NR_FILE_PAGES) +
3675f82a7a86SYosry Ahmed global_node_page_state(NR_ANON_MAPPED);
3676ce00a967SJohannes Weiner if (swap)
3677f82a7a86SYosry Ahmed val += total_swap_pages - get_nr_swap_pages();
36783e32cb2eSJohannes Weiner } else {
36793e32cb2eSJohannes Weiner if (!swap)
36803e32cb2eSJohannes Weiner val = page_counter_read(&memcg->memory);
36813e32cb2eSJohannes Weiner else
36823e32cb2eSJohannes Weiner val = page_counter_read(&memcg->memsw);
36833e32cb2eSJohannes Weiner }
3684c12176d3SMichal Hocko return val;
3685ce00a967SJohannes Weiner }
3686ce00a967SJohannes Weiner
36873e32cb2eSJohannes Weiner enum {
36883e32cb2eSJohannes Weiner RES_USAGE,
36893e32cb2eSJohannes Weiner RES_LIMIT,
36903e32cb2eSJohannes Weiner RES_MAX_USAGE,
36913e32cb2eSJohannes Weiner RES_FAILCNT,
36923e32cb2eSJohannes Weiner RES_SOFT_LIMIT,
36933e32cb2eSJohannes Weiner };
3694ce00a967SJohannes Weiner
mem_cgroup_read_u64(struct cgroup_subsys_state * css,struct cftype * cft)3695791badbdSTejun Heo static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
3696791badbdSTejun Heo struct cftype *cft)
36978cdea7c0SBalbir Singh {
3698182446d0STejun Heo struct mem_cgroup *memcg = mem_cgroup_from_css(css);
36993e32cb2eSJohannes Weiner struct page_counter *counter;
3700af36f906STejun Heo
37013e32cb2eSJohannes Weiner switch (MEMFILE_TYPE(cft->private)) {
37028c7c6e34SKAMEZAWA Hiroyuki case _MEM:
37033e32cb2eSJohannes Weiner counter = &memcg->memory;
3704510fc4e1SGlauber Costa break;
37053e32cb2eSJohannes Weiner case _MEMSWAP:
37063e32cb2eSJohannes Weiner counter = &memcg->memsw;
37073e32cb2eSJohannes Weiner break;
37083e32cb2eSJohannes Weiner case _KMEM:
37093e32cb2eSJohannes Weiner counter = &memcg->kmem;
37103e32cb2eSJohannes Weiner break;
3711d55f90bfSVladimir Davydov case _TCP:
37120db15298SJohannes Weiner counter = &memcg->tcpmem;
3713d55f90bfSVladimir Davydov break;
37143e32cb2eSJohannes Weiner default:
37153e32cb2eSJohannes Weiner BUG();
37163e32cb2eSJohannes Weiner }
37173e32cb2eSJohannes Weiner
37183e32cb2eSJohannes Weiner switch (MEMFILE_ATTR(cft->private)) {
37193e32cb2eSJohannes Weiner case RES_USAGE:
37203e32cb2eSJohannes Weiner if (counter == &memcg->memory)
3721c12176d3SMichal Hocko return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
37223e32cb2eSJohannes Weiner if (counter == &memcg->memsw)
3723c12176d3SMichal Hocko return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
37243e32cb2eSJohannes Weiner return (u64)page_counter_read(counter) * PAGE_SIZE;
37253e32cb2eSJohannes Weiner case RES_LIMIT:
3726bbec2e15SRoman Gushchin return (u64)counter->max * PAGE_SIZE;
37273e32cb2eSJohannes Weiner case RES_MAX_USAGE:
37283e32cb2eSJohannes Weiner return (u64)counter->watermark * PAGE_SIZE;
37293e32cb2eSJohannes Weiner case RES_FAILCNT:
37303e32cb2eSJohannes Weiner return counter->failcnt;
37313e32cb2eSJohannes Weiner case RES_SOFT_LIMIT:
37322178e20cSYue Zhao return (u64)READ_ONCE(memcg->soft_limit) * PAGE_SIZE;
37338c7c6e34SKAMEZAWA Hiroyuki default:
37348c7c6e34SKAMEZAWA Hiroyuki BUG();
37358c7c6e34SKAMEZAWA Hiroyuki }
37368cdea7c0SBalbir Singh }
3737510fc4e1SGlauber Costa
37386b0ba2abSFlorian Schmidt /*
37396b0ba2abSFlorian Schmidt * This function doesn't do anything useful. Its only job is to provide a read
37406b0ba2abSFlorian Schmidt * handler for a file so that cgroup_file_mode() will add read permissions.
37416b0ba2abSFlorian Schmidt */
mem_cgroup_dummy_seq_show(__always_unused struct seq_file * m,__always_unused void * v)37426b0ba2abSFlorian Schmidt static int mem_cgroup_dummy_seq_show(__always_unused struct seq_file *m,
37436b0ba2abSFlorian Schmidt __always_unused void *v)
37446b0ba2abSFlorian Schmidt {
37456b0ba2abSFlorian Schmidt return -EINVAL;
37466b0ba2abSFlorian Schmidt }
37476b0ba2abSFlorian Schmidt
374884c07d11SKirill Tkhai #ifdef CONFIG_MEMCG_KMEM
memcg_online_kmem(struct mem_cgroup * memcg)3749567e9ab2SJohannes Weiner static int memcg_online_kmem(struct mem_cgroup *memcg)
3750d6441637SVladimir Davydov {
3751bf4f0599SRoman Gushchin struct obj_cgroup *objcg;
3752d6441637SVladimir Davydov
37539c94bef9SXiang Yang if (mem_cgroup_kmem_disabled())
3754b313aeeeSVladimir Davydov return 0;
3755b313aeeeSVladimir Davydov
3756da0efe30SMuchun Song if (unlikely(mem_cgroup_is_root(memcg)))
3757da0efe30SMuchun Song return 0;
3758d6441637SVladimir Davydov
3759bf4f0599SRoman Gushchin objcg = obj_cgroup_alloc();
3760f9c69d63SMuchun Song if (!objcg)
3761bf4f0599SRoman Gushchin return -ENOMEM;
3762f9c69d63SMuchun Song
3763bf4f0599SRoman Gushchin objcg->memcg = memcg;
3764bf4f0599SRoman Gushchin rcu_assign_pointer(memcg->objcg, objcg);
3765bf4f0599SRoman Gushchin
3766f7a449f7SRoman Gushchin static_branch_enable(&memcg_kmem_online_key);
3767d648bcc7SRoman Gushchin
3768f9c69d63SMuchun Song memcg->kmemcg_id = memcg->id.id;
37690b8f73e1SJohannes Weiner
37700b8f73e1SJohannes Weiner return 0;
3771d6441637SVladimir Davydov }
3772d6441637SVladimir Davydov
memcg_offline_kmem(struct mem_cgroup * memcg)37738e0a8912SJohannes Weiner static void memcg_offline_kmem(struct mem_cgroup *memcg)
37748e0a8912SJohannes Weiner {
377564268868SMuchun Song struct mem_cgroup *parent;
37768e0a8912SJohannes Weiner
37779c94bef9SXiang Yang if (mem_cgroup_kmem_disabled())
3778da0efe30SMuchun Song return;
3779da0efe30SMuchun Song
3780da0efe30SMuchun Song if (unlikely(mem_cgroup_is_root(memcg)))
37818e0a8912SJohannes Weiner return;
37829855609bSRoman Gushchin
37838e0a8912SJohannes Weiner parent = parent_mem_cgroup(memcg);
37848e0a8912SJohannes Weiner if (!parent)
37858e0a8912SJohannes Weiner parent = root_mem_cgroup;
37868e0a8912SJohannes Weiner
3787bf4f0599SRoman Gushchin memcg_reparent_objcgs(memcg, parent);
3788fb2f2b0aSRoman Gushchin
37895abc1e37SMuchun Song /*
379064268868SMuchun Song * After we have finished memcg_reparent_objcgs(), all list_lrus
379164268868SMuchun Song * corresponding to this cgroup are guaranteed to remain empty.
379264268868SMuchun Song * The ordering is imposed by list_lru_node->lock taken by
37931f391eb2SMuchun Song * memcg_reparent_list_lrus().
37948e0a8912SJohannes Weiner */
37951f391eb2SMuchun Song memcg_reparent_list_lrus(memcg, parent);
37968e0a8912SJohannes Weiner }
3797d6441637SVladimir Davydov #else
memcg_online_kmem(struct mem_cgroup * memcg)37980b8f73e1SJohannes Weiner static int memcg_online_kmem(struct mem_cgroup *memcg)
3799127424c8SJohannes Weiner {
3800127424c8SJohannes Weiner return 0;
3801127424c8SJohannes Weiner }
memcg_offline_kmem(struct mem_cgroup * memcg)3802127424c8SJohannes Weiner static void memcg_offline_kmem(struct mem_cgroup *memcg)
3803127424c8SJohannes Weiner {
3804127424c8SJohannes Weiner }
380584c07d11SKirill Tkhai #endif /* CONFIG_MEMCG_KMEM */
3806127424c8SJohannes Weiner
memcg_update_tcp_max(struct mem_cgroup * memcg,unsigned long max)3807bbec2e15SRoman Gushchin static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max)
3808d55f90bfSVladimir Davydov {
3809d55f90bfSVladimir Davydov int ret;
3810d55f90bfSVladimir Davydov
3811bbec2e15SRoman Gushchin mutex_lock(&memcg_max_mutex);
3812d55f90bfSVladimir Davydov
3813bbec2e15SRoman Gushchin ret = page_counter_set_max(&memcg->tcpmem, max);
3814d55f90bfSVladimir Davydov if (ret)
3815d55f90bfSVladimir Davydov goto out;
3816d55f90bfSVladimir Davydov
38170db15298SJohannes Weiner if (!memcg->tcpmem_active) {
3818d55f90bfSVladimir Davydov /*
3819d55f90bfSVladimir Davydov * The active flag needs to be written after the static_key
3820d55f90bfSVladimir Davydov * update. This is what guarantees that the socket activation
38212d758073SJohannes Weiner * function is the last one to run. See mem_cgroup_sk_alloc()
38222d758073SJohannes Weiner * for details, and note that we don't mark any socket as
38232d758073SJohannes Weiner * belonging to this memcg until that flag is up.
3824d55f90bfSVladimir Davydov *
3825d55f90bfSVladimir Davydov * We need to do this, because static_keys will span multiple
3826d55f90bfSVladimir Davydov * sites, but we can't control their order. If we mark a socket
3827d55f90bfSVladimir Davydov * as accounted, but the accounting functions are not patched in
3828d55f90bfSVladimir Davydov * yet, we'll lose accounting.
3829d55f90bfSVladimir Davydov *
38302d758073SJohannes Weiner * We never race with the readers in mem_cgroup_sk_alloc(),
3831d55f90bfSVladimir Davydov * because when this value change, the code to process it is not
3832d55f90bfSVladimir Davydov * patched in yet.
3833d55f90bfSVladimir Davydov */
3834d55f90bfSVladimir Davydov static_branch_inc(&memcg_sockets_enabled_key);
38350db15298SJohannes Weiner memcg->tcpmem_active = true;
3836d55f90bfSVladimir Davydov }
3837d55f90bfSVladimir Davydov out:
3838bbec2e15SRoman Gushchin mutex_unlock(&memcg_max_mutex);
3839d55f90bfSVladimir Davydov return ret;
3840d55f90bfSVladimir Davydov }
3841d55f90bfSVladimir Davydov
3842628f4235SKAMEZAWA Hiroyuki /*
3843628f4235SKAMEZAWA Hiroyuki * The user of this function is...
3844628f4235SKAMEZAWA Hiroyuki * RES_LIMIT.
3845628f4235SKAMEZAWA Hiroyuki */
mem_cgroup_write(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)3846451af504STejun Heo static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
3847451af504STejun Heo char *buf, size_t nbytes, loff_t off)
38488cdea7c0SBalbir Singh {
3849451af504STejun Heo struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
38503e32cb2eSJohannes Weiner unsigned long nr_pages;
3851628f4235SKAMEZAWA Hiroyuki int ret;
3852628f4235SKAMEZAWA Hiroyuki
3853451af504STejun Heo buf = strstrip(buf);
3854650c5e56SJohannes Weiner ret = page_counter_memparse(buf, "-1", &nr_pages);
38553e32cb2eSJohannes Weiner if (ret)
38563e32cb2eSJohannes Weiner return ret;
3857af36f906STejun Heo
38583e32cb2eSJohannes Weiner switch (MEMFILE_ATTR(of_cft(of)->private)) {
3859628f4235SKAMEZAWA Hiroyuki case RES_LIMIT:
38604b3bde4cSBalbir Singh if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
38614b3bde4cSBalbir Singh ret = -EINVAL;
38624b3bde4cSBalbir Singh break;
38634b3bde4cSBalbir Singh }
38643e32cb2eSJohannes Weiner switch (MEMFILE_TYPE(of_cft(of)->private)) {
38653e32cb2eSJohannes Weiner case _MEM:
3866bbec2e15SRoman Gushchin ret = mem_cgroup_resize_max(memcg, nr_pages, false);
38678c7c6e34SKAMEZAWA Hiroyuki break;
38683e32cb2eSJohannes Weiner case _MEMSWAP:
3869bbec2e15SRoman Gushchin ret = mem_cgroup_resize_max(memcg, nr_pages, true);
38703e32cb2eSJohannes Weiner break;
38714597648fSMichal Hocko case _KMEM:
38724597648fSMichal Hocko pr_warn_once("kmem.limit_in_bytes is deprecated and will be removed. "
38734597648fSMichal Hocko "Writing any value to this file has no effect. "
38744597648fSMichal Hocko "Please report your usecase to linux-mm@kvack.org if you "
38754597648fSMichal Hocko "depend on this functionality.\n");
38764597648fSMichal Hocko ret = 0;
38774597648fSMichal Hocko break;
3878d55f90bfSVladimir Davydov case _TCP:
3879bbec2e15SRoman Gushchin ret = memcg_update_tcp_max(memcg, nr_pages);
3880d55f90bfSVladimir Davydov break;
38813e32cb2eSJohannes Weiner }
3882628f4235SKAMEZAWA Hiroyuki break;
3883296c81d8SBalbir Singh case RES_SOFT_LIMIT:
38842343e88dSSebastian Andrzej Siewior if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
38852343e88dSSebastian Andrzej Siewior ret = -EOPNOTSUPP;
38862343e88dSSebastian Andrzej Siewior } else {
38872178e20cSYue Zhao WRITE_ONCE(memcg->soft_limit, nr_pages);
38883e32cb2eSJohannes Weiner ret = 0;
38892343e88dSSebastian Andrzej Siewior }
3890628f4235SKAMEZAWA Hiroyuki break;
3891628f4235SKAMEZAWA Hiroyuki }
3892451af504STejun Heo return ret ?: nbytes;
38938cdea7c0SBalbir Singh }
38948cdea7c0SBalbir Singh
mem_cgroup_reset(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)38956770c64eSTejun Heo static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
38966770c64eSTejun Heo size_t nbytes, loff_t off)
3897c84872e1SPavel Emelyanov {
38986770c64eSTejun Heo struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
38993e32cb2eSJohannes Weiner struct page_counter *counter;
3900c84872e1SPavel Emelyanov
39013e32cb2eSJohannes Weiner switch (MEMFILE_TYPE(of_cft(of)->private)) {
39023e32cb2eSJohannes Weiner case _MEM:
39033e32cb2eSJohannes Weiner counter = &memcg->memory;
39043e32cb2eSJohannes Weiner break;
39053e32cb2eSJohannes Weiner case _MEMSWAP:
39063e32cb2eSJohannes Weiner counter = &memcg->memsw;
39073e32cb2eSJohannes Weiner break;
39083e32cb2eSJohannes Weiner case _KMEM:
39093e32cb2eSJohannes Weiner counter = &memcg->kmem;
39103e32cb2eSJohannes Weiner break;
3911d55f90bfSVladimir Davydov case _TCP:
39120db15298SJohannes Weiner counter = &memcg->tcpmem;
3913d55f90bfSVladimir Davydov break;
39143e32cb2eSJohannes Weiner default:
39153e32cb2eSJohannes Weiner BUG();
39163e32cb2eSJohannes Weiner }
3917af36f906STejun Heo
39183e32cb2eSJohannes Weiner switch (MEMFILE_ATTR(of_cft(of)->private)) {
391929f2a4daSPavel Emelyanov case RES_MAX_USAGE:
39203e32cb2eSJohannes Weiner page_counter_reset_watermark(counter);
392129f2a4daSPavel Emelyanov break;
392229f2a4daSPavel Emelyanov case RES_FAILCNT:
39233e32cb2eSJohannes Weiner counter->failcnt = 0;
392429f2a4daSPavel Emelyanov break;
39253e32cb2eSJohannes Weiner default:
39263e32cb2eSJohannes Weiner BUG();
392729f2a4daSPavel Emelyanov }
3928f64c3f54SBalbir Singh
39296770c64eSTejun Heo return nbytes;
3930c84872e1SPavel Emelyanov }
3931c84872e1SPavel Emelyanov
mem_cgroup_move_charge_read(struct cgroup_subsys_state * css,struct cftype * cft)3932182446d0STejun Heo static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
39337dc74be0SDaisuke Nishimura struct cftype *cft)
39347dc74be0SDaisuke Nishimura {
3935182446d0STejun Heo return mem_cgroup_from_css(css)->move_charge_at_immigrate;
39367dc74be0SDaisuke Nishimura }
39377dc74be0SDaisuke Nishimura
393802491447SDaisuke Nishimura #ifdef CONFIG_MMU
mem_cgroup_move_charge_write(struct cgroup_subsys_state * css,struct cftype * cft,u64 val)3939182446d0STejun Heo static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
39407dc74be0SDaisuke Nishimura struct cftype *cft, u64 val)
39417dc74be0SDaisuke Nishimura {
3942182446d0STejun Heo struct mem_cgroup *memcg = mem_cgroup_from_css(css);
39437dc74be0SDaisuke Nishimura
3944da34a848SJohannes Weiner pr_warn_once("Cgroup memory moving (move_charge_at_immigrate) is deprecated. "
3945da34a848SJohannes Weiner "Please report your usecase to linux-mm@kvack.org if you "
3946da34a848SJohannes Weiner "depend on this functionality.\n");
3947da34a848SJohannes Weiner
39481dfab5abSJohannes Weiner if (val & ~MOVE_MASK)
39497dc74be0SDaisuke Nishimura return -EINVAL;
39507dc74be0SDaisuke Nishimura
3951ee5e8472SGlauber Costa /*
3952ee5e8472SGlauber Costa * No kind of locking is needed in here, because ->can_attach() will
3953ee5e8472SGlauber Costa * check this value once in the beginning of the process, and then carry
3954ee5e8472SGlauber Costa * on with stale data. This means that changes to this value will only
3955ee5e8472SGlauber Costa * affect task migrations starting after the change.
3956ee5e8472SGlauber Costa */
3957ee5e8472SGlauber Costa memcg->move_charge_at_immigrate = val;
39587dc74be0SDaisuke Nishimura return 0;
39597dc74be0SDaisuke Nishimura }
396002491447SDaisuke Nishimura #else
mem_cgroup_move_charge_write(struct cgroup_subsys_state * css,struct cftype * cft,u64 val)3961182446d0STejun Heo static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
396202491447SDaisuke Nishimura struct cftype *cft, u64 val)
396302491447SDaisuke Nishimura {
396402491447SDaisuke Nishimura return -ENOSYS;
396502491447SDaisuke Nishimura }
396602491447SDaisuke Nishimura #endif
39677dc74be0SDaisuke Nishimura
3968406eb0c9SYing Han #ifdef CONFIG_NUMA
3969113b7dfdSJohannes Weiner
3970113b7dfdSJohannes Weiner #define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))
3971113b7dfdSJohannes Weiner #define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))
3972113b7dfdSJohannes Weiner #define LRU_ALL ((1 << NR_LRU_LISTS) - 1)
3973113b7dfdSJohannes Weiner
mem_cgroup_node_nr_lru_pages(struct mem_cgroup * memcg,int nid,unsigned int lru_mask,bool tree)3974113b7dfdSJohannes Weiner static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
3975dd8657b6SShakeel Butt int nid, unsigned int lru_mask, bool tree)
3976113b7dfdSJohannes Weiner {
3977867e5e1dSJohannes Weiner struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
3978113b7dfdSJohannes Weiner unsigned long nr = 0;
3979113b7dfdSJohannes Weiner enum lru_list lru;
3980113b7dfdSJohannes Weiner
3981113b7dfdSJohannes Weiner VM_BUG_ON((unsigned)nid >= nr_node_ids);
3982113b7dfdSJohannes Weiner
3983113b7dfdSJohannes Weiner for_each_lru(lru) {
3984113b7dfdSJohannes Weiner if (!(BIT(lru) & lru_mask))
3985113b7dfdSJohannes Weiner continue;
3986dd8657b6SShakeel Butt if (tree)
3987dd8657b6SShakeel Butt nr += lruvec_page_state(lruvec, NR_LRU_BASE + lru);
3988dd8657b6SShakeel Butt else
3989205b20ccSJohannes Weiner nr += lruvec_page_state_local(lruvec, NR_LRU_BASE + lru);
3990113b7dfdSJohannes Weiner }
3991113b7dfdSJohannes Weiner return nr;
3992113b7dfdSJohannes Weiner }
3993113b7dfdSJohannes Weiner
mem_cgroup_nr_lru_pages(struct mem_cgroup * memcg,unsigned int lru_mask,bool tree)3994113b7dfdSJohannes Weiner static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
3995dd8657b6SShakeel Butt unsigned int lru_mask,
3996dd8657b6SShakeel Butt bool tree)
3997113b7dfdSJohannes Weiner {
3998113b7dfdSJohannes Weiner unsigned long nr = 0;
3999113b7dfdSJohannes Weiner enum lru_list lru;
4000113b7dfdSJohannes Weiner
4001113b7dfdSJohannes Weiner for_each_lru(lru) {
4002113b7dfdSJohannes Weiner if (!(BIT(lru) & lru_mask))
4003113b7dfdSJohannes Weiner continue;
4004dd8657b6SShakeel Butt if (tree)
4005dd8657b6SShakeel Butt nr += memcg_page_state(memcg, NR_LRU_BASE + lru);
4006dd8657b6SShakeel Butt else
4007205b20ccSJohannes Weiner nr += memcg_page_state_local(memcg, NR_LRU_BASE + lru);
4008113b7dfdSJohannes Weiner }
4009113b7dfdSJohannes Weiner return nr;
4010113b7dfdSJohannes Weiner }
4011113b7dfdSJohannes Weiner
memcg_numa_stat_show(struct seq_file * m,void * v)40122da8ca82STejun Heo static int memcg_numa_stat_show(struct seq_file *m, void *v)
4013406eb0c9SYing Han {
401425485de6SGreg Thelen struct numa_stat {
401525485de6SGreg Thelen const char *name;
401625485de6SGreg Thelen unsigned int lru_mask;
401725485de6SGreg Thelen };
401825485de6SGreg Thelen
401925485de6SGreg Thelen static const struct numa_stat stats[] = {
402025485de6SGreg Thelen { "total", LRU_ALL },
402125485de6SGreg Thelen { "file", LRU_ALL_FILE },
402225485de6SGreg Thelen { "anon", LRU_ALL_ANON },
402325485de6SGreg Thelen { "unevictable", BIT(LRU_UNEVICTABLE) },
402425485de6SGreg Thelen };
402525485de6SGreg Thelen const struct numa_stat *stat;
4026406eb0c9SYing Han int nid;
4027aa9694bbSChris Down struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
4028406eb0c9SYing Han
4029fd25a9e0SShakeel Butt mem_cgroup_flush_stats();
40302d146aa3SJohannes Weiner
403125485de6SGreg Thelen for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
4032dd8657b6SShakeel Butt seq_printf(m, "%s=%lu", stat->name,
4033dd8657b6SShakeel Butt mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
4034dd8657b6SShakeel Butt false));
4035dd8657b6SShakeel Butt for_each_node_state(nid, N_MEMORY)
4036dd8657b6SShakeel Butt seq_printf(m, " N%d=%lu", nid,
4037dd8657b6SShakeel Butt mem_cgroup_node_nr_lru_pages(memcg, nid,
4038dd8657b6SShakeel Butt stat->lru_mask, false));
4039406eb0c9SYing Han seq_putc(m, '\n');
404025485de6SGreg Thelen }
4041406eb0c9SYing Han
4042071aee13SYing Han for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
4043406eb0c9SYing Han
4044dd8657b6SShakeel Butt seq_printf(m, "hierarchical_%s=%lu", stat->name,
4045dd8657b6SShakeel Butt mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
4046dd8657b6SShakeel Butt true));
4047dd8657b6SShakeel Butt for_each_node_state(nid, N_MEMORY)
4048dd8657b6SShakeel Butt seq_printf(m, " N%d=%lu", nid,
4049dd8657b6SShakeel Butt mem_cgroup_node_nr_lru_pages(memcg, nid,
4050dd8657b6SShakeel Butt stat->lru_mask, true));
4051406eb0c9SYing Han seq_putc(m, '\n');
4052071aee13SYing Han }
4053406eb0c9SYing Han
4054406eb0c9SYing Han return 0;
4055406eb0c9SYing Han }
4056406eb0c9SYing Han #endif /* CONFIG_NUMA */
4057406eb0c9SYing Han
4058c8713d0bSJohannes Weiner static const unsigned int memcg1_stats[] = {
40590d1c2072SJohannes Weiner NR_FILE_PAGES,
4060be5d0a74SJohannes Weiner NR_ANON_MAPPED,
4061468c3982SJohannes Weiner #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4062468c3982SJohannes Weiner NR_ANON_THPS,
4063468c3982SJohannes Weiner #endif
4064c8713d0bSJohannes Weiner NR_SHMEM,
4065c8713d0bSJohannes Weiner NR_FILE_MAPPED,
4066c8713d0bSJohannes Weiner NR_FILE_DIRTY,
4067c8713d0bSJohannes Weiner NR_WRITEBACK,
4068e09b0b61SYang Shi WORKINGSET_REFAULT_ANON,
4069e09b0b61SYang Shi WORKINGSET_REFAULT_FILE,
4070c8713d0bSJohannes Weiner MEMCG_SWAP,
4071c8713d0bSJohannes Weiner };
4072c8713d0bSJohannes Weiner
4073c8713d0bSJohannes Weiner static const char *const memcg1_stat_names[] = {
4074c8713d0bSJohannes Weiner "cache",
4075c8713d0bSJohannes Weiner "rss",
4076468c3982SJohannes Weiner #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4077c8713d0bSJohannes Weiner "rss_huge",
4078468c3982SJohannes Weiner #endif
4079c8713d0bSJohannes Weiner "shmem",
4080c8713d0bSJohannes Weiner "mapped_file",
4081c8713d0bSJohannes Weiner "dirty",
4082c8713d0bSJohannes Weiner "writeback",
4083e09b0b61SYang Shi "workingset_refault_anon",
4084e09b0b61SYang Shi "workingset_refault_file",
4085c8713d0bSJohannes Weiner "swap",
4086c8713d0bSJohannes Weiner };
4087c8713d0bSJohannes Weiner
4088df0e53d0SJohannes Weiner /* Universal VM events cgroup1 shows, original sort order */
40898dd53fd3SGreg Thelen static const unsigned int memcg1_events[] = {
4090df0e53d0SJohannes Weiner PGPGIN,
4091df0e53d0SJohannes Weiner PGPGOUT,
4092df0e53d0SJohannes Weiner PGFAULT,
4093df0e53d0SJohannes Weiner PGMAJFAULT,
4094df0e53d0SJohannes Weiner };
4095df0e53d0SJohannes Weiner
memcg1_stat_format(struct mem_cgroup * memcg,struct seq_buf * s)4096dddb44ffSYosry Ahmed static void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s)
4097d2ceb9b7SKAMEZAWA Hiroyuki {
40983e32cb2eSJohannes Weiner unsigned long memory, memsw;
4099af7c4b0eSJohannes Weiner struct mem_cgroup *mi;
4100af7c4b0eSJohannes Weiner unsigned int i;
4101d2ceb9b7SKAMEZAWA Hiroyuki
410271cd3113SJohannes Weiner BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats));
410370bc068cSRickard Strandqvist
4104fd25a9e0SShakeel Butt mem_cgroup_flush_stats();
41052d146aa3SJohannes Weiner
410671cd3113SJohannes Weiner for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
4107468c3982SJohannes Weiner unsigned long nr;
4108468c3982SJohannes Weiner
410971cd3113SJohannes Weiner if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
41101dd3a273SDaisuke Nishimura continue;
4111468c3982SJohannes Weiner nr = memcg_page_state_local(memcg, memcg1_stats[i]);
4112dddb44ffSYosry Ahmed seq_buf_printf(s, "%s %lu\n", memcg1_stat_names[i],
4113e09b0b61SYang Shi nr * memcg_page_state_unit(memcg1_stats[i]));
41141dd3a273SDaisuke Nishimura }
41156d12e2d8SKAMEZAWA Hiroyuki
4116df0e53d0SJohannes Weiner for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
4117dddb44ffSYosry Ahmed seq_buf_printf(s, "%s %lu\n", vm_event_name(memcg1_events[i]),
4118205b20ccSJohannes Weiner memcg_events_local(memcg, memcg1_events[i]));
4119af7c4b0eSJohannes Weiner
4120af7c4b0eSJohannes Weiner for (i = 0; i < NR_LRU_LISTS; i++)
4121dddb44ffSYosry Ahmed seq_buf_printf(s, "%s %lu\n", lru_list_name(i),
4122205b20ccSJohannes Weiner memcg_page_state_local(memcg, NR_LRU_BASE + i) *
412321d89d15SJohannes Weiner PAGE_SIZE);
4124af7c4b0eSJohannes Weiner
412514067bb3SKAMEZAWA Hiroyuki /* Hierarchical information */
41263e32cb2eSJohannes Weiner memory = memsw = PAGE_COUNTER_MAX;
41273e32cb2eSJohannes Weiner for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
412815b42562SChris Down memory = min(memory, READ_ONCE(mi->memory.max));
412915b42562SChris Down memsw = min(memsw, READ_ONCE(mi->memsw.max));
41303e32cb2eSJohannes Weiner }
4131dddb44ffSYosry Ahmed seq_buf_printf(s, "hierarchical_memory_limit %llu\n",
41323e32cb2eSJohannes Weiner (u64)memory * PAGE_SIZE);
41337941d214SJohannes Weiner if (do_memsw_account())
4134dddb44ffSYosry Ahmed seq_buf_printf(s, "hierarchical_memsw_limit %llu\n",
41353e32cb2eSJohannes Weiner (u64)memsw * PAGE_SIZE);
41367f016ee8SKOSAKI Motohiro
41378de7ecc6SShakeel Butt for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
41387de2e9f1Szhongjiang-ali unsigned long nr;
41397de2e9f1Szhongjiang-ali
414071cd3113SJohannes Weiner if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
41411dd3a273SDaisuke Nishimura continue;
41427de2e9f1Szhongjiang-ali nr = memcg_page_state(memcg, memcg1_stats[i]);
4143dddb44ffSYosry Ahmed seq_buf_printf(s, "total_%s %llu\n", memcg1_stat_names[i],
4144e09b0b61SYang Shi (u64)nr * memcg_page_state_unit(memcg1_stats[i]));
4145af7c4b0eSJohannes Weiner }
4146af7c4b0eSJohannes Weiner
41478de7ecc6SShakeel Butt for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
4148dddb44ffSYosry Ahmed seq_buf_printf(s, "total_%s %llu\n",
4149ebc5d83dSKonstantin Khlebnikov vm_event_name(memcg1_events[i]),
4150dd923990SYafang Shao (u64)memcg_events(memcg, memcg1_events[i]));
4151af7c4b0eSJohannes Weiner
41528de7ecc6SShakeel Butt for (i = 0; i < NR_LRU_LISTS; i++)
4153dddb44ffSYosry Ahmed seq_buf_printf(s, "total_%s %llu\n", lru_list_name(i),
415442a30035SJohannes Weiner (u64)memcg_page_state(memcg, NR_LRU_BASE + i) *
415542a30035SJohannes Weiner PAGE_SIZE);
415614067bb3SKAMEZAWA Hiroyuki
41577f016ee8SKOSAKI Motohiro #ifdef CONFIG_DEBUG_VM
41587f016ee8SKOSAKI Motohiro {
4159ef8f2327SMel Gorman pg_data_t *pgdat;
4160ef8f2327SMel Gorman struct mem_cgroup_per_node *mz;
41611431d4d1SJohannes Weiner unsigned long anon_cost = 0;
41621431d4d1SJohannes Weiner unsigned long file_cost = 0;
41637f016ee8SKOSAKI Motohiro
4164ef8f2327SMel Gorman for_each_online_pgdat(pgdat) {
4165a3747b53SJohannes Weiner mz = memcg->nodeinfo[pgdat->node_id];
41667f016ee8SKOSAKI Motohiro
41671431d4d1SJohannes Weiner anon_cost += mz->lruvec.anon_cost;
41681431d4d1SJohannes Weiner file_cost += mz->lruvec.file_cost;
41697f016ee8SKOSAKI Motohiro }
4170dddb44ffSYosry Ahmed seq_buf_printf(s, "anon_cost %lu\n", anon_cost);
4171dddb44ffSYosry Ahmed seq_buf_printf(s, "file_cost %lu\n", file_cost);
41727f016ee8SKOSAKI Motohiro }
41737f016ee8SKOSAKI Motohiro #endif
4174d2ceb9b7SKAMEZAWA Hiroyuki }
4175d2ceb9b7SKAMEZAWA Hiroyuki
mem_cgroup_swappiness_read(struct cgroup_subsys_state * css,struct cftype * cft)4176182446d0STejun Heo static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
4177182446d0STejun Heo struct cftype *cft)
4178a7885eb8SKOSAKI Motohiro {
4179182446d0STejun Heo struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4180a7885eb8SKOSAKI Motohiro
41811f4c025bSKAMEZAWA Hiroyuki return mem_cgroup_swappiness(memcg);
4182a7885eb8SKOSAKI Motohiro }
4183a7885eb8SKOSAKI Motohiro
mem_cgroup_swappiness_write(struct cgroup_subsys_state * css,struct cftype * cft,u64 val)4184182446d0STejun Heo static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
4185182446d0STejun Heo struct cftype *cft, u64 val)
4186a7885eb8SKOSAKI Motohiro {
4187182446d0STejun Heo struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4188068b38c1SLi Zefan
418937bc3cb9SBaolin Wang if (val > 200)
4190a7885eb8SKOSAKI Motohiro return -EINVAL;
4191a7885eb8SKOSAKI Motohiro
4192a4792030SShakeel Butt if (!mem_cgroup_is_root(memcg))
419382b3aa26SYue Zhao WRITE_ONCE(memcg->swappiness, val);
41943dae7fecSJohannes Weiner else
419582b3aa26SYue Zhao WRITE_ONCE(vm_swappiness, val);
4196068b38c1SLi Zefan
4197a7885eb8SKOSAKI Motohiro return 0;
4198a7885eb8SKOSAKI Motohiro }
4199a7885eb8SKOSAKI Motohiro
__mem_cgroup_threshold(struct mem_cgroup * memcg,bool swap)42002e72b634SKirill A. Shutemov static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
42012e72b634SKirill A. Shutemov {
42022e72b634SKirill A. Shutemov struct mem_cgroup_threshold_ary *t;
42033e32cb2eSJohannes Weiner unsigned long usage;
42042e72b634SKirill A. Shutemov int i;
42052e72b634SKirill A. Shutemov
42062e72b634SKirill A. Shutemov rcu_read_lock();
42072e72b634SKirill A. Shutemov if (!swap)
42082c488db2SKirill A. Shutemov t = rcu_dereference(memcg->thresholds.primary);
42092e72b634SKirill A. Shutemov else
42102c488db2SKirill A. Shutemov t = rcu_dereference(memcg->memsw_thresholds.primary);
42112e72b634SKirill A. Shutemov
42122e72b634SKirill A. Shutemov if (!t)
42132e72b634SKirill A. Shutemov goto unlock;
42142e72b634SKirill A. Shutemov
4215ce00a967SJohannes Weiner usage = mem_cgroup_usage(memcg, swap);
42162e72b634SKirill A. Shutemov
42172e72b634SKirill A. Shutemov /*
4218748dad36SSha Zhengju * current_threshold points to threshold just below or equal to usage.
42192e72b634SKirill A. Shutemov * If it's not true, a threshold was crossed after last
42202e72b634SKirill A. Shutemov * call of __mem_cgroup_threshold().
42212e72b634SKirill A. Shutemov */
42225407a562SPhil Carmody i = t->current_threshold;
42232e72b634SKirill A. Shutemov
42242e72b634SKirill A. Shutemov /*
42252e72b634SKirill A. Shutemov * Iterate backward over array of thresholds starting from
42262e72b634SKirill A. Shutemov * current_threshold and check if a threshold is crossed.
42272e72b634SKirill A. Shutemov * If none of thresholds below usage is crossed, we read
42282e72b634SKirill A. Shutemov * only one element of the array here.
42292e72b634SKirill A. Shutemov */
42302e72b634SKirill A. Shutemov for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
42312e72b634SKirill A. Shutemov eventfd_signal(t->entries[i].eventfd, 1);
42322e72b634SKirill A. Shutemov
42332e72b634SKirill A. Shutemov /* i = current_threshold + 1 */
42342e72b634SKirill A. Shutemov i++;
42352e72b634SKirill A. Shutemov
42362e72b634SKirill A. Shutemov /*
42372e72b634SKirill A. Shutemov * Iterate forward over array of thresholds starting from
42382e72b634SKirill A. Shutemov * current_threshold+1 and check if a threshold is crossed.
42392e72b634SKirill A. Shutemov * If none of thresholds above usage is crossed, we read
42402e72b634SKirill A. Shutemov * only one element of the array here.
42412e72b634SKirill A. Shutemov */
42422e72b634SKirill A. Shutemov for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
42432e72b634SKirill A. Shutemov eventfd_signal(t->entries[i].eventfd, 1);
42442e72b634SKirill A. Shutemov
42452e72b634SKirill A. Shutemov /* Update current_threshold */
42465407a562SPhil Carmody t->current_threshold = i - 1;
42472e72b634SKirill A. Shutemov unlock:
42482e72b634SKirill A. Shutemov rcu_read_unlock();
42492e72b634SKirill A. Shutemov }
42502e72b634SKirill A. Shutemov
mem_cgroup_threshold(struct mem_cgroup * memcg)42512e72b634SKirill A. Shutemov static void mem_cgroup_threshold(struct mem_cgroup *memcg)
42522e72b634SKirill A. Shutemov {
4253ad4ca5f4SKirill A. Shutemov while (memcg) {
42542e72b634SKirill A. Shutemov __mem_cgroup_threshold(memcg, false);
42557941d214SJohannes Weiner if (do_memsw_account())
42562e72b634SKirill A. Shutemov __mem_cgroup_threshold(memcg, true);
4257ad4ca5f4SKirill A. Shutemov
4258ad4ca5f4SKirill A. Shutemov memcg = parent_mem_cgroup(memcg);
4259ad4ca5f4SKirill A. Shutemov }
42602e72b634SKirill A. Shutemov }
42612e72b634SKirill A. Shutemov
compare_thresholds(const void * a,const void * b)42622e72b634SKirill A. Shutemov static int compare_thresholds(const void *a, const void *b)
42632e72b634SKirill A. Shutemov {
42642e72b634SKirill A. Shutemov const struct mem_cgroup_threshold *_a = a;
42652e72b634SKirill A. Shutemov const struct mem_cgroup_threshold *_b = b;
42662e72b634SKirill A. Shutemov
42672bff24a3SGreg Thelen if (_a->threshold > _b->threshold)
42682bff24a3SGreg Thelen return 1;
42692bff24a3SGreg Thelen
42702bff24a3SGreg Thelen if (_a->threshold < _b->threshold)
42712bff24a3SGreg Thelen return -1;
42722bff24a3SGreg Thelen
42732bff24a3SGreg Thelen return 0;
42742e72b634SKirill A. Shutemov }
42752e72b634SKirill A. Shutemov
mem_cgroup_oom_notify_cb(struct mem_cgroup * memcg)4276c0ff4b85SRaghavendra K T static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
42779490ff27SKAMEZAWA Hiroyuki {
42789490ff27SKAMEZAWA Hiroyuki struct mem_cgroup_eventfd_list *ev;
42799490ff27SKAMEZAWA Hiroyuki
42802bcf2e92SMichal Hocko spin_lock(&memcg_oom_lock);
42812bcf2e92SMichal Hocko
4282c0ff4b85SRaghavendra K T list_for_each_entry(ev, &memcg->oom_notify, list)
42839490ff27SKAMEZAWA Hiroyuki eventfd_signal(ev->eventfd, 1);
42842bcf2e92SMichal Hocko
42852bcf2e92SMichal Hocko spin_unlock(&memcg_oom_lock);
42869490ff27SKAMEZAWA Hiroyuki return 0;
42879490ff27SKAMEZAWA Hiroyuki }
42889490ff27SKAMEZAWA Hiroyuki
mem_cgroup_oom_notify(struct mem_cgroup * memcg)4289c0ff4b85SRaghavendra K T static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
42909490ff27SKAMEZAWA Hiroyuki {
42917d74b06fSKAMEZAWA Hiroyuki struct mem_cgroup *iter;
42927d74b06fSKAMEZAWA Hiroyuki
4293c0ff4b85SRaghavendra K T for_each_mem_cgroup_tree(iter, memcg)
42947d74b06fSKAMEZAWA Hiroyuki mem_cgroup_oom_notify_cb(iter);
42959490ff27SKAMEZAWA Hiroyuki }
42969490ff27SKAMEZAWA Hiroyuki
__mem_cgroup_usage_register_event(struct mem_cgroup * memcg,struct eventfd_ctx * eventfd,const char * args,enum res_type type)429759b6f873STejun Heo static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
4298347c4a87STejun Heo struct eventfd_ctx *eventfd, const char *args, enum res_type type)
42992e72b634SKirill A. Shutemov {
43002c488db2SKirill A. Shutemov struct mem_cgroup_thresholds *thresholds;
43012c488db2SKirill A. Shutemov struct mem_cgroup_threshold_ary *new;
43023e32cb2eSJohannes Weiner unsigned long threshold;
43033e32cb2eSJohannes Weiner unsigned long usage;
43042c488db2SKirill A. Shutemov int i, size, ret;
43052e72b634SKirill A. Shutemov
4306650c5e56SJohannes Weiner ret = page_counter_memparse(args, "-1", &threshold);
43072e72b634SKirill A. Shutemov if (ret)
43082e72b634SKirill A. Shutemov return ret;
43092e72b634SKirill A. Shutemov
43102e72b634SKirill A. Shutemov mutex_lock(&memcg->thresholds_lock);
43112c488db2SKirill A. Shutemov
431205b84301SJohannes Weiner if (type == _MEM) {
43132c488db2SKirill A. Shutemov thresholds = &memcg->thresholds;
4314ce00a967SJohannes Weiner usage = mem_cgroup_usage(memcg, false);
431505b84301SJohannes Weiner } else if (type == _MEMSWAP) {
43162c488db2SKirill A. Shutemov thresholds = &memcg->memsw_thresholds;
4317ce00a967SJohannes Weiner usage = mem_cgroup_usage(memcg, true);
431805b84301SJohannes Weiner } else
43192e72b634SKirill A. Shutemov BUG();
43202e72b634SKirill A. Shutemov
43212e72b634SKirill A. Shutemov /* Check if a threshold crossed before adding a new one */
43222c488db2SKirill A. Shutemov if (thresholds->primary)
43232e72b634SKirill A. Shutemov __mem_cgroup_threshold(memcg, type == _MEMSWAP);
43242e72b634SKirill A. Shutemov
43252c488db2SKirill A. Shutemov size = thresholds->primary ? thresholds->primary->size + 1 : 1;
43262e72b634SKirill A. Shutemov
43272e72b634SKirill A. Shutemov /* Allocate memory for new array of thresholds */
432867b8046fSGustavo A. R. Silva new = kmalloc(struct_size(new, entries, size), GFP_KERNEL);
43292c488db2SKirill A. Shutemov if (!new) {
43302e72b634SKirill A. Shutemov ret = -ENOMEM;
43312e72b634SKirill A. Shutemov goto unlock;
43322e72b634SKirill A. Shutemov }
43332c488db2SKirill A. Shutemov new->size = size;
43342e72b634SKirill A. Shutemov
43352e72b634SKirill A. Shutemov /* Copy thresholds (if any) to new array */
4336e90342e6SGustavo A. R. Silva if (thresholds->primary)
4337e90342e6SGustavo A. R. Silva memcpy(new->entries, thresholds->primary->entries,
4338e90342e6SGustavo A. R. Silva flex_array_size(new, entries, size - 1));
43392c488db2SKirill A. Shutemov
43402e72b634SKirill A. Shutemov /* Add new threshold */
43412c488db2SKirill A. Shutemov new->entries[size - 1].eventfd = eventfd;
43422c488db2SKirill A. Shutemov new->entries[size - 1].threshold = threshold;
43432e72b634SKirill A. Shutemov
43442e72b634SKirill A. Shutemov /* Sort thresholds. Registering of new threshold isn't time-critical */
434561e604e6SGustavo A. R. Silva sort(new->entries, size, sizeof(*new->entries),
43462e72b634SKirill A. Shutemov compare_thresholds, NULL);
43472e72b634SKirill A. Shutemov
43482e72b634SKirill A. Shutemov /* Find current threshold */
43492c488db2SKirill A. Shutemov new->current_threshold = -1;
43502e72b634SKirill A. Shutemov for (i = 0; i < size; i++) {
4351748dad36SSha Zhengju if (new->entries[i].threshold <= usage) {
43522e72b634SKirill A. Shutemov /*
43532c488db2SKirill A. Shutemov * new->current_threshold will not be used until
43542c488db2SKirill A. Shutemov * rcu_assign_pointer(), so it's safe to increment
43552e72b634SKirill A. Shutemov * it here.
43562e72b634SKirill A. Shutemov */
43572c488db2SKirill A. Shutemov ++new->current_threshold;
4358748dad36SSha Zhengju } else
4359748dad36SSha Zhengju break;
43602e72b634SKirill A. Shutemov }
43612e72b634SKirill A. Shutemov
43622c488db2SKirill A. Shutemov /* Free old spare buffer and save old primary buffer as spare */
43632c488db2SKirill A. Shutemov kfree(thresholds->spare);
43642c488db2SKirill A. Shutemov thresholds->spare = thresholds->primary;
43652c488db2SKirill A. Shutemov
43662c488db2SKirill A. Shutemov rcu_assign_pointer(thresholds->primary, new);
43672e72b634SKirill A. Shutemov
4368907860edSKirill A. Shutemov /* To be sure that nobody uses thresholds */
43692e72b634SKirill A. Shutemov synchronize_rcu();
43702e72b634SKirill A. Shutemov
43712e72b634SKirill A. Shutemov unlock:
43722e72b634SKirill A. Shutemov mutex_unlock(&memcg->thresholds_lock);
43732e72b634SKirill A. Shutemov
43742e72b634SKirill A. Shutemov return ret;
43752e72b634SKirill A. Shutemov }
43762e72b634SKirill A. Shutemov
mem_cgroup_usage_register_event(struct mem_cgroup * memcg,struct eventfd_ctx * eventfd,const char * args)437759b6f873STejun Heo static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
4378347c4a87STejun Heo struct eventfd_ctx *eventfd, const char *args)
43792e72b634SKirill A. Shutemov {
438059b6f873STejun Heo return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
4381347c4a87STejun Heo }
4382347c4a87STejun Heo
memsw_cgroup_usage_register_event(struct mem_cgroup * memcg,struct eventfd_ctx * eventfd,const char * args)438359b6f873STejun Heo static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
4384347c4a87STejun Heo struct eventfd_ctx *eventfd, const char *args)
4385347c4a87STejun Heo {
438659b6f873STejun Heo return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
4387347c4a87STejun Heo }
4388347c4a87STejun Heo
__mem_cgroup_usage_unregister_event(struct mem_cgroup * memcg,struct eventfd_ctx * eventfd,enum res_type type)438959b6f873STejun Heo static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4390347c4a87STejun Heo struct eventfd_ctx *eventfd, enum res_type type)
43912e72b634SKirill A. Shutemov {
43922c488db2SKirill A. Shutemov struct mem_cgroup_thresholds *thresholds;
43932c488db2SKirill A. Shutemov struct mem_cgroup_threshold_ary *new;
43943e32cb2eSJohannes Weiner unsigned long usage;
43957d36665aSChunguang Xu int i, j, size, entries;
43962e72b634SKirill A. Shutemov
43972e72b634SKirill A. Shutemov mutex_lock(&memcg->thresholds_lock);
439805b84301SJohannes Weiner
439905b84301SJohannes Weiner if (type == _MEM) {
44002c488db2SKirill A. Shutemov thresholds = &memcg->thresholds;
4401ce00a967SJohannes Weiner usage = mem_cgroup_usage(memcg, false);
440205b84301SJohannes Weiner } else if (type == _MEMSWAP) {
44032c488db2SKirill A. Shutemov thresholds = &memcg->memsw_thresholds;
4404ce00a967SJohannes Weiner usage = mem_cgroup_usage(memcg, true);
440505b84301SJohannes Weiner } else
44062e72b634SKirill A. Shutemov BUG();
44072e72b634SKirill A. Shutemov
4408371528caSAnton Vorontsov if (!thresholds->primary)
4409371528caSAnton Vorontsov goto unlock;
4410371528caSAnton Vorontsov
44112e72b634SKirill A. Shutemov /* Check if a threshold crossed before removing */
44122e72b634SKirill A. Shutemov __mem_cgroup_threshold(memcg, type == _MEMSWAP);
44132e72b634SKirill A. Shutemov
44142e72b634SKirill A. Shutemov /* Calculate new number of threshold */
44157d36665aSChunguang Xu size = entries = 0;
44162c488db2SKirill A. Shutemov for (i = 0; i < thresholds->primary->size; i++) {
44172c488db2SKirill A. Shutemov if (thresholds->primary->entries[i].eventfd != eventfd)
44182e72b634SKirill A. Shutemov size++;
44197d36665aSChunguang Xu else
44207d36665aSChunguang Xu entries++;
44212e72b634SKirill A. Shutemov }
44222e72b634SKirill A. Shutemov
44232c488db2SKirill A. Shutemov new = thresholds->spare;
4424907860edSKirill A. Shutemov
44257d36665aSChunguang Xu /* If no items related to eventfd have been cleared, nothing to do */
44267d36665aSChunguang Xu if (!entries)
44277d36665aSChunguang Xu goto unlock;
44287d36665aSChunguang Xu
44292e72b634SKirill A. Shutemov /* Set thresholds array to NULL if we don't have thresholds */
44302e72b634SKirill A. Shutemov if (!size) {
44312c488db2SKirill A. Shutemov kfree(new);
44322c488db2SKirill A. Shutemov new = NULL;
4433907860edSKirill A. Shutemov goto swap_buffers;
44342e72b634SKirill A. Shutemov }
44352e72b634SKirill A. Shutemov
44362c488db2SKirill A. Shutemov new->size = size;
44372e72b634SKirill A. Shutemov
44382e72b634SKirill A. Shutemov /* Copy thresholds and find current threshold */
44392c488db2SKirill A. Shutemov new->current_threshold = -1;
44402c488db2SKirill A. Shutemov for (i = 0, j = 0; i < thresholds->primary->size; i++) {
44412c488db2SKirill A. Shutemov if (thresholds->primary->entries[i].eventfd == eventfd)
44422e72b634SKirill A. Shutemov continue;
44432e72b634SKirill A. Shutemov
44442c488db2SKirill A. Shutemov new->entries[j] = thresholds->primary->entries[i];
4445748dad36SSha Zhengju if (new->entries[j].threshold <= usage) {
44462e72b634SKirill A. Shutemov /*
44472c488db2SKirill A. Shutemov * new->current_threshold will not be used
44482e72b634SKirill A. Shutemov * until rcu_assign_pointer(), so it's safe to increment
44492e72b634SKirill A. Shutemov * it here.
44502e72b634SKirill A. Shutemov */
44512c488db2SKirill A. Shutemov ++new->current_threshold;
44522e72b634SKirill A. Shutemov }
44532e72b634SKirill A. Shutemov j++;
44542e72b634SKirill A. Shutemov }
44552e72b634SKirill A. Shutemov
4456907860edSKirill A. Shutemov swap_buffers:
44572c488db2SKirill A. Shutemov /* Swap primary and spare array */
44582c488db2SKirill A. Shutemov thresholds->spare = thresholds->primary;
44598c757763SSha Zhengju
44602c488db2SKirill A. Shutemov rcu_assign_pointer(thresholds->primary, new);
44612e72b634SKirill A. Shutemov
4462907860edSKirill A. Shutemov /* To be sure that nobody uses thresholds */
44632e72b634SKirill A. Shutemov synchronize_rcu();
44646611d8d7SMartijn Coenen
44656611d8d7SMartijn Coenen /* If all events are unregistered, free the spare array */
44666611d8d7SMartijn Coenen if (!new) {
44676611d8d7SMartijn Coenen kfree(thresholds->spare);
44686611d8d7SMartijn Coenen thresholds->spare = NULL;
44696611d8d7SMartijn Coenen }
4470371528caSAnton Vorontsov unlock:
44712e72b634SKirill A. Shutemov mutex_unlock(&memcg->thresholds_lock);
44722e72b634SKirill A. Shutemov }
4473c1e862c1SKAMEZAWA Hiroyuki
mem_cgroup_usage_unregister_event(struct mem_cgroup * memcg,struct eventfd_ctx * eventfd)447459b6f873STejun Heo static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4475347c4a87STejun Heo struct eventfd_ctx *eventfd)
44769490ff27SKAMEZAWA Hiroyuki {
447759b6f873STejun Heo return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
4478347c4a87STejun Heo }
44799490ff27SKAMEZAWA Hiroyuki
memsw_cgroup_usage_unregister_event(struct mem_cgroup * memcg,struct eventfd_ctx * eventfd)448059b6f873STejun Heo static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4481347c4a87STejun Heo struct eventfd_ctx *eventfd)
4482347c4a87STejun Heo {
448359b6f873STejun Heo return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
4484347c4a87STejun Heo }
4485347c4a87STejun Heo
mem_cgroup_oom_register_event(struct mem_cgroup * memcg,struct eventfd_ctx * eventfd,const char * args)448659b6f873STejun Heo static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
4487347c4a87STejun Heo struct eventfd_ctx *eventfd, const char *args)
44889490ff27SKAMEZAWA Hiroyuki {
44899490ff27SKAMEZAWA Hiroyuki struct mem_cgroup_eventfd_list *event;
44909490ff27SKAMEZAWA Hiroyuki
44919490ff27SKAMEZAWA Hiroyuki event = kmalloc(sizeof(*event), GFP_KERNEL);
44929490ff27SKAMEZAWA Hiroyuki if (!event)
44939490ff27SKAMEZAWA Hiroyuki return -ENOMEM;
44949490ff27SKAMEZAWA Hiroyuki
44951af8efe9SMichal Hocko spin_lock(&memcg_oom_lock);
44969490ff27SKAMEZAWA Hiroyuki
44979490ff27SKAMEZAWA Hiroyuki event->eventfd = eventfd;
44989490ff27SKAMEZAWA Hiroyuki list_add(&event->list, &memcg->oom_notify);
44999490ff27SKAMEZAWA Hiroyuki
45009490ff27SKAMEZAWA Hiroyuki /* already in OOM ? */
4501c2b42d3cSTejun Heo if (memcg->under_oom)
45029490ff27SKAMEZAWA Hiroyuki eventfd_signal(eventfd, 1);
45031af8efe9SMichal Hocko spin_unlock(&memcg_oom_lock);
45049490ff27SKAMEZAWA Hiroyuki
45059490ff27SKAMEZAWA Hiroyuki return 0;
45069490ff27SKAMEZAWA Hiroyuki }
45079490ff27SKAMEZAWA Hiroyuki
mem_cgroup_oom_unregister_event(struct mem_cgroup * memcg,struct eventfd_ctx * eventfd)450859b6f873STejun Heo static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
4509347c4a87STejun Heo struct eventfd_ctx *eventfd)
45109490ff27SKAMEZAWA Hiroyuki {
45119490ff27SKAMEZAWA Hiroyuki struct mem_cgroup_eventfd_list *ev, *tmp;
45129490ff27SKAMEZAWA Hiroyuki
45131af8efe9SMichal Hocko spin_lock(&memcg_oom_lock);
45149490ff27SKAMEZAWA Hiroyuki
4515c0ff4b85SRaghavendra K T list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
45169490ff27SKAMEZAWA Hiroyuki if (ev->eventfd == eventfd) {
45179490ff27SKAMEZAWA Hiroyuki list_del(&ev->list);
45189490ff27SKAMEZAWA Hiroyuki kfree(ev);
45199490ff27SKAMEZAWA Hiroyuki }
45209490ff27SKAMEZAWA Hiroyuki }
45219490ff27SKAMEZAWA Hiroyuki
45221af8efe9SMichal Hocko spin_unlock(&memcg_oom_lock);
45239490ff27SKAMEZAWA Hiroyuki }
45249490ff27SKAMEZAWA Hiroyuki
mem_cgroup_oom_control_read(struct seq_file * sf,void * v)45252da8ca82STejun Heo static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
45263c11ecf4SKAMEZAWA Hiroyuki {
4527aa9694bbSChris Down struct mem_cgroup *memcg = mem_cgroup_from_seq(sf);
45283c11ecf4SKAMEZAWA Hiroyuki
452917c56de6SYue Zhao seq_printf(sf, "oom_kill_disable %d\n", READ_ONCE(memcg->oom_kill_disable));
4530c2b42d3cSTejun Heo seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
4531fe6bdfc8SRoman Gushchin seq_printf(sf, "oom_kill %lu\n",
4532fe6bdfc8SRoman Gushchin atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL]));
45333c11ecf4SKAMEZAWA Hiroyuki return 0;
45343c11ecf4SKAMEZAWA Hiroyuki }
45353c11ecf4SKAMEZAWA Hiroyuki
mem_cgroup_oom_control_write(struct cgroup_subsys_state * css,struct cftype * cft,u64 val)4536182446d0STejun Heo static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
45373c11ecf4SKAMEZAWA Hiroyuki struct cftype *cft, u64 val)
45383c11ecf4SKAMEZAWA Hiroyuki {
4539182446d0STejun Heo struct mem_cgroup *memcg = mem_cgroup_from_css(css);
45403c11ecf4SKAMEZAWA Hiroyuki
45413c11ecf4SKAMEZAWA Hiroyuki /* cannot set to root cgroup and only 0 and 1 are allowed */
4542a4792030SShakeel Butt if (mem_cgroup_is_root(memcg) || !((val == 0) || (val == 1)))
45433c11ecf4SKAMEZAWA Hiroyuki return -EINVAL;
45443c11ecf4SKAMEZAWA Hiroyuki
454517c56de6SYue Zhao WRITE_ONCE(memcg->oom_kill_disable, val);
45464d845ebfSKAMEZAWA Hiroyuki if (!val)
4547c0ff4b85SRaghavendra K T memcg_oom_recover(memcg);
45483dae7fecSJohannes Weiner
45493c11ecf4SKAMEZAWA Hiroyuki return 0;
45503c11ecf4SKAMEZAWA Hiroyuki }
45513c11ecf4SKAMEZAWA Hiroyuki
455252ebea74STejun Heo #ifdef CONFIG_CGROUP_WRITEBACK
455352ebea74STejun Heo
45543a8e9ac8STejun Heo #include <trace/events/writeback.h>
45553a8e9ac8STejun Heo
memcg_wb_domain_init(struct mem_cgroup * memcg,gfp_t gfp)4556841710aaSTejun Heo static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
4557841710aaSTejun Heo {
4558841710aaSTejun Heo return wb_domain_init(&memcg->cgwb_domain, gfp);
4559841710aaSTejun Heo }
4560841710aaSTejun Heo
memcg_wb_domain_exit(struct mem_cgroup * memcg)4561841710aaSTejun Heo static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
4562841710aaSTejun Heo {
4563841710aaSTejun Heo wb_domain_exit(&memcg->cgwb_domain);
4564841710aaSTejun Heo }
4565841710aaSTejun Heo
memcg_wb_domain_size_changed(struct mem_cgroup * memcg)45662529bb3aSTejun Heo static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
45672529bb3aSTejun Heo {
45682529bb3aSTejun Heo wb_domain_size_changed(&memcg->cgwb_domain);
45692529bb3aSTejun Heo }
45702529bb3aSTejun Heo
mem_cgroup_wb_domain(struct bdi_writeback * wb)4571841710aaSTejun Heo struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
4572841710aaSTejun Heo {
4573841710aaSTejun Heo struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4574841710aaSTejun Heo
4575841710aaSTejun Heo if (!memcg->css.parent)
4576841710aaSTejun Heo return NULL;
4577841710aaSTejun Heo
4578841710aaSTejun Heo return &memcg->cgwb_domain;
4579841710aaSTejun Heo }
4580841710aaSTejun Heo
4581c2aa723aSTejun Heo /**
4582c2aa723aSTejun Heo * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
4583c2aa723aSTejun Heo * @wb: bdi_writeback in question
4584c5edf9cdSTejun Heo * @pfilepages: out parameter for number of file pages
4585c5edf9cdSTejun Heo * @pheadroom: out parameter for number of allocatable pages according to memcg
4586c2aa723aSTejun Heo * @pdirty: out parameter for number of dirty pages
4587c2aa723aSTejun Heo * @pwriteback: out parameter for number of pages under writeback
4588c2aa723aSTejun Heo *
4589c5edf9cdSTejun Heo * Determine the numbers of file, headroom, dirty, and writeback pages in
4590c5edf9cdSTejun Heo * @wb's memcg. File, dirty and writeback are self-explanatory. Headroom
4591c5edf9cdSTejun Heo * is a bit more involved.
4592c2aa723aSTejun Heo *
4593c5edf9cdSTejun Heo * A memcg's headroom is "min(max, high) - used". In the hierarchy, the
4594c5edf9cdSTejun Heo * headroom is calculated as the lowest headroom of itself and the
4595c5edf9cdSTejun Heo * ancestors. Note that this doesn't consider the actual amount of
4596c5edf9cdSTejun Heo * available memory in the system. The caller should further cap
4597c5edf9cdSTejun Heo * *@pheadroom accordingly.
4598c2aa723aSTejun Heo */
mem_cgroup_wb_stats(struct bdi_writeback * wb,unsigned long * pfilepages,unsigned long * pheadroom,unsigned long * pdirty,unsigned long * pwriteback)4599c5edf9cdSTejun Heo void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
4600c5edf9cdSTejun Heo unsigned long *pheadroom, unsigned long *pdirty,
4601c5edf9cdSTejun Heo unsigned long *pwriteback)
4602c2aa723aSTejun Heo {
4603c2aa723aSTejun Heo struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4604c2aa723aSTejun Heo struct mem_cgroup *parent;
4605c2aa723aSTejun Heo
4606190409caSYosry Ahmed mem_cgroup_flush_stats();
4607c2aa723aSTejun Heo
46082d146aa3SJohannes Weiner *pdirty = memcg_page_state(memcg, NR_FILE_DIRTY);
46092d146aa3SJohannes Weiner *pwriteback = memcg_page_state(memcg, NR_WRITEBACK);
46102d146aa3SJohannes Weiner *pfilepages = memcg_page_state(memcg, NR_INACTIVE_FILE) +
46112d146aa3SJohannes Weiner memcg_page_state(memcg, NR_ACTIVE_FILE);
46122d146aa3SJohannes Weiner
4613c5edf9cdSTejun Heo *pheadroom = PAGE_COUNTER_MAX;
4614c2aa723aSTejun Heo while ((parent = parent_mem_cgroup(memcg))) {
461515b42562SChris Down unsigned long ceiling = min(READ_ONCE(memcg->memory.max),
4616d1663a90SJakub Kicinski READ_ONCE(memcg->memory.high));
4617c2aa723aSTejun Heo unsigned long used = page_counter_read(&memcg->memory);
4618c2aa723aSTejun Heo
4619c5edf9cdSTejun Heo *pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
4620c2aa723aSTejun Heo memcg = parent;
4621c2aa723aSTejun Heo }
4622c2aa723aSTejun Heo }
4623c2aa723aSTejun Heo
462497b27821STejun Heo /*
462597b27821STejun Heo * Foreign dirty flushing
462697b27821STejun Heo *
462797b27821STejun Heo * There's an inherent mismatch between memcg and writeback. The former
4628f0953a1bSIngo Molnar * tracks ownership per-page while the latter per-inode. This was a
462997b27821STejun Heo * deliberate design decision because honoring per-page ownership in the
463097b27821STejun Heo * writeback path is complicated, may lead to higher CPU and IO overheads
463197b27821STejun Heo * and deemed unnecessary given that write-sharing an inode across
463297b27821STejun Heo * different cgroups isn't a common use-case.
463397b27821STejun Heo *
463497b27821STejun Heo * Combined with inode majority-writer ownership switching, this works well
463597b27821STejun Heo * enough in most cases but there are some pathological cases. For
463697b27821STejun Heo * example, let's say there are two cgroups A and B which keep writing to
463797b27821STejun Heo * different but confined parts of the same inode. B owns the inode and
463897b27821STejun Heo * A's memory is limited far below B's. A's dirty ratio can rise enough to
463997b27821STejun Heo * trigger balance_dirty_pages() sleeps but B's can be low enough to avoid
464097b27821STejun Heo * triggering background writeback. A will be slowed down without a way to
464197b27821STejun Heo * make writeback of the dirty pages happen.
464297b27821STejun Heo *
4643f0953a1bSIngo Molnar * Conditions like the above can lead to a cgroup getting repeatedly and
464497b27821STejun Heo * severely throttled after making some progress after each
4645f0953a1bSIngo Molnar * dirty_expire_interval while the underlying IO device is almost
464697b27821STejun Heo * completely idle.
464797b27821STejun Heo *
464897b27821STejun Heo * Solving this problem completely requires matching the ownership tracking
464997b27821STejun Heo * granularities between memcg and writeback in either direction. However,
465097b27821STejun Heo * the more egregious behaviors can be avoided by simply remembering the
465197b27821STejun Heo * most recent foreign dirtying events and initiating remote flushes on
465297b27821STejun Heo * them when local writeback isn't enough to keep the memory clean enough.
465397b27821STejun Heo *
465497b27821STejun Heo * The following two functions implement such mechanism. When a foreign
465597b27821STejun Heo * page - a page whose memcg and writeback ownerships don't match - is
465697b27821STejun Heo * dirtied, mem_cgroup_track_foreign_dirty() records the inode owning
465797b27821STejun Heo * bdi_writeback on the page owning memcg. When balance_dirty_pages()
465897b27821STejun Heo * decides that the memcg needs to sleep due to high dirty ratio, it calls
465997b27821STejun Heo * mem_cgroup_flush_foreign() which queues writeback on the recorded
466097b27821STejun Heo * foreign bdi_writebacks which haven't expired. Both the numbers of
466197b27821STejun Heo * recorded bdi_writebacks and concurrent in-flight foreign writebacks are
466297b27821STejun Heo * limited to MEMCG_CGWB_FRN_CNT.
466397b27821STejun Heo *
466497b27821STejun Heo * The mechanism only remembers IDs and doesn't hold any object references.
466597b27821STejun Heo * As being wrong occasionally doesn't matter, updates and accesses to the
466697b27821STejun Heo * records are lockless and racy.
466797b27821STejun Heo */
mem_cgroup_track_foreign_dirty_slowpath(struct folio * folio,struct bdi_writeback * wb)46689d8053fcSMatthew Wilcox (Oracle) void mem_cgroup_track_foreign_dirty_slowpath(struct folio *folio,
466997b27821STejun Heo struct bdi_writeback *wb)
467097b27821STejun Heo {
46719d8053fcSMatthew Wilcox (Oracle) struct mem_cgroup *memcg = folio_memcg(folio);
467297b27821STejun Heo struct memcg_cgwb_frn *frn;
467397b27821STejun Heo u64 now = get_jiffies_64();
467497b27821STejun Heo u64 oldest_at = now;
467597b27821STejun Heo int oldest = -1;
467697b27821STejun Heo int i;
467797b27821STejun Heo
46789d8053fcSMatthew Wilcox (Oracle) trace_track_foreign_dirty(folio, wb);
46793a8e9ac8STejun Heo
468097b27821STejun Heo /*
468197b27821STejun Heo * Pick the slot to use. If there is already a slot for @wb, keep
468297b27821STejun Heo * using it. If not replace the oldest one which isn't being
468397b27821STejun Heo * written out.
468497b27821STejun Heo */
468597b27821STejun Heo for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
468697b27821STejun Heo frn = &memcg->cgwb_frn[i];
468797b27821STejun Heo if (frn->bdi_id == wb->bdi->id &&
468897b27821STejun Heo frn->memcg_id == wb->memcg_css->id)
468997b27821STejun Heo break;
469097b27821STejun Heo if (time_before64(frn->at, oldest_at) &&
469197b27821STejun Heo atomic_read(&frn->done.cnt) == 1) {
469297b27821STejun Heo oldest = i;
469397b27821STejun Heo oldest_at = frn->at;
469497b27821STejun Heo }
469597b27821STejun Heo }
469697b27821STejun Heo
469797b27821STejun Heo if (i < MEMCG_CGWB_FRN_CNT) {
469897b27821STejun Heo /*
469997b27821STejun Heo * Re-using an existing one. Update timestamp lazily to
470097b27821STejun Heo * avoid making the cacheline hot. We want them to be
470197b27821STejun Heo * reasonably up-to-date and significantly shorter than
470297b27821STejun Heo * dirty_expire_interval as that's what expires the record.
470397b27821STejun Heo * Use the shorter of 1s and dirty_expire_interval / 8.
470497b27821STejun Heo */
470597b27821STejun Heo unsigned long update_intv =
470697b27821STejun Heo min_t(unsigned long, HZ,
470797b27821STejun Heo msecs_to_jiffies(dirty_expire_interval * 10) / 8);
470897b27821STejun Heo
470997b27821STejun Heo if (time_before64(frn->at, now - update_intv))
471097b27821STejun Heo frn->at = now;
471197b27821STejun Heo } else if (oldest >= 0) {
471297b27821STejun Heo /* replace the oldest free one */
471397b27821STejun Heo frn = &memcg->cgwb_frn[oldest];
471497b27821STejun Heo frn->bdi_id = wb->bdi->id;
471597b27821STejun Heo frn->memcg_id = wb->memcg_css->id;
471697b27821STejun Heo frn->at = now;
471797b27821STejun Heo }
471897b27821STejun Heo }
471997b27821STejun Heo
472097b27821STejun Heo /* issue foreign writeback flushes for recorded foreign dirtying events */
mem_cgroup_flush_foreign(struct bdi_writeback * wb)472197b27821STejun Heo void mem_cgroup_flush_foreign(struct bdi_writeback *wb)
472297b27821STejun Heo {
472397b27821STejun Heo struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
472497b27821STejun Heo unsigned long intv = msecs_to_jiffies(dirty_expire_interval * 10);
472597b27821STejun Heo u64 now = jiffies_64;
472697b27821STejun Heo int i;
472797b27821STejun Heo
472897b27821STejun Heo for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
472997b27821STejun Heo struct memcg_cgwb_frn *frn = &memcg->cgwb_frn[i];
473097b27821STejun Heo
473197b27821STejun Heo /*
473297b27821STejun Heo * If the record is older than dirty_expire_interval,
473397b27821STejun Heo * writeback on it has already started. No need to kick it
473497b27821STejun Heo * off again. Also, don't start a new one if there's
473597b27821STejun Heo * already one in flight.
473697b27821STejun Heo */
473797b27821STejun Heo if (time_after64(frn->at, now - intv) &&
473897b27821STejun Heo atomic_read(&frn->done.cnt) == 1) {
473997b27821STejun Heo frn->at = 0;
47403a8e9ac8STejun Heo trace_flush_foreign(wb, frn->bdi_id, frn->memcg_id);
47417490a2d2SShakeel Butt cgroup_writeback_by_id(frn->bdi_id, frn->memcg_id,
474297b27821STejun Heo WB_REASON_FOREIGN_FLUSH,
474397b27821STejun Heo &frn->done);
474497b27821STejun Heo }
474597b27821STejun Heo }
474697b27821STejun Heo }
474797b27821STejun Heo
4748841710aaSTejun Heo #else /* CONFIG_CGROUP_WRITEBACK */
4749841710aaSTejun Heo
memcg_wb_domain_init(struct mem_cgroup * memcg,gfp_t gfp)4750841710aaSTejun Heo static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
4751841710aaSTejun Heo {
4752841710aaSTejun Heo return 0;
4753841710aaSTejun Heo }
4754841710aaSTejun Heo
memcg_wb_domain_exit(struct mem_cgroup * memcg)4755841710aaSTejun Heo static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
4756841710aaSTejun Heo {
4757841710aaSTejun Heo }
4758841710aaSTejun Heo
memcg_wb_domain_size_changed(struct mem_cgroup * memcg)47592529bb3aSTejun Heo static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
47602529bb3aSTejun Heo {
47612529bb3aSTejun Heo }
47622529bb3aSTejun Heo
476352ebea74STejun Heo #endif /* CONFIG_CGROUP_WRITEBACK */
476452ebea74STejun Heo
476579bd9814STejun Heo /*
47663bc942f3STejun Heo * DO NOT USE IN NEW FILES.
47673bc942f3STejun Heo *
47683bc942f3STejun Heo * "cgroup.event_control" implementation.
47693bc942f3STejun Heo *
47703bc942f3STejun Heo * This is way over-engineered. It tries to support fully configurable
47713bc942f3STejun Heo * events for each user. Such level of flexibility is completely
47723bc942f3STejun Heo * unnecessary especially in the light of the planned unified hierarchy.
47733bc942f3STejun Heo *
47743bc942f3STejun Heo * Please deprecate this and replace with something simpler if at all
47753bc942f3STejun Heo * possible.
47763bc942f3STejun Heo */
47773bc942f3STejun Heo
47783bc942f3STejun Heo /*
477979bd9814STejun Heo * Unregister event and free resources.
478079bd9814STejun Heo *
478179bd9814STejun Heo * Gets called from workqueue.
478279bd9814STejun Heo */
memcg_event_remove(struct work_struct * work)47833bc942f3STejun Heo static void memcg_event_remove(struct work_struct *work)
478479bd9814STejun Heo {
47853bc942f3STejun Heo struct mem_cgroup_event *event =
47863bc942f3STejun Heo container_of(work, struct mem_cgroup_event, remove);
478759b6f873STejun Heo struct mem_cgroup *memcg = event->memcg;
478879bd9814STejun Heo
478979bd9814STejun Heo remove_wait_queue(event->wqh, &event->wait);
479079bd9814STejun Heo
479159b6f873STejun Heo event->unregister_event(memcg, event->eventfd);
479279bd9814STejun Heo
479379bd9814STejun Heo /* Notify userspace the event is going away. */
479479bd9814STejun Heo eventfd_signal(event->eventfd, 1);
479579bd9814STejun Heo
479679bd9814STejun Heo eventfd_ctx_put(event->eventfd);
479779bd9814STejun Heo kfree(event);
479859b6f873STejun Heo css_put(&memcg->css);
479979bd9814STejun Heo }
480079bd9814STejun Heo
480179bd9814STejun Heo /*
4802a9a08845SLinus Torvalds * Gets called on EPOLLHUP on eventfd when user closes it.
480379bd9814STejun Heo *
480479bd9814STejun Heo * Called with wqh->lock held and interrupts disabled.
480579bd9814STejun Heo */
memcg_event_wake(wait_queue_entry_t * wait,unsigned mode,int sync,void * key)4806ac6424b9SIngo Molnar static int memcg_event_wake(wait_queue_entry_t *wait, unsigned mode,
480779bd9814STejun Heo int sync, void *key)
480879bd9814STejun Heo {
48093bc942f3STejun Heo struct mem_cgroup_event *event =
48103bc942f3STejun Heo container_of(wait, struct mem_cgroup_event, wait);
481159b6f873STejun Heo struct mem_cgroup *memcg = event->memcg;
48123ad6f93eSAl Viro __poll_t flags = key_to_poll(key);
481379bd9814STejun Heo
4814a9a08845SLinus Torvalds if (flags & EPOLLHUP) {
481579bd9814STejun Heo /*
481679bd9814STejun Heo * If the event has been detached at cgroup removal, we
481779bd9814STejun Heo * can simply return knowing the other side will cleanup
481879bd9814STejun Heo * for us.
481979bd9814STejun Heo *
482079bd9814STejun Heo * We can't race against event freeing since the other
482179bd9814STejun Heo * side will require wqh->lock via remove_wait_queue(),
482279bd9814STejun Heo * which we hold.
482379bd9814STejun Heo */
4824fba94807STejun Heo spin_lock(&memcg->event_list_lock);
482579bd9814STejun Heo if (!list_empty(&event->list)) {
482679bd9814STejun Heo list_del_init(&event->list);
482779bd9814STejun Heo /*
482879bd9814STejun Heo * We are in atomic context, but cgroup_event_remove()
482979bd9814STejun Heo * may sleep, so we have to call it in workqueue.
483079bd9814STejun Heo */
483179bd9814STejun Heo schedule_work(&event->remove);
483279bd9814STejun Heo }
4833fba94807STejun Heo spin_unlock(&memcg->event_list_lock);
483479bd9814STejun Heo }
483579bd9814STejun Heo
483679bd9814STejun Heo return 0;
483779bd9814STejun Heo }
483879bd9814STejun Heo
memcg_event_ptable_queue_proc(struct file * file,wait_queue_head_t * wqh,poll_table * pt)48393bc942f3STejun Heo static void memcg_event_ptable_queue_proc(struct file *file,
484079bd9814STejun Heo wait_queue_head_t *wqh, poll_table *pt)
484179bd9814STejun Heo {
48423bc942f3STejun Heo struct mem_cgroup_event *event =
48433bc942f3STejun Heo container_of(pt, struct mem_cgroup_event, pt);
484479bd9814STejun Heo
484579bd9814STejun Heo event->wqh = wqh;
484679bd9814STejun Heo add_wait_queue(wqh, &event->wait);
484779bd9814STejun Heo }
484879bd9814STejun Heo
484979bd9814STejun Heo /*
48503bc942f3STejun Heo * DO NOT USE IN NEW FILES.
48513bc942f3STejun Heo *
485279bd9814STejun Heo * Parse input and register new cgroup event handler.
485379bd9814STejun Heo *
485479bd9814STejun Heo * Input must be in format '<event_fd> <control_fd> <args>'.
485579bd9814STejun Heo * Interpretation of args is defined by control file implementation.
485679bd9814STejun Heo */
memcg_write_event_control(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)4857451af504STejun Heo static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
4858451af504STejun Heo char *buf, size_t nbytes, loff_t off)
485979bd9814STejun Heo {
4860451af504STejun Heo struct cgroup_subsys_state *css = of_css(of);
4861fba94807STejun Heo struct mem_cgroup *memcg = mem_cgroup_from_css(css);
48623bc942f3STejun Heo struct mem_cgroup_event *event;
486379bd9814STejun Heo struct cgroup_subsys_state *cfile_css;
486479bd9814STejun Heo unsigned int efd, cfd;
486579bd9814STejun Heo struct fd efile;
486679bd9814STejun Heo struct fd cfile;
48674a7ba45bSTejun Heo struct dentry *cdentry;
4868fba94807STejun Heo const char *name;
486979bd9814STejun Heo char *endp;
487079bd9814STejun Heo int ret;
487179bd9814STejun Heo
48722343e88dSSebastian Andrzej Siewior if (IS_ENABLED(CONFIG_PREEMPT_RT))
48732343e88dSSebastian Andrzej Siewior return -EOPNOTSUPP;
48742343e88dSSebastian Andrzej Siewior
4875451af504STejun Heo buf = strstrip(buf);
4876451af504STejun Heo
4877451af504STejun Heo efd = simple_strtoul(buf, &endp, 10);
487879bd9814STejun Heo if (*endp != ' ')
487979bd9814STejun Heo return -EINVAL;
4880451af504STejun Heo buf = endp + 1;
488179bd9814STejun Heo
4882451af504STejun Heo cfd = simple_strtoul(buf, &endp, 10);
488379bd9814STejun Heo if ((*endp != ' ') && (*endp != '\0'))
488479bd9814STejun Heo return -EINVAL;
4885451af504STejun Heo buf = endp + 1;
488679bd9814STejun Heo
488779bd9814STejun Heo event = kzalloc(sizeof(*event), GFP_KERNEL);
488879bd9814STejun Heo if (!event)
488979bd9814STejun Heo return -ENOMEM;
489079bd9814STejun Heo
489159b6f873STejun Heo event->memcg = memcg;
489279bd9814STejun Heo INIT_LIST_HEAD(&event->list);
48933bc942f3STejun Heo init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
48943bc942f3STejun Heo init_waitqueue_func_entry(&event->wait, memcg_event_wake);
48953bc942f3STejun Heo INIT_WORK(&event->remove, memcg_event_remove);
489679bd9814STejun Heo
489779bd9814STejun Heo efile = fdget(efd);
489879bd9814STejun Heo if (!efile.file) {
489979bd9814STejun Heo ret = -EBADF;
490079bd9814STejun Heo goto out_kfree;
490179bd9814STejun Heo }
490279bd9814STejun Heo
490379bd9814STejun Heo event->eventfd = eventfd_ctx_fileget(efile.file);
490479bd9814STejun Heo if (IS_ERR(event->eventfd)) {
490579bd9814STejun Heo ret = PTR_ERR(event->eventfd);
490679bd9814STejun Heo goto out_put_efile;
490779bd9814STejun Heo }
490879bd9814STejun Heo
490979bd9814STejun Heo cfile = fdget(cfd);
491079bd9814STejun Heo if (!cfile.file) {
491179bd9814STejun Heo ret = -EBADF;
491279bd9814STejun Heo goto out_put_eventfd;
491379bd9814STejun Heo }
491479bd9814STejun Heo
491579bd9814STejun Heo /* the process need read permission on control file */
491679bd9814STejun Heo /* AV: shouldn't we check that it's been opened for read instead? */
491702f92b38SChristian Brauner ret = file_permission(cfile.file, MAY_READ);
491879bd9814STejun Heo if (ret < 0)
491979bd9814STejun Heo goto out_put_cfile;
492079bd9814STejun Heo
492179bd9814STejun Heo /*
49224a7ba45bSTejun Heo * The control file must be a regular cgroup1 file. As a regular cgroup
49234a7ba45bSTejun Heo * file can't be renamed, it's safe to access its name afterwards.
49244a7ba45bSTejun Heo */
49254a7ba45bSTejun Heo cdentry = cfile.file->f_path.dentry;
49264a7ba45bSTejun Heo if (cdentry->d_sb->s_type != &cgroup_fs_type || !d_is_reg(cdentry)) {
49274a7ba45bSTejun Heo ret = -EINVAL;
49284a7ba45bSTejun Heo goto out_put_cfile;
49294a7ba45bSTejun Heo }
49304a7ba45bSTejun Heo
49314a7ba45bSTejun Heo /*
4932fba94807STejun Heo * Determine the event callbacks and set them in @event. This used
4933fba94807STejun Heo * to be done via struct cftype but cgroup core no longer knows
4934fba94807STejun Heo * about these events. The following is crude but the whole thing
4935fba94807STejun Heo * is for compatibility anyway.
49363bc942f3STejun Heo *
49373bc942f3STejun Heo * DO NOT ADD NEW FILES.
4938fba94807STejun Heo */
49394a7ba45bSTejun Heo name = cdentry->d_name.name;
4940fba94807STejun Heo
4941fba94807STejun Heo if (!strcmp(name, "memory.usage_in_bytes")) {
4942fba94807STejun Heo event->register_event = mem_cgroup_usage_register_event;
4943fba94807STejun Heo event->unregister_event = mem_cgroup_usage_unregister_event;
4944fba94807STejun Heo } else if (!strcmp(name, "memory.oom_control")) {
4945fba94807STejun Heo event->register_event = mem_cgroup_oom_register_event;
4946fba94807STejun Heo event->unregister_event = mem_cgroup_oom_unregister_event;
4947fba94807STejun Heo } else if (!strcmp(name, "memory.pressure_level")) {
4948fba94807STejun Heo event->register_event = vmpressure_register_event;
4949fba94807STejun Heo event->unregister_event = vmpressure_unregister_event;
4950fba94807STejun Heo } else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
4951347c4a87STejun Heo event->register_event = memsw_cgroup_usage_register_event;
4952347c4a87STejun Heo event->unregister_event = memsw_cgroup_usage_unregister_event;
4953fba94807STejun Heo } else {
4954fba94807STejun Heo ret = -EINVAL;
4955fba94807STejun Heo goto out_put_cfile;
4956fba94807STejun Heo }
4957fba94807STejun Heo
4958fba94807STejun Heo /*
4959b5557c4cSTejun Heo * Verify @cfile should belong to @css. Also, remaining events are
4960b5557c4cSTejun Heo * automatically removed on cgroup destruction but the removal is
4961b5557c4cSTejun Heo * asynchronous, so take an extra ref on @css.
496279bd9814STejun Heo */
49634a7ba45bSTejun Heo cfile_css = css_tryget_online_from_dir(cdentry->d_parent,
4964073219e9STejun Heo &memory_cgrp_subsys);
496579bd9814STejun Heo ret = -EINVAL;
49665a17f543STejun Heo if (IS_ERR(cfile_css))
496779bd9814STejun Heo goto out_put_cfile;
49685a17f543STejun Heo if (cfile_css != css) {
49695a17f543STejun Heo css_put(cfile_css);
49705a17f543STejun Heo goto out_put_cfile;
49715a17f543STejun Heo }
497279bd9814STejun Heo
4973451af504STejun Heo ret = event->register_event(memcg, event->eventfd, buf);
497479bd9814STejun Heo if (ret)
497579bd9814STejun Heo goto out_put_css;
497679bd9814STejun Heo
49779965ed17SChristoph Hellwig vfs_poll(efile.file, &event->pt);
497879bd9814STejun Heo
49794ba9515dSShakeel Butt spin_lock_irq(&memcg->event_list_lock);
4980fba94807STejun Heo list_add(&event->list, &memcg->event_list);
49814ba9515dSShakeel Butt spin_unlock_irq(&memcg->event_list_lock);
498279bd9814STejun Heo
498379bd9814STejun Heo fdput(cfile);
498479bd9814STejun Heo fdput(efile);
498579bd9814STejun Heo
4986451af504STejun Heo return nbytes;
498779bd9814STejun Heo
498879bd9814STejun Heo out_put_css:
4989b5557c4cSTejun Heo css_put(css);
499079bd9814STejun Heo out_put_cfile:
499179bd9814STejun Heo fdput(cfile);
499279bd9814STejun Heo out_put_eventfd:
499379bd9814STejun Heo eventfd_ctx_put(event->eventfd);
499479bd9814STejun Heo out_put_efile:
499579bd9814STejun Heo fdput(efile);
499679bd9814STejun Heo out_kfree:
499779bd9814STejun Heo kfree(event);
499879bd9814STejun Heo
499979bd9814STejun Heo return ret;
500079bd9814STejun Heo }
500179bd9814STejun Heo
5002c29b5b3dSMuchun Song #if defined(CONFIG_MEMCG_KMEM) && (defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG))
mem_cgroup_slab_show(struct seq_file * m,void * p)5003c29b5b3dSMuchun Song static int mem_cgroup_slab_show(struct seq_file *m, void *p)
5004c29b5b3dSMuchun Song {
5005c29b5b3dSMuchun Song /*
5006c29b5b3dSMuchun Song * Deprecated.
5007df4ae285SYang Yang * Please, take a look at tools/cgroup/memcg_slabinfo.py .
5008c29b5b3dSMuchun Song */
5009c29b5b3dSMuchun Song return 0;
5010c29b5b3dSMuchun Song }
5011c29b5b3dSMuchun Song #endif
5012c29b5b3dSMuchun Song
5013dddb44ffSYosry Ahmed static int memory_stat_show(struct seq_file *m, void *v);
5014dddb44ffSYosry Ahmed
5015241994edSJohannes Weiner static struct cftype mem_cgroup_legacy_files[] = {
50168cdea7c0SBalbir Singh {
50170eea1030SBalbir Singh .name = "usage_in_bytes",
50188c7c6e34SKAMEZAWA Hiroyuki .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
5019791badbdSTejun Heo .read_u64 = mem_cgroup_read_u64,
50208cdea7c0SBalbir Singh },
50218cdea7c0SBalbir Singh {
5022c84872e1SPavel Emelyanov .name = "max_usage_in_bytes",
50238c7c6e34SKAMEZAWA Hiroyuki .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
50246770c64eSTejun Heo .write = mem_cgroup_reset,
5025791badbdSTejun Heo .read_u64 = mem_cgroup_read_u64,
5026c84872e1SPavel Emelyanov },
5027c84872e1SPavel Emelyanov {
50280eea1030SBalbir Singh .name = "limit_in_bytes",
50298c7c6e34SKAMEZAWA Hiroyuki .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
5030451af504STejun Heo .write = mem_cgroup_write,
5031791badbdSTejun Heo .read_u64 = mem_cgroup_read_u64,
50328cdea7c0SBalbir Singh },
50338cdea7c0SBalbir Singh {
5034296c81d8SBalbir Singh .name = "soft_limit_in_bytes",
5035296c81d8SBalbir Singh .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
5036451af504STejun Heo .write = mem_cgroup_write,
5037791badbdSTejun Heo .read_u64 = mem_cgroup_read_u64,
5038296c81d8SBalbir Singh },
5039296c81d8SBalbir Singh {
50408cdea7c0SBalbir Singh .name = "failcnt",
50418c7c6e34SKAMEZAWA Hiroyuki .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
50426770c64eSTejun Heo .write = mem_cgroup_reset,
5043791badbdSTejun Heo .read_u64 = mem_cgroup_read_u64,
50448cdea7c0SBalbir Singh },
50458697d331SBalbir Singh {
5046d2ceb9b7SKAMEZAWA Hiroyuki .name = "stat",
5047dddb44ffSYosry Ahmed .seq_show = memory_stat_show,
5048d2ceb9b7SKAMEZAWA Hiroyuki },
5049c1e862c1SKAMEZAWA Hiroyuki {
5050c1e862c1SKAMEZAWA Hiroyuki .name = "force_empty",
50516770c64eSTejun Heo .write = mem_cgroup_force_empty_write,
5052c1e862c1SKAMEZAWA Hiroyuki },
505318f59ea7SBalbir Singh {
505418f59ea7SBalbir Singh .name = "use_hierarchy",
505518f59ea7SBalbir Singh .write_u64 = mem_cgroup_hierarchy_write,
505618f59ea7SBalbir Singh .read_u64 = mem_cgroup_hierarchy_read,
505718f59ea7SBalbir Singh },
5058a7885eb8SKOSAKI Motohiro {
50593bc942f3STejun Heo .name = "cgroup.event_control", /* XXX: for compat */
5060451af504STejun Heo .write = memcg_write_event_control,
50617dbdb199STejun Heo .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
506279bd9814STejun Heo },
506379bd9814STejun Heo {
5064a7885eb8SKOSAKI Motohiro .name = "swappiness",
5065a7885eb8SKOSAKI Motohiro .read_u64 = mem_cgroup_swappiness_read,
5066a7885eb8SKOSAKI Motohiro .write_u64 = mem_cgroup_swappiness_write,
5067a7885eb8SKOSAKI Motohiro },
50687dc74be0SDaisuke Nishimura {
50697dc74be0SDaisuke Nishimura .name = "move_charge_at_immigrate",
50707dc74be0SDaisuke Nishimura .read_u64 = mem_cgroup_move_charge_read,
50717dc74be0SDaisuke Nishimura .write_u64 = mem_cgroup_move_charge_write,
50727dc74be0SDaisuke Nishimura },
50739490ff27SKAMEZAWA Hiroyuki {
50749490ff27SKAMEZAWA Hiroyuki .name = "oom_control",
50752da8ca82STejun Heo .seq_show = mem_cgroup_oom_control_read,
50763c11ecf4SKAMEZAWA Hiroyuki .write_u64 = mem_cgroup_oom_control_write,
50779490ff27SKAMEZAWA Hiroyuki },
507870ddf637SAnton Vorontsov {
507970ddf637SAnton Vorontsov .name = "pressure_level",
50806b0ba2abSFlorian Schmidt .seq_show = mem_cgroup_dummy_seq_show,
508170ddf637SAnton Vorontsov },
5082406eb0c9SYing Han #ifdef CONFIG_NUMA
5083406eb0c9SYing Han {
5084406eb0c9SYing Han .name = "numa_stat",
50852da8ca82STejun Heo .seq_show = memcg_numa_stat_show,
5086406eb0c9SYing Han },
5087406eb0c9SYing Han #endif
5088510fc4e1SGlauber Costa {
50894597648fSMichal Hocko .name = "kmem.limit_in_bytes",
50904597648fSMichal Hocko .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
50914597648fSMichal Hocko .write = mem_cgroup_write,
50924597648fSMichal Hocko .read_u64 = mem_cgroup_read_u64,
50934597648fSMichal Hocko },
50944597648fSMichal Hocko {
5095510fc4e1SGlauber Costa .name = "kmem.usage_in_bytes",
5096510fc4e1SGlauber Costa .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
5097791badbdSTejun Heo .read_u64 = mem_cgroup_read_u64,
5098510fc4e1SGlauber Costa },
5099510fc4e1SGlauber Costa {
5100510fc4e1SGlauber Costa .name = "kmem.failcnt",
5101510fc4e1SGlauber Costa .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
51026770c64eSTejun Heo .write = mem_cgroup_reset,
5103791badbdSTejun Heo .read_u64 = mem_cgroup_read_u64,
5104510fc4e1SGlauber Costa },
5105510fc4e1SGlauber Costa {
5106510fc4e1SGlauber Costa .name = "kmem.max_usage_in_bytes",
5107510fc4e1SGlauber Costa .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
51086770c64eSTejun Heo .write = mem_cgroup_reset,
5109791badbdSTejun Heo .read_u64 = mem_cgroup_read_u64,
5110510fc4e1SGlauber Costa },
5111a87425a3SYafang Shao #if defined(CONFIG_MEMCG_KMEM) && \
5112a87425a3SYafang Shao (defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG))
5113749c5415SGlauber Costa {
5114749c5415SGlauber Costa .name = "kmem.slabinfo",
5115c29b5b3dSMuchun Song .seq_show = mem_cgroup_slab_show,
5116749c5415SGlauber Costa },
5117749c5415SGlauber Costa #endif
5118d55f90bfSVladimir Davydov {
5119d55f90bfSVladimir Davydov .name = "kmem.tcp.limit_in_bytes",
5120d55f90bfSVladimir Davydov .private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
5121d55f90bfSVladimir Davydov .write = mem_cgroup_write,
5122d55f90bfSVladimir Davydov .read_u64 = mem_cgroup_read_u64,
5123d55f90bfSVladimir Davydov },
5124d55f90bfSVladimir Davydov {
5125d55f90bfSVladimir Davydov .name = "kmem.tcp.usage_in_bytes",
5126d55f90bfSVladimir Davydov .private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
5127d55f90bfSVladimir Davydov .read_u64 = mem_cgroup_read_u64,
5128d55f90bfSVladimir Davydov },
5129d55f90bfSVladimir Davydov {
5130d55f90bfSVladimir Davydov .name = "kmem.tcp.failcnt",
5131d55f90bfSVladimir Davydov .private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
5132d55f90bfSVladimir Davydov .write = mem_cgroup_reset,
5133d55f90bfSVladimir Davydov .read_u64 = mem_cgroup_read_u64,
5134d55f90bfSVladimir Davydov },
5135d55f90bfSVladimir Davydov {
5136d55f90bfSVladimir Davydov .name = "kmem.tcp.max_usage_in_bytes",
5137d55f90bfSVladimir Davydov .private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
5138d55f90bfSVladimir Davydov .write = mem_cgroup_reset,
5139d55f90bfSVladimir Davydov .read_u64 = mem_cgroup_read_u64,
5140d55f90bfSVladimir Davydov },
51416bc10349STejun Heo { }, /* terminate */
5142af36f906STejun Heo };
51438c7c6e34SKAMEZAWA Hiroyuki
514473f576c0SJohannes Weiner /*
514573f576c0SJohannes Weiner * Private memory cgroup IDR
514673f576c0SJohannes Weiner *
514773f576c0SJohannes Weiner * Swap-out records and page cache shadow entries need to store memcg
514873f576c0SJohannes Weiner * references in constrained space, so we maintain an ID space that is
514973f576c0SJohannes Weiner * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of
515073f576c0SJohannes Weiner * memory-controlled cgroups to 64k.
515173f576c0SJohannes Weiner *
5152b8f2935fSEthon Paul * However, there usually are many references to the offline CSS after
515373f576c0SJohannes Weiner * the cgroup has been destroyed, such as page cache or reclaimable
515473f576c0SJohannes Weiner * slab objects, that don't need to hang on to the ID. We want to keep
515573f576c0SJohannes Weiner * those dead CSS from occupying IDs, or we might quickly exhaust the
515673f576c0SJohannes Weiner * relatively small ID space and prevent the creation of new cgroups
515773f576c0SJohannes Weiner * even when there are much fewer than 64k cgroups - possibly none.
515873f576c0SJohannes Weiner *
515973f576c0SJohannes Weiner * Maintain a private 16-bit ID space for memcg, and allow the ID to
516073f576c0SJohannes Weiner * be freed and recycled when it's no longer needed, which is usually
516173f576c0SJohannes Weiner * when the CSS is offlined.
516273f576c0SJohannes Weiner *
516373f576c0SJohannes Weiner * The only exception to that are records of swapped out tmpfs/shmem
516473f576c0SJohannes Weiner * pages that need to be attributed to live ancestors on swapin. But
516573f576c0SJohannes Weiner * those references are manageable from userspace.
516673f576c0SJohannes Weiner */
516773f576c0SJohannes Weiner
516860b1e24cSMiaohe Lin #define MEM_CGROUP_ID_MAX ((1UL << MEM_CGROUP_ID_SHIFT) - 1)
516973f576c0SJohannes Weiner static DEFINE_IDR(mem_cgroup_idr);
517073f576c0SJohannes Weiner
mem_cgroup_id_remove(struct mem_cgroup * memcg)51717e97de0bSKirill Tkhai static void mem_cgroup_id_remove(struct mem_cgroup *memcg)
51727e97de0bSKirill Tkhai {
51737e97de0bSKirill Tkhai if (memcg->id.id > 0) {
51747e97de0bSKirill Tkhai idr_remove(&mem_cgroup_idr, memcg->id.id);
51757e97de0bSKirill Tkhai memcg->id.id = 0;
51767e97de0bSKirill Tkhai }
51777e97de0bSKirill Tkhai }
51787e97de0bSKirill Tkhai
mem_cgroup_id_get_many(struct mem_cgroup * memcg,unsigned int n)5179c1514c0aSVincenzo Frascino static void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg,
5180c1514c0aSVincenzo Frascino unsigned int n)
518173f576c0SJohannes Weiner {
51821c2d479aSKirill Tkhai refcount_add(n, &memcg->id.ref);
518373f576c0SJohannes Weiner }
518473f576c0SJohannes Weiner
mem_cgroup_id_put_many(struct mem_cgroup * memcg,unsigned int n)5185615d66c3SVladimir Davydov static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
518673f576c0SJohannes Weiner {
51871c2d479aSKirill Tkhai if (refcount_sub_and_test(n, &memcg->id.ref)) {
51887e97de0bSKirill Tkhai mem_cgroup_id_remove(memcg);
518973f576c0SJohannes Weiner
519073f576c0SJohannes Weiner /* Memcg ID pins CSS */
519173f576c0SJohannes Weiner css_put(&memcg->css);
519273f576c0SJohannes Weiner }
519373f576c0SJohannes Weiner }
519473f576c0SJohannes Weiner
mem_cgroup_id_put(struct mem_cgroup * memcg)5195615d66c3SVladimir Davydov static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
5196615d66c3SVladimir Davydov {
5197615d66c3SVladimir Davydov mem_cgroup_id_put_many(memcg, 1);
5198615d66c3SVladimir Davydov }
5199615d66c3SVladimir Davydov
520073f576c0SJohannes Weiner /**
520173f576c0SJohannes Weiner * mem_cgroup_from_id - look up a memcg from a memcg id
520273f576c0SJohannes Weiner * @id: the memcg id to look up
520373f576c0SJohannes Weiner *
520473f576c0SJohannes Weiner * Caller must hold rcu_read_lock().
520573f576c0SJohannes Weiner */
mem_cgroup_from_id(unsigned short id)520673f576c0SJohannes Weiner struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
520773f576c0SJohannes Weiner {
520873f576c0SJohannes Weiner WARN_ON_ONCE(!rcu_read_lock_held());
520973f576c0SJohannes Weiner return idr_find(&mem_cgroup_idr, id);
521073f576c0SJohannes Weiner }
521173f576c0SJohannes Weiner
5212c15187a4SRoman Gushchin #ifdef CONFIG_SHRINKER_DEBUG
mem_cgroup_get_from_ino(unsigned long ino)5213c15187a4SRoman Gushchin struct mem_cgroup *mem_cgroup_get_from_ino(unsigned long ino)
5214c15187a4SRoman Gushchin {
5215c15187a4SRoman Gushchin struct cgroup *cgrp;
5216c15187a4SRoman Gushchin struct cgroup_subsys_state *css;
5217c15187a4SRoman Gushchin struct mem_cgroup *memcg;
5218c15187a4SRoman Gushchin
5219c15187a4SRoman Gushchin cgrp = cgroup_get_from_id(ino);
5220fa7e439cSMichal Koutný if (IS_ERR(cgrp))
5221c0f2df49STejun Heo return ERR_CAST(cgrp);
5222c15187a4SRoman Gushchin
5223c15187a4SRoman Gushchin css = cgroup_get_e_css(cgrp, &memory_cgrp_subsys);
5224c15187a4SRoman Gushchin if (css)
5225c15187a4SRoman Gushchin memcg = container_of(css, struct mem_cgroup, css);
5226c15187a4SRoman Gushchin else
5227c15187a4SRoman Gushchin memcg = ERR_PTR(-ENOENT);
5228c15187a4SRoman Gushchin
5229c15187a4SRoman Gushchin cgroup_put(cgrp);
5230c15187a4SRoman Gushchin
5231c15187a4SRoman Gushchin return memcg;
5232c15187a4SRoman Gushchin }
5233c15187a4SRoman Gushchin #endif
5234c15187a4SRoman Gushchin
alloc_mem_cgroup_per_node_info(struct mem_cgroup * memcg,int node)5235ef8f2327SMel Gorman static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
52366d12e2d8SKAMEZAWA Hiroyuki {
52376d12e2d8SKAMEZAWA Hiroyuki struct mem_cgroup_per_node *pn;
52388c9bb398SWei Yang
52398c9bb398SWei Yang pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, node);
52406d12e2d8SKAMEZAWA Hiroyuki if (!pn)
52416d12e2d8SKAMEZAWA Hiroyuki return 1;
52421ecaab2bSKAMEZAWA Hiroyuki
52437e1c0d6fSShakeel Butt pn->lruvec_stats_percpu = alloc_percpu_gfp(struct lruvec_stats_percpu,
52443e38e0aaSRoman Gushchin GFP_KERNEL_ACCOUNT);
52457e1c0d6fSShakeel Butt if (!pn->lruvec_stats_percpu) {
524600f3ca2cSJohannes Weiner kfree(pn);
524700f3ca2cSJohannes Weiner return 1;
524800f3ca2cSJohannes Weiner }
524900f3ca2cSJohannes Weiner
5250ef8f2327SMel Gorman lruvec_init(&pn->lruvec);
5251ef8f2327SMel Gorman pn->memcg = memcg;
5252ef8f2327SMel Gorman
525354f72fe0SJohannes Weiner memcg->nodeinfo[node] = pn;
52546d12e2d8SKAMEZAWA Hiroyuki return 0;
52556d12e2d8SKAMEZAWA Hiroyuki }
52566d12e2d8SKAMEZAWA Hiroyuki
free_mem_cgroup_per_node_info(struct mem_cgroup * memcg,int node)5257ef8f2327SMel Gorman static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
52581ecaab2bSKAMEZAWA Hiroyuki {
525900f3ca2cSJohannes Weiner struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
526000f3ca2cSJohannes Weiner
52614eaf431fSMichal Hocko if (!pn)
52624eaf431fSMichal Hocko return;
52634eaf431fSMichal Hocko
52647e1c0d6fSShakeel Butt free_percpu(pn->lruvec_stats_percpu);
526500f3ca2cSJohannes Weiner kfree(pn);
52661ecaab2bSKAMEZAWA Hiroyuki }
52671ecaab2bSKAMEZAWA Hiroyuki
__mem_cgroup_free(struct mem_cgroup * memcg)526840e952f9STahsin Erdogan static void __mem_cgroup_free(struct mem_cgroup *memcg)
52690b8f73e1SJohannes Weiner {
52700b8f73e1SJohannes Weiner int node;
52710b8f73e1SJohannes Weiner
52720b8f73e1SJohannes Weiner for_each_node(node)
5273ef8f2327SMel Gorman free_mem_cgroup_per_node_info(memcg, node);
5274410f8e82SShakeel Butt kfree(memcg->vmstats);
5275871789d4SChris Down free_percpu(memcg->vmstats_percpu);
52760b8f73e1SJohannes Weiner kfree(memcg);
52770b8f73e1SJohannes Weiner }
52780b8f73e1SJohannes Weiner
mem_cgroup_free(struct mem_cgroup * memcg)527940e952f9STahsin Erdogan static void mem_cgroup_free(struct mem_cgroup *memcg)
528040e952f9STahsin Erdogan {
5281ec1c86b2SYu Zhao lru_gen_exit_memcg(memcg);
528240e952f9STahsin Erdogan memcg_wb_domain_exit(memcg);
528340e952f9STahsin Erdogan __mem_cgroup_free(memcg);
528440e952f9STahsin Erdogan }
528540e952f9STahsin Erdogan
mem_cgroup_alloc(void)528633327948SKAMEZAWA Hiroyuki static struct mem_cgroup *mem_cgroup_alloc(void)
528733327948SKAMEZAWA Hiroyuki {
5288d79154bbSHugh Dickins struct mem_cgroup *memcg;
52890b8f73e1SJohannes Weiner int node;
529097b27821STejun Heo int __maybe_unused i;
529111d67612SYafang Shao long error = -ENOMEM;
529233327948SKAMEZAWA Hiroyuki
529306b2c3b0SWang Weiyang memcg = kzalloc(struct_size(memcg, nodeinfo, nr_node_ids), GFP_KERNEL);
5294d79154bbSHugh Dickins if (!memcg)
529511d67612SYafang Shao return ERR_PTR(error);
5296e7bbcdf3SDan Carpenter
529773f576c0SJohannes Weiner memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
5298be740503SMuchun Song 1, MEM_CGROUP_ID_MAX + 1, GFP_KERNEL);
529911d67612SYafang Shao if (memcg->id.id < 0) {
530011d67612SYafang Shao error = memcg->id.id;
530173f576c0SJohannes Weiner goto fail;
530211d67612SYafang Shao }
530373f576c0SJohannes Weiner
5304410f8e82SShakeel Butt memcg->vmstats = kzalloc(sizeof(struct memcg_vmstats), GFP_KERNEL);
5305410f8e82SShakeel Butt if (!memcg->vmstats)
5306410f8e82SShakeel Butt goto fail;
5307410f8e82SShakeel Butt
53083e38e0aaSRoman Gushchin memcg->vmstats_percpu = alloc_percpu_gfp(struct memcg_vmstats_percpu,
53093e38e0aaSRoman Gushchin GFP_KERNEL_ACCOUNT);
5310871789d4SChris Down if (!memcg->vmstats_percpu)
53110b8f73e1SJohannes Weiner goto fail;
53128cdea7c0SBalbir Singh
53133ed28fa1SBob Liu for_each_node(node)
5314ef8f2327SMel Gorman if (alloc_mem_cgroup_per_node_info(memcg, node))
53150b8f73e1SJohannes Weiner goto fail;
5316f64c3f54SBalbir Singh
53170b8f73e1SJohannes Weiner if (memcg_wb_domain_init(memcg, GFP_KERNEL))
53180b8f73e1SJohannes Weiner goto fail;
531928dbc4b6SBalbir Singh
5320f7e1cb6eSJohannes Weiner INIT_WORK(&memcg->high_work, high_work_func);
5321d142e3e6SGlauber Costa INIT_LIST_HEAD(&memcg->oom_notify);
5322d142e3e6SGlauber Costa mutex_init(&memcg->thresholds_lock);
5323d142e3e6SGlauber Costa spin_lock_init(&memcg->move_lock);
532470ddf637SAnton Vorontsov vmpressure_init(&memcg->vmpressure);
5325fba94807STejun Heo INIT_LIST_HEAD(&memcg->event_list);
5326fba94807STejun Heo spin_lock_init(&memcg->event_list_lock);
5327d886f4e4SJohannes Weiner memcg->socket_pressure = jiffies;
532884c07d11SKirill Tkhai #ifdef CONFIG_MEMCG_KMEM
5329900a38f0SVladimir Davydov memcg->kmemcg_id = -1;
5330bf4f0599SRoman Gushchin INIT_LIST_HEAD(&memcg->objcg_list);
5331900a38f0SVladimir Davydov #endif
533252ebea74STejun Heo #ifdef CONFIG_CGROUP_WRITEBACK
533352ebea74STejun Heo INIT_LIST_HEAD(&memcg->cgwb_list);
533497b27821STejun Heo for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
533597b27821STejun Heo memcg->cgwb_frn[i].done =
533697b27821STejun Heo __WB_COMPLETION_INIT(&memcg_cgwb_frn_waitq);
533752ebea74STejun Heo #endif
533887eaceb3SYang Shi #ifdef CONFIG_TRANSPARENT_HUGEPAGE
533987eaceb3SYang Shi spin_lock_init(&memcg->deferred_split_queue.split_queue_lock);
534087eaceb3SYang Shi INIT_LIST_HEAD(&memcg->deferred_split_queue.split_queue);
534187eaceb3SYang Shi memcg->deferred_split_queue.split_queue_len = 0;
534287eaceb3SYang Shi #endif
5343ec1c86b2SYu Zhao lru_gen_init_memcg(memcg);
53440b8f73e1SJohannes Weiner return memcg;
53450b8f73e1SJohannes Weiner fail:
53467e97de0bSKirill Tkhai mem_cgroup_id_remove(memcg);
534740e952f9STahsin Erdogan __mem_cgroup_free(memcg);
534811d67612SYafang Shao return ERR_PTR(error);
5349d142e3e6SGlauber Costa }
5350d142e3e6SGlauber Costa
53510b8f73e1SJohannes Weiner static struct cgroup_subsys_state * __ref
mem_cgroup_css_alloc(struct cgroup_subsys_state * parent_css)53520b8f73e1SJohannes Weiner mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
5353d142e3e6SGlauber Costa {
53540b8f73e1SJohannes Weiner struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
5355b87d8cefSRoman Gushchin struct mem_cgroup *memcg, *old_memcg;
5356d142e3e6SGlauber Costa
5357b87d8cefSRoman Gushchin old_memcg = set_active_memcg(parent);
53580b8f73e1SJohannes Weiner memcg = mem_cgroup_alloc();
5359b87d8cefSRoman Gushchin set_active_memcg(old_memcg);
536011d67612SYafang Shao if (IS_ERR(memcg))
536111d67612SYafang Shao return ERR_CAST(memcg);
53624219b2daSLi Zefan
5363d1663a90SJakub Kicinski page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
53642178e20cSYue Zhao WRITE_ONCE(memcg->soft_limit, PAGE_COUNTER_MAX);
5365f4840ccfSJohannes Weiner #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP)
5366f4840ccfSJohannes Weiner memcg->zswap_max = PAGE_COUNTER_MAX;
5367f4840ccfSJohannes Weiner #endif
53684b82ab4fSJakub Kicinski page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
53690b8f73e1SJohannes Weiner if (parent) {
537082b3aa26SYue Zhao WRITE_ONCE(memcg->swappiness, mem_cgroup_swappiness(parent));
537117c56de6SYue Zhao WRITE_ONCE(memcg->oom_kill_disable, READ_ONCE(parent->oom_kill_disable));
5372bef8620cSRoman Gushchin
53730b8f73e1SJohannes Weiner page_counter_init(&memcg->memory, &parent->memory);
537437e84351SVladimir Davydov page_counter_init(&memcg->swap, &parent->swap);
53753e32cb2eSJohannes Weiner page_counter_init(&memcg->kmem, &parent->kmem);
53760db15298SJohannes Weiner page_counter_init(&memcg->tcpmem, &parent->tcpmem);
537718f59ea7SBalbir Singh } else {
53788278f1c7SShakeel Butt init_memcg_events();
5379bef8620cSRoman Gushchin page_counter_init(&memcg->memory, NULL);
5380bef8620cSRoman Gushchin page_counter_init(&memcg->swap, NULL);
5381bef8620cSRoman Gushchin page_counter_init(&memcg->kmem, NULL);
5382bef8620cSRoman Gushchin page_counter_init(&memcg->tcpmem, NULL);
5383d6441637SVladimir Davydov
53840b8f73e1SJohannes Weiner root_mem_cgroup = memcg;
53850b8f73e1SJohannes Weiner return &memcg->css;
53860b8f73e1SJohannes Weiner }
53870b8f73e1SJohannes Weiner
5388f7e1cb6eSJohannes Weiner if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
5389ef12947cSJohannes Weiner static_branch_inc(&memcg_sockets_enabled_key);
5390f7e1cb6eSJohannes Weiner
5391b6c1a8afSYafang Shao #if defined(CONFIG_MEMCG_KMEM)
5392b6c1a8afSYafang Shao if (!cgroup_memory_nobpf)
5393b6c1a8afSYafang Shao static_branch_inc(&memcg_bpf_enabled_key);
5394b6c1a8afSYafang Shao #endif
5395b6c1a8afSYafang Shao
53960b8f73e1SJohannes Weiner return &memcg->css;
53970b8f73e1SJohannes Weiner }
53980b8f73e1SJohannes Weiner
mem_cgroup_css_online(struct cgroup_subsys_state * css)539973f576c0SJohannes Weiner static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
54000b8f73e1SJohannes Weiner {
540158fa2a55SVladimir Davydov struct mem_cgroup *memcg = mem_cgroup_from_css(css);
540258fa2a55SVladimir Davydov
5403da0efe30SMuchun Song if (memcg_online_kmem(memcg))
5404da0efe30SMuchun Song goto remove_id;
5405da0efe30SMuchun Song
54060a4465d3SKirill Tkhai /*
5407e4262c4fSYang Shi * A memcg must be visible for expand_shrinker_info()
54080a4465d3SKirill Tkhai * by the time the maps are allocated. So, we allocate maps
54090a4465d3SKirill Tkhai * here, when for_each_mem_cgroup() can't skip it.
54100a4465d3SKirill Tkhai */
5411da0efe30SMuchun Song if (alloc_shrinker_info(memcg))
5412da0efe30SMuchun Song goto offline_kmem;
54130a4465d3SKirill Tkhai
5414aa48e47eSShakeel Butt if (unlikely(mem_cgroup_is_root(memcg)))
5415aa48e47eSShakeel Butt queue_delayed_work(system_unbound_wq, &stats_flush_dwork,
5416396faf88SMiaohe Lin FLUSH_TIME);
5417e4dde56cSYu Zhao lru_gen_online_memcg(memcg);
54186f0df8e1SJohannes Weiner
54196f0df8e1SJohannes Weiner /* Online state pins memcg ID, memcg ID pins CSS */
54206f0df8e1SJohannes Weiner refcount_set(&memcg->id.ref, 1);
54216f0df8e1SJohannes Weiner css_get(css);
54226f0df8e1SJohannes Weiner
54236f0df8e1SJohannes Weiner /*
54246f0df8e1SJohannes Weiner * Ensure mem_cgroup_from_id() works once we're fully online.
54256f0df8e1SJohannes Weiner *
54266f0df8e1SJohannes Weiner * We could do this earlier and require callers to filter with
54276f0df8e1SJohannes Weiner * css_tryget_online(). But right now there are no users that
54286f0df8e1SJohannes Weiner * need earlier access, and the workingset code relies on the
54296f0df8e1SJohannes Weiner * cgroup tree linkage (mem_cgroup_get_nr_swap_pages()). So
54306f0df8e1SJohannes Weiner * publish it here at the end of onlining. This matches the
54316f0df8e1SJohannes Weiner * regular ID destruction during offlining.
54326f0df8e1SJohannes Weiner */
54336f0df8e1SJohannes Weiner idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
54346f0df8e1SJohannes Weiner
54352f7dd7a4SJohannes Weiner return 0;
5436da0efe30SMuchun Song offline_kmem:
5437da0efe30SMuchun Song memcg_offline_kmem(memcg);
5438da0efe30SMuchun Song remove_id:
5439da0efe30SMuchun Song mem_cgroup_id_remove(memcg);
5440da0efe30SMuchun Song return -ENOMEM;
54418cdea7c0SBalbir Singh }
54428cdea7c0SBalbir Singh
mem_cgroup_css_offline(struct cgroup_subsys_state * css)5443eb95419bSTejun Heo static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
5444df878fb0SKAMEZAWA Hiroyuki {
5445eb95419bSTejun Heo struct mem_cgroup *memcg = mem_cgroup_from_css(css);
54463bc942f3STejun Heo struct mem_cgroup_event *event, *tmp;
544779bd9814STejun Heo
544879bd9814STejun Heo /*
544979bd9814STejun Heo * Unregister events and notify userspace.
545079bd9814STejun Heo * Notify userspace about cgroup removing only after rmdir of cgroup
545179bd9814STejun Heo * directory to avoid race between userspace and kernelspace.
545279bd9814STejun Heo */
54534ba9515dSShakeel Butt spin_lock_irq(&memcg->event_list_lock);
5454fba94807STejun Heo list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
545579bd9814STejun Heo list_del_init(&event->list);
545679bd9814STejun Heo schedule_work(&event->remove);
545779bd9814STejun Heo }
54584ba9515dSShakeel Butt spin_unlock_irq(&memcg->event_list_lock);
5459ec64f515SKAMEZAWA Hiroyuki
5460bf8d5d52SRoman Gushchin page_counter_set_min(&memcg->memory, 0);
546123067153SRoman Gushchin page_counter_set_low(&memcg->memory, 0);
546263677c74SRoman Gushchin
5463567e9ab2SJohannes Weiner memcg_offline_kmem(memcg);
5464a178015cSYang Shi reparent_shrinker_deferred(memcg);
546552ebea74STejun Heo wb_memcg_offline(memcg);
5466e4dde56cSYu Zhao lru_gen_offline_memcg(memcg);
546773f576c0SJohannes Weiner
5468591edfb1SRoman Gushchin drain_all_stock(memcg);
5469591edfb1SRoman Gushchin
547073f576c0SJohannes Weiner mem_cgroup_id_put(memcg);
5471df878fb0SKAMEZAWA Hiroyuki }
5472df878fb0SKAMEZAWA Hiroyuki
mem_cgroup_css_released(struct cgroup_subsys_state * css)54736df38689SVladimir Davydov static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
54746df38689SVladimir Davydov {
54756df38689SVladimir Davydov struct mem_cgroup *memcg = mem_cgroup_from_css(css);
54766df38689SVladimir Davydov
54776df38689SVladimir Davydov invalidate_reclaim_iterators(memcg);
5478e4dde56cSYu Zhao lru_gen_release_memcg(memcg);
54796df38689SVladimir Davydov }
54806df38689SVladimir Davydov
mem_cgroup_css_free(struct cgroup_subsys_state * css)5481eb95419bSTejun Heo static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
54828cdea7c0SBalbir Singh {
5483eb95419bSTejun Heo struct mem_cgroup *memcg = mem_cgroup_from_css(css);
548497b27821STejun Heo int __maybe_unused i;
5485c268e994SDaisuke Nishimura
548697b27821STejun Heo #ifdef CONFIG_CGROUP_WRITEBACK
548797b27821STejun Heo for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
548897b27821STejun Heo wb_wait_for_completion(&memcg->cgwb_frn[i].done);
548997b27821STejun Heo #endif
5490f7e1cb6eSJohannes Weiner if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
5491ef12947cSJohannes Weiner static_branch_dec(&memcg_sockets_enabled_key);
54923893e302SJohannes Weiner
54930db15298SJohannes Weiner if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
5494d55f90bfSVladimir Davydov static_branch_dec(&memcg_sockets_enabled_key);
54953893e302SJohannes Weiner
5496b6c1a8afSYafang Shao #if defined(CONFIG_MEMCG_KMEM)
5497b6c1a8afSYafang Shao if (!cgroup_memory_nobpf)
5498b6c1a8afSYafang Shao static_branch_dec(&memcg_bpf_enabled_key);
5499b6c1a8afSYafang Shao #endif
5500b6c1a8afSYafang Shao
55010b8f73e1SJohannes Weiner vmpressure_cleanup(&memcg->vmpressure);
55020b8f73e1SJohannes Weiner cancel_work_sync(&memcg->high_work);
55030b8f73e1SJohannes Weiner mem_cgroup_remove_from_trees(memcg);
5504e4262c4fSYang Shi free_shrinker_info(memcg);
55050b8f73e1SJohannes Weiner mem_cgroup_free(memcg);
55068cdea7c0SBalbir Singh }
55078cdea7c0SBalbir Singh
55081ced953bSTejun Heo /**
55091ced953bSTejun Heo * mem_cgroup_css_reset - reset the states of a mem_cgroup
55101ced953bSTejun Heo * @css: the target css
55111ced953bSTejun Heo *
55121ced953bSTejun Heo * Reset the states of the mem_cgroup associated with @css. This is
55131ced953bSTejun Heo * invoked when the userland requests disabling on the default hierarchy
55141ced953bSTejun Heo * but the memcg is pinned through dependency. The memcg should stop
55151ced953bSTejun Heo * applying policies and should revert to the vanilla state as it may be
55161ced953bSTejun Heo * made visible again.
55171ced953bSTejun Heo *
55181ced953bSTejun Heo * The current implementation only resets the essential configurations.
55191ced953bSTejun Heo * This needs to be expanded to cover all the visible parts.
55201ced953bSTejun Heo */
mem_cgroup_css_reset(struct cgroup_subsys_state * css)55211ced953bSTejun Heo static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
55221ced953bSTejun Heo {
55231ced953bSTejun Heo struct mem_cgroup *memcg = mem_cgroup_from_css(css);
55241ced953bSTejun Heo
5525bbec2e15SRoman Gushchin page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX);
5526bbec2e15SRoman Gushchin page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX);
5527bbec2e15SRoman Gushchin page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX);
5528bbec2e15SRoman Gushchin page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX);
5529bf8d5d52SRoman Gushchin page_counter_set_min(&memcg->memory, 0);
553023067153SRoman Gushchin page_counter_set_low(&memcg->memory, 0);
5531d1663a90SJakub Kicinski page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
55322178e20cSYue Zhao WRITE_ONCE(memcg->soft_limit, PAGE_COUNTER_MAX);
55334b82ab4fSJakub Kicinski page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
55342529bb3aSTejun Heo memcg_wb_domain_size_changed(memcg);
55351ced953bSTejun Heo }
55361ced953bSTejun Heo
mem_cgroup_css_rstat_flush(struct cgroup_subsys_state * css,int cpu)55372d146aa3SJohannes Weiner static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu)
55382d146aa3SJohannes Weiner {
55392d146aa3SJohannes Weiner struct mem_cgroup *memcg = mem_cgroup_from_css(css);
55402d146aa3SJohannes Weiner struct mem_cgroup *parent = parent_mem_cgroup(memcg);
55412d146aa3SJohannes Weiner struct memcg_vmstats_percpu *statc;
5542f82e6bf9SYosry Ahmed long delta, delta_cpu, v;
55437e1c0d6fSShakeel Butt int i, nid;
55442d146aa3SJohannes Weiner
55452d146aa3SJohannes Weiner statc = per_cpu_ptr(memcg->vmstats_percpu, cpu);
55462d146aa3SJohannes Weiner
55472d146aa3SJohannes Weiner for (i = 0; i < MEMCG_NR_STAT; i++) {
55482d146aa3SJohannes Weiner /*
55492d146aa3SJohannes Weiner * Collect the aggregated propagation counts of groups
55502d146aa3SJohannes Weiner * below us. We're in a per-cpu loop here and this is
55512d146aa3SJohannes Weiner * a global counter, so the first cycle will get them.
55522d146aa3SJohannes Weiner */
5553410f8e82SShakeel Butt delta = memcg->vmstats->state_pending[i];
55542d146aa3SJohannes Weiner if (delta)
5555410f8e82SShakeel Butt memcg->vmstats->state_pending[i] = 0;
55562d146aa3SJohannes Weiner
55572d146aa3SJohannes Weiner /* Add CPU changes on this level since the last flush */
5558f82e6bf9SYosry Ahmed delta_cpu = 0;
55592d146aa3SJohannes Weiner v = READ_ONCE(statc->state[i]);
55602d146aa3SJohannes Weiner if (v != statc->state_prev[i]) {
5561f82e6bf9SYosry Ahmed delta_cpu = v - statc->state_prev[i];
5562f82e6bf9SYosry Ahmed delta += delta_cpu;
55632d146aa3SJohannes Weiner statc->state_prev[i] = v;
55642d146aa3SJohannes Weiner }
55652d146aa3SJohannes Weiner
55662d146aa3SJohannes Weiner /* Aggregate counts on this level and propagate upwards */
5567f82e6bf9SYosry Ahmed if (delta_cpu)
5568f82e6bf9SYosry Ahmed memcg->vmstats->state_local[i] += delta_cpu;
5569f82e6bf9SYosry Ahmed
5570f82e6bf9SYosry Ahmed if (delta) {
5571410f8e82SShakeel Butt memcg->vmstats->state[i] += delta;
55722d146aa3SJohannes Weiner if (parent)
5573410f8e82SShakeel Butt parent->vmstats->state_pending[i] += delta;
55742d146aa3SJohannes Weiner }
5575f82e6bf9SYosry Ahmed }
55762d146aa3SJohannes Weiner
55778278f1c7SShakeel Butt for (i = 0; i < NR_MEMCG_EVENTS; i++) {
5578410f8e82SShakeel Butt delta = memcg->vmstats->events_pending[i];
55792d146aa3SJohannes Weiner if (delta)
5580410f8e82SShakeel Butt memcg->vmstats->events_pending[i] = 0;
55812d146aa3SJohannes Weiner
5582f82e6bf9SYosry Ahmed delta_cpu = 0;
55832d146aa3SJohannes Weiner v = READ_ONCE(statc->events[i]);
55842d146aa3SJohannes Weiner if (v != statc->events_prev[i]) {
5585f82e6bf9SYosry Ahmed delta_cpu = v - statc->events_prev[i];
5586f82e6bf9SYosry Ahmed delta += delta_cpu;
55872d146aa3SJohannes Weiner statc->events_prev[i] = v;
55882d146aa3SJohannes Weiner }
55892d146aa3SJohannes Weiner
5590f82e6bf9SYosry Ahmed if (delta_cpu)
5591f82e6bf9SYosry Ahmed memcg->vmstats->events_local[i] += delta_cpu;
55922d146aa3SJohannes Weiner
5593f82e6bf9SYosry Ahmed if (delta) {
5594410f8e82SShakeel Butt memcg->vmstats->events[i] += delta;
55952d146aa3SJohannes Weiner if (parent)
5596410f8e82SShakeel Butt parent->vmstats->events_pending[i] += delta;
55972d146aa3SJohannes Weiner }
5598f82e6bf9SYosry Ahmed }
55997e1c0d6fSShakeel Butt
56007e1c0d6fSShakeel Butt for_each_node_state(nid, N_MEMORY) {
56017e1c0d6fSShakeel Butt struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid];
56027e1c0d6fSShakeel Butt struct mem_cgroup_per_node *ppn = NULL;
56037e1c0d6fSShakeel Butt struct lruvec_stats_percpu *lstatc;
56047e1c0d6fSShakeel Butt
56057e1c0d6fSShakeel Butt if (parent)
56067e1c0d6fSShakeel Butt ppn = parent->nodeinfo[nid];
56077e1c0d6fSShakeel Butt
56087e1c0d6fSShakeel Butt lstatc = per_cpu_ptr(pn->lruvec_stats_percpu, cpu);
56097e1c0d6fSShakeel Butt
56107e1c0d6fSShakeel Butt for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
56117e1c0d6fSShakeel Butt delta = pn->lruvec_stats.state_pending[i];
56127e1c0d6fSShakeel Butt if (delta)
56137e1c0d6fSShakeel Butt pn->lruvec_stats.state_pending[i] = 0;
56147e1c0d6fSShakeel Butt
5615f82e6bf9SYosry Ahmed delta_cpu = 0;
56167e1c0d6fSShakeel Butt v = READ_ONCE(lstatc->state[i]);
56177e1c0d6fSShakeel Butt if (v != lstatc->state_prev[i]) {
5618f82e6bf9SYosry Ahmed delta_cpu = v - lstatc->state_prev[i];
5619f82e6bf9SYosry Ahmed delta += delta_cpu;
56207e1c0d6fSShakeel Butt lstatc->state_prev[i] = v;
56217e1c0d6fSShakeel Butt }
56227e1c0d6fSShakeel Butt
5623f82e6bf9SYosry Ahmed if (delta_cpu)
5624f82e6bf9SYosry Ahmed pn->lruvec_stats.state_local[i] += delta_cpu;
56257e1c0d6fSShakeel Butt
5626f82e6bf9SYosry Ahmed if (delta) {
56277e1c0d6fSShakeel Butt pn->lruvec_stats.state[i] += delta;
56287e1c0d6fSShakeel Butt if (ppn)
56297e1c0d6fSShakeel Butt ppn->lruvec_stats.state_pending[i] += delta;
56307e1c0d6fSShakeel Butt }
56317e1c0d6fSShakeel Butt }
56322d146aa3SJohannes Weiner }
5633f82e6bf9SYosry Ahmed }
56342d146aa3SJohannes Weiner
563502491447SDaisuke Nishimura #ifdef CONFIG_MMU
56367dc74be0SDaisuke Nishimura /* Handlers for move charge at task migration. */
mem_cgroup_do_precharge(unsigned long count)5637854ffa8dSDaisuke Nishimura static int mem_cgroup_do_precharge(unsigned long count)
56387dc74be0SDaisuke Nishimura {
563905b84301SJohannes Weiner int ret;
56409476db97SJohannes Weiner
5641d0164adcSMel Gorman /* Try a single bulk charge without reclaim first, kswapd may wake */
5642d0164adcSMel Gorman ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
56439476db97SJohannes Weiner if (!ret) {
5644854ffa8dSDaisuke Nishimura mc.precharge += count;
5645854ffa8dSDaisuke Nishimura return ret;
5646854ffa8dSDaisuke Nishimura }
56479476db97SJohannes Weiner
56483674534bSDavid Rientjes /* Try charges one by one with reclaim, but do not retry */
5649854ffa8dSDaisuke Nishimura while (count--) {
56503674534bSDavid Rientjes ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
565138c5d72fSKAMEZAWA Hiroyuki if (ret)
565238c5d72fSKAMEZAWA Hiroyuki return ret;
56534ffef5feSDaisuke Nishimura mc.precharge++;
56549476db97SJohannes Weiner cond_resched();
5655854ffa8dSDaisuke Nishimura }
56569476db97SJohannes Weiner return 0;
56574ffef5feSDaisuke Nishimura }
56584ffef5feSDaisuke Nishimura
56594ffef5feSDaisuke Nishimura union mc_target {
56604ffef5feSDaisuke Nishimura struct page *page;
566102491447SDaisuke Nishimura swp_entry_t ent;
56624ffef5feSDaisuke Nishimura };
56634ffef5feSDaisuke Nishimura
56644ffef5feSDaisuke Nishimura enum mc_target_type {
56658d32ff84SNaoya Horiguchi MC_TARGET_NONE = 0,
56664ffef5feSDaisuke Nishimura MC_TARGET_PAGE,
566702491447SDaisuke Nishimura MC_TARGET_SWAP,
5668c733a828SJérôme Glisse MC_TARGET_DEVICE,
56694ffef5feSDaisuke Nishimura };
56704ffef5feSDaisuke Nishimura
mc_handle_present_pte(struct vm_area_struct * vma,unsigned long addr,pte_t ptent)567190254a65SDaisuke Nishimura static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
567290254a65SDaisuke Nishimura unsigned long addr, pte_t ptent)
567390254a65SDaisuke Nishimura {
567425b2995aSChristoph Hellwig struct page *page = vm_normal_page(vma, addr, ptent);
567590254a65SDaisuke Nishimura
567658f341f7SMiaohe Lin if (!page)
567790254a65SDaisuke Nishimura return NULL;
567890254a65SDaisuke Nishimura if (PageAnon(page)) {
56791dfab5abSJohannes Weiner if (!(mc.flags & MOVE_ANON))
568090254a65SDaisuke Nishimura return NULL;
56811dfab5abSJohannes Weiner } else {
56821dfab5abSJohannes Weiner if (!(mc.flags & MOVE_FILE))
568390254a65SDaisuke Nishimura return NULL;
56841dfab5abSJohannes Weiner }
568558f341f7SMiaohe Lin get_page(page);
568690254a65SDaisuke Nishimura
568790254a65SDaisuke Nishimura return page;
568890254a65SDaisuke Nishimura }
568990254a65SDaisuke Nishimura
5690c733a828SJérôme Glisse #if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE)
mc_handle_swap_pte(struct vm_area_struct * vma,pte_t ptent,swp_entry_t * entry)569190254a65SDaisuke Nishimura static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
569248406ef8SLi RongQing pte_t ptent, swp_entry_t *entry)
569390254a65SDaisuke Nishimura {
569490254a65SDaisuke Nishimura struct page *page = NULL;
569590254a65SDaisuke Nishimura swp_entry_t ent = pte_to_swp_entry(ptent);
569690254a65SDaisuke Nishimura
56979a137153SRalph Campbell if (!(mc.flags & MOVE_ANON))
569890254a65SDaisuke Nishimura return NULL;
5699c733a828SJérôme Glisse
5700c733a828SJérôme Glisse /*
570127674ef6SChristoph Hellwig * Handle device private pages that are not accessible by the CPU, but
570227674ef6SChristoph Hellwig * stored as special swap entries in the page table.
5703c733a828SJérôme Glisse */
5704c733a828SJérôme Glisse if (is_device_private_entry(ent)) {
5705af5cdaf8SAlistair Popple page = pfn_swap_entry_to_page(ent);
570627674ef6SChristoph Hellwig if (!get_page_unless_zero(page))
5707c733a828SJérôme Glisse return NULL;
5708c733a828SJérôme Glisse return page;
5709c733a828SJérôme Glisse }
5710c733a828SJérôme Glisse
57119a137153SRalph Campbell if (non_swap_entry(ent))
57129a137153SRalph Campbell return NULL;
57139a137153SRalph Campbell
57144b91355eSKAMEZAWA Hiroyuki /*
5715cb691e2fSMatthew Wilcox (Oracle) * Because swap_cache_get_folio() updates some statistics counter,
57164b91355eSKAMEZAWA Hiroyuki * we call find_get_page() with swapper_space directly.
57174b91355eSKAMEZAWA Hiroyuki */
5718f6ab1f7fSHuang Ying page = find_get_page(swap_address_space(ent), swp_offset(ent));
571990254a65SDaisuke Nishimura entry->val = ent.val;
572090254a65SDaisuke Nishimura
572190254a65SDaisuke Nishimura return page;
572290254a65SDaisuke Nishimura }
57234b91355eSKAMEZAWA Hiroyuki #else
mc_handle_swap_pte(struct vm_area_struct * vma,pte_t ptent,swp_entry_t * entry)57244b91355eSKAMEZAWA Hiroyuki static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
572548406ef8SLi RongQing pte_t ptent, swp_entry_t *entry)
57264b91355eSKAMEZAWA Hiroyuki {
57274b91355eSKAMEZAWA Hiroyuki return NULL;
57284b91355eSKAMEZAWA Hiroyuki }
57294b91355eSKAMEZAWA Hiroyuki #endif
573090254a65SDaisuke Nishimura
mc_handle_file_pte(struct vm_area_struct * vma,unsigned long addr,pte_t ptent)573187946a72SDaisuke Nishimura static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
573248384b0bSPeter Xu unsigned long addr, pte_t ptent)
573387946a72SDaisuke Nishimura {
5734524984ffSMatthew Wilcox (Oracle) unsigned long index;
5735524984ffSMatthew Wilcox (Oracle) struct folio *folio;
5736524984ffSMatthew Wilcox (Oracle)
573787946a72SDaisuke Nishimura if (!vma->vm_file) /* anonymous vma */
573887946a72SDaisuke Nishimura return NULL;
57391dfab5abSJohannes Weiner if (!(mc.flags & MOVE_FILE))
574087946a72SDaisuke Nishimura return NULL;
574187946a72SDaisuke Nishimura
5742524984ffSMatthew Wilcox (Oracle) /* folio is moved even if it's not RSS of this task(page-faulted). */
5743aa3b1895SHugh Dickins /* shmem/tmpfs may report page out on swap: account for that too. */
5744524984ffSMatthew Wilcox (Oracle) index = linear_page_index(vma, addr);
5745524984ffSMatthew Wilcox (Oracle) folio = filemap_get_incore_folio(vma->vm_file->f_mapping, index);
574666dabbb6SChristoph Hellwig if (IS_ERR(folio))
5747524984ffSMatthew Wilcox (Oracle) return NULL;
5748524984ffSMatthew Wilcox (Oracle) return folio_file_page(folio, index);
574987946a72SDaisuke Nishimura }
575087946a72SDaisuke Nishimura
5751b1b0deabSChen Gang /**
5752b1b0deabSChen Gang * mem_cgroup_move_account - move account of the page
5753b1b0deabSChen Gang * @page: the page
575425843c2bSLi RongQing * @compound: charge the page as compound or small page
5755b1b0deabSChen Gang * @from: mem_cgroup which the page is moved from.
5756b1b0deabSChen Gang * @to: mem_cgroup which the page is moved to. @from != @to.
5757b1b0deabSChen Gang *
57584e0cf05fSJohannes Weiner * The page must be locked and not on the LRU.
5759b1b0deabSChen Gang *
5760b1b0deabSChen Gang * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
5761b1b0deabSChen Gang * from old cgroup.
5762b1b0deabSChen Gang */
mem_cgroup_move_account(struct page * page,bool compound,struct mem_cgroup * from,struct mem_cgroup * to)5763b1b0deabSChen Gang static int mem_cgroup_move_account(struct page *page,
5764f627c2f5SKirill A. Shutemov bool compound,
5765b1b0deabSChen Gang struct mem_cgroup *from,
5766b1b0deabSChen Gang struct mem_cgroup *to)
5767b1b0deabSChen Gang {
5768fcce4672SMatthew Wilcox (Oracle) struct folio *folio = page_folio(page);
5769ae8af438SKonstantin Khlebnikov struct lruvec *from_vec, *to_vec;
5770ae8af438SKonstantin Khlebnikov struct pglist_data *pgdat;
5771fcce4672SMatthew Wilcox (Oracle) unsigned int nr_pages = compound ? folio_nr_pages(folio) : 1;
57728e88bd2dSMatthew Wilcox (Oracle) int nid, ret;
5773b1b0deabSChen Gang
5774b1b0deabSChen Gang VM_BUG_ON(from == to);
57754e0cf05fSJohannes Weiner VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
5776fcce4672SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
57779c325215SMatthew Wilcox (Oracle) VM_BUG_ON(compound && !folio_test_large(folio));
5778b1b0deabSChen Gang
5779b1b0deabSChen Gang ret = -EINVAL;
5780fcce4672SMatthew Wilcox (Oracle) if (folio_memcg(folio) != from)
57814e0cf05fSJohannes Weiner goto out;
5782b1b0deabSChen Gang
5783fcce4672SMatthew Wilcox (Oracle) pgdat = folio_pgdat(folio);
5784867e5e1dSJohannes Weiner from_vec = mem_cgroup_lruvec(from, pgdat);
5785867e5e1dSJohannes Weiner to_vec = mem_cgroup_lruvec(to, pgdat);
5786ae8af438SKonstantin Khlebnikov
5787fcce4672SMatthew Wilcox (Oracle) folio_memcg_lock(folio);
5788b1b0deabSChen Gang
5789fcce4672SMatthew Wilcox (Oracle) if (folio_test_anon(folio)) {
5790fcce4672SMatthew Wilcox (Oracle) if (folio_mapped(folio)) {
5791be5d0a74SJohannes Weiner __mod_lruvec_state(from_vec, NR_ANON_MAPPED, -nr_pages);
5792be5d0a74SJohannes Weiner __mod_lruvec_state(to_vec, NR_ANON_MAPPED, nr_pages);
57936199277bSMatthew Wilcox (Oracle) if (folio_test_pmd_mappable(folio)) {
579469473e5dSMuchun Song __mod_lruvec_state(from_vec, NR_ANON_THPS,
579569473e5dSMuchun Song -nr_pages);
579669473e5dSMuchun Song __mod_lruvec_state(to_vec, NR_ANON_THPS,
579769473e5dSMuchun Song nr_pages);
5798468c3982SJohannes Weiner }
5799be5d0a74SJohannes Weiner }
5800be5d0a74SJohannes Weiner } else {
58010d1c2072SJohannes Weiner __mod_lruvec_state(from_vec, NR_FILE_PAGES, -nr_pages);
58020d1c2072SJohannes Weiner __mod_lruvec_state(to_vec, NR_FILE_PAGES, nr_pages);
58030d1c2072SJohannes Weiner
5804fcce4672SMatthew Wilcox (Oracle) if (folio_test_swapbacked(folio)) {
58050d1c2072SJohannes Weiner __mod_lruvec_state(from_vec, NR_SHMEM, -nr_pages);
58060d1c2072SJohannes Weiner __mod_lruvec_state(to_vec, NR_SHMEM, nr_pages);
58070d1c2072SJohannes Weiner }
58080d1c2072SJohannes Weiner
5809fcce4672SMatthew Wilcox (Oracle) if (folio_mapped(folio)) {
5810ae8af438SKonstantin Khlebnikov __mod_lruvec_state(from_vec, NR_FILE_MAPPED, -nr_pages);
5811ae8af438SKonstantin Khlebnikov __mod_lruvec_state(to_vec, NR_FILE_MAPPED, nr_pages);
5812b1b0deabSChen Gang }
5813b1b0deabSChen Gang
5814fcce4672SMatthew Wilcox (Oracle) if (folio_test_dirty(folio)) {
5815fcce4672SMatthew Wilcox (Oracle) struct address_space *mapping = folio_mapping(folio);
5816c4843a75SGreg Thelen
5817f56753acSChristoph Hellwig if (mapping_can_writeback(mapping)) {
581849e50d27SJohannes Weiner __mod_lruvec_state(from_vec, NR_FILE_DIRTY,
581949e50d27SJohannes Weiner -nr_pages);
582049e50d27SJohannes Weiner __mod_lruvec_state(to_vec, NR_FILE_DIRTY,
582149e50d27SJohannes Weiner nr_pages);
582249e50d27SJohannes Weiner }
5823c4843a75SGreg Thelen }
5824c4843a75SGreg Thelen }
5825c4843a75SGreg Thelen
5826c449deb2SHugh Dickins #ifdef CONFIG_SWAP
5827c449deb2SHugh Dickins if (folio_test_swapcache(folio)) {
5828c449deb2SHugh Dickins __mod_lruvec_state(from_vec, NR_SWAPCACHE, -nr_pages);
5829c449deb2SHugh Dickins __mod_lruvec_state(to_vec, NR_SWAPCACHE, nr_pages);
5830c449deb2SHugh Dickins }
5831c449deb2SHugh Dickins #endif
5832fcce4672SMatthew Wilcox (Oracle) if (folio_test_writeback(folio)) {
5833ae8af438SKonstantin Khlebnikov __mod_lruvec_state(from_vec, NR_WRITEBACK, -nr_pages);
5834ae8af438SKonstantin Khlebnikov __mod_lruvec_state(to_vec, NR_WRITEBACK, nr_pages);
5835b1b0deabSChen Gang }
5836b1b0deabSChen Gang
5837b1b0deabSChen Gang /*
5838abb242f5SJohannes Weiner * All state has been migrated, let's switch to the new memcg.
5839abb242f5SJohannes Weiner *
5840bcfe06bfSRoman Gushchin * It is safe to change page's memcg here because the page
5841abb242f5SJohannes Weiner * is referenced, charged, isolated, and locked: we can't race
5842abb242f5SJohannes Weiner * with (un)charging, migration, LRU putback, or anything else
5843bcfe06bfSRoman Gushchin * that would rely on a stable page's memory cgroup.
5844abb242f5SJohannes Weiner *
58456c77b607SKefeng Wang * Note that folio_memcg_lock is a memcg lock, not a page lock,
5846bcfe06bfSRoman Gushchin * to save space. As soon as we switch page's memory cgroup to a
5847abb242f5SJohannes Weiner * new memcg that isn't locked, the above state can change
5848abb242f5SJohannes Weiner * concurrently again. Make sure we're truly done with it.
5849b1b0deabSChen Gang */
5850abb242f5SJohannes Weiner smp_mb();
5851b1b0deabSChen Gang
58521a3e1f40SJohannes Weiner css_get(&to->css);
58531a3e1f40SJohannes Weiner css_put(&from->css);
58541a3e1f40SJohannes Weiner
5855fcce4672SMatthew Wilcox (Oracle) folio->memcg_data = (unsigned long)to;
585687eaceb3SYang Shi
5857f70ad448SMatthew Wilcox (Oracle) __folio_memcg_unlock(from);
5858b1b0deabSChen Gang
5859b1b0deabSChen Gang ret = 0;
5860fcce4672SMatthew Wilcox (Oracle) nid = folio_nid(folio);
5861b1b0deabSChen Gang
5862b1b0deabSChen Gang local_irq_disable();
58636e0110c2SMatthew Wilcox (Oracle) mem_cgroup_charge_statistics(to, nr_pages);
58648e88bd2dSMatthew Wilcox (Oracle) memcg_check_events(to, nid);
58656e0110c2SMatthew Wilcox (Oracle) mem_cgroup_charge_statistics(from, -nr_pages);
58668e88bd2dSMatthew Wilcox (Oracle) memcg_check_events(from, nid);
5867b1b0deabSChen Gang local_irq_enable();
5868b1b0deabSChen Gang out:
5869b1b0deabSChen Gang return ret;
5870b1b0deabSChen Gang }
5871b1b0deabSChen Gang
58727cf7806cSLi RongQing /**
58737cf7806cSLi RongQing * get_mctgt_type - get target type of moving charge
58747cf7806cSLi RongQing * @vma: the vma the pte to be checked belongs
58757cf7806cSLi RongQing * @addr: the address corresponding to the pte to be checked
58767cf7806cSLi RongQing * @ptent: the pte to be checked
58777cf7806cSLi RongQing * @target: the pointer the target page or swap ent will be stored(can be NULL)
58787cf7806cSLi RongQing *
5879853f62a3SMatthew Wilcox (Oracle) * Context: Called with pte lock held.
5880853f62a3SMatthew Wilcox (Oracle) * Return:
5881853f62a3SMatthew Wilcox (Oracle) * * MC_TARGET_NONE - If the pte is not a target for move charge.
5882853f62a3SMatthew Wilcox (Oracle) * * MC_TARGET_PAGE - If the page corresponding to this pte is a target for
5883853f62a3SMatthew Wilcox (Oracle) * move charge. If @target is not NULL, the page is stored in target->page
5884853f62a3SMatthew Wilcox (Oracle) * with extra refcnt taken (Caller should release it).
5885853f62a3SMatthew Wilcox (Oracle) * * MC_TARGET_SWAP - If the swap entry corresponding to this pte is a
5886853f62a3SMatthew Wilcox (Oracle) * target for charge migration. If @target is not NULL, the entry is
5887853f62a3SMatthew Wilcox (Oracle) * stored in target->ent.
5888853f62a3SMatthew Wilcox (Oracle) * * MC_TARGET_DEVICE - Like MC_TARGET_PAGE but page is device memory and
5889853f62a3SMatthew Wilcox (Oracle) * thus not on the lru. For now such page is charged like a regular page
5890853f62a3SMatthew Wilcox (Oracle) * would be as it is just special memory taking the place of a regular page.
5891c733a828SJérôme Glisse * See Documentations/vm/hmm.txt and include/linux/hmm.h
58927cf7806cSLi RongQing */
get_mctgt_type(struct vm_area_struct * vma,unsigned long addr,pte_t ptent,union mc_target * target)58938d32ff84SNaoya Horiguchi static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
58944ffef5feSDaisuke Nishimura unsigned long addr, pte_t ptent, union mc_target *target)
58954ffef5feSDaisuke Nishimura {
589602491447SDaisuke Nishimura struct page *page = NULL;
58978d32ff84SNaoya Horiguchi enum mc_target_type ret = MC_TARGET_NONE;
589802491447SDaisuke Nishimura swp_entry_t ent = { .val = 0 };
58994ffef5feSDaisuke Nishimura
590090254a65SDaisuke Nishimura if (pte_present(ptent))
590190254a65SDaisuke Nishimura page = mc_handle_present_pte(vma, addr, ptent);
59025c041f5dSPeter Xu else if (pte_none_mostly(ptent))
59035c041f5dSPeter Xu /*
59045c041f5dSPeter Xu * PTE markers should be treated as a none pte here, separated
59055c041f5dSPeter Xu * from other swap handling below.
59065c041f5dSPeter Xu */
59075c041f5dSPeter Xu page = mc_handle_file_pte(vma, addr, ptent);
590890254a65SDaisuke Nishimura else if (is_swap_pte(ptent))
590948406ef8SLi RongQing page = mc_handle_swap_pte(vma, ptent, &ent);
591090254a65SDaisuke Nishimura
59114e0cf05fSJohannes Weiner if (target && page) {
59124e0cf05fSJohannes Weiner if (!trylock_page(page)) {
59134e0cf05fSJohannes Weiner put_page(page);
59144e0cf05fSJohannes Weiner return ret;
59154e0cf05fSJohannes Weiner }
59164e0cf05fSJohannes Weiner /*
59174e0cf05fSJohannes Weiner * page_mapped() must be stable during the move. This
59184e0cf05fSJohannes Weiner * pte is locked, so if it's present, the page cannot
59194e0cf05fSJohannes Weiner * become unmapped. If it isn't, we have only partial
59204e0cf05fSJohannes Weiner * control over the mapped state: the page lock will
59214e0cf05fSJohannes Weiner * prevent new faults against pagecache and swapcache,
59224e0cf05fSJohannes Weiner * so an unmapped page cannot become mapped. However,
59234e0cf05fSJohannes Weiner * if the page is already mapped elsewhere, it can
59244e0cf05fSJohannes Weiner * unmap, and there is nothing we can do about it.
59254e0cf05fSJohannes Weiner * Alas, skip moving the page in this case.
59264e0cf05fSJohannes Weiner */
59274e0cf05fSJohannes Weiner if (!pte_present(ptent) && page_mapped(page)) {
59284e0cf05fSJohannes Weiner unlock_page(page);
59294e0cf05fSJohannes Weiner put_page(page);
59304e0cf05fSJohannes Weiner return ret;
59314e0cf05fSJohannes Weiner }
59324e0cf05fSJohannes Weiner }
59334e0cf05fSJohannes Weiner
593490254a65SDaisuke Nishimura if (!page && !ent.val)
59358d32ff84SNaoya Horiguchi return ret;
593602491447SDaisuke Nishimura if (page) {
59374ffef5feSDaisuke Nishimura /*
59380a31bc97SJohannes Weiner * Do only loose check w/o serialization.
59391306a85aSJohannes Weiner * mem_cgroup_move_account() checks the page is valid or
59400a31bc97SJohannes Weiner * not under LRU exclusion.
59414ffef5feSDaisuke Nishimura */
5942bcfe06bfSRoman Gushchin if (page_memcg(page) == mc.from) {
59434ffef5feSDaisuke Nishimura ret = MC_TARGET_PAGE;
5944f25cbb7aSAlex Sierra if (is_device_private_page(page) ||
5945f25cbb7aSAlex Sierra is_device_coherent_page(page))
5946c733a828SJérôme Glisse ret = MC_TARGET_DEVICE;
59474ffef5feSDaisuke Nishimura if (target)
59484ffef5feSDaisuke Nishimura target->page = page;
59494ffef5feSDaisuke Nishimura }
59504e0cf05fSJohannes Weiner if (!ret || !target) {
59514e0cf05fSJohannes Weiner if (target)
59524e0cf05fSJohannes Weiner unlock_page(page);
59534ffef5feSDaisuke Nishimura put_page(page);
595402491447SDaisuke Nishimura }
59554e0cf05fSJohannes Weiner }
59563e14a57bSHuang Ying /*
59573e14a57bSHuang Ying * There is a swap entry and a page doesn't exist or isn't charged.
59583e14a57bSHuang Ying * But we cannot move a tail-page in a THP.
59593e14a57bSHuang Ying */
59603e14a57bSHuang Ying if (ent.val && !ret && (!page || !PageTransCompound(page)) &&
596134c00c31SLi Zefan mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
596202491447SDaisuke Nishimura ret = MC_TARGET_SWAP;
596302491447SDaisuke Nishimura if (target)
596402491447SDaisuke Nishimura target->ent = ent;
596502491447SDaisuke Nishimura }
59664ffef5feSDaisuke Nishimura return ret;
59674ffef5feSDaisuke Nishimura }
59684ffef5feSDaisuke Nishimura
596912724850SNaoya Horiguchi #ifdef CONFIG_TRANSPARENT_HUGEPAGE
597012724850SNaoya Horiguchi /*
5971d6810d73SHuang Ying * We don't consider PMD mapped swapping or file mapped pages because THP does
5972d6810d73SHuang Ying * not support them for now.
597312724850SNaoya Horiguchi * Caller should make sure that pmd_trans_huge(pmd) is true.
597412724850SNaoya Horiguchi */
get_mctgt_type_thp(struct vm_area_struct * vma,unsigned long addr,pmd_t pmd,union mc_target * target)597512724850SNaoya Horiguchi static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
597612724850SNaoya Horiguchi unsigned long addr, pmd_t pmd, union mc_target *target)
597712724850SNaoya Horiguchi {
597812724850SNaoya Horiguchi struct page *page = NULL;
597912724850SNaoya Horiguchi enum mc_target_type ret = MC_TARGET_NONE;
598012724850SNaoya Horiguchi
598184c3fc4eSZi Yan if (unlikely(is_swap_pmd(pmd))) {
598284c3fc4eSZi Yan VM_BUG_ON(thp_migration_supported() &&
598384c3fc4eSZi Yan !is_pmd_migration_entry(pmd));
598484c3fc4eSZi Yan return ret;
598584c3fc4eSZi Yan }
598612724850SNaoya Horiguchi page = pmd_page(pmd);
5987309381feSSasha Levin VM_BUG_ON_PAGE(!page || !PageHead(page), page);
59881dfab5abSJohannes Weiner if (!(mc.flags & MOVE_ANON))
598912724850SNaoya Horiguchi return ret;
5990bcfe06bfSRoman Gushchin if (page_memcg(page) == mc.from) {
599112724850SNaoya Horiguchi ret = MC_TARGET_PAGE;
599212724850SNaoya Horiguchi if (target) {
599312724850SNaoya Horiguchi get_page(page);
59944e0cf05fSJohannes Weiner if (!trylock_page(page)) {
59954e0cf05fSJohannes Weiner put_page(page);
59964e0cf05fSJohannes Weiner return MC_TARGET_NONE;
59974e0cf05fSJohannes Weiner }
599812724850SNaoya Horiguchi target->page = page;
599912724850SNaoya Horiguchi }
600012724850SNaoya Horiguchi }
600112724850SNaoya Horiguchi return ret;
600212724850SNaoya Horiguchi }
600312724850SNaoya Horiguchi #else
get_mctgt_type_thp(struct vm_area_struct * vma,unsigned long addr,pmd_t pmd,union mc_target * target)600412724850SNaoya Horiguchi static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
600512724850SNaoya Horiguchi unsigned long addr, pmd_t pmd, union mc_target *target)
600612724850SNaoya Horiguchi {
600712724850SNaoya Horiguchi return MC_TARGET_NONE;
600812724850SNaoya Horiguchi }
600912724850SNaoya Horiguchi #endif
601012724850SNaoya Horiguchi
mem_cgroup_count_precharge_pte_range(pmd_t * pmd,unsigned long addr,unsigned long end,struct mm_walk * walk)60114ffef5feSDaisuke Nishimura static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
60124ffef5feSDaisuke Nishimura unsigned long addr, unsigned long end,
60134ffef5feSDaisuke Nishimura struct mm_walk *walk)
60144ffef5feSDaisuke Nishimura {
601526bcd64aSNaoya Horiguchi struct vm_area_struct *vma = walk->vma;
60164ffef5feSDaisuke Nishimura pte_t *pte;
60174ffef5feSDaisuke Nishimura spinlock_t *ptl;
60184ffef5feSDaisuke Nishimura
6019b6ec57f4SKirill A. Shutemov ptl = pmd_trans_huge_lock(pmd, vma);
6020b6ec57f4SKirill A. Shutemov if (ptl) {
6021c733a828SJérôme Glisse /*
6022c733a828SJérôme Glisse * Note their can not be MC_TARGET_DEVICE for now as we do not
602325b2995aSChristoph Hellwig * support transparent huge page with MEMORY_DEVICE_PRIVATE but
602425b2995aSChristoph Hellwig * this might change.
6025c733a828SJérôme Glisse */
602612724850SNaoya Horiguchi if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
602712724850SNaoya Horiguchi mc.precharge += HPAGE_PMD_NR;
6028bf929152SKirill A. Shutemov spin_unlock(ptl);
60291a5a9906SAndrea Arcangeli return 0;
603012724850SNaoya Horiguchi }
603103319327SDave Hansen
60324ffef5feSDaisuke Nishimura pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
603304dee9e8SHugh Dickins if (!pte)
603404dee9e8SHugh Dickins return 0;
60354ffef5feSDaisuke Nishimura for (; addr != end; pte++, addr += PAGE_SIZE)
6036c33c7948SRyan Roberts if (get_mctgt_type(vma, addr, ptep_get(pte), NULL))
60374ffef5feSDaisuke Nishimura mc.precharge++; /* increment precharge temporarily */
60384ffef5feSDaisuke Nishimura pte_unmap_unlock(pte - 1, ptl);
60394ffef5feSDaisuke Nishimura cond_resched();
60404ffef5feSDaisuke Nishimura
60414ffef5feSDaisuke Nishimura return 0;
60424ffef5feSDaisuke Nishimura }
60434ffef5feSDaisuke Nishimura
60447b86ac33SChristoph Hellwig static const struct mm_walk_ops precharge_walk_ops = {
60457b86ac33SChristoph Hellwig .pmd_entry = mem_cgroup_count_precharge_pte_range,
604649b06385SSuren Baghdasaryan .walk_lock = PGWALK_RDLOCK,
60477b86ac33SChristoph Hellwig };
60487b86ac33SChristoph Hellwig
mem_cgroup_count_precharge(struct mm_struct * mm)60494ffef5feSDaisuke Nishimura static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
60504ffef5feSDaisuke Nishimura {
60514ffef5feSDaisuke Nishimura unsigned long precharge;
60524ffef5feSDaisuke Nishimura
6053d8ed45c5SMichel Lespinasse mmap_read_lock(mm);
6054ba0aff8eSLiam R. Howlett walk_page_range(mm, 0, ULONG_MAX, &precharge_walk_ops, NULL);
6055d8ed45c5SMichel Lespinasse mmap_read_unlock(mm);
60564ffef5feSDaisuke Nishimura
60574ffef5feSDaisuke Nishimura precharge = mc.precharge;
60584ffef5feSDaisuke Nishimura mc.precharge = 0;
60594ffef5feSDaisuke Nishimura
60604ffef5feSDaisuke Nishimura return precharge;
60614ffef5feSDaisuke Nishimura }
60624ffef5feSDaisuke Nishimura
mem_cgroup_precharge_mc(struct mm_struct * mm)60634ffef5feSDaisuke Nishimura static int mem_cgroup_precharge_mc(struct mm_struct *mm)
60644ffef5feSDaisuke Nishimura {
6065dfe076b0SDaisuke Nishimura unsigned long precharge = mem_cgroup_count_precharge(mm);
6066dfe076b0SDaisuke Nishimura
6067dfe076b0SDaisuke Nishimura VM_BUG_ON(mc.moving_task);
6068dfe076b0SDaisuke Nishimura mc.moving_task = current;
6069dfe076b0SDaisuke Nishimura return mem_cgroup_do_precharge(precharge);
60704ffef5feSDaisuke Nishimura }
60714ffef5feSDaisuke Nishimura
6072dfe076b0SDaisuke Nishimura /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
__mem_cgroup_clear_mc(void)6073dfe076b0SDaisuke Nishimura static void __mem_cgroup_clear_mc(void)
60744ffef5feSDaisuke Nishimura {
60752bd9bb20SKAMEZAWA Hiroyuki struct mem_cgroup *from = mc.from;
60762bd9bb20SKAMEZAWA Hiroyuki struct mem_cgroup *to = mc.to;
60772bd9bb20SKAMEZAWA Hiroyuki
60784ffef5feSDaisuke Nishimura /* we must uncharge all the leftover precharges from mc.to */
6079854ffa8dSDaisuke Nishimura if (mc.precharge) {
608000501b53SJohannes Weiner cancel_charge(mc.to, mc.precharge);
6081854ffa8dSDaisuke Nishimura mc.precharge = 0;
6082854ffa8dSDaisuke Nishimura }
6083854ffa8dSDaisuke Nishimura /*
6084854ffa8dSDaisuke Nishimura * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
6085854ffa8dSDaisuke Nishimura * we must uncharge here.
6086854ffa8dSDaisuke Nishimura */
6087854ffa8dSDaisuke Nishimura if (mc.moved_charge) {
608800501b53SJohannes Weiner cancel_charge(mc.from, mc.moved_charge);
6089854ffa8dSDaisuke Nishimura mc.moved_charge = 0;
60904ffef5feSDaisuke Nishimura }
6091483c30b5SDaisuke Nishimura /* we must fixup refcnts and charges */
6092483c30b5SDaisuke Nishimura if (mc.moved_swap) {
6093483c30b5SDaisuke Nishimura /* uncharge swap account from the old cgroup */
6094ce00a967SJohannes Weiner if (!mem_cgroup_is_root(mc.from))
60953e32cb2eSJohannes Weiner page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
6096483c30b5SDaisuke Nishimura
6097615d66c3SVladimir Davydov mem_cgroup_id_put_many(mc.from, mc.moved_swap);
6098615d66c3SVladimir Davydov
6099483c30b5SDaisuke Nishimura /*
61003e32cb2eSJohannes Weiner * we charged both to->memory and to->memsw, so we
61013e32cb2eSJohannes Weiner * should uncharge to->memory.
6102483c30b5SDaisuke Nishimura */
6103ce00a967SJohannes Weiner if (!mem_cgroup_is_root(mc.to))
61043e32cb2eSJohannes Weiner page_counter_uncharge(&mc.to->memory, mc.moved_swap);
6105483c30b5SDaisuke Nishimura
6106483c30b5SDaisuke Nishimura mc.moved_swap = 0;
6107483c30b5SDaisuke Nishimura }
6108dfe076b0SDaisuke Nishimura memcg_oom_recover(from);
6109dfe076b0SDaisuke Nishimura memcg_oom_recover(to);
6110dfe076b0SDaisuke Nishimura wake_up_all(&mc.waitq);
6111b1dd693eSDaisuke Nishimura }
6112dfe076b0SDaisuke Nishimura
mem_cgroup_clear_mc(void)6113dfe076b0SDaisuke Nishimura static void mem_cgroup_clear_mc(void)
6114dfe076b0SDaisuke Nishimura {
6115264a0ae1STejun Heo struct mm_struct *mm = mc.mm;
6116264a0ae1STejun Heo
6117dfe076b0SDaisuke Nishimura /*
6118dfe076b0SDaisuke Nishimura * we must clear moving_task before waking up waiters at the end of
6119dfe076b0SDaisuke Nishimura * task migration.
6120dfe076b0SDaisuke Nishimura */
6121dfe076b0SDaisuke Nishimura mc.moving_task = NULL;
6122dfe076b0SDaisuke Nishimura __mem_cgroup_clear_mc();
61232bd9bb20SKAMEZAWA Hiroyuki spin_lock(&mc.lock);
61244ffef5feSDaisuke Nishimura mc.from = NULL;
61254ffef5feSDaisuke Nishimura mc.to = NULL;
6126264a0ae1STejun Heo mc.mm = NULL;
61272bd9bb20SKAMEZAWA Hiroyuki spin_unlock(&mc.lock);
6128264a0ae1STejun Heo
6129264a0ae1STejun Heo mmput(mm);
61307dc74be0SDaisuke Nishimura }
61317dc74be0SDaisuke Nishimura
mem_cgroup_can_attach(struct cgroup_taskset * tset)61321f7dd3e5STejun Heo static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
61337dc74be0SDaisuke Nishimura {
61341f7dd3e5STejun Heo struct cgroup_subsys_state *css;
6135eed67d75SRoss Zwisler struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
61369f2115f9STejun Heo struct mem_cgroup *from;
61374530eddbSTejun Heo struct task_struct *leader, *p;
61389f2115f9STejun Heo struct mm_struct *mm;
61391dfab5abSJohannes Weiner unsigned long move_flags;
61409f2115f9STejun Heo int ret = 0;
61417dc74be0SDaisuke Nishimura
61421f7dd3e5STejun Heo /* charge immigration isn't supported on the default hierarchy */
61431f7dd3e5STejun Heo if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
61449f2115f9STejun Heo return 0;
61459f2115f9STejun Heo
61464530eddbSTejun Heo /*
61474530eddbSTejun Heo * Multi-process migrations only happen on the default hierarchy
61484530eddbSTejun Heo * where charge immigration is not used. Perform charge
61494530eddbSTejun Heo * immigration if @tset contains a leader and whine if there are
61504530eddbSTejun Heo * multiple.
61514530eddbSTejun Heo */
61524530eddbSTejun Heo p = NULL;
61531f7dd3e5STejun Heo cgroup_taskset_for_each_leader(leader, css, tset) {
61544530eddbSTejun Heo WARN_ON_ONCE(p);
61554530eddbSTejun Heo p = leader;
61561f7dd3e5STejun Heo memcg = mem_cgroup_from_css(css);
61574530eddbSTejun Heo }
61584530eddbSTejun Heo if (!p)
61594530eddbSTejun Heo return 0;
61604530eddbSTejun Heo
61611f7dd3e5STejun Heo /*
6162f0953a1bSIngo Molnar * We are now committed to this value whatever it is. Changes in this
61631f7dd3e5STejun Heo * tunable will only affect upcoming migrations, not the current one.
61641f7dd3e5STejun Heo * So we need to save it, and keep it going.
61651f7dd3e5STejun Heo */
61661f7dd3e5STejun Heo move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
61671f7dd3e5STejun Heo if (!move_flags)
61681f7dd3e5STejun Heo return 0;
61691f7dd3e5STejun Heo
61709f2115f9STejun Heo from = mem_cgroup_from_task(p);
61717dc74be0SDaisuke Nishimura
6172c0ff4b85SRaghavendra K T VM_BUG_ON(from == memcg);
61737dc74be0SDaisuke Nishimura
61747dc74be0SDaisuke Nishimura mm = get_task_mm(p);
61757dc74be0SDaisuke Nishimura if (!mm)
61767dc74be0SDaisuke Nishimura return 0;
61777dc74be0SDaisuke Nishimura /* We move charges only when we move a owner of the mm */
61784ffef5feSDaisuke Nishimura if (mm->owner == p) {
61794ffef5feSDaisuke Nishimura VM_BUG_ON(mc.from);
61804ffef5feSDaisuke Nishimura VM_BUG_ON(mc.to);
61814ffef5feSDaisuke Nishimura VM_BUG_ON(mc.precharge);
6182854ffa8dSDaisuke Nishimura VM_BUG_ON(mc.moved_charge);
6183483c30b5SDaisuke Nishimura VM_BUG_ON(mc.moved_swap);
6184247b1447SJohannes Weiner
61852bd9bb20SKAMEZAWA Hiroyuki spin_lock(&mc.lock);
6186264a0ae1STejun Heo mc.mm = mm;
61874ffef5feSDaisuke Nishimura mc.from = from;
6188c0ff4b85SRaghavendra K T mc.to = memcg;
61891dfab5abSJohannes Weiner mc.flags = move_flags;
61902bd9bb20SKAMEZAWA Hiroyuki spin_unlock(&mc.lock);
6191dfe076b0SDaisuke Nishimura /* We set mc.moving_task later */
61927dc74be0SDaisuke Nishimura
61934ffef5feSDaisuke Nishimura ret = mem_cgroup_precharge_mc(mm);
61944ffef5feSDaisuke Nishimura if (ret)
61954ffef5feSDaisuke Nishimura mem_cgroup_clear_mc();
6196264a0ae1STejun Heo } else {
61977dc74be0SDaisuke Nishimura mmput(mm);
6198264a0ae1STejun Heo }
61997dc74be0SDaisuke Nishimura return ret;
62007dc74be0SDaisuke Nishimura }
62017dc74be0SDaisuke Nishimura
mem_cgroup_cancel_attach(struct cgroup_taskset * tset)62021f7dd3e5STejun Heo static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
62037dc74be0SDaisuke Nishimura {
62044e2f245dSJohannes Weiner if (mc.to)
62054ffef5feSDaisuke Nishimura mem_cgroup_clear_mc();
62067dc74be0SDaisuke Nishimura }
62077dc74be0SDaisuke Nishimura
mem_cgroup_move_charge_pte_range(pmd_t * pmd,unsigned long addr,unsigned long end,struct mm_walk * walk)62084ffef5feSDaisuke Nishimura static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
62094ffef5feSDaisuke Nishimura unsigned long addr, unsigned long end,
62104ffef5feSDaisuke Nishimura struct mm_walk *walk)
62117dc74be0SDaisuke Nishimura {
62124ffef5feSDaisuke Nishimura int ret = 0;
621326bcd64aSNaoya Horiguchi struct vm_area_struct *vma = walk->vma;
62144ffef5feSDaisuke Nishimura pte_t *pte;
62154ffef5feSDaisuke Nishimura spinlock_t *ptl;
621612724850SNaoya Horiguchi enum mc_target_type target_type;
621712724850SNaoya Horiguchi union mc_target target;
621812724850SNaoya Horiguchi struct page *page;
62194ffef5feSDaisuke Nishimura
6220b6ec57f4SKirill A. Shutemov ptl = pmd_trans_huge_lock(pmd, vma);
6221b6ec57f4SKirill A. Shutemov if (ptl) {
622262ade86aSHugh Dickins if (mc.precharge < HPAGE_PMD_NR) {
6223bf929152SKirill A. Shutemov spin_unlock(ptl);
62241a5a9906SAndrea Arcangeli return 0;
622512724850SNaoya Horiguchi }
622612724850SNaoya Horiguchi target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
622712724850SNaoya Horiguchi if (target_type == MC_TARGET_PAGE) {
622812724850SNaoya Horiguchi page = target.page;
6229f7f9c00dSBaolin Wang if (isolate_lru_page(page)) {
6230f627c2f5SKirill A. Shutemov if (!mem_cgroup_move_account(page, true,
62311306a85aSJohannes Weiner mc.from, mc.to)) {
623212724850SNaoya Horiguchi mc.precharge -= HPAGE_PMD_NR;
623312724850SNaoya Horiguchi mc.moved_charge += HPAGE_PMD_NR;
623412724850SNaoya Horiguchi }
623512724850SNaoya Horiguchi putback_lru_page(page);
623612724850SNaoya Horiguchi }
62374e0cf05fSJohannes Weiner unlock_page(page);
623812724850SNaoya Horiguchi put_page(page);
6239c733a828SJérôme Glisse } else if (target_type == MC_TARGET_DEVICE) {
6240c733a828SJérôme Glisse page = target.page;
6241c733a828SJérôme Glisse if (!mem_cgroup_move_account(page, true,
6242c733a828SJérôme Glisse mc.from, mc.to)) {
6243c733a828SJérôme Glisse mc.precharge -= HPAGE_PMD_NR;
6244c733a828SJérôme Glisse mc.moved_charge += HPAGE_PMD_NR;
6245c733a828SJérôme Glisse }
62464e0cf05fSJohannes Weiner unlock_page(page);
6247c733a828SJérôme Glisse put_page(page);
624812724850SNaoya Horiguchi }
6249bf929152SKirill A. Shutemov spin_unlock(ptl);
625012724850SNaoya Horiguchi return 0;
625112724850SNaoya Horiguchi }
625212724850SNaoya Horiguchi
62534ffef5feSDaisuke Nishimura retry:
62544ffef5feSDaisuke Nishimura pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
625504dee9e8SHugh Dickins if (!pte)
625604dee9e8SHugh Dickins return 0;
62574ffef5feSDaisuke Nishimura for (; addr != end; addr += PAGE_SIZE) {
6258c33c7948SRyan Roberts pte_t ptent = ptep_get(pte++);
6259c733a828SJérôme Glisse bool device = false;
626002491447SDaisuke Nishimura swp_entry_t ent;
62614ffef5feSDaisuke Nishimura
62624ffef5feSDaisuke Nishimura if (!mc.precharge)
62634ffef5feSDaisuke Nishimura break;
62644ffef5feSDaisuke Nishimura
62658d32ff84SNaoya Horiguchi switch (get_mctgt_type(vma, addr, ptent, &target)) {
6266c733a828SJérôme Glisse case MC_TARGET_DEVICE:
6267c733a828SJérôme Glisse device = true;
6268e4a9bc58SJoe Perches fallthrough;
62694ffef5feSDaisuke Nishimura case MC_TARGET_PAGE:
62704ffef5feSDaisuke Nishimura page = target.page;
627153f9263bSKirill A. Shutemov /*
627253f9263bSKirill A. Shutemov * We can have a part of the split pmd here. Moving it
627353f9263bSKirill A. Shutemov * can be done but it would be too convoluted so simply
627453f9263bSKirill A. Shutemov * ignore such a partial THP and keep it in original
627553f9263bSKirill A. Shutemov * memcg. There should be somebody mapping the head.
627653f9263bSKirill A. Shutemov */
627753f9263bSKirill A. Shutemov if (PageTransCompound(page))
627853f9263bSKirill A. Shutemov goto put;
6279f7f9c00dSBaolin Wang if (!device && !isolate_lru_page(page))
62804ffef5feSDaisuke Nishimura goto put;
6281f627c2f5SKirill A. Shutemov if (!mem_cgroup_move_account(page, false,
6282f627c2f5SKirill A. Shutemov mc.from, mc.to)) {
62834ffef5feSDaisuke Nishimura mc.precharge--;
6284854ffa8dSDaisuke Nishimura /* we uncharge from mc.from later. */
6285854ffa8dSDaisuke Nishimura mc.moved_charge++;
62864ffef5feSDaisuke Nishimura }
6287c733a828SJérôme Glisse if (!device)
62884ffef5feSDaisuke Nishimura putback_lru_page(page);
62894e0cf05fSJohannes Weiner put: /* get_mctgt_type() gets & locks the page */
62904e0cf05fSJohannes Weiner unlock_page(page);
62914ffef5feSDaisuke Nishimura put_page(page);
62924ffef5feSDaisuke Nishimura break;
629302491447SDaisuke Nishimura case MC_TARGET_SWAP:
629402491447SDaisuke Nishimura ent = target.ent;
6295e91cbb42SHugh Dickins if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
629602491447SDaisuke Nishimura mc.precharge--;
62978d22a935SHugh Dickins mem_cgroup_id_get_many(mc.to, 1);
62988d22a935SHugh Dickins /* we fixup other refcnts and charges later. */
6299483c30b5SDaisuke Nishimura mc.moved_swap++;
6300483c30b5SDaisuke Nishimura }
630102491447SDaisuke Nishimura break;
63024ffef5feSDaisuke Nishimura default:
63034ffef5feSDaisuke Nishimura break;
63044ffef5feSDaisuke Nishimura }
63054ffef5feSDaisuke Nishimura }
63064ffef5feSDaisuke Nishimura pte_unmap_unlock(pte - 1, ptl);
63074ffef5feSDaisuke Nishimura cond_resched();
63084ffef5feSDaisuke Nishimura
63094ffef5feSDaisuke Nishimura if (addr != end) {
63104ffef5feSDaisuke Nishimura /*
63114ffef5feSDaisuke Nishimura * We have consumed all precharges we got in can_attach().
63124ffef5feSDaisuke Nishimura * We try charge one by one, but don't do any additional
63134ffef5feSDaisuke Nishimura * charges to mc.to if we have failed in charge once in attach()
63144ffef5feSDaisuke Nishimura * phase.
63154ffef5feSDaisuke Nishimura */
6316854ffa8dSDaisuke Nishimura ret = mem_cgroup_do_precharge(1);
63174ffef5feSDaisuke Nishimura if (!ret)
63184ffef5feSDaisuke Nishimura goto retry;
63194ffef5feSDaisuke Nishimura }
63204ffef5feSDaisuke Nishimura
63214ffef5feSDaisuke Nishimura return ret;
63224ffef5feSDaisuke Nishimura }
63234ffef5feSDaisuke Nishimura
63247b86ac33SChristoph Hellwig static const struct mm_walk_ops charge_walk_ops = {
632526bcd64aSNaoya Horiguchi .pmd_entry = mem_cgroup_move_charge_pte_range,
632649b06385SSuren Baghdasaryan .walk_lock = PGWALK_RDLOCK,
632726bcd64aSNaoya Horiguchi };
63284ffef5feSDaisuke Nishimura
mem_cgroup_move_charge(void)63297b86ac33SChristoph Hellwig static void mem_cgroup_move_charge(void)
63307b86ac33SChristoph Hellwig {
63314ffef5feSDaisuke Nishimura lru_add_drain_all();
6332312722cbSJohannes Weiner /*
63336c77b607SKefeng Wang * Signal folio_memcg_lock() to take the memcg's move_lock
633481f8c3a4SJohannes Weiner * while we're moving its pages to another memcg. Then wait
633581f8c3a4SJohannes Weiner * for already started RCU-only updates to finish.
6336312722cbSJohannes Weiner */
6337312722cbSJohannes Weiner atomic_inc(&mc.from->moving_account);
6338312722cbSJohannes Weiner synchronize_rcu();
6339dfe076b0SDaisuke Nishimura retry:
6340d8ed45c5SMichel Lespinasse if (unlikely(!mmap_read_trylock(mc.mm))) {
6341dfe076b0SDaisuke Nishimura /*
6342c1e8d7c6SMichel Lespinasse * Someone who are holding the mmap_lock might be waiting in
6343dfe076b0SDaisuke Nishimura * waitq. So we cancel all extra charges, wake up all waiters,
6344dfe076b0SDaisuke Nishimura * and retry. Because we cancel precharges, we might not be able
6345dfe076b0SDaisuke Nishimura * to move enough charges, but moving charge is a best-effort
6346dfe076b0SDaisuke Nishimura * feature anyway, so it wouldn't be a big problem.
6347dfe076b0SDaisuke Nishimura */
6348dfe076b0SDaisuke Nishimura __mem_cgroup_clear_mc();
6349dfe076b0SDaisuke Nishimura cond_resched();
6350dfe076b0SDaisuke Nishimura goto retry;
6351dfe076b0SDaisuke Nishimura }
63524ffef5feSDaisuke Nishimura /*
635326bcd64aSNaoya Horiguchi * When we have consumed all precharges and failed in doing
635426bcd64aSNaoya Horiguchi * additional charge, the page walk just aborts.
63554ffef5feSDaisuke Nishimura */
6356ba0aff8eSLiam R. Howlett walk_page_range(mc.mm, 0, ULONG_MAX, &charge_walk_ops, NULL);
6357d8ed45c5SMichel Lespinasse mmap_read_unlock(mc.mm);
6358312722cbSJohannes Weiner atomic_dec(&mc.from->moving_account);
63597dc74be0SDaisuke Nishimura }
63607dc74be0SDaisuke Nishimura
mem_cgroup_move_task(void)6361264a0ae1STejun Heo static void mem_cgroup_move_task(void)
636267e465a7SBalbir Singh {
6363264a0ae1STejun Heo if (mc.to) {
6364264a0ae1STejun Heo mem_cgroup_move_charge();
63654ffef5feSDaisuke Nishimura mem_cgroup_clear_mc();
636667e465a7SBalbir Singh }
6367264a0ae1STejun Heo }
63685cfb80a7SDaisuke Nishimura #else /* !CONFIG_MMU */
mem_cgroup_can_attach(struct cgroup_taskset * tset)63691f7dd3e5STejun Heo static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
63705cfb80a7SDaisuke Nishimura {
63715cfb80a7SDaisuke Nishimura return 0;
63725cfb80a7SDaisuke Nishimura }
mem_cgroup_cancel_attach(struct cgroup_taskset * tset)63731f7dd3e5STejun Heo static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
63745cfb80a7SDaisuke Nishimura {
63755cfb80a7SDaisuke Nishimura }
mem_cgroup_move_task(void)6376264a0ae1STejun Heo static void mem_cgroup_move_task(void)
63775cfb80a7SDaisuke Nishimura {
63785cfb80a7SDaisuke Nishimura }
63795cfb80a7SDaisuke Nishimura #endif
638067e465a7SBalbir Singh
6381bd74fdaeSYu Zhao #ifdef CONFIG_LRU_GEN
mem_cgroup_attach(struct cgroup_taskset * tset)6382bd74fdaeSYu Zhao static void mem_cgroup_attach(struct cgroup_taskset *tset)
6383bd74fdaeSYu Zhao {
6384bd74fdaeSYu Zhao struct task_struct *task;
6385bd74fdaeSYu Zhao struct cgroup_subsys_state *css;
6386bd74fdaeSYu Zhao
6387bd74fdaeSYu Zhao /* find the first leader if there is any */
6388bd74fdaeSYu Zhao cgroup_taskset_for_each_leader(task, css, tset)
6389bd74fdaeSYu Zhao break;
6390bd74fdaeSYu Zhao
6391bd74fdaeSYu Zhao if (!task)
6392bd74fdaeSYu Zhao return;
6393bd74fdaeSYu Zhao
6394bd74fdaeSYu Zhao task_lock(task);
6395bd74fdaeSYu Zhao if (task->mm && READ_ONCE(task->mm->owner) == task)
6396bd74fdaeSYu Zhao lru_gen_migrate_mm(task->mm);
6397bd74fdaeSYu Zhao task_unlock(task);
6398bd74fdaeSYu Zhao }
6399bd74fdaeSYu Zhao #else
mem_cgroup_attach(struct cgroup_taskset * tset)6400bd74fdaeSYu Zhao static void mem_cgroup_attach(struct cgroup_taskset *tset)
6401bd74fdaeSYu Zhao {
6402bd74fdaeSYu Zhao }
6403bd74fdaeSYu Zhao #endif /* CONFIG_LRU_GEN */
6404bd74fdaeSYu Zhao
seq_puts_memcg_tunable(struct seq_file * m,unsigned long value)6405677dc973SChris Down static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value)
6406677dc973SChris Down {
6407677dc973SChris Down if (value == PAGE_COUNTER_MAX)
6408677dc973SChris Down seq_puts(m, "max\n");
6409677dc973SChris Down else
6410677dc973SChris Down seq_printf(m, "%llu\n", (u64)value * PAGE_SIZE);
6411677dc973SChris Down
6412677dc973SChris Down return 0;
6413677dc973SChris Down }
6414677dc973SChris Down
memory_current_read(struct cgroup_subsys_state * css,struct cftype * cft)6415241994edSJohannes Weiner static u64 memory_current_read(struct cgroup_subsys_state *css,
6416241994edSJohannes Weiner struct cftype *cft)
6417241994edSJohannes Weiner {
6418f5fc3c5dSJohannes Weiner struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6419f5fc3c5dSJohannes Weiner
6420f5fc3c5dSJohannes Weiner return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
6421241994edSJohannes Weiner }
6422241994edSJohannes Weiner
memory_peak_read(struct cgroup_subsys_state * css,struct cftype * cft)64238e20d4b3SGanesan Rajagopal static u64 memory_peak_read(struct cgroup_subsys_state *css,
64248e20d4b3SGanesan Rajagopal struct cftype *cft)
64258e20d4b3SGanesan Rajagopal {
64268e20d4b3SGanesan Rajagopal struct mem_cgroup *memcg = mem_cgroup_from_css(css);
64278e20d4b3SGanesan Rajagopal
64288e20d4b3SGanesan Rajagopal return (u64)memcg->memory.watermark * PAGE_SIZE;
64298e20d4b3SGanesan Rajagopal }
64308e20d4b3SGanesan Rajagopal
memory_min_show(struct seq_file * m,void * v)6431bf8d5d52SRoman Gushchin static int memory_min_show(struct seq_file *m, void *v)
6432bf8d5d52SRoman Gushchin {
6433677dc973SChris Down return seq_puts_memcg_tunable(m,
6434677dc973SChris Down READ_ONCE(mem_cgroup_from_seq(m)->memory.min));
6435bf8d5d52SRoman Gushchin }
6436bf8d5d52SRoman Gushchin
memory_min_write(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)6437bf8d5d52SRoman Gushchin static ssize_t memory_min_write(struct kernfs_open_file *of,
6438bf8d5d52SRoman Gushchin char *buf, size_t nbytes, loff_t off)
6439bf8d5d52SRoman Gushchin {
6440bf8d5d52SRoman Gushchin struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6441bf8d5d52SRoman Gushchin unsigned long min;
6442bf8d5d52SRoman Gushchin int err;
6443bf8d5d52SRoman Gushchin
6444bf8d5d52SRoman Gushchin buf = strstrip(buf);
6445bf8d5d52SRoman Gushchin err = page_counter_memparse(buf, "max", &min);
6446bf8d5d52SRoman Gushchin if (err)
6447bf8d5d52SRoman Gushchin return err;
6448bf8d5d52SRoman Gushchin
6449bf8d5d52SRoman Gushchin page_counter_set_min(&memcg->memory, min);
6450bf8d5d52SRoman Gushchin
6451bf8d5d52SRoman Gushchin return nbytes;
6452bf8d5d52SRoman Gushchin }
6453bf8d5d52SRoman Gushchin
memory_low_show(struct seq_file * m,void * v)6454241994edSJohannes Weiner static int memory_low_show(struct seq_file *m, void *v)
6455241994edSJohannes Weiner {
6456677dc973SChris Down return seq_puts_memcg_tunable(m,
6457677dc973SChris Down READ_ONCE(mem_cgroup_from_seq(m)->memory.low));
6458241994edSJohannes Weiner }
6459241994edSJohannes Weiner
memory_low_write(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)6460241994edSJohannes Weiner static ssize_t memory_low_write(struct kernfs_open_file *of,
6461241994edSJohannes Weiner char *buf, size_t nbytes, loff_t off)
6462241994edSJohannes Weiner {
6463241994edSJohannes Weiner struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6464241994edSJohannes Weiner unsigned long low;
6465241994edSJohannes Weiner int err;
6466241994edSJohannes Weiner
6467241994edSJohannes Weiner buf = strstrip(buf);
6468d2973697SJohannes Weiner err = page_counter_memparse(buf, "max", &low);
6469241994edSJohannes Weiner if (err)
6470241994edSJohannes Weiner return err;
6471241994edSJohannes Weiner
647223067153SRoman Gushchin page_counter_set_low(&memcg->memory, low);
6473241994edSJohannes Weiner
6474241994edSJohannes Weiner return nbytes;
6475241994edSJohannes Weiner }
6476241994edSJohannes Weiner
memory_high_show(struct seq_file * m,void * v)6477241994edSJohannes Weiner static int memory_high_show(struct seq_file *m, void *v)
6478241994edSJohannes Weiner {
6479d1663a90SJakub Kicinski return seq_puts_memcg_tunable(m,
6480d1663a90SJakub Kicinski READ_ONCE(mem_cgroup_from_seq(m)->memory.high));
6481241994edSJohannes Weiner }
6482241994edSJohannes Weiner
memory_high_write(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)6483241994edSJohannes Weiner static ssize_t memory_high_write(struct kernfs_open_file *of,
6484241994edSJohannes Weiner char *buf, size_t nbytes, loff_t off)
6485241994edSJohannes Weiner {
6486241994edSJohannes Weiner struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6487d977aa93SChris Down unsigned int nr_retries = MAX_RECLAIM_RETRIES;
64888c8c383cSJohannes Weiner bool drained = false;
6489241994edSJohannes Weiner unsigned long high;
6490241994edSJohannes Weiner int err;
6491241994edSJohannes Weiner
6492241994edSJohannes Weiner buf = strstrip(buf);
6493d2973697SJohannes Weiner err = page_counter_memparse(buf, "max", &high);
6494241994edSJohannes Weiner if (err)
6495241994edSJohannes Weiner return err;
6496241994edSJohannes Weiner
6497e82553c1SJohannes Weiner page_counter_set_high(&memcg->memory, high);
6498e82553c1SJohannes Weiner
64998c8c383cSJohannes Weiner for (;;) {
65008c8c383cSJohannes Weiner unsigned long nr_pages = page_counter_read(&memcg->memory);
65018c8c383cSJohannes Weiner unsigned long reclaimed;
65028c8c383cSJohannes Weiner
65038c8c383cSJohannes Weiner if (nr_pages <= high)
65048c8c383cSJohannes Weiner break;
65058c8c383cSJohannes Weiner
65068c8c383cSJohannes Weiner if (signal_pending(current))
65078c8c383cSJohannes Weiner break;
65088c8c383cSJohannes Weiner
65098c8c383cSJohannes Weiner if (!drained) {
65108c8c383cSJohannes Weiner drain_all_stock(memcg);
65118c8c383cSJohannes Weiner drained = true;
65128c8c383cSJohannes Weiner continue;
65138c8c383cSJohannes Weiner }
65148c8c383cSJohannes Weiner
65158c8c383cSJohannes Weiner reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
651655ab834aSMichal Hocko GFP_KERNEL, MEMCG_RECLAIM_MAY_SWAP);
6517588083bbSJohannes Weiner
65188c8c383cSJohannes Weiner if (!reclaimed && !nr_retries--)
65198c8c383cSJohannes Weiner break;
65208c8c383cSJohannes Weiner }
65218c8c383cSJohannes Weiner
652219ce33acSJohannes Weiner memcg_wb_domain_size_changed(memcg);
6523241994edSJohannes Weiner return nbytes;
6524241994edSJohannes Weiner }
6525241994edSJohannes Weiner
memory_max_show(struct seq_file * m,void * v)6526241994edSJohannes Weiner static int memory_max_show(struct seq_file *m, void *v)
6527241994edSJohannes Weiner {
6528677dc973SChris Down return seq_puts_memcg_tunable(m,
6529677dc973SChris Down READ_ONCE(mem_cgroup_from_seq(m)->memory.max));
6530241994edSJohannes Weiner }
6531241994edSJohannes Weiner
memory_max_write(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)6532241994edSJohannes Weiner static ssize_t memory_max_write(struct kernfs_open_file *of,
6533241994edSJohannes Weiner char *buf, size_t nbytes, loff_t off)
6534241994edSJohannes Weiner {
6535241994edSJohannes Weiner struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6536d977aa93SChris Down unsigned int nr_reclaims = MAX_RECLAIM_RETRIES;
6537b6e6edcfSJohannes Weiner bool drained = false;
6538241994edSJohannes Weiner unsigned long max;
6539241994edSJohannes Weiner int err;
6540241994edSJohannes Weiner
6541241994edSJohannes Weiner buf = strstrip(buf);
6542d2973697SJohannes Weiner err = page_counter_memparse(buf, "max", &max);
6543241994edSJohannes Weiner if (err)
6544241994edSJohannes Weiner return err;
6545241994edSJohannes Weiner
6546bbec2e15SRoman Gushchin xchg(&memcg->memory.max, max);
6547b6e6edcfSJohannes Weiner
6548b6e6edcfSJohannes Weiner for (;;) {
6549b6e6edcfSJohannes Weiner unsigned long nr_pages = page_counter_read(&memcg->memory);
6550b6e6edcfSJohannes Weiner
6551b6e6edcfSJohannes Weiner if (nr_pages <= max)
6552b6e6edcfSJohannes Weiner break;
6553b6e6edcfSJohannes Weiner
65547249c9f0SJohannes Weiner if (signal_pending(current))
6555b6e6edcfSJohannes Weiner break;
6556b6e6edcfSJohannes Weiner
6557b6e6edcfSJohannes Weiner if (!drained) {
6558b6e6edcfSJohannes Weiner drain_all_stock(memcg);
6559b6e6edcfSJohannes Weiner drained = true;
6560b6e6edcfSJohannes Weiner continue;
6561b6e6edcfSJohannes Weiner }
6562b6e6edcfSJohannes Weiner
6563b6e6edcfSJohannes Weiner if (nr_reclaims) {
6564b6e6edcfSJohannes Weiner if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
656555ab834aSMichal Hocko GFP_KERNEL, MEMCG_RECLAIM_MAY_SWAP))
6566b6e6edcfSJohannes Weiner nr_reclaims--;
6567b6e6edcfSJohannes Weiner continue;
6568b6e6edcfSJohannes Weiner }
6569b6e6edcfSJohannes Weiner
6570e27be240SJohannes Weiner memcg_memory_event(memcg, MEMCG_OOM);
6571b6e6edcfSJohannes Weiner if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
6572b6e6edcfSJohannes Weiner break;
6573b6e6edcfSJohannes Weiner }
6574241994edSJohannes Weiner
65752529bb3aSTejun Heo memcg_wb_domain_size_changed(memcg);
6576241994edSJohannes Weiner return nbytes;
6577241994edSJohannes Weiner }
6578241994edSJohannes Weiner
__memory_events_show(struct seq_file * m,atomic_long_t * events)65791e577f97SShakeel Butt static void __memory_events_show(struct seq_file *m, atomic_long_t *events)
65801e577f97SShakeel Butt {
65811e577f97SShakeel Butt seq_printf(m, "low %lu\n", atomic_long_read(&events[MEMCG_LOW]));
65821e577f97SShakeel Butt seq_printf(m, "high %lu\n", atomic_long_read(&events[MEMCG_HIGH]));
65831e577f97SShakeel Butt seq_printf(m, "max %lu\n", atomic_long_read(&events[MEMCG_MAX]));
65841e577f97SShakeel Butt seq_printf(m, "oom %lu\n", atomic_long_read(&events[MEMCG_OOM]));
65851e577f97SShakeel Butt seq_printf(m, "oom_kill %lu\n",
65861e577f97SShakeel Butt atomic_long_read(&events[MEMCG_OOM_KILL]));
6587b6bf9abbSDan Schatzberg seq_printf(m, "oom_group_kill %lu\n",
6588b6bf9abbSDan Schatzberg atomic_long_read(&events[MEMCG_OOM_GROUP_KILL]));
65891e577f97SShakeel Butt }
65901e577f97SShakeel Butt
memory_events_show(struct seq_file * m,void * v)6591241994edSJohannes Weiner static int memory_events_show(struct seq_file *m, void *v)
6592241994edSJohannes Weiner {
6593aa9694bbSChris Down struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6594241994edSJohannes Weiner
65951e577f97SShakeel Butt __memory_events_show(m, memcg->memory_events);
65961e577f97SShakeel Butt return 0;
65971e577f97SShakeel Butt }
6598241994edSJohannes Weiner
memory_events_local_show(struct seq_file * m,void * v)65991e577f97SShakeel Butt static int memory_events_local_show(struct seq_file *m, void *v)
66001e577f97SShakeel Butt {
66011e577f97SShakeel Butt struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
66021e577f97SShakeel Butt
66031e577f97SShakeel Butt __memory_events_show(m, memcg->memory_events_local);
6604241994edSJohannes Weiner return 0;
6605241994edSJohannes Weiner }
6606241994edSJohannes Weiner
memory_stat_show(struct seq_file * m,void * v)6607587d9f72SJohannes Weiner static int memory_stat_show(struct seq_file *m, void *v)
6608587d9f72SJohannes Weiner {
6609aa9694bbSChris Down struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
661068aaee14STetsuo Handa char *buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
66115b42360cSYosry Ahmed struct seq_buf s;
6612587d9f72SJohannes Weiner
6613c8713d0bSJohannes Weiner if (!buf)
6614c8713d0bSJohannes Weiner return -ENOMEM;
66155b42360cSYosry Ahmed seq_buf_init(&s, buf, PAGE_SIZE);
66165b42360cSYosry Ahmed memory_stat_format(memcg, &s);
6617c8713d0bSJohannes Weiner seq_puts(m, buf);
6618c8713d0bSJohannes Weiner kfree(buf);
6619587d9f72SJohannes Weiner return 0;
6620587d9f72SJohannes Weiner }
6621587d9f72SJohannes Weiner
66225f9a4f4aSMuchun Song #ifdef CONFIG_NUMA
lruvec_page_state_output(struct lruvec * lruvec,int item)6623fff66b79SMuchun Song static inline unsigned long lruvec_page_state_output(struct lruvec *lruvec,
6624fff66b79SMuchun Song int item)
6625fff66b79SMuchun Song {
6626fff66b79SMuchun Song return lruvec_page_state(lruvec, item) * memcg_page_state_unit(item);
6627fff66b79SMuchun Song }
6628fff66b79SMuchun Song
memory_numa_stat_show(struct seq_file * m,void * v)66295f9a4f4aSMuchun Song static int memory_numa_stat_show(struct seq_file *m, void *v)
66305f9a4f4aSMuchun Song {
66315f9a4f4aSMuchun Song int i;
66325f9a4f4aSMuchun Song struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
66335f9a4f4aSMuchun Song
6634fd25a9e0SShakeel Butt mem_cgroup_flush_stats();
66357e1c0d6fSShakeel Butt
66365f9a4f4aSMuchun Song for (i = 0; i < ARRAY_SIZE(memory_stats); i++) {
66375f9a4f4aSMuchun Song int nid;
66385f9a4f4aSMuchun Song
66395f9a4f4aSMuchun Song if (memory_stats[i].idx >= NR_VM_NODE_STAT_ITEMS)
66405f9a4f4aSMuchun Song continue;
66415f9a4f4aSMuchun Song
66425f9a4f4aSMuchun Song seq_printf(m, "%s", memory_stats[i].name);
66435f9a4f4aSMuchun Song for_each_node_state(nid, N_MEMORY) {
66445f9a4f4aSMuchun Song u64 size;
66455f9a4f4aSMuchun Song struct lruvec *lruvec;
66465f9a4f4aSMuchun Song
66475f9a4f4aSMuchun Song lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
6648fff66b79SMuchun Song size = lruvec_page_state_output(lruvec,
6649fff66b79SMuchun Song memory_stats[i].idx);
66505f9a4f4aSMuchun Song seq_printf(m, " N%d=%llu", nid, size);
66515f9a4f4aSMuchun Song }
66525f9a4f4aSMuchun Song seq_putc(m, '\n');
66535f9a4f4aSMuchun Song }
66545f9a4f4aSMuchun Song
66555f9a4f4aSMuchun Song return 0;
66565f9a4f4aSMuchun Song }
66575f9a4f4aSMuchun Song #endif
66585f9a4f4aSMuchun Song
memory_oom_group_show(struct seq_file * m,void * v)66593d8b38ebSRoman Gushchin static int memory_oom_group_show(struct seq_file *m, void *v)
66603d8b38ebSRoman Gushchin {
6661aa9694bbSChris Down struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
66623d8b38ebSRoman Gushchin
6663eaf7b66bSYue Zhao seq_printf(m, "%d\n", READ_ONCE(memcg->oom_group));
66643d8b38ebSRoman Gushchin
66653d8b38ebSRoman Gushchin return 0;
66663d8b38ebSRoman Gushchin }
66673d8b38ebSRoman Gushchin
memory_oom_group_write(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)66683d8b38ebSRoman Gushchin static ssize_t memory_oom_group_write(struct kernfs_open_file *of,
66693d8b38ebSRoman Gushchin char *buf, size_t nbytes, loff_t off)
66703d8b38ebSRoman Gushchin {
66713d8b38ebSRoman Gushchin struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
66723d8b38ebSRoman Gushchin int ret, oom_group;
66733d8b38ebSRoman Gushchin
66743d8b38ebSRoman Gushchin buf = strstrip(buf);
66753d8b38ebSRoman Gushchin if (!buf)
66763d8b38ebSRoman Gushchin return -EINVAL;
66773d8b38ebSRoman Gushchin
66783d8b38ebSRoman Gushchin ret = kstrtoint(buf, 0, &oom_group);
66793d8b38ebSRoman Gushchin if (ret)
66803d8b38ebSRoman Gushchin return ret;
66813d8b38ebSRoman Gushchin
66823d8b38ebSRoman Gushchin if (oom_group != 0 && oom_group != 1)
66833d8b38ebSRoman Gushchin return -EINVAL;
66843d8b38ebSRoman Gushchin
6685eaf7b66bSYue Zhao WRITE_ONCE(memcg->oom_group, oom_group);
66863d8b38ebSRoman Gushchin
66873d8b38ebSRoman Gushchin return nbytes;
66883d8b38ebSRoman Gushchin }
66893d8b38ebSRoman Gushchin
memory_reclaim(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)669094968384SShakeel Butt static ssize_t memory_reclaim(struct kernfs_open_file *of, char *buf,
669194968384SShakeel Butt size_t nbytes, loff_t off)
669294968384SShakeel Butt {
669394968384SShakeel Butt struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
669494968384SShakeel Butt unsigned int nr_retries = MAX_RECLAIM_RETRIES;
669594968384SShakeel Butt unsigned long nr_to_reclaim, nr_reclaimed = 0;
669655ab834aSMichal Hocko unsigned int reclaim_options;
669755ab834aSMichal Hocko int err;
669894968384SShakeel Butt
669994968384SShakeel Butt buf = strstrip(buf);
670055ab834aSMichal Hocko err = page_counter_memparse(buf, "", &nr_to_reclaim);
670155ab834aSMichal Hocko if (err)
670255ab834aSMichal Hocko return err;
670394968384SShakeel Butt
670455ab834aSMichal Hocko reclaim_options = MEMCG_RECLAIM_MAY_SWAP | MEMCG_RECLAIM_PROACTIVE;
670594968384SShakeel Butt while (nr_reclaimed < nr_to_reclaim) {
670694968384SShakeel Butt unsigned long reclaimed;
670794968384SShakeel Butt
670894968384SShakeel Butt if (signal_pending(current))
670994968384SShakeel Butt return -EINTR;
671094968384SShakeel Butt
671194968384SShakeel Butt /*
671294968384SShakeel Butt * This is the final attempt, drain percpu lru caches in the
671394968384SShakeel Butt * hope of introducing more evictable pages for
671494968384SShakeel Butt * try_to_free_mem_cgroup_pages().
671594968384SShakeel Butt */
671694968384SShakeel Butt if (!nr_retries)
671794968384SShakeel Butt lru_add_drain_all();
671894968384SShakeel Butt
671994968384SShakeel Butt reclaimed = try_to_free_mem_cgroup_pages(memcg,
67200388536aSEfly Young min(nr_to_reclaim - nr_reclaimed, SWAP_CLUSTER_MAX),
672155ab834aSMichal Hocko GFP_KERNEL, reclaim_options);
672294968384SShakeel Butt
672394968384SShakeel Butt if (!reclaimed && !nr_retries--)
672494968384SShakeel Butt return -EAGAIN;
672594968384SShakeel Butt
672694968384SShakeel Butt nr_reclaimed += reclaimed;
672794968384SShakeel Butt }
672894968384SShakeel Butt
672994968384SShakeel Butt return nbytes;
673094968384SShakeel Butt }
673194968384SShakeel Butt
6732241994edSJohannes Weiner static struct cftype memory_files[] = {
6733241994edSJohannes Weiner {
6734241994edSJohannes Weiner .name = "current",
6735f5fc3c5dSJohannes Weiner .flags = CFTYPE_NOT_ON_ROOT,
6736241994edSJohannes Weiner .read_u64 = memory_current_read,
6737241994edSJohannes Weiner },
6738241994edSJohannes Weiner {
67398e20d4b3SGanesan Rajagopal .name = "peak",
67408e20d4b3SGanesan Rajagopal .flags = CFTYPE_NOT_ON_ROOT,
67418e20d4b3SGanesan Rajagopal .read_u64 = memory_peak_read,
67428e20d4b3SGanesan Rajagopal },
67438e20d4b3SGanesan Rajagopal {
6744bf8d5d52SRoman Gushchin .name = "min",
6745bf8d5d52SRoman Gushchin .flags = CFTYPE_NOT_ON_ROOT,
6746bf8d5d52SRoman Gushchin .seq_show = memory_min_show,
6747bf8d5d52SRoman Gushchin .write = memory_min_write,
6748bf8d5d52SRoman Gushchin },
6749bf8d5d52SRoman Gushchin {
6750241994edSJohannes Weiner .name = "low",
6751241994edSJohannes Weiner .flags = CFTYPE_NOT_ON_ROOT,
6752241994edSJohannes Weiner .seq_show = memory_low_show,
6753241994edSJohannes Weiner .write = memory_low_write,
6754241994edSJohannes Weiner },
6755241994edSJohannes Weiner {
6756241994edSJohannes Weiner .name = "high",
6757241994edSJohannes Weiner .flags = CFTYPE_NOT_ON_ROOT,
6758241994edSJohannes Weiner .seq_show = memory_high_show,
6759241994edSJohannes Weiner .write = memory_high_write,
6760241994edSJohannes Weiner },
6761241994edSJohannes Weiner {
6762241994edSJohannes Weiner .name = "max",
6763241994edSJohannes Weiner .flags = CFTYPE_NOT_ON_ROOT,
6764241994edSJohannes Weiner .seq_show = memory_max_show,
6765241994edSJohannes Weiner .write = memory_max_write,
6766241994edSJohannes Weiner },
6767241994edSJohannes Weiner {
6768241994edSJohannes Weiner .name = "events",
6769241994edSJohannes Weiner .flags = CFTYPE_NOT_ON_ROOT,
6770472912a2STejun Heo .file_offset = offsetof(struct mem_cgroup, events_file),
6771241994edSJohannes Weiner .seq_show = memory_events_show,
6772241994edSJohannes Weiner },
6773587d9f72SJohannes Weiner {
67741e577f97SShakeel Butt .name = "events.local",
67751e577f97SShakeel Butt .flags = CFTYPE_NOT_ON_ROOT,
67761e577f97SShakeel Butt .file_offset = offsetof(struct mem_cgroup, events_local_file),
67771e577f97SShakeel Butt .seq_show = memory_events_local_show,
67781e577f97SShakeel Butt },
67791e577f97SShakeel Butt {
6780587d9f72SJohannes Weiner .name = "stat",
6781587d9f72SJohannes Weiner .seq_show = memory_stat_show,
6782587d9f72SJohannes Weiner },
67835f9a4f4aSMuchun Song #ifdef CONFIG_NUMA
67845f9a4f4aSMuchun Song {
67855f9a4f4aSMuchun Song .name = "numa_stat",
67865f9a4f4aSMuchun Song .seq_show = memory_numa_stat_show,
67875f9a4f4aSMuchun Song },
67885f9a4f4aSMuchun Song #endif
67893d8b38ebSRoman Gushchin {
67903d8b38ebSRoman Gushchin .name = "oom.group",
67913d8b38ebSRoman Gushchin .flags = CFTYPE_NOT_ON_ROOT | CFTYPE_NS_DELEGATABLE,
67923d8b38ebSRoman Gushchin .seq_show = memory_oom_group_show,
67933d8b38ebSRoman Gushchin .write = memory_oom_group_write,
67943d8b38ebSRoman Gushchin },
679594968384SShakeel Butt {
679694968384SShakeel Butt .name = "reclaim",
679794968384SShakeel Butt .flags = CFTYPE_NS_DELEGATABLE,
679894968384SShakeel Butt .write = memory_reclaim,
679994968384SShakeel Butt },
6800241994edSJohannes Weiner { } /* terminate */
6801241994edSJohannes Weiner };
6802241994edSJohannes Weiner
6803073219e9STejun Heo struct cgroup_subsys memory_cgrp_subsys = {
680492fb9748STejun Heo .css_alloc = mem_cgroup_css_alloc,
6805d142e3e6SGlauber Costa .css_online = mem_cgroup_css_online,
680692fb9748STejun Heo .css_offline = mem_cgroup_css_offline,
68076df38689SVladimir Davydov .css_released = mem_cgroup_css_released,
680892fb9748STejun Heo .css_free = mem_cgroup_css_free,
68091ced953bSTejun Heo .css_reset = mem_cgroup_css_reset,
68102d146aa3SJohannes Weiner .css_rstat_flush = mem_cgroup_css_rstat_flush,
68117dc74be0SDaisuke Nishimura .can_attach = mem_cgroup_can_attach,
6812bd74fdaeSYu Zhao .attach = mem_cgroup_attach,
68137dc74be0SDaisuke Nishimura .cancel_attach = mem_cgroup_cancel_attach,
6814264a0ae1STejun Heo .post_attach = mem_cgroup_move_task,
6815241994edSJohannes Weiner .dfl_cftypes = memory_files,
6816241994edSJohannes Weiner .legacy_cftypes = mem_cgroup_legacy_files,
68176d12e2d8SKAMEZAWA Hiroyuki .early_init = 0,
68188cdea7c0SBalbir Singh };
6819c077719bSKAMEZAWA Hiroyuki
6820bc50bcc6SJohannes Weiner /*
6821bc50bcc6SJohannes Weiner * This function calculates an individual cgroup's effective
6822bc50bcc6SJohannes Weiner * protection which is derived from its own memory.min/low, its
6823bc50bcc6SJohannes Weiner * parent's and siblings' settings, as well as the actual memory
6824bc50bcc6SJohannes Weiner * distribution in the tree.
6825bc50bcc6SJohannes Weiner *
6826bc50bcc6SJohannes Weiner * The following rules apply to the effective protection values:
6827bc50bcc6SJohannes Weiner *
6828bc50bcc6SJohannes Weiner * 1. At the first level of reclaim, effective protection is equal to
6829bc50bcc6SJohannes Weiner * the declared protection in memory.min and memory.low.
6830bc50bcc6SJohannes Weiner *
6831bc50bcc6SJohannes Weiner * 2. To enable safe delegation of the protection configuration, at
6832bc50bcc6SJohannes Weiner * subsequent levels the effective protection is capped to the
6833bc50bcc6SJohannes Weiner * parent's effective protection.
6834bc50bcc6SJohannes Weiner *
6835bc50bcc6SJohannes Weiner * 3. To make complex and dynamic subtrees easier to configure, the
6836bc50bcc6SJohannes Weiner * user is allowed to overcommit the declared protection at a given
6837bc50bcc6SJohannes Weiner * level. If that is the case, the parent's effective protection is
6838bc50bcc6SJohannes Weiner * distributed to the children in proportion to how much protection
6839bc50bcc6SJohannes Weiner * they have declared and how much of it they are utilizing.
6840bc50bcc6SJohannes Weiner *
6841bc50bcc6SJohannes Weiner * This makes distribution proportional, but also work-conserving:
6842bc50bcc6SJohannes Weiner * if one cgroup claims much more protection than it uses memory,
6843bc50bcc6SJohannes Weiner * the unused remainder is available to its siblings.
6844bc50bcc6SJohannes Weiner *
6845bc50bcc6SJohannes Weiner * 4. Conversely, when the declared protection is undercommitted at a
6846bc50bcc6SJohannes Weiner * given level, the distribution of the larger parental protection
6847bc50bcc6SJohannes Weiner * budget is NOT proportional. A cgroup's protection from a sibling
6848bc50bcc6SJohannes Weiner * is capped to its own memory.min/low setting.
6849bc50bcc6SJohannes Weiner *
68508a931f80SJohannes Weiner * 5. However, to allow protecting recursive subtrees from each other
68518a931f80SJohannes Weiner * without having to declare each individual cgroup's fixed share
68528a931f80SJohannes Weiner * of the ancestor's claim to protection, any unutilized -
68538a931f80SJohannes Weiner * "floating" - protection from up the tree is distributed in
68548a931f80SJohannes Weiner * proportion to each cgroup's *usage*. This makes the protection
68558a931f80SJohannes Weiner * neutral wrt sibling cgroups and lets them compete freely over
68568a931f80SJohannes Weiner * the shared parental protection budget, but it protects the
68578a931f80SJohannes Weiner * subtree as a whole from neighboring subtrees.
68588a931f80SJohannes Weiner *
68598a931f80SJohannes Weiner * Note that 4. and 5. are not in conflict: 4. is about protecting
68608a931f80SJohannes Weiner * against immediate siblings whereas 5. is about protecting against
68618a931f80SJohannes Weiner * neighboring subtrees.
6862bc50bcc6SJohannes Weiner */
effective_protection(unsigned long usage,unsigned long parent_usage,unsigned long setting,unsigned long parent_effective,unsigned long siblings_protected)6863bc50bcc6SJohannes Weiner static unsigned long effective_protection(unsigned long usage,
68648a931f80SJohannes Weiner unsigned long parent_usage,
6865bc50bcc6SJohannes Weiner unsigned long setting,
6866bc50bcc6SJohannes Weiner unsigned long parent_effective,
6867bc50bcc6SJohannes Weiner unsigned long siblings_protected)
6868bc50bcc6SJohannes Weiner {
6869bc50bcc6SJohannes Weiner unsigned long protected;
68708a931f80SJohannes Weiner unsigned long ep;
6871bc50bcc6SJohannes Weiner
6872bc50bcc6SJohannes Weiner protected = min(usage, setting);
6873bc50bcc6SJohannes Weiner /*
6874bc50bcc6SJohannes Weiner * If all cgroups at this level combined claim and use more
687508e0f49eSHaifeng Xu * protection than what the parent affords them, distribute
6876bc50bcc6SJohannes Weiner * shares in proportion to utilization.
6877bc50bcc6SJohannes Weiner *
6878bc50bcc6SJohannes Weiner * We are using actual utilization rather than the statically
6879bc50bcc6SJohannes Weiner * claimed protection in order to be work-conserving: claimed
6880bc50bcc6SJohannes Weiner * but unused protection is available to siblings that would
6881bc50bcc6SJohannes Weiner * otherwise get a smaller chunk than what they claimed.
6882bc50bcc6SJohannes Weiner */
6883bc50bcc6SJohannes Weiner if (siblings_protected > parent_effective)
6884bc50bcc6SJohannes Weiner return protected * parent_effective / siblings_protected;
6885bc50bcc6SJohannes Weiner
6886bc50bcc6SJohannes Weiner /*
6887bc50bcc6SJohannes Weiner * Ok, utilized protection of all children is within what the
6888bc50bcc6SJohannes Weiner * parent affords them, so we know whatever this child claims
6889bc50bcc6SJohannes Weiner * and utilizes is effectively protected.
6890bc50bcc6SJohannes Weiner *
6891bc50bcc6SJohannes Weiner * If there is unprotected usage beyond this value, reclaim
6892bc50bcc6SJohannes Weiner * will apply pressure in proportion to that amount.
6893bc50bcc6SJohannes Weiner *
6894bc50bcc6SJohannes Weiner * If there is unutilized protection, the cgroup will be fully
6895bc50bcc6SJohannes Weiner * shielded from reclaim, but we do return a smaller value for
6896bc50bcc6SJohannes Weiner * protection than what the group could enjoy in theory. This
6897bc50bcc6SJohannes Weiner * is okay. With the overcommit distribution above, effective
6898bc50bcc6SJohannes Weiner * protection is always dependent on how memory is actually
6899bc50bcc6SJohannes Weiner * consumed among the siblings anyway.
6900bc50bcc6SJohannes Weiner */
69018a931f80SJohannes Weiner ep = protected;
69028a931f80SJohannes Weiner
69038a931f80SJohannes Weiner /*
69048a931f80SJohannes Weiner * If the children aren't claiming (all of) the protection
69058a931f80SJohannes Weiner * afforded to them by the parent, distribute the remainder in
69068a931f80SJohannes Weiner * proportion to the (unprotected) memory of each cgroup. That
69078a931f80SJohannes Weiner * way, cgroups that aren't explicitly prioritized wrt each
69088a931f80SJohannes Weiner * other compete freely over the allowance, but they are
69098a931f80SJohannes Weiner * collectively protected from neighboring trees.
69108a931f80SJohannes Weiner *
69118a931f80SJohannes Weiner * We're using unprotected memory for the weight so that if
69128a931f80SJohannes Weiner * some cgroups DO claim explicit protection, we don't protect
69138a931f80SJohannes Weiner * the same bytes twice.
6914cd324edcSJohannes Weiner *
6915cd324edcSJohannes Weiner * Check both usage and parent_usage against the respective
6916cd324edcSJohannes Weiner * protected values. One should imply the other, but they
6917cd324edcSJohannes Weiner * aren't read atomically - make sure the division is sane.
69188a931f80SJohannes Weiner */
69198a931f80SJohannes Weiner if (!(cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT))
69208a931f80SJohannes Weiner return ep;
6921cd324edcSJohannes Weiner if (parent_effective > siblings_protected &&
6922cd324edcSJohannes Weiner parent_usage > siblings_protected &&
6923cd324edcSJohannes Weiner usage > protected) {
69248a931f80SJohannes Weiner unsigned long unclaimed;
69258a931f80SJohannes Weiner
69268a931f80SJohannes Weiner unclaimed = parent_effective - siblings_protected;
69278a931f80SJohannes Weiner unclaimed *= usage - protected;
69288a931f80SJohannes Weiner unclaimed /= parent_usage - siblings_protected;
69298a931f80SJohannes Weiner
69308a931f80SJohannes Weiner ep += unclaimed;
69318a931f80SJohannes Weiner }
69328a931f80SJohannes Weiner
69338a931f80SJohannes Weiner return ep;
6934bc50bcc6SJohannes Weiner }
6935bc50bcc6SJohannes Weiner
6936241994edSJohannes Weiner /**
693705395718SMel Gorman * mem_cgroup_calculate_protection - check if memory consumption is in the normal range
693834c81057SSean Christopherson * @root: the top ancestor of the sub-tree being checked
6939241994edSJohannes Weiner * @memcg: the memory cgroup to check
6940241994edSJohannes Weiner *
694123067153SRoman Gushchin * WARNING: This function is not stateless! It can only be used as part
694223067153SRoman Gushchin * of a top-down tree iteration, not for isolated queries.
6943241994edSJohannes Weiner */
mem_cgroup_calculate_protection(struct mem_cgroup * root,struct mem_cgroup * memcg)694445c7f7e1SChris Down void mem_cgroup_calculate_protection(struct mem_cgroup *root,
6945bf8d5d52SRoman Gushchin struct mem_cgroup *memcg)
6946241994edSJohannes Weiner {
69478a931f80SJohannes Weiner unsigned long usage, parent_usage;
694823067153SRoman Gushchin struct mem_cgroup *parent;
694923067153SRoman Gushchin
6950241994edSJohannes Weiner if (mem_cgroup_disabled())
695145c7f7e1SChris Down return;
6952241994edSJohannes Weiner
695334c81057SSean Christopherson if (!root)
695434c81057SSean Christopherson root = root_mem_cgroup;
695522f7496fSYafang Shao
695622f7496fSYafang Shao /*
695722f7496fSYafang Shao * Effective values of the reclaim targets are ignored so they
695822f7496fSYafang Shao * can be stale. Have a look at mem_cgroup_protection for more
695922f7496fSYafang Shao * details.
696022f7496fSYafang Shao * TODO: calculation should be more robust so that we do not need
696122f7496fSYafang Shao * that special casing.
696222f7496fSYafang Shao */
696334c81057SSean Christopherson if (memcg == root)
696445c7f7e1SChris Down return;
6965241994edSJohannes Weiner
696623067153SRoman Gushchin usage = page_counter_read(&memcg->memory);
6967bf8d5d52SRoman Gushchin if (!usage)
696845c7f7e1SChris Down return;
696934c81057SSean Christopherson
6970bf8d5d52SRoman Gushchin parent = parent_mem_cgroup(memcg);
6971df2a4196SRoman Gushchin
6972bc50bcc6SJohannes Weiner if (parent == root) {
6973c3d53200SChris Down memcg->memory.emin = READ_ONCE(memcg->memory.min);
697403960e33SChris Down memcg->memory.elow = READ_ONCE(memcg->memory.low);
697545c7f7e1SChris Down return;
6976bf8d5d52SRoman Gushchin }
6977bf8d5d52SRoman Gushchin
69788a931f80SJohannes Weiner parent_usage = page_counter_read(&parent->memory);
69798a931f80SJohannes Weiner
6980b3a7822eSChris Down WRITE_ONCE(memcg->memory.emin, effective_protection(usage, parent_usage,
6981c3d53200SChris Down READ_ONCE(memcg->memory.min),
6982c3d53200SChris Down READ_ONCE(parent->memory.emin),
6983b3a7822eSChris Down atomic_long_read(&parent->memory.children_min_usage)));
698423067153SRoman Gushchin
6985b3a7822eSChris Down WRITE_ONCE(memcg->memory.elow, effective_protection(usage, parent_usage,
698603960e33SChris Down READ_ONCE(memcg->memory.low),
698703960e33SChris Down READ_ONCE(parent->memory.elow),
6988b3a7822eSChris Down atomic_long_read(&parent->memory.children_low_usage)));
6989241994edSJohannes Weiner }
6990241994edSJohannes Weiner
charge_memcg(struct folio * folio,struct mem_cgroup * memcg,gfp_t gfp)69918f425e4eSMatthew Wilcox (Oracle) static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg,
69928f425e4eSMatthew Wilcox (Oracle) gfp_t gfp)
69930add0c77SShakeel Butt {
6994118f2875SMatthew Wilcox (Oracle) long nr_pages = folio_nr_pages(folio);
69950add0c77SShakeel Butt int ret;
69960add0c77SShakeel Butt
69970add0c77SShakeel Butt ret = try_charge(memcg, gfp, nr_pages);
69980add0c77SShakeel Butt if (ret)
69990add0c77SShakeel Butt goto out;
70000add0c77SShakeel Butt
70010add0c77SShakeel Butt css_get(&memcg->css);
7002118f2875SMatthew Wilcox (Oracle) commit_charge(folio, memcg);
70030add0c77SShakeel Butt
70040add0c77SShakeel Butt local_irq_disable();
70056e0110c2SMatthew Wilcox (Oracle) mem_cgroup_charge_statistics(memcg, nr_pages);
70068f425e4eSMatthew Wilcox (Oracle) memcg_check_events(memcg, folio_nid(folio));
70070add0c77SShakeel Butt local_irq_enable();
70080add0c77SShakeel Butt out:
70090add0c77SShakeel Butt return ret;
70100add0c77SShakeel Butt }
70110add0c77SShakeel Butt
__mem_cgroup_charge(struct folio * folio,struct mm_struct * mm,gfp_t gfp)70128f425e4eSMatthew Wilcox (Oracle) int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp)
701300501b53SJohannes Weiner {
70140add0c77SShakeel Butt struct mem_cgroup *memcg;
70150add0c77SShakeel Butt int ret;
701600501b53SJohannes Weiner
70170add0c77SShakeel Butt memcg = get_mem_cgroup_from_mm(mm);
70188f425e4eSMatthew Wilcox (Oracle) ret = charge_memcg(folio, memcg, gfp);
70190add0c77SShakeel Butt css_put(&memcg->css);
70202d1c4980SJohannes Weiner
70210add0c77SShakeel Butt return ret;
7022e993d905SVladimir Davydov }
702300501b53SJohannes Weiner
70240add0c77SShakeel Butt /**
702565995918SMatthew Wilcox (Oracle) * mem_cgroup_swapin_charge_folio - Charge a newly allocated folio for swapin.
702665995918SMatthew Wilcox (Oracle) * @folio: folio to charge.
70270add0c77SShakeel Butt * @mm: mm context of the victim
70280add0c77SShakeel Butt * @gfp: reclaim mode
702965995918SMatthew Wilcox (Oracle) * @entry: swap entry for which the folio is allocated
70300add0c77SShakeel Butt *
703165995918SMatthew Wilcox (Oracle) * This function charges a folio allocated for swapin. Please call this before
703265995918SMatthew Wilcox (Oracle) * adding the folio to the swapcache.
70330add0c77SShakeel Butt *
70340add0c77SShakeel Butt * Returns 0 on success. Otherwise, an error code is returned.
70350add0c77SShakeel Butt */
mem_cgroup_swapin_charge_folio(struct folio * folio,struct mm_struct * mm,gfp_t gfp,swp_entry_t entry)703665995918SMatthew Wilcox (Oracle) int mem_cgroup_swapin_charge_folio(struct folio *folio, struct mm_struct *mm,
70370add0c77SShakeel Butt gfp_t gfp, swp_entry_t entry)
70380add0c77SShakeel Butt {
70390add0c77SShakeel Butt struct mem_cgroup *memcg;
70400add0c77SShakeel Butt unsigned short id;
70410add0c77SShakeel Butt int ret;
70420add0c77SShakeel Butt
70430add0c77SShakeel Butt if (mem_cgroup_disabled())
70440add0c77SShakeel Butt return 0;
70450add0c77SShakeel Butt
70460add0c77SShakeel Butt id = lookup_swap_cgroup_id(entry);
70470add0c77SShakeel Butt rcu_read_lock();
70480add0c77SShakeel Butt memcg = mem_cgroup_from_id(id);
70490add0c77SShakeel Butt if (!memcg || !css_tryget_online(&memcg->css))
705000501b53SJohannes Weiner memcg = get_mem_cgroup_from_mm(mm);
70510add0c77SShakeel Butt rcu_read_unlock();
705200501b53SJohannes Weiner
70538f425e4eSMatthew Wilcox (Oracle) ret = charge_memcg(folio, memcg, gfp);
705400501b53SJohannes Weiner
70550add0c77SShakeel Butt css_put(&memcg->css);
70560add0c77SShakeel Butt return ret;
70570add0c77SShakeel Butt }
70586abb5a86SJohannes Weiner
70590add0c77SShakeel Butt /*
70600add0c77SShakeel Butt * mem_cgroup_swapin_uncharge_swap - uncharge swap slot
70610add0c77SShakeel Butt * @entry: swap entry for which the page is charged
70620add0c77SShakeel Butt *
70630add0c77SShakeel Butt * Call this function after successfully adding the charged page to swapcache.
70640add0c77SShakeel Butt *
70650add0c77SShakeel Butt * Note: This function assumes the page for which swap slot is being uncharged
70660add0c77SShakeel Butt * is order 0 page.
70670add0c77SShakeel Butt */
mem_cgroup_swapin_uncharge_swap(swp_entry_t entry)70680add0c77SShakeel Butt void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry)
70690add0c77SShakeel Butt {
7070cae3af62SMuchun Song /*
7071cae3af62SMuchun Song * Cgroup1's unified memory+swap counter has been charged with the
7072cae3af62SMuchun Song * new swapcache page, finish the transfer by uncharging the swap
7073cae3af62SMuchun Song * slot. The swap slot would also get uncharged when it dies, but
7074cae3af62SMuchun Song * it can stick around indefinitely and we'd count the page twice
7075cae3af62SMuchun Song * the entire time.
7076cae3af62SMuchun Song *
7077cae3af62SMuchun Song * Cgroup2 has separate resource counters for memory and swap,
7078cae3af62SMuchun Song * so this is a non-issue here. Memory and swap charge lifetimes
7079cae3af62SMuchun Song * correspond 1:1 to page and swap slot lifetimes: we charge the
7080cae3af62SMuchun Song * page to memory here, and uncharge swap when the slot is freed.
7081cae3af62SMuchun Song */
70820add0c77SShakeel Butt if (!mem_cgroup_disabled() && do_memsw_account()) {
708300501b53SJohannes Weiner /*
708400501b53SJohannes Weiner * The swap entry might not get freed for a long time,
708500501b53SJohannes Weiner * let's not wait for it. The page already received a
708600501b53SJohannes Weiner * memory+swap charge, drop the swap entry duplicate.
708700501b53SJohannes Weiner */
70880add0c77SShakeel Butt mem_cgroup_uncharge_swap(entry, 1);
708900501b53SJohannes Weiner }
70903fea5a49SJohannes Weiner }
70913fea5a49SJohannes Weiner
7092a9d5adeeSJérôme Glisse struct uncharge_gather {
7093a9d5adeeSJérôme Glisse struct mem_cgroup *memcg;
7094b4e0b68fSMuchun Song unsigned long nr_memory;
7095a9d5adeeSJérôme Glisse unsigned long pgpgout;
7096a9d5adeeSJérôme Glisse unsigned long nr_kmem;
70978e88bd2dSMatthew Wilcox (Oracle) int nid;
7098a9d5adeeSJérôme Glisse };
7099a9d5adeeSJérôme Glisse
uncharge_gather_clear(struct uncharge_gather * ug)7100a9d5adeeSJérôme Glisse static inline void uncharge_gather_clear(struct uncharge_gather *ug)
7101747db954SJohannes Weiner {
7102a9d5adeeSJérôme Glisse memset(ug, 0, sizeof(*ug));
7103a9d5adeeSJérôme Glisse }
7104a9d5adeeSJérôme Glisse
uncharge_batch(const struct uncharge_gather * ug)7105a9d5adeeSJérôme Glisse static void uncharge_batch(const struct uncharge_gather *ug)
7106a9d5adeeSJérôme Glisse {
7107747db954SJohannes Weiner unsigned long flags;
7108747db954SJohannes Weiner
7109b4e0b68fSMuchun Song if (ug->nr_memory) {
7110b4e0b68fSMuchun Song page_counter_uncharge(&ug->memcg->memory, ug->nr_memory);
71117941d214SJohannes Weiner if (do_memsw_account())
7112b4e0b68fSMuchun Song page_counter_uncharge(&ug->memcg->memsw, ug->nr_memory);
7113a8c49af3SYosry Ahmed if (ug->nr_kmem)
7114a8c49af3SYosry Ahmed memcg_account_kmem(ug->memcg, -ug->nr_kmem);
7115a9d5adeeSJérôme Glisse memcg_oom_recover(ug->memcg);
7116ce00a967SJohannes Weiner }
7117747db954SJohannes Weiner
7118747db954SJohannes Weiner local_irq_save(flags);
7119c9019e9bSJohannes Weiner __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);
7120b4e0b68fSMuchun Song __this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_memory);
71218e88bd2dSMatthew Wilcox (Oracle) memcg_check_events(ug->memcg, ug->nid);
7122747db954SJohannes Weiner local_irq_restore(flags);
7123f1796544SMichal Hocko
7124c4ed6ebfSMatthew Wilcox (Oracle) /* drop reference from uncharge_folio */
7125f1796544SMichal Hocko css_put(&ug->memcg->css);
7126747db954SJohannes Weiner }
7127747db954SJohannes Weiner
uncharge_folio(struct folio * folio,struct uncharge_gather * ug)7128c4ed6ebfSMatthew Wilcox (Oracle) static void uncharge_folio(struct folio *folio, struct uncharge_gather *ug)
7129747db954SJohannes Weiner {
7130c4ed6ebfSMatthew Wilcox (Oracle) long nr_pages;
7131b4e0b68fSMuchun Song struct mem_cgroup *memcg;
7132b4e0b68fSMuchun Song struct obj_cgroup *objcg;
71339f762dbeSJohannes Weiner
7134c4ed6ebfSMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
7135747db954SJohannes Weiner
7136747db954SJohannes Weiner /*
7137747db954SJohannes Weiner * Nobody should be changing or seriously looking at
7138c4ed6ebfSMatthew Wilcox (Oracle) * folio memcg or objcg at this point, we have fully
7139c4ed6ebfSMatthew Wilcox (Oracle) * exclusive access to the folio.
7140747db954SJohannes Weiner */
7141fead2b86SMichal Hocko if (folio_memcg_kmem(folio)) {
71421b7e4464SMatthew Wilcox (Oracle) objcg = __folio_objcg(folio);
7143b4e0b68fSMuchun Song /*
7144b4e0b68fSMuchun Song * This get matches the put at the end of the function and
7145b4e0b68fSMuchun Song * kmem pages do not hold memcg references anymore.
7146b4e0b68fSMuchun Song */
7147b4e0b68fSMuchun Song memcg = get_mem_cgroup_from_objcg(objcg);
7148b4e0b68fSMuchun Song } else {
71491b7e4464SMatthew Wilcox (Oracle) memcg = __folio_memcg(folio);
7150b4e0b68fSMuchun Song }
7151747db954SJohannes Weiner
7152b4e0b68fSMuchun Song if (!memcg)
7153b4e0b68fSMuchun Song return;
7154b4e0b68fSMuchun Song
7155b4e0b68fSMuchun Song if (ug->memcg != memcg) {
7156a9d5adeeSJérôme Glisse if (ug->memcg) {
7157a9d5adeeSJérôme Glisse uncharge_batch(ug);
7158a9d5adeeSJérôme Glisse uncharge_gather_clear(ug);
7159747db954SJohannes Weiner }
7160b4e0b68fSMuchun Song ug->memcg = memcg;
7161c4ed6ebfSMatthew Wilcox (Oracle) ug->nid = folio_nid(folio);
7162f1796544SMichal Hocko
7163f1796544SMichal Hocko /* pairs with css_put in uncharge_batch */
7164b4e0b68fSMuchun Song css_get(&memcg->css);
7165747db954SJohannes Weiner }
7166747db954SJohannes Weiner
7167c4ed6ebfSMatthew Wilcox (Oracle) nr_pages = folio_nr_pages(folio);
71689f762dbeSJohannes Weiner
7169fead2b86SMichal Hocko if (folio_memcg_kmem(folio)) {
7170b4e0b68fSMuchun Song ug->nr_memory += nr_pages;
71719f762dbeSJohannes Weiner ug->nr_kmem += nr_pages;
7172b4e0b68fSMuchun Song
7173c4ed6ebfSMatthew Wilcox (Oracle) folio->memcg_data = 0;
7174b4e0b68fSMuchun Song obj_cgroup_put(objcg);
7175b4e0b68fSMuchun Song } else {
7176b4e0b68fSMuchun Song /* LRU pages aren't accounted at the root level */
7177b4e0b68fSMuchun Song if (!mem_cgroup_is_root(memcg))
7178b4e0b68fSMuchun Song ug->nr_memory += nr_pages;
717918b2db3bSRoman Gushchin ug->pgpgout++;
7180747db954SJohannes Weiner
7181c4ed6ebfSMatthew Wilcox (Oracle) folio->memcg_data = 0;
7182b4e0b68fSMuchun Song }
7183b4e0b68fSMuchun Song
7184b4e0b68fSMuchun Song css_put(&memcg->css);
7185a9d5adeeSJérôme Glisse }
7186a9d5adeeSJérôme Glisse
__mem_cgroup_uncharge(struct folio * folio)7187bbc6b703SMatthew Wilcox (Oracle) void __mem_cgroup_uncharge(struct folio *folio)
71880a31bc97SJohannes Weiner {
7189a9d5adeeSJérôme Glisse struct uncharge_gather ug;
7190a9d5adeeSJérôme Glisse
7191bbc6b703SMatthew Wilcox (Oracle) /* Don't touch folio->lru of any random page, pre-check: */
7192bbc6b703SMatthew Wilcox (Oracle) if (!folio_memcg(folio))
71930a31bc97SJohannes Weiner return;
71940a31bc97SJohannes Weiner
7195a9d5adeeSJérôme Glisse uncharge_gather_clear(&ug);
7196bbc6b703SMatthew Wilcox (Oracle) uncharge_folio(folio, &ug);
7197a9d5adeeSJérôme Glisse uncharge_batch(&ug);
71980a31bc97SJohannes Weiner }
7199747db954SJohannes Weiner
7200747db954SJohannes Weiner /**
72012c8d8f97SSuren Baghdasaryan * __mem_cgroup_uncharge_list - uncharge a list of page
7202747db954SJohannes Weiner * @page_list: list of pages to uncharge
7203747db954SJohannes Weiner *
7204747db954SJohannes Weiner * Uncharge a list of pages previously charged with
72052c8d8f97SSuren Baghdasaryan * __mem_cgroup_charge().
72060a31bc97SJohannes Weiner */
__mem_cgroup_uncharge_list(struct list_head * page_list)72072c8d8f97SSuren Baghdasaryan void __mem_cgroup_uncharge_list(struct list_head *page_list)
7208747db954SJohannes Weiner {
7209c41a40b6SMuchun Song struct uncharge_gather ug;
7210c4ed6ebfSMatthew Wilcox (Oracle) struct folio *folio;
7211c41a40b6SMuchun Song
7212c41a40b6SMuchun Song uncharge_gather_clear(&ug);
7213c4ed6ebfSMatthew Wilcox (Oracle) list_for_each_entry(folio, page_list, lru)
7214c4ed6ebfSMatthew Wilcox (Oracle) uncharge_folio(folio, &ug);
7215c41a40b6SMuchun Song if (ug.memcg)
7216c41a40b6SMuchun Song uncharge_batch(&ug);
72170a31bc97SJohannes Weiner }
72180a31bc97SJohannes Weiner
72190a31bc97SJohannes Weiner /**
7220d21bba2bSMatthew Wilcox (Oracle) * mem_cgroup_migrate - Charge a folio's replacement.
7221d21bba2bSMatthew Wilcox (Oracle) * @old: Currently circulating folio.
7222d21bba2bSMatthew Wilcox (Oracle) * @new: Replacement folio.
72230a31bc97SJohannes Weiner *
7224d21bba2bSMatthew Wilcox (Oracle) * Charge @new as a replacement folio for @old. @old will
72256a93ca8fSJohannes Weiner * be uncharged upon free.
72260a31bc97SJohannes Weiner *
7227d21bba2bSMatthew Wilcox (Oracle) * Both folios must be locked, @new->mapping must be set up.
72280a31bc97SJohannes Weiner */
mem_cgroup_migrate(struct folio * old,struct folio * new)7229d21bba2bSMatthew Wilcox (Oracle) void mem_cgroup_migrate(struct folio *old, struct folio *new)
72300a31bc97SJohannes Weiner {
723129833315SJohannes Weiner struct mem_cgroup *memcg;
7232d21bba2bSMatthew Wilcox (Oracle) long nr_pages = folio_nr_pages(new);
7233d93c4130STejun Heo unsigned long flags;
72340a31bc97SJohannes Weiner
7235d21bba2bSMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(!folio_test_locked(old), old);
7236d21bba2bSMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(!folio_test_locked(new), new);
7237d21bba2bSMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(folio_test_anon(old) != folio_test_anon(new), new);
7238d21bba2bSMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(folio_nr_pages(old) != nr_pages, new);
72390a31bc97SJohannes Weiner
72400a31bc97SJohannes Weiner if (mem_cgroup_disabled())
72410a31bc97SJohannes Weiner return;
72420a31bc97SJohannes Weiner
7243d21bba2bSMatthew Wilcox (Oracle) /* Page cache replacement: new folio already charged? */
7244d21bba2bSMatthew Wilcox (Oracle) if (folio_memcg(new))
72450a31bc97SJohannes Weiner return;
72460a31bc97SJohannes Weiner
7247d21bba2bSMatthew Wilcox (Oracle) memcg = folio_memcg(old);
7248d21bba2bSMatthew Wilcox (Oracle) VM_WARN_ON_ONCE_FOLIO(!memcg, old);
724929833315SJohannes Weiner if (!memcg)
72500a31bc97SJohannes Weiner return;
72510a31bc97SJohannes Weiner
725244b7a8d3SJohannes Weiner /* Force-charge the new page. The old one will be freed soon */
72538dc87c7dSMuchun Song if (!mem_cgroup_is_root(memcg)) {
725444b7a8d3SJohannes Weiner page_counter_charge(&memcg->memory, nr_pages);
725544b7a8d3SJohannes Weiner if (do_memsw_account())
725644b7a8d3SJohannes Weiner page_counter_charge(&memcg->memsw, nr_pages);
72578dc87c7dSMuchun Song }
72580a31bc97SJohannes Weiner
72591a3e1f40SJohannes Weiner css_get(&memcg->css);
7260d21bba2bSMatthew Wilcox (Oracle) commit_charge(new, memcg);
726144b7a8d3SJohannes Weiner
7262d93c4130STejun Heo local_irq_save(flags);
72636e0110c2SMatthew Wilcox (Oracle) mem_cgroup_charge_statistics(memcg, nr_pages);
7264d21bba2bSMatthew Wilcox (Oracle) memcg_check_events(memcg, folio_nid(new));
7265d93c4130STejun Heo local_irq_restore(flags);
72660a31bc97SJohannes Weiner }
72670a31bc97SJohannes Weiner
7268ef12947cSJohannes Weiner DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
726911092087SJohannes Weiner EXPORT_SYMBOL(memcg_sockets_enabled_key);
727011092087SJohannes Weiner
mem_cgroup_sk_alloc(struct sock * sk)72712d758073SJohannes Weiner void mem_cgroup_sk_alloc(struct sock *sk)
727211092087SJohannes Weiner {
727311092087SJohannes Weiner struct mem_cgroup *memcg;
727411092087SJohannes Weiner
72752d758073SJohannes Weiner if (!mem_cgroup_sockets_enabled)
72762d758073SJohannes Weiner return;
72772d758073SJohannes Weiner
7278e876ecc6SShakeel Butt /* Do not associate the sock with unrelated interrupted task's memcg. */
7279086f694aSShakeel Butt if (!in_task())
7280e876ecc6SShakeel Butt return;
7281e876ecc6SShakeel Butt
728211092087SJohannes Weiner rcu_read_lock();
728311092087SJohannes Weiner memcg = mem_cgroup_from_task(current);
72847848ed62SKamalesh Babulal if (mem_cgroup_is_root(memcg))
7285f7e1cb6eSJohannes Weiner goto out;
72860db15298SJohannes Weiner if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
7287f7e1cb6eSJohannes Weiner goto out;
72888965aa28SShakeel Butt if (css_tryget(&memcg->css))
728911092087SJohannes Weiner sk->sk_memcg = memcg;
7290f7e1cb6eSJohannes Weiner out:
729111092087SJohannes Weiner rcu_read_unlock();
729211092087SJohannes Weiner }
729311092087SJohannes Weiner
mem_cgroup_sk_free(struct sock * sk)72942d758073SJohannes Weiner void mem_cgroup_sk_free(struct sock *sk)
729511092087SJohannes Weiner {
72962d758073SJohannes Weiner if (sk->sk_memcg)
729711092087SJohannes Weiner css_put(&sk->sk_memcg->css);
729811092087SJohannes Weiner }
729911092087SJohannes Weiner
730011092087SJohannes Weiner /**
730111092087SJohannes Weiner * mem_cgroup_charge_skmem - charge socket memory
730211092087SJohannes Weiner * @memcg: memcg to charge
730311092087SJohannes Weiner * @nr_pages: number of pages to charge
73044b1327beSWei Wang * @gfp_mask: reclaim mode
730511092087SJohannes Weiner *
730611092087SJohannes Weiner * Charges @nr_pages to @memcg. Returns %true if the charge fit within
73074b1327beSWei Wang * @memcg's configured limit, %false if it doesn't.
730811092087SJohannes Weiner */
mem_cgroup_charge_skmem(struct mem_cgroup * memcg,unsigned int nr_pages,gfp_t gfp_mask)73094b1327beSWei Wang bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages,
73104b1327beSWei Wang gfp_t gfp_mask)
731111092087SJohannes Weiner {
7312f7e1cb6eSJohannes Weiner if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
73130db15298SJohannes Weiner struct page_counter *fail;
731411092087SJohannes Weiner
73150db15298SJohannes Weiner if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
73160db15298SJohannes Weiner memcg->tcpmem_pressure = 0;
731711092087SJohannes Weiner return true;
731811092087SJohannes Weiner }
73190db15298SJohannes Weiner memcg->tcpmem_pressure = 1;
73204b1327beSWei Wang if (gfp_mask & __GFP_NOFAIL) {
73214b1327beSWei Wang page_counter_charge(&memcg->tcpmem, nr_pages);
73224b1327beSWei Wang return true;
73234b1327beSWei Wang }
732411092087SJohannes Weiner return false;
732511092087SJohannes Weiner }
7326d886f4e4SJohannes Weiner
73274b1327beSWei Wang if (try_charge(memcg, gfp_mask, nr_pages) == 0) {
7328c9019e9bSJohannes Weiner mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);
7329f7e1cb6eSJohannes Weiner return true;
73304b1327beSWei Wang }
7331f7e1cb6eSJohannes Weiner
7332f7e1cb6eSJohannes Weiner return false;
7333f7e1cb6eSJohannes Weiner }
733411092087SJohannes Weiner
733511092087SJohannes Weiner /**
733611092087SJohannes Weiner * mem_cgroup_uncharge_skmem - uncharge socket memory
7337b7701a5fSMike Rapoport * @memcg: memcg to uncharge
7338b7701a5fSMike Rapoport * @nr_pages: number of pages to uncharge
733911092087SJohannes Weiner */
mem_cgroup_uncharge_skmem(struct mem_cgroup * memcg,unsigned int nr_pages)734011092087SJohannes Weiner void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
734111092087SJohannes Weiner {
7342f7e1cb6eSJohannes Weiner if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
73430db15298SJohannes Weiner page_counter_uncharge(&memcg->tcpmem, nr_pages);
7344f7e1cb6eSJohannes Weiner return;
7345f7e1cb6eSJohannes Weiner }
7346d886f4e4SJohannes Weiner
7347c9019e9bSJohannes Weiner mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
7348b2807f07SJohannes Weiner
7349475d0487SRoman Gushchin refill_stock(memcg, nr_pages);
735011092087SJohannes Weiner }
735111092087SJohannes Weiner
cgroup_memory(char * s)7352f7e1cb6eSJohannes Weiner static int __init cgroup_memory(char *s)
7353f7e1cb6eSJohannes Weiner {
7354f7e1cb6eSJohannes Weiner char *token;
7355f7e1cb6eSJohannes Weiner
7356f7e1cb6eSJohannes Weiner while ((token = strsep(&s, ",")) != NULL) {
7357f7e1cb6eSJohannes Weiner if (!*token)
7358f7e1cb6eSJohannes Weiner continue;
7359f7e1cb6eSJohannes Weiner if (!strcmp(token, "nosocket"))
7360f7e1cb6eSJohannes Weiner cgroup_memory_nosocket = true;
736104823c83SVladimir Davydov if (!strcmp(token, "nokmem"))
736204823c83SVladimir Davydov cgroup_memory_nokmem = true;
7363b6c1a8afSYafang Shao if (!strcmp(token, "nobpf"))
7364b6c1a8afSYafang Shao cgroup_memory_nobpf = true;
7365f7e1cb6eSJohannes Weiner }
7366460a79e1SRandy Dunlap return 1;
7367f7e1cb6eSJohannes Weiner }
7368f7e1cb6eSJohannes Weiner __setup("cgroup.memory=", cgroup_memory);
736911092087SJohannes Weiner
73702d11085eSMichal Hocko /*
73711081312fSMichal Hocko * subsys_initcall() for memory controller.
73721081312fSMichal Hocko *
7373308167fcSSebastian Andrzej Siewior * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this
7374308167fcSSebastian Andrzej Siewior * context because of lock dependencies (cgroup_lock -> cpu hotplug) but
7375308167fcSSebastian Andrzej Siewior * basically everything that doesn't depend on a specific mem_cgroup structure
7376308167fcSSebastian Andrzej Siewior * should be initialized from here.
73772d11085eSMichal Hocko */
mem_cgroup_init(void)73782d11085eSMichal Hocko static int __init mem_cgroup_init(void)
73792d11085eSMichal Hocko {
738095a045f6SJohannes Weiner int cpu, node;
738195a045f6SJohannes Weiner
7382f3344adfSMuchun Song /*
7383f3344adfSMuchun Song * Currently s32 type (can refer to struct batched_lruvec_stat) is
7384f3344adfSMuchun Song * used for per-memcg-per-cpu caching of per-node statistics. In order
7385f3344adfSMuchun Song * to work fine, we should make sure that the overfill threshold can't
7386f3344adfSMuchun Song * exceed S32_MAX / PAGE_SIZE.
7387f3344adfSMuchun Song */
7388f3344adfSMuchun Song BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S32_MAX / PAGE_SIZE);
7389f3344adfSMuchun Song
7390308167fcSSebastian Andrzej Siewior cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
7391308167fcSSebastian Andrzej Siewior memcg_hotplug_cpu_dead);
739295a045f6SJohannes Weiner
739395a045f6SJohannes Weiner for_each_possible_cpu(cpu)
739495a045f6SJohannes Weiner INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
739595a045f6SJohannes Weiner drain_local_stock);
739695a045f6SJohannes Weiner
739795a045f6SJohannes Weiner for_each_node(node) {
739895a045f6SJohannes Weiner struct mem_cgroup_tree_per_node *rtpn;
739995a045f6SJohannes Weiner
740091f0dcceSHaifeng Xu rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, node);
740195a045f6SJohannes Weiner
7402ef8f2327SMel Gorman rtpn->rb_root = RB_ROOT;
7403fa90b2fdSDavidlohr Bueso rtpn->rb_rightmost = NULL;
7404ef8f2327SMel Gorman spin_lock_init(&rtpn->lock);
740595a045f6SJohannes Weiner soft_limit_tree.rb_tree_per_node[node] = rtpn;
740695a045f6SJohannes Weiner }
740795a045f6SJohannes Weiner
74082d11085eSMichal Hocko return 0;
74092d11085eSMichal Hocko }
74102d11085eSMichal Hocko subsys_initcall(mem_cgroup_init);
741121afa38eSJohannes Weiner
7412e55b9f96SJohannes Weiner #ifdef CONFIG_SWAP
mem_cgroup_id_get_online(struct mem_cgroup * memcg)7413358c07fcSArnd Bergmann static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
7414358c07fcSArnd Bergmann {
74151c2d479aSKirill Tkhai while (!refcount_inc_not_zero(&memcg->id.ref)) {
7416358c07fcSArnd Bergmann /*
7417358c07fcSArnd Bergmann * The root cgroup cannot be destroyed, so it's refcount must
7418358c07fcSArnd Bergmann * always be >= 1.
7419358c07fcSArnd Bergmann */
74207848ed62SKamalesh Babulal if (WARN_ON_ONCE(mem_cgroup_is_root(memcg))) {
7421358c07fcSArnd Bergmann VM_BUG_ON(1);
7422358c07fcSArnd Bergmann break;
7423358c07fcSArnd Bergmann }
7424358c07fcSArnd Bergmann memcg = parent_mem_cgroup(memcg);
7425358c07fcSArnd Bergmann if (!memcg)
7426358c07fcSArnd Bergmann memcg = root_mem_cgroup;
7427358c07fcSArnd Bergmann }
7428358c07fcSArnd Bergmann return memcg;
7429358c07fcSArnd Bergmann }
7430358c07fcSArnd Bergmann
743121afa38eSJohannes Weiner /**
743221afa38eSJohannes Weiner * mem_cgroup_swapout - transfer a memsw charge to swap
74333ecb0087SMatthew Wilcox (Oracle) * @folio: folio whose memsw charge to transfer
743421afa38eSJohannes Weiner * @entry: swap entry to move the charge to
743521afa38eSJohannes Weiner *
74363ecb0087SMatthew Wilcox (Oracle) * Transfer the memsw charge of @folio to @entry.
743721afa38eSJohannes Weiner */
mem_cgroup_swapout(struct folio * folio,swp_entry_t entry)74383ecb0087SMatthew Wilcox (Oracle) void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry)
743921afa38eSJohannes Weiner {
74401f47b61fSVladimir Davydov struct mem_cgroup *memcg, *swap_memcg;
7441d6810d73SHuang Ying unsigned int nr_entries;
744221afa38eSJohannes Weiner unsigned short oldid;
744321afa38eSJohannes Weiner
74443ecb0087SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
74453ecb0087SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
744621afa38eSJohannes Weiner
744776358ab5SAlex Shi if (mem_cgroup_disabled())
744876358ab5SAlex Shi return;
744976358ab5SAlex Shi
7450b94c4e94SJohannes Weiner if (!do_memsw_account())
745121afa38eSJohannes Weiner return;
745221afa38eSJohannes Weiner
74533ecb0087SMatthew Wilcox (Oracle) memcg = folio_memcg(folio);
745421afa38eSJohannes Weiner
74553ecb0087SMatthew Wilcox (Oracle) VM_WARN_ON_ONCE_FOLIO(!memcg, folio);
745621afa38eSJohannes Weiner if (!memcg)
745721afa38eSJohannes Weiner return;
745821afa38eSJohannes Weiner
74591f47b61fSVladimir Davydov /*
74601f47b61fSVladimir Davydov * In case the memcg owning these pages has been offlined and doesn't
74611f47b61fSVladimir Davydov * have an ID allocated to it anymore, charge the closest online
74621f47b61fSVladimir Davydov * ancestor for the swap instead and transfer the memory+swap charge.
74631f47b61fSVladimir Davydov */
74641f47b61fSVladimir Davydov swap_memcg = mem_cgroup_id_get_online(memcg);
74653ecb0087SMatthew Wilcox (Oracle) nr_entries = folio_nr_pages(folio);
7466d6810d73SHuang Ying /* Get references for the tail pages, too */
7467d6810d73SHuang Ying if (nr_entries > 1)
7468d6810d73SHuang Ying mem_cgroup_id_get_many(swap_memcg, nr_entries - 1);
7469d6810d73SHuang Ying oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg),
7470d6810d73SHuang Ying nr_entries);
74713ecb0087SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(oldid, folio);
7472c9019e9bSJohannes Weiner mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);
747321afa38eSJohannes Weiner
74743ecb0087SMatthew Wilcox (Oracle) folio->memcg_data = 0;
747521afa38eSJohannes Weiner
747621afa38eSJohannes Weiner if (!mem_cgroup_is_root(memcg))
7477d6810d73SHuang Ying page_counter_uncharge(&memcg->memory, nr_entries);
747821afa38eSJohannes Weiner
7479b25806dcSJohannes Weiner if (memcg != swap_memcg) {
74801f47b61fSVladimir Davydov if (!mem_cgroup_is_root(swap_memcg))
7481d6810d73SHuang Ying page_counter_charge(&swap_memcg->memsw, nr_entries);
7482d6810d73SHuang Ying page_counter_uncharge(&memcg->memsw, nr_entries);
74831f47b61fSVladimir Davydov }
74841f47b61fSVladimir Davydov
7485ce9ce665SSebastian Andrzej Siewior /*
7486ce9ce665SSebastian Andrzej Siewior * Interrupts should be disabled here because the caller holds the
7487b93b0163SMatthew Wilcox * i_pages lock which is taken with interrupts-off. It is
7488ce9ce665SSebastian Andrzej Siewior * important here to have the interrupts disabled because it is the
7489b93b0163SMatthew Wilcox * only synchronisation we have for updating the per-CPU variables.
7490ce9ce665SSebastian Andrzej Siewior */
7491be3e67b5SSebastian Andrzej Siewior memcg_stats_lock();
74926e0110c2SMatthew Wilcox (Oracle) mem_cgroup_charge_statistics(memcg, -nr_entries);
7493be3e67b5SSebastian Andrzej Siewior memcg_stats_unlock();
74943ecb0087SMatthew Wilcox (Oracle) memcg_check_events(memcg, folio_nid(folio));
749573f576c0SJohannes Weiner
74961a3e1f40SJohannes Weiner css_put(&memcg->css);
749721afa38eSJohannes Weiner }
749821afa38eSJohannes Weiner
749938d8b4e6SHuang Ying /**
7500e2e3fdc7SMatthew Wilcox (Oracle) * __mem_cgroup_try_charge_swap - try charging swap space for a folio
7501e2e3fdc7SMatthew Wilcox (Oracle) * @folio: folio being added to swap
750237e84351SVladimir Davydov * @entry: swap entry to charge
750337e84351SVladimir Davydov *
7504e2e3fdc7SMatthew Wilcox (Oracle) * Try to charge @folio's memcg for the swap space at @entry.
750537e84351SVladimir Davydov *
750637e84351SVladimir Davydov * Returns 0 on success, -ENOMEM on failure.
750737e84351SVladimir Davydov */
__mem_cgroup_try_charge_swap(struct folio * folio,swp_entry_t entry)7508e2e3fdc7SMatthew Wilcox (Oracle) int __mem_cgroup_try_charge_swap(struct folio *folio, swp_entry_t entry)
750937e84351SVladimir Davydov {
7510e2e3fdc7SMatthew Wilcox (Oracle) unsigned int nr_pages = folio_nr_pages(folio);
751137e84351SVladimir Davydov struct page_counter *counter;
751238d8b4e6SHuang Ying struct mem_cgroup *memcg;
751337e84351SVladimir Davydov unsigned short oldid;
751437e84351SVladimir Davydov
7515b94c4e94SJohannes Weiner if (do_memsw_account())
751637e84351SVladimir Davydov return 0;
751737e84351SVladimir Davydov
7518e2e3fdc7SMatthew Wilcox (Oracle) memcg = folio_memcg(folio);
751937e84351SVladimir Davydov
7520e2e3fdc7SMatthew Wilcox (Oracle) VM_WARN_ON_ONCE_FOLIO(!memcg, folio);
752137e84351SVladimir Davydov if (!memcg)
752237e84351SVladimir Davydov return 0;
752337e84351SVladimir Davydov
7524f3a53a3aSTejun Heo if (!entry.val) {
7525f3a53a3aSTejun Heo memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
7526bb98f2c5STejun Heo return 0;
7527f3a53a3aSTejun Heo }
7528bb98f2c5STejun Heo
75291f47b61fSVladimir Davydov memcg = mem_cgroup_id_get_online(memcg);
753037e84351SVladimir Davydov
7531b25806dcSJohannes Weiner if (!mem_cgroup_is_root(memcg) &&
753238d8b4e6SHuang Ying !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
7533f3a53a3aSTejun Heo memcg_memory_event(memcg, MEMCG_SWAP_MAX);
7534f3a53a3aSTejun Heo memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
75351f47b61fSVladimir Davydov mem_cgroup_id_put(memcg);
75361f47b61fSVladimir Davydov return -ENOMEM;
75371f47b61fSVladimir Davydov }
75381f47b61fSVladimir Davydov
753938d8b4e6SHuang Ying /* Get references for the tail pages, too */
754038d8b4e6SHuang Ying if (nr_pages > 1)
754138d8b4e6SHuang Ying mem_cgroup_id_get_many(memcg, nr_pages - 1);
754238d8b4e6SHuang Ying oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages);
7543e2e3fdc7SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(oldid, folio);
7544c9019e9bSJohannes Weiner mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
754537e84351SVladimir Davydov
754637e84351SVladimir Davydov return 0;
754737e84351SVladimir Davydov }
754837e84351SVladimir Davydov
754921afa38eSJohannes Weiner /**
755001c4b28cSSuren Baghdasaryan * __mem_cgroup_uncharge_swap - uncharge swap space
755121afa38eSJohannes Weiner * @entry: swap entry to uncharge
755238d8b4e6SHuang Ying * @nr_pages: the amount of swap space to uncharge
755321afa38eSJohannes Weiner */
__mem_cgroup_uncharge_swap(swp_entry_t entry,unsigned int nr_pages)755401c4b28cSSuren Baghdasaryan void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
755521afa38eSJohannes Weiner {
755621afa38eSJohannes Weiner struct mem_cgroup *memcg;
755721afa38eSJohannes Weiner unsigned short id;
755821afa38eSJohannes Weiner
755938d8b4e6SHuang Ying id = swap_cgroup_record(entry, 0, nr_pages);
756021afa38eSJohannes Weiner rcu_read_lock();
7561adbe427bSVladimir Davydov memcg = mem_cgroup_from_id(id);
756221afa38eSJohannes Weiner if (memcg) {
7563b25806dcSJohannes Weiner if (!mem_cgroup_is_root(memcg)) {
7564b94c4e94SJohannes Weiner if (do_memsw_account())
756538d8b4e6SHuang Ying page_counter_uncharge(&memcg->memsw, nr_pages);
7566b94c4e94SJohannes Weiner else
7567b94c4e94SJohannes Weiner page_counter_uncharge(&memcg->swap, nr_pages);
756837e84351SVladimir Davydov }
7569c9019e9bSJohannes Weiner mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
757038d8b4e6SHuang Ying mem_cgroup_id_put_many(memcg, nr_pages);
757121afa38eSJohannes Weiner }
757221afa38eSJohannes Weiner rcu_read_unlock();
757321afa38eSJohannes Weiner }
757421afa38eSJohannes Weiner
mem_cgroup_get_nr_swap_pages(struct mem_cgroup * memcg)7575d8b38438SVladimir Davydov long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
7576d8b38438SVladimir Davydov {
7577d8b38438SVladimir Davydov long nr_swap_pages = get_nr_swap_pages();
7578d8b38438SVladimir Davydov
7579b25806dcSJohannes Weiner if (mem_cgroup_disabled() || do_memsw_account())
7580d8b38438SVladimir Davydov return nr_swap_pages;
75817848ed62SKamalesh Babulal for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg))
7582d8b38438SVladimir Davydov nr_swap_pages = min_t(long, nr_swap_pages,
7583bbec2e15SRoman Gushchin READ_ONCE(memcg->swap.max) -
7584d8b38438SVladimir Davydov page_counter_read(&memcg->swap));
7585d8b38438SVladimir Davydov return nr_swap_pages;
7586d8b38438SVladimir Davydov }
7587d8b38438SVladimir Davydov
mem_cgroup_swap_full(struct folio * folio)75889202d527SMatthew Wilcox (Oracle) bool mem_cgroup_swap_full(struct folio *folio)
75895ccc5abaSVladimir Davydov {
75905ccc5abaSVladimir Davydov struct mem_cgroup *memcg;
75915ccc5abaSVladimir Davydov
75929202d527SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
75935ccc5abaSVladimir Davydov
75945ccc5abaSVladimir Davydov if (vm_swap_full())
75955ccc5abaSVladimir Davydov return true;
7596b25806dcSJohannes Weiner if (do_memsw_account())
75975ccc5abaSVladimir Davydov return false;
75985ccc5abaSVladimir Davydov
75999202d527SMatthew Wilcox (Oracle) memcg = folio_memcg(folio);
76005ccc5abaSVladimir Davydov if (!memcg)
76015ccc5abaSVladimir Davydov return false;
76025ccc5abaSVladimir Davydov
76037848ed62SKamalesh Babulal for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) {
76044b82ab4fSJakub Kicinski unsigned long usage = page_counter_read(&memcg->swap);
76054b82ab4fSJakub Kicinski
76064b82ab4fSJakub Kicinski if (usage * 2 >= READ_ONCE(memcg->swap.high) ||
76074b82ab4fSJakub Kicinski usage * 2 >= READ_ONCE(memcg->swap.max))
76085ccc5abaSVladimir Davydov return true;
76094b82ab4fSJakub Kicinski }
76105ccc5abaSVladimir Davydov
76115ccc5abaSVladimir Davydov return false;
76125ccc5abaSVladimir Davydov }
76135ccc5abaSVladimir Davydov
setup_swap_account(char * s)7614eccb52e7SJohannes Weiner static int __init setup_swap_account(char *s)
761521afa38eSJohannes Weiner {
7616*8350888bSJohannes Weiner bool res;
7617*8350888bSJohannes Weiner
7618*8350888bSJohannes Weiner if (!kstrtobool(s, &res) && !res)
7619*8350888bSJohannes Weiner pr_warn_once("The swapaccount=0 commandline option is deprecated "
7620*8350888bSJohannes Weiner "in favor of configuring swap control via cgroupfs. "
7621b25806dcSJohannes Weiner "Please report your usecase to linux-mm@kvack.org if you "
7622b25806dcSJohannes Weiner "depend on this functionality.\n");
762321afa38eSJohannes Weiner return 1;
762421afa38eSJohannes Weiner }
7625eccb52e7SJohannes Weiner __setup("swapaccount=", setup_swap_account);
762621afa38eSJohannes Weiner
swap_current_read(struct cgroup_subsys_state * css,struct cftype * cft)762737e84351SVladimir Davydov static u64 swap_current_read(struct cgroup_subsys_state *css,
762837e84351SVladimir Davydov struct cftype *cft)
762937e84351SVladimir Davydov {
763037e84351SVladimir Davydov struct mem_cgroup *memcg = mem_cgroup_from_css(css);
763137e84351SVladimir Davydov
763237e84351SVladimir Davydov return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
763337e84351SVladimir Davydov }
763437e84351SVladimir Davydov
swap_peak_read(struct cgroup_subsys_state * css,struct cftype * cft)7635e0e0b412SLars R. Damerow static u64 swap_peak_read(struct cgroup_subsys_state *css,
7636e0e0b412SLars R. Damerow struct cftype *cft)
7637e0e0b412SLars R. Damerow {
7638e0e0b412SLars R. Damerow struct mem_cgroup *memcg = mem_cgroup_from_css(css);
7639e0e0b412SLars R. Damerow
7640e0e0b412SLars R. Damerow return (u64)memcg->swap.watermark * PAGE_SIZE;
7641e0e0b412SLars R. Damerow }
7642e0e0b412SLars R. Damerow
swap_high_show(struct seq_file * m,void * v)76434b82ab4fSJakub Kicinski static int swap_high_show(struct seq_file *m, void *v)
76444b82ab4fSJakub Kicinski {
76454b82ab4fSJakub Kicinski return seq_puts_memcg_tunable(m,
76464b82ab4fSJakub Kicinski READ_ONCE(mem_cgroup_from_seq(m)->swap.high));
76474b82ab4fSJakub Kicinski }
76484b82ab4fSJakub Kicinski
swap_high_write(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)76494b82ab4fSJakub Kicinski static ssize_t swap_high_write(struct kernfs_open_file *of,
76504b82ab4fSJakub Kicinski char *buf, size_t nbytes, loff_t off)
76514b82ab4fSJakub Kicinski {
76524b82ab4fSJakub Kicinski struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
76534b82ab4fSJakub Kicinski unsigned long high;
76544b82ab4fSJakub Kicinski int err;
76554b82ab4fSJakub Kicinski
76564b82ab4fSJakub Kicinski buf = strstrip(buf);
76574b82ab4fSJakub Kicinski err = page_counter_memparse(buf, "max", &high);
76584b82ab4fSJakub Kicinski if (err)
76594b82ab4fSJakub Kicinski return err;
76604b82ab4fSJakub Kicinski
76614b82ab4fSJakub Kicinski page_counter_set_high(&memcg->swap, high);
76624b82ab4fSJakub Kicinski
76634b82ab4fSJakub Kicinski return nbytes;
76644b82ab4fSJakub Kicinski }
76654b82ab4fSJakub Kicinski
swap_max_show(struct seq_file * m,void * v)766637e84351SVladimir Davydov static int swap_max_show(struct seq_file *m, void *v)
766737e84351SVladimir Davydov {
7668677dc973SChris Down return seq_puts_memcg_tunable(m,
7669677dc973SChris Down READ_ONCE(mem_cgroup_from_seq(m)->swap.max));
767037e84351SVladimir Davydov }
767137e84351SVladimir Davydov
swap_max_write(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)767237e84351SVladimir Davydov static ssize_t swap_max_write(struct kernfs_open_file *of,
767337e84351SVladimir Davydov char *buf, size_t nbytes, loff_t off)
767437e84351SVladimir Davydov {
767537e84351SVladimir Davydov struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
767637e84351SVladimir Davydov unsigned long max;
767737e84351SVladimir Davydov int err;
767837e84351SVladimir Davydov
767937e84351SVladimir Davydov buf = strstrip(buf);
768037e84351SVladimir Davydov err = page_counter_memparse(buf, "max", &max);
768137e84351SVladimir Davydov if (err)
768237e84351SVladimir Davydov return err;
768337e84351SVladimir Davydov
7684be09102bSTejun Heo xchg(&memcg->swap.max, max);
768537e84351SVladimir Davydov
768637e84351SVladimir Davydov return nbytes;
768737e84351SVladimir Davydov }
768837e84351SVladimir Davydov
swap_events_show(struct seq_file * m,void * v)7689f3a53a3aSTejun Heo static int swap_events_show(struct seq_file *m, void *v)
7690f3a53a3aSTejun Heo {
7691aa9694bbSChris Down struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
7692f3a53a3aSTejun Heo
76934b82ab4fSJakub Kicinski seq_printf(m, "high %lu\n",
76944b82ab4fSJakub Kicinski atomic_long_read(&memcg->memory_events[MEMCG_SWAP_HIGH]));
7695f3a53a3aSTejun Heo seq_printf(m, "max %lu\n",
7696f3a53a3aSTejun Heo atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]));
7697f3a53a3aSTejun Heo seq_printf(m, "fail %lu\n",
7698f3a53a3aSTejun Heo atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL]));
7699f3a53a3aSTejun Heo
7700f3a53a3aSTejun Heo return 0;
7701f3a53a3aSTejun Heo }
7702f3a53a3aSTejun Heo
770337e84351SVladimir Davydov static struct cftype swap_files[] = {
770437e84351SVladimir Davydov {
770537e84351SVladimir Davydov .name = "swap.current",
770637e84351SVladimir Davydov .flags = CFTYPE_NOT_ON_ROOT,
770737e84351SVladimir Davydov .read_u64 = swap_current_read,
770837e84351SVladimir Davydov },
770937e84351SVladimir Davydov {
77104b82ab4fSJakub Kicinski .name = "swap.high",
77114b82ab4fSJakub Kicinski .flags = CFTYPE_NOT_ON_ROOT,
77124b82ab4fSJakub Kicinski .seq_show = swap_high_show,
77134b82ab4fSJakub Kicinski .write = swap_high_write,
77144b82ab4fSJakub Kicinski },
77154b82ab4fSJakub Kicinski {
771637e84351SVladimir Davydov .name = "swap.max",
771737e84351SVladimir Davydov .flags = CFTYPE_NOT_ON_ROOT,
771837e84351SVladimir Davydov .seq_show = swap_max_show,
771937e84351SVladimir Davydov .write = swap_max_write,
772037e84351SVladimir Davydov },
7721f3a53a3aSTejun Heo {
7722e0e0b412SLars R. Damerow .name = "swap.peak",
7723e0e0b412SLars R. Damerow .flags = CFTYPE_NOT_ON_ROOT,
7724e0e0b412SLars R. Damerow .read_u64 = swap_peak_read,
7725e0e0b412SLars R. Damerow },
7726e0e0b412SLars R. Damerow {
7727f3a53a3aSTejun Heo .name = "swap.events",
7728f3a53a3aSTejun Heo .flags = CFTYPE_NOT_ON_ROOT,
7729f3a53a3aSTejun Heo .file_offset = offsetof(struct mem_cgroup, swap_events_file),
7730f3a53a3aSTejun Heo .seq_show = swap_events_show,
7731f3a53a3aSTejun Heo },
773237e84351SVladimir Davydov { } /* terminate */
773337e84351SVladimir Davydov };
773437e84351SVladimir Davydov
7735eccb52e7SJohannes Weiner static struct cftype memsw_files[] = {
773621afa38eSJohannes Weiner {
773721afa38eSJohannes Weiner .name = "memsw.usage_in_bytes",
773821afa38eSJohannes Weiner .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
773921afa38eSJohannes Weiner .read_u64 = mem_cgroup_read_u64,
774021afa38eSJohannes Weiner },
774121afa38eSJohannes Weiner {
774221afa38eSJohannes Weiner .name = "memsw.max_usage_in_bytes",
774321afa38eSJohannes Weiner .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
774421afa38eSJohannes Weiner .write = mem_cgroup_reset,
774521afa38eSJohannes Weiner .read_u64 = mem_cgroup_read_u64,
774621afa38eSJohannes Weiner },
774721afa38eSJohannes Weiner {
774821afa38eSJohannes Weiner .name = "memsw.limit_in_bytes",
774921afa38eSJohannes Weiner .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
775021afa38eSJohannes Weiner .write = mem_cgroup_write,
775121afa38eSJohannes Weiner .read_u64 = mem_cgroup_read_u64,
775221afa38eSJohannes Weiner },
775321afa38eSJohannes Weiner {
775421afa38eSJohannes Weiner .name = "memsw.failcnt",
775521afa38eSJohannes Weiner .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
775621afa38eSJohannes Weiner .write = mem_cgroup_reset,
775721afa38eSJohannes Weiner .read_u64 = mem_cgroup_read_u64,
775821afa38eSJohannes Weiner },
775921afa38eSJohannes Weiner { }, /* terminate */
776021afa38eSJohannes Weiner };
776121afa38eSJohannes Weiner
7762f4840ccfSJohannes Weiner #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP)
7763f4840ccfSJohannes Weiner /**
7764f4840ccfSJohannes Weiner * obj_cgroup_may_zswap - check if this cgroup can zswap
7765f4840ccfSJohannes Weiner * @objcg: the object cgroup
7766f4840ccfSJohannes Weiner *
7767f4840ccfSJohannes Weiner * Check if the hierarchical zswap limit has been reached.
7768f4840ccfSJohannes Weiner *
7769f4840ccfSJohannes Weiner * This doesn't check for specific headroom, and it is not atomic
7770f4840ccfSJohannes Weiner * either. But with zswap, the size of the allocation is only known
7771f4840ccfSJohannes Weiner * once compression has occured, and this optimistic pre-check avoids
7772f4840ccfSJohannes Weiner * spending cycles on compression when there is already no room left
7773f4840ccfSJohannes Weiner * or zswap is disabled altogether somewhere in the hierarchy.
7774f4840ccfSJohannes Weiner */
obj_cgroup_may_zswap(struct obj_cgroup * objcg)7775f4840ccfSJohannes Weiner bool obj_cgroup_may_zswap(struct obj_cgroup *objcg)
7776f4840ccfSJohannes Weiner {
7777f4840ccfSJohannes Weiner struct mem_cgroup *memcg, *original_memcg;
7778f4840ccfSJohannes Weiner bool ret = true;
7779f4840ccfSJohannes Weiner
7780f4840ccfSJohannes Weiner if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
7781f4840ccfSJohannes Weiner return true;
7782f4840ccfSJohannes Weiner
7783f4840ccfSJohannes Weiner original_memcg = get_mem_cgroup_from_objcg(objcg);
77847848ed62SKamalesh Babulal for (memcg = original_memcg; !mem_cgroup_is_root(memcg);
7785f4840ccfSJohannes Weiner memcg = parent_mem_cgroup(memcg)) {
7786f4840ccfSJohannes Weiner unsigned long max = READ_ONCE(memcg->zswap_max);
7787f4840ccfSJohannes Weiner unsigned long pages;
7788f4840ccfSJohannes Weiner
7789f4840ccfSJohannes Weiner if (max == PAGE_COUNTER_MAX)
7790f4840ccfSJohannes Weiner continue;
7791f4840ccfSJohannes Weiner if (max == 0) {
7792f4840ccfSJohannes Weiner ret = false;
7793f4840ccfSJohannes Weiner break;
7794f4840ccfSJohannes Weiner }
7795f4840ccfSJohannes Weiner
7796f4840ccfSJohannes Weiner cgroup_rstat_flush(memcg->css.cgroup);
7797f4840ccfSJohannes Weiner pages = memcg_page_state(memcg, MEMCG_ZSWAP_B) / PAGE_SIZE;
7798f4840ccfSJohannes Weiner if (pages < max)
7799f4840ccfSJohannes Weiner continue;
7800f4840ccfSJohannes Weiner ret = false;
7801f4840ccfSJohannes Weiner break;
7802f4840ccfSJohannes Weiner }
7803f4840ccfSJohannes Weiner mem_cgroup_put(original_memcg);
7804f4840ccfSJohannes Weiner return ret;
7805f4840ccfSJohannes Weiner }
7806f4840ccfSJohannes Weiner
7807f4840ccfSJohannes Weiner /**
7808f4840ccfSJohannes Weiner * obj_cgroup_charge_zswap - charge compression backend memory
7809f4840ccfSJohannes Weiner * @objcg: the object cgroup
7810f4840ccfSJohannes Weiner * @size: size of compressed object
7811f4840ccfSJohannes Weiner *
78123a1060c2SMiaohe Lin * This forces the charge after obj_cgroup_may_zswap() allowed
7813f4840ccfSJohannes Weiner * compression and storage in zwap for this cgroup to go ahead.
7814f4840ccfSJohannes Weiner */
obj_cgroup_charge_zswap(struct obj_cgroup * objcg,size_t size)7815f4840ccfSJohannes Weiner void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size)
7816f4840ccfSJohannes Weiner {
7817f4840ccfSJohannes Weiner struct mem_cgroup *memcg;
7818f4840ccfSJohannes Weiner
7819f4840ccfSJohannes Weiner if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
7820f4840ccfSJohannes Weiner return;
7821f4840ccfSJohannes Weiner
7822f4840ccfSJohannes Weiner VM_WARN_ON_ONCE(!(current->flags & PF_MEMALLOC));
7823f4840ccfSJohannes Weiner
7824f4840ccfSJohannes Weiner /* PF_MEMALLOC context, charging must succeed */
7825f4840ccfSJohannes Weiner if (obj_cgroup_charge(objcg, GFP_KERNEL, size))
7826f4840ccfSJohannes Weiner VM_WARN_ON_ONCE(1);
7827f4840ccfSJohannes Weiner
7828f4840ccfSJohannes Weiner rcu_read_lock();
7829f4840ccfSJohannes Weiner memcg = obj_cgroup_memcg(objcg);
7830f4840ccfSJohannes Weiner mod_memcg_state(memcg, MEMCG_ZSWAP_B, size);
7831f4840ccfSJohannes Weiner mod_memcg_state(memcg, MEMCG_ZSWAPPED, 1);
7832f4840ccfSJohannes Weiner rcu_read_unlock();
7833f4840ccfSJohannes Weiner }
7834f4840ccfSJohannes Weiner
7835f4840ccfSJohannes Weiner /**
7836f4840ccfSJohannes Weiner * obj_cgroup_uncharge_zswap - uncharge compression backend memory
7837f4840ccfSJohannes Weiner * @objcg: the object cgroup
7838f4840ccfSJohannes Weiner * @size: size of compressed object
7839f4840ccfSJohannes Weiner *
7840f4840ccfSJohannes Weiner * Uncharges zswap memory on page in.
7841f4840ccfSJohannes Weiner */
obj_cgroup_uncharge_zswap(struct obj_cgroup * objcg,size_t size)7842f4840ccfSJohannes Weiner void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size)
7843f4840ccfSJohannes Weiner {
7844f4840ccfSJohannes Weiner struct mem_cgroup *memcg;
7845f4840ccfSJohannes Weiner
7846f4840ccfSJohannes Weiner if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
7847f4840ccfSJohannes Weiner return;
7848f4840ccfSJohannes Weiner
7849f4840ccfSJohannes Weiner obj_cgroup_uncharge(objcg, size);
7850f4840ccfSJohannes Weiner
7851f4840ccfSJohannes Weiner rcu_read_lock();
7852f4840ccfSJohannes Weiner memcg = obj_cgroup_memcg(objcg);
7853f4840ccfSJohannes Weiner mod_memcg_state(memcg, MEMCG_ZSWAP_B, -size);
7854f4840ccfSJohannes Weiner mod_memcg_state(memcg, MEMCG_ZSWAPPED, -1);
7855f4840ccfSJohannes Weiner rcu_read_unlock();
7856f4840ccfSJohannes Weiner }
7857f4840ccfSJohannes Weiner
zswap_current_read(struct cgroup_subsys_state * css,struct cftype * cft)7858f4840ccfSJohannes Weiner static u64 zswap_current_read(struct cgroup_subsys_state *css,
7859f4840ccfSJohannes Weiner struct cftype *cft)
7860f4840ccfSJohannes Weiner {
7861f4840ccfSJohannes Weiner cgroup_rstat_flush(css->cgroup);
7862f4840ccfSJohannes Weiner return memcg_page_state(mem_cgroup_from_css(css), MEMCG_ZSWAP_B);
7863f4840ccfSJohannes Weiner }
7864f4840ccfSJohannes Weiner
zswap_max_show(struct seq_file * m,void * v)7865f4840ccfSJohannes Weiner static int zswap_max_show(struct seq_file *m, void *v)
7866f4840ccfSJohannes Weiner {
7867f4840ccfSJohannes Weiner return seq_puts_memcg_tunable(m,
7868f4840ccfSJohannes Weiner READ_ONCE(mem_cgroup_from_seq(m)->zswap_max));
7869f4840ccfSJohannes Weiner }
7870f4840ccfSJohannes Weiner
zswap_max_write(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)7871f4840ccfSJohannes Weiner static ssize_t zswap_max_write(struct kernfs_open_file *of,
7872f4840ccfSJohannes Weiner char *buf, size_t nbytes, loff_t off)
7873f4840ccfSJohannes Weiner {
7874f4840ccfSJohannes Weiner struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7875f4840ccfSJohannes Weiner unsigned long max;
7876f4840ccfSJohannes Weiner int err;
7877f4840ccfSJohannes Weiner
7878f4840ccfSJohannes Weiner buf = strstrip(buf);
7879f4840ccfSJohannes Weiner err = page_counter_memparse(buf, "max", &max);
7880f4840ccfSJohannes Weiner if (err)
7881f4840ccfSJohannes Weiner return err;
7882f4840ccfSJohannes Weiner
7883f4840ccfSJohannes Weiner xchg(&memcg->zswap_max, max);
7884f4840ccfSJohannes Weiner
7885f4840ccfSJohannes Weiner return nbytes;
7886f4840ccfSJohannes Weiner }
7887f4840ccfSJohannes Weiner
7888f4840ccfSJohannes Weiner static struct cftype zswap_files[] = {
7889f4840ccfSJohannes Weiner {
7890f4840ccfSJohannes Weiner .name = "zswap.current",
7891f4840ccfSJohannes Weiner .flags = CFTYPE_NOT_ON_ROOT,
7892f4840ccfSJohannes Weiner .read_u64 = zswap_current_read,
7893f4840ccfSJohannes Weiner },
7894f4840ccfSJohannes Weiner {
7895f4840ccfSJohannes Weiner .name = "zswap.max",
7896f4840ccfSJohannes Weiner .flags = CFTYPE_NOT_ON_ROOT,
7897f4840ccfSJohannes Weiner .seq_show = zswap_max_show,
7898f4840ccfSJohannes Weiner .write = zswap_max_write,
7899f4840ccfSJohannes Weiner },
7900f4840ccfSJohannes Weiner { } /* terminate */
7901f4840ccfSJohannes Weiner };
7902f4840ccfSJohannes Weiner #endif /* CONFIG_MEMCG_KMEM && CONFIG_ZSWAP */
7903f4840ccfSJohannes Weiner
mem_cgroup_swap_init(void)790421afa38eSJohannes Weiner static int __init mem_cgroup_swap_init(void)
790521afa38eSJohannes Weiner {
79062d1c4980SJohannes Weiner if (mem_cgroup_disabled())
7907eccb52e7SJohannes Weiner return 0;
7908eccb52e7SJohannes Weiner
7909eccb52e7SJohannes Weiner WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, swap_files));
7910eccb52e7SJohannes Weiner WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys, memsw_files));
7911f4840ccfSJohannes Weiner #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP)
7912f4840ccfSJohannes Weiner WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, zswap_files));
7913f4840ccfSJohannes Weiner #endif
791421afa38eSJohannes Weiner return 0;
791521afa38eSJohannes Weiner }
7916b25806dcSJohannes Weiner subsys_initcall(mem_cgroup_swap_init);
791721afa38eSJohannes Weiner
7918e55b9f96SJohannes Weiner #endif /* CONFIG_SWAP */
7919