xref: /openbmc/linux/mm/memcontrol.c (revision dbf563ee)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* memcontrol.c - Memory Controller
3  *
4  * Copyright IBM Corporation, 2007
5  * Author Balbir Singh <balbir@linux.vnet.ibm.com>
6  *
7  * Copyright 2007 OpenVZ SWsoft Inc
8  * Author: Pavel Emelianov <xemul@openvz.org>
9  *
10  * Memory thresholds
11  * Copyright (C) 2009 Nokia Corporation
12  * Author: Kirill A. Shutemov
13  *
14  * Kernel Memory Controller
15  * Copyright (C) 2012 Parallels Inc. and Google Inc.
16  * Authors: Glauber Costa and Suleiman Souhlal
17  *
18  * Native page reclaim
19  * Charge lifetime sanitation
20  * Lockless page tracking & accounting
21  * Unified hierarchy configuration model
22  * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
23  */
24 
25 #include <linux/page_counter.h>
26 #include <linux/memcontrol.h>
27 #include <linux/cgroup.h>
28 #include <linux/pagewalk.h>
29 #include <linux/sched/mm.h>
30 #include <linux/shmem_fs.h>
31 #include <linux/hugetlb.h>
32 #include <linux/pagemap.h>
33 #include <linux/vm_event_item.h>
34 #include <linux/smp.h>
35 #include <linux/page-flags.h>
36 #include <linux/backing-dev.h>
37 #include <linux/bit_spinlock.h>
38 #include <linux/rcupdate.h>
39 #include <linux/limits.h>
40 #include <linux/export.h>
41 #include <linux/mutex.h>
42 #include <linux/rbtree.h>
43 #include <linux/slab.h>
44 #include <linux/swap.h>
45 #include <linux/swapops.h>
46 #include <linux/spinlock.h>
47 #include <linux/eventfd.h>
48 #include <linux/poll.h>
49 #include <linux/sort.h>
50 #include <linux/fs.h>
51 #include <linux/seq_file.h>
52 #include <linux/vmpressure.h>
53 #include <linux/mm_inline.h>
54 #include <linux/swap_cgroup.h>
55 #include <linux/cpu.h>
56 #include <linux/oom.h>
57 #include <linux/lockdep.h>
58 #include <linux/file.h>
59 #include <linux/tracehook.h>
60 #include <linux/psi.h>
61 #include <linux/seq_buf.h>
62 #include "internal.h"
63 #include <net/sock.h>
64 #include <net/ip.h>
65 #include "slab.h"
66 
67 #include <linux/uaccess.h>
68 
69 #include <trace/events/vmscan.h>
70 
71 struct cgroup_subsys memory_cgrp_subsys __read_mostly;
72 EXPORT_SYMBOL(memory_cgrp_subsys);
73 
74 struct mem_cgroup *root_mem_cgroup __read_mostly;
75 
76 /* Socket memory accounting disabled? */
77 static bool cgroup_memory_nosocket;
78 
79 /* Kernel memory accounting disabled? */
80 static bool cgroup_memory_nokmem;
81 
82 /* Whether the swap controller is active */
83 #ifdef CONFIG_MEMCG_SWAP
84 bool cgroup_memory_noswap __read_mostly;
85 #else
86 #define cgroup_memory_noswap		1
87 #endif
88 
89 #ifdef CONFIG_CGROUP_WRITEBACK
90 static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq);
91 #endif
92 
93 /* Whether legacy memory+swap accounting is active */
94 static bool do_memsw_account(void)
95 {
96 	return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_noswap;
97 }
98 
99 #define THRESHOLDS_EVENTS_TARGET 128
100 #define SOFTLIMIT_EVENTS_TARGET 1024
101 
102 /*
103  * Cgroups above their limits are maintained in a RB-Tree, independent of
104  * their hierarchy representation
105  */
106 
107 struct mem_cgroup_tree_per_node {
108 	struct rb_root rb_root;
109 	struct rb_node *rb_rightmost;
110 	spinlock_t lock;
111 };
112 
113 struct mem_cgroup_tree {
114 	struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
115 };
116 
117 static struct mem_cgroup_tree soft_limit_tree __read_mostly;
118 
119 /* for OOM */
120 struct mem_cgroup_eventfd_list {
121 	struct list_head list;
122 	struct eventfd_ctx *eventfd;
123 };
124 
125 /*
126  * cgroup_event represents events which userspace want to receive.
127  */
128 struct mem_cgroup_event {
129 	/*
130 	 * memcg which the event belongs to.
131 	 */
132 	struct mem_cgroup *memcg;
133 	/*
134 	 * eventfd to signal userspace about the event.
135 	 */
136 	struct eventfd_ctx *eventfd;
137 	/*
138 	 * Each of these stored in a list by the cgroup.
139 	 */
140 	struct list_head list;
141 	/*
142 	 * register_event() callback will be used to add new userspace
143 	 * waiter for changes related to this event.  Use eventfd_signal()
144 	 * on eventfd to send notification to userspace.
145 	 */
146 	int (*register_event)(struct mem_cgroup *memcg,
147 			      struct eventfd_ctx *eventfd, const char *args);
148 	/*
149 	 * unregister_event() callback will be called when userspace closes
150 	 * the eventfd or on cgroup removing.  This callback must be set,
151 	 * if you want provide notification functionality.
152 	 */
153 	void (*unregister_event)(struct mem_cgroup *memcg,
154 				 struct eventfd_ctx *eventfd);
155 	/*
156 	 * All fields below needed to unregister event when
157 	 * userspace closes eventfd.
158 	 */
159 	poll_table pt;
160 	wait_queue_head_t *wqh;
161 	wait_queue_entry_t wait;
162 	struct work_struct remove;
163 };
164 
165 static void mem_cgroup_threshold(struct mem_cgroup *memcg);
166 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
167 
168 /* Stuffs for move charges at task migration. */
169 /*
170  * Types of charges to be moved.
171  */
172 #define MOVE_ANON	0x1U
173 #define MOVE_FILE	0x2U
174 #define MOVE_MASK	(MOVE_ANON | MOVE_FILE)
175 
176 /* "mc" and its members are protected by cgroup_mutex */
177 static struct move_charge_struct {
178 	spinlock_t	  lock; /* for from, to */
179 	struct mm_struct  *mm;
180 	struct mem_cgroup *from;
181 	struct mem_cgroup *to;
182 	unsigned long flags;
183 	unsigned long precharge;
184 	unsigned long moved_charge;
185 	unsigned long moved_swap;
186 	struct task_struct *moving_task;	/* a task moving charges */
187 	wait_queue_head_t waitq;		/* a waitq for other context */
188 } mc = {
189 	.lock = __SPIN_LOCK_UNLOCKED(mc.lock),
190 	.waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
191 };
192 
193 /*
194  * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
195  * limit reclaim to prevent infinite loops, if they ever occur.
196  */
197 #define	MEM_CGROUP_MAX_RECLAIM_LOOPS		100
198 #define	MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS	2
199 
200 enum charge_type {
201 	MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
202 	MEM_CGROUP_CHARGE_TYPE_ANON,
203 	MEM_CGROUP_CHARGE_TYPE_SWAPOUT,	/* for accounting swapcache */
204 	MEM_CGROUP_CHARGE_TYPE_DROP,	/* a page was unused swap cache */
205 	NR_CHARGE_TYPE,
206 };
207 
208 /* for encoding cft->private value on file */
209 enum res_type {
210 	_MEM,
211 	_MEMSWAP,
212 	_OOM_TYPE,
213 	_KMEM,
214 	_TCP,
215 };
216 
217 #define MEMFILE_PRIVATE(x, val)	((x) << 16 | (val))
218 #define MEMFILE_TYPE(val)	((val) >> 16 & 0xffff)
219 #define MEMFILE_ATTR(val)	((val) & 0xffff)
220 /* Used for OOM nofiier */
221 #define OOM_CONTROL		(0)
222 
223 /*
224  * Iteration constructs for visiting all cgroups (under a tree).  If
225  * loops are exited prematurely (break), mem_cgroup_iter_break() must
226  * be used for reference counting.
227  */
228 #define for_each_mem_cgroup_tree(iter, root)		\
229 	for (iter = mem_cgroup_iter(root, NULL, NULL);	\
230 	     iter != NULL;				\
231 	     iter = mem_cgroup_iter(root, iter, NULL))
232 
233 #define for_each_mem_cgroup(iter)			\
234 	for (iter = mem_cgroup_iter(NULL, NULL, NULL);	\
235 	     iter != NULL;				\
236 	     iter = mem_cgroup_iter(NULL, iter, NULL))
237 
238 static inline bool should_force_charge(void)
239 {
240 	return tsk_is_oom_victim(current) || fatal_signal_pending(current) ||
241 		(current->flags & PF_EXITING);
242 }
243 
244 /* Some nice accessors for the vmpressure. */
245 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
246 {
247 	if (!memcg)
248 		memcg = root_mem_cgroup;
249 	return &memcg->vmpressure;
250 }
251 
252 struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
253 {
254 	return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
255 }
256 
257 #ifdef CONFIG_MEMCG_KMEM
258 extern spinlock_t css_set_lock;
259 
260 static void obj_cgroup_release(struct percpu_ref *ref)
261 {
262 	struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt);
263 	struct mem_cgroup *memcg;
264 	unsigned int nr_bytes;
265 	unsigned int nr_pages;
266 	unsigned long flags;
267 
268 	/*
269 	 * At this point all allocated objects are freed, and
270 	 * objcg->nr_charged_bytes can't have an arbitrary byte value.
271 	 * However, it can be PAGE_SIZE or (x * PAGE_SIZE).
272 	 *
273 	 * The following sequence can lead to it:
274 	 * 1) CPU0: objcg == stock->cached_objcg
275 	 * 2) CPU1: we do a small allocation (e.g. 92 bytes),
276 	 *          PAGE_SIZE bytes are charged
277 	 * 3) CPU1: a process from another memcg is allocating something,
278 	 *          the stock if flushed,
279 	 *          objcg->nr_charged_bytes = PAGE_SIZE - 92
280 	 * 5) CPU0: we do release this object,
281 	 *          92 bytes are added to stock->nr_bytes
282 	 * 6) CPU0: stock is flushed,
283 	 *          92 bytes are added to objcg->nr_charged_bytes
284 	 *
285 	 * In the result, nr_charged_bytes == PAGE_SIZE.
286 	 * This page will be uncharged in obj_cgroup_release().
287 	 */
288 	nr_bytes = atomic_read(&objcg->nr_charged_bytes);
289 	WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1));
290 	nr_pages = nr_bytes >> PAGE_SHIFT;
291 
292 	spin_lock_irqsave(&css_set_lock, flags);
293 	memcg = obj_cgroup_memcg(objcg);
294 	if (nr_pages)
295 		__memcg_kmem_uncharge(memcg, nr_pages);
296 	list_del(&objcg->list);
297 	mem_cgroup_put(memcg);
298 	spin_unlock_irqrestore(&css_set_lock, flags);
299 
300 	percpu_ref_exit(ref);
301 	kfree_rcu(objcg, rcu);
302 }
303 
304 static struct obj_cgroup *obj_cgroup_alloc(void)
305 {
306 	struct obj_cgroup *objcg;
307 	int ret;
308 
309 	objcg = kzalloc(sizeof(struct obj_cgroup), GFP_KERNEL);
310 	if (!objcg)
311 		return NULL;
312 
313 	ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0,
314 			      GFP_KERNEL);
315 	if (ret) {
316 		kfree(objcg);
317 		return NULL;
318 	}
319 	INIT_LIST_HEAD(&objcg->list);
320 	return objcg;
321 }
322 
323 static void memcg_reparent_objcgs(struct mem_cgroup *memcg,
324 				  struct mem_cgroup *parent)
325 {
326 	struct obj_cgroup *objcg, *iter;
327 
328 	objcg = rcu_replace_pointer(memcg->objcg, NULL, true);
329 
330 	spin_lock_irq(&css_set_lock);
331 
332 	/* Move active objcg to the parent's list */
333 	xchg(&objcg->memcg, parent);
334 	css_get(&parent->css);
335 	list_add(&objcg->list, &parent->objcg_list);
336 
337 	/* Move already reparented objcgs to the parent's list */
338 	list_for_each_entry(iter, &memcg->objcg_list, list) {
339 		css_get(&parent->css);
340 		xchg(&iter->memcg, parent);
341 		css_put(&memcg->css);
342 	}
343 	list_splice(&memcg->objcg_list, &parent->objcg_list);
344 
345 	spin_unlock_irq(&css_set_lock);
346 
347 	percpu_ref_kill(&objcg->refcnt);
348 }
349 
350 /*
351  * This will be used as a shrinker list's index.
352  * The main reason for not using cgroup id for this:
353  *  this works better in sparse environments, where we have a lot of memcgs,
354  *  but only a few kmem-limited. Or also, if we have, for instance, 200
355  *  memcgs, and none but the 200th is kmem-limited, we'd have to have a
356  *  200 entry array for that.
357  *
358  * The current size of the caches array is stored in memcg_nr_cache_ids. It
359  * will double each time we have to increase it.
360  */
361 static DEFINE_IDA(memcg_cache_ida);
362 int memcg_nr_cache_ids;
363 
364 /* Protects memcg_nr_cache_ids */
365 static DECLARE_RWSEM(memcg_cache_ids_sem);
366 
367 void memcg_get_cache_ids(void)
368 {
369 	down_read(&memcg_cache_ids_sem);
370 }
371 
372 void memcg_put_cache_ids(void)
373 {
374 	up_read(&memcg_cache_ids_sem);
375 }
376 
377 /*
378  * MIN_SIZE is different than 1, because we would like to avoid going through
379  * the alloc/free process all the time. In a small machine, 4 kmem-limited
380  * cgroups is a reasonable guess. In the future, it could be a parameter or
381  * tunable, but that is strictly not necessary.
382  *
383  * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get
384  * this constant directly from cgroup, but it is understandable that this is
385  * better kept as an internal representation in cgroup.c. In any case, the
386  * cgrp_id space is not getting any smaller, and we don't have to necessarily
387  * increase ours as well if it increases.
388  */
389 #define MEMCG_CACHES_MIN_SIZE 4
390 #define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
391 
392 /*
393  * A lot of the calls to the cache allocation functions are expected to be
394  * inlined by the compiler. Since the calls to memcg_slab_pre_alloc_hook() are
395  * conditional to this static branch, we'll have to allow modules that does
396  * kmem_cache_alloc and the such to see this symbol as well
397  */
398 DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);
399 EXPORT_SYMBOL(memcg_kmem_enabled_key);
400 #endif
401 
402 static int memcg_shrinker_map_size;
403 static DEFINE_MUTEX(memcg_shrinker_map_mutex);
404 
405 static void memcg_free_shrinker_map_rcu(struct rcu_head *head)
406 {
407 	kvfree(container_of(head, struct memcg_shrinker_map, rcu));
408 }
409 
410 static int memcg_expand_one_shrinker_map(struct mem_cgroup *memcg,
411 					 int size, int old_size)
412 {
413 	struct memcg_shrinker_map *new, *old;
414 	int nid;
415 
416 	lockdep_assert_held(&memcg_shrinker_map_mutex);
417 
418 	for_each_node(nid) {
419 		old = rcu_dereference_protected(
420 			mem_cgroup_nodeinfo(memcg, nid)->shrinker_map, true);
421 		/* Not yet online memcg */
422 		if (!old)
423 			return 0;
424 
425 		new = kvmalloc_node(sizeof(*new) + size, GFP_KERNEL, nid);
426 		if (!new)
427 			return -ENOMEM;
428 
429 		/* Set all old bits, clear all new bits */
430 		memset(new->map, (int)0xff, old_size);
431 		memset((void *)new->map + old_size, 0, size - old_size);
432 
433 		rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, new);
434 		call_rcu(&old->rcu, memcg_free_shrinker_map_rcu);
435 	}
436 
437 	return 0;
438 }
439 
440 static void memcg_free_shrinker_maps(struct mem_cgroup *memcg)
441 {
442 	struct mem_cgroup_per_node *pn;
443 	struct memcg_shrinker_map *map;
444 	int nid;
445 
446 	if (mem_cgroup_is_root(memcg))
447 		return;
448 
449 	for_each_node(nid) {
450 		pn = mem_cgroup_nodeinfo(memcg, nid);
451 		map = rcu_dereference_protected(pn->shrinker_map, true);
452 		if (map)
453 			kvfree(map);
454 		rcu_assign_pointer(pn->shrinker_map, NULL);
455 	}
456 }
457 
458 static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg)
459 {
460 	struct memcg_shrinker_map *map;
461 	int nid, size, ret = 0;
462 
463 	if (mem_cgroup_is_root(memcg))
464 		return 0;
465 
466 	mutex_lock(&memcg_shrinker_map_mutex);
467 	size = memcg_shrinker_map_size;
468 	for_each_node(nid) {
469 		map = kvzalloc_node(sizeof(*map) + size, GFP_KERNEL, nid);
470 		if (!map) {
471 			memcg_free_shrinker_maps(memcg);
472 			ret = -ENOMEM;
473 			break;
474 		}
475 		rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, map);
476 	}
477 	mutex_unlock(&memcg_shrinker_map_mutex);
478 
479 	return ret;
480 }
481 
482 int memcg_expand_shrinker_maps(int new_id)
483 {
484 	int size, old_size, ret = 0;
485 	struct mem_cgroup *memcg;
486 
487 	size = DIV_ROUND_UP(new_id + 1, BITS_PER_LONG) * sizeof(unsigned long);
488 	old_size = memcg_shrinker_map_size;
489 	if (size <= old_size)
490 		return 0;
491 
492 	mutex_lock(&memcg_shrinker_map_mutex);
493 	if (!root_mem_cgroup)
494 		goto unlock;
495 
496 	for_each_mem_cgroup(memcg) {
497 		if (mem_cgroup_is_root(memcg))
498 			continue;
499 		ret = memcg_expand_one_shrinker_map(memcg, size, old_size);
500 		if (ret) {
501 			mem_cgroup_iter_break(NULL, memcg);
502 			goto unlock;
503 		}
504 	}
505 unlock:
506 	if (!ret)
507 		memcg_shrinker_map_size = size;
508 	mutex_unlock(&memcg_shrinker_map_mutex);
509 	return ret;
510 }
511 
512 void memcg_set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
513 {
514 	if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
515 		struct memcg_shrinker_map *map;
516 
517 		rcu_read_lock();
518 		map = rcu_dereference(memcg->nodeinfo[nid]->shrinker_map);
519 		/* Pairs with smp mb in shrink_slab() */
520 		smp_mb__before_atomic();
521 		set_bit(shrinker_id, map->map);
522 		rcu_read_unlock();
523 	}
524 }
525 
526 /**
527  * mem_cgroup_css_from_page - css of the memcg associated with a page
528  * @page: page of interest
529  *
530  * If memcg is bound to the default hierarchy, css of the memcg associated
531  * with @page is returned.  The returned css remains associated with @page
532  * until it is released.
533  *
534  * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
535  * is returned.
536  */
537 struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
538 {
539 	struct mem_cgroup *memcg;
540 
541 	memcg = page->mem_cgroup;
542 
543 	if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
544 		memcg = root_mem_cgroup;
545 
546 	return &memcg->css;
547 }
548 
549 /**
550  * page_cgroup_ino - return inode number of the memcg a page is charged to
551  * @page: the page
552  *
553  * Look up the closest online ancestor of the memory cgroup @page is charged to
554  * and return its inode number or 0 if @page is not charged to any cgroup. It
555  * is safe to call this function without holding a reference to @page.
556  *
557  * Note, this function is inherently racy, because there is nothing to prevent
558  * the cgroup inode from getting torn down and potentially reallocated a moment
559  * after page_cgroup_ino() returns, so it only should be used by callers that
560  * do not care (such as procfs interfaces).
561  */
562 ino_t page_cgroup_ino(struct page *page)
563 {
564 	struct mem_cgroup *memcg;
565 	unsigned long ino = 0;
566 
567 	rcu_read_lock();
568 	memcg = page->mem_cgroup;
569 
570 	/*
571 	 * The lowest bit set means that memcg isn't a valid
572 	 * memcg pointer, but a obj_cgroups pointer.
573 	 * In this case the page is shared and doesn't belong
574 	 * to any specific memory cgroup.
575 	 */
576 	if ((unsigned long) memcg & 0x1UL)
577 		memcg = NULL;
578 
579 	while (memcg && !(memcg->css.flags & CSS_ONLINE))
580 		memcg = parent_mem_cgroup(memcg);
581 	if (memcg)
582 		ino = cgroup_ino(memcg->css.cgroup);
583 	rcu_read_unlock();
584 	return ino;
585 }
586 
587 static struct mem_cgroup_per_node *
588 mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page)
589 {
590 	int nid = page_to_nid(page);
591 
592 	return memcg->nodeinfo[nid];
593 }
594 
595 static struct mem_cgroup_tree_per_node *
596 soft_limit_tree_node(int nid)
597 {
598 	return soft_limit_tree.rb_tree_per_node[nid];
599 }
600 
601 static struct mem_cgroup_tree_per_node *
602 soft_limit_tree_from_page(struct page *page)
603 {
604 	int nid = page_to_nid(page);
605 
606 	return soft_limit_tree.rb_tree_per_node[nid];
607 }
608 
609 static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
610 					 struct mem_cgroup_tree_per_node *mctz,
611 					 unsigned long new_usage_in_excess)
612 {
613 	struct rb_node **p = &mctz->rb_root.rb_node;
614 	struct rb_node *parent = NULL;
615 	struct mem_cgroup_per_node *mz_node;
616 	bool rightmost = true;
617 
618 	if (mz->on_tree)
619 		return;
620 
621 	mz->usage_in_excess = new_usage_in_excess;
622 	if (!mz->usage_in_excess)
623 		return;
624 	while (*p) {
625 		parent = *p;
626 		mz_node = rb_entry(parent, struct mem_cgroup_per_node,
627 					tree_node);
628 		if (mz->usage_in_excess < mz_node->usage_in_excess) {
629 			p = &(*p)->rb_left;
630 			rightmost = false;
631 		}
632 
633 		/*
634 		 * We can't avoid mem cgroups that are over their soft
635 		 * limit by the same amount
636 		 */
637 		else if (mz->usage_in_excess >= mz_node->usage_in_excess)
638 			p = &(*p)->rb_right;
639 	}
640 
641 	if (rightmost)
642 		mctz->rb_rightmost = &mz->tree_node;
643 
644 	rb_link_node(&mz->tree_node, parent, p);
645 	rb_insert_color(&mz->tree_node, &mctz->rb_root);
646 	mz->on_tree = true;
647 }
648 
649 static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
650 					 struct mem_cgroup_tree_per_node *mctz)
651 {
652 	if (!mz->on_tree)
653 		return;
654 
655 	if (&mz->tree_node == mctz->rb_rightmost)
656 		mctz->rb_rightmost = rb_prev(&mz->tree_node);
657 
658 	rb_erase(&mz->tree_node, &mctz->rb_root);
659 	mz->on_tree = false;
660 }
661 
662 static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
663 				       struct mem_cgroup_tree_per_node *mctz)
664 {
665 	unsigned long flags;
666 
667 	spin_lock_irqsave(&mctz->lock, flags);
668 	__mem_cgroup_remove_exceeded(mz, mctz);
669 	spin_unlock_irqrestore(&mctz->lock, flags);
670 }
671 
672 static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
673 {
674 	unsigned long nr_pages = page_counter_read(&memcg->memory);
675 	unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
676 	unsigned long excess = 0;
677 
678 	if (nr_pages > soft_limit)
679 		excess = nr_pages - soft_limit;
680 
681 	return excess;
682 }
683 
684 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
685 {
686 	unsigned long excess;
687 	struct mem_cgroup_per_node *mz;
688 	struct mem_cgroup_tree_per_node *mctz;
689 
690 	mctz = soft_limit_tree_from_page(page);
691 	if (!mctz)
692 		return;
693 	/*
694 	 * Necessary to update all ancestors when hierarchy is used.
695 	 * because their event counter is not touched.
696 	 */
697 	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
698 		mz = mem_cgroup_page_nodeinfo(memcg, page);
699 		excess = soft_limit_excess(memcg);
700 		/*
701 		 * We have to update the tree if mz is on RB-tree or
702 		 * mem is over its softlimit.
703 		 */
704 		if (excess || mz->on_tree) {
705 			unsigned long flags;
706 
707 			spin_lock_irqsave(&mctz->lock, flags);
708 			/* if on-tree, remove it */
709 			if (mz->on_tree)
710 				__mem_cgroup_remove_exceeded(mz, mctz);
711 			/*
712 			 * Insert again. mz->usage_in_excess will be updated.
713 			 * If excess is 0, no tree ops.
714 			 */
715 			__mem_cgroup_insert_exceeded(mz, mctz, excess);
716 			spin_unlock_irqrestore(&mctz->lock, flags);
717 		}
718 	}
719 }
720 
721 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
722 {
723 	struct mem_cgroup_tree_per_node *mctz;
724 	struct mem_cgroup_per_node *mz;
725 	int nid;
726 
727 	for_each_node(nid) {
728 		mz = mem_cgroup_nodeinfo(memcg, nid);
729 		mctz = soft_limit_tree_node(nid);
730 		if (mctz)
731 			mem_cgroup_remove_exceeded(mz, mctz);
732 	}
733 }
734 
735 static struct mem_cgroup_per_node *
736 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
737 {
738 	struct mem_cgroup_per_node *mz;
739 
740 retry:
741 	mz = NULL;
742 	if (!mctz->rb_rightmost)
743 		goto done;		/* Nothing to reclaim from */
744 
745 	mz = rb_entry(mctz->rb_rightmost,
746 		      struct mem_cgroup_per_node, tree_node);
747 	/*
748 	 * Remove the node now but someone else can add it back,
749 	 * we will to add it back at the end of reclaim to its correct
750 	 * position in the tree.
751 	 */
752 	__mem_cgroup_remove_exceeded(mz, mctz);
753 	if (!soft_limit_excess(mz->memcg) ||
754 	    !css_tryget(&mz->memcg->css))
755 		goto retry;
756 done:
757 	return mz;
758 }
759 
760 static struct mem_cgroup_per_node *
761 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
762 {
763 	struct mem_cgroup_per_node *mz;
764 
765 	spin_lock_irq(&mctz->lock);
766 	mz = __mem_cgroup_largest_soft_limit_node(mctz);
767 	spin_unlock_irq(&mctz->lock);
768 	return mz;
769 }
770 
771 /**
772  * __mod_memcg_state - update cgroup memory statistics
773  * @memcg: the memory cgroup
774  * @idx: the stat item - can be enum memcg_stat_item or enum node_stat_item
775  * @val: delta to add to the counter, can be negative
776  */
777 void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val)
778 {
779 	long x, threshold = MEMCG_CHARGE_BATCH;
780 
781 	if (mem_cgroup_disabled())
782 		return;
783 
784 	if (memcg_stat_item_in_bytes(idx))
785 		threshold <<= PAGE_SHIFT;
786 
787 	x = val + __this_cpu_read(memcg->vmstats_percpu->stat[idx]);
788 	if (unlikely(abs(x) > threshold)) {
789 		struct mem_cgroup *mi;
790 
791 		/*
792 		 * Batch local counters to keep them in sync with
793 		 * the hierarchical ones.
794 		 */
795 		__this_cpu_add(memcg->vmstats_local->stat[idx], x);
796 		for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
797 			atomic_long_add(x, &mi->vmstats[idx]);
798 		x = 0;
799 	}
800 	__this_cpu_write(memcg->vmstats_percpu->stat[idx], x);
801 }
802 
803 static struct mem_cgroup_per_node *
804 parent_nodeinfo(struct mem_cgroup_per_node *pn, int nid)
805 {
806 	struct mem_cgroup *parent;
807 
808 	parent = parent_mem_cgroup(pn->memcg);
809 	if (!parent)
810 		return NULL;
811 	return mem_cgroup_nodeinfo(parent, nid);
812 }
813 
814 void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
815 			      int val)
816 {
817 	struct mem_cgroup_per_node *pn;
818 	struct mem_cgroup *memcg;
819 	long x, threshold = MEMCG_CHARGE_BATCH;
820 
821 	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
822 	memcg = pn->memcg;
823 
824 	/* Update memcg */
825 	__mod_memcg_state(memcg, idx, val);
826 
827 	/* Update lruvec */
828 	__this_cpu_add(pn->lruvec_stat_local->count[idx], val);
829 
830 	if (vmstat_item_in_bytes(idx))
831 		threshold <<= PAGE_SHIFT;
832 
833 	x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]);
834 	if (unlikely(abs(x) > threshold)) {
835 		pg_data_t *pgdat = lruvec_pgdat(lruvec);
836 		struct mem_cgroup_per_node *pi;
837 
838 		for (pi = pn; pi; pi = parent_nodeinfo(pi, pgdat->node_id))
839 			atomic_long_add(x, &pi->lruvec_stat[idx]);
840 		x = 0;
841 	}
842 	__this_cpu_write(pn->lruvec_stat_cpu->count[idx], x);
843 }
844 
845 /**
846  * __mod_lruvec_state - update lruvec memory statistics
847  * @lruvec: the lruvec
848  * @idx: the stat item
849  * @val: delta to add to the counter, can be negative
850  *
851  * The lruvec is the intersection of the NUMA node and a cgroup. This
852  * function updates the all three counters that are affected by a
853  * change of state at this level: per-node, per-cgroup, per-lruvec.
854  */
855 void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
856 			int val)
857 {
858 	/* Update node */
859 	__mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
860 
861 	/* Update memcg and lruvec */
862 	if (!mem_cgroup_disabled())
863 		__mod_memcg_lruvec_state(lruvec, idx, val);
864 }
865 
866 void __mod_lruvec_slab_state(void *p, enum node_stat_item idx, int val)
867 {
868 	pg_data_t *pgdat = page_pgdat(virt_to_page(p));
869 	struct mem_cgroup *memcg;
870 	struct lruvec *lruvec;
871 
872 	rcu_read_lock();
873 	memcg = mem_cgroup_from_obj(p);
874 
875 	/* Untracked pages have no memcg, no lruvec. Update only the node */
876 	if (!memcg || memcg == root_mem_cgroup) {
877 		__mod_node_page_state(pgdat, idx, val);
878 	} else {
879 		lruvec = mem_cgroup_lruvec(memcg, pgdat);
880 		__mod_lruvec_state(lruvec, idx, val);
881 	}
882 	rcu_read_unlock();
883 }
884 
885 void mod_memcg_obj_state(void *p, int idx, int val)
886 {
887 	struct mem_cgroup *memcg;
888 
889 	rcu_read_lock();
890 	memcg = mem_cgroup_from_obj(p);
891 	if (memcg)
892 		mod_memcg_state(memcg, idx, val);
893 	rcu_read_unlock();
894 }
895 
896 /**
897  * __count_memcg_events - account VM events in a cgroup
898  * @memcg: the memory cgroup
899  * @idx: the event item
900  * @count: the number of events that occured
901  */
902 void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
903 			  unsigned long count)
904 {
905 	unsigned long x;
906 
907 	if (mem_cgroup_disabled())
908 		return;
909 
910 	x = count + __this_cpu_read(memcg->vmstats_percpu->events[idx]);
911 	if (unlikely(x > MEMCG_CHARGE_BATCH)) {
912 		struct mem_cgroup *mi;
913 
914 		/*
915 		 * Batch local counters to keep them in sync with
916 		 * the hierarchical ones.
917 		 */
918 		__this_cpu_add(memcg->vmstats_local->events[idx], x);
919 		for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
920 			atomic_long_add(x, &mi->vmevents[idx]);
921 		x = 0;
922 	}
923 	__this_cpu_write(memcg->vmstats_percpu->events[idx], x);
924 }
925 
926 static unsigned long memcg_events(struct mem_cgroup *memcg, int event)
927 {
928 	return atomic_long_read(&memcg->vmevents[event]);
929 }
930 
931 static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event)
932 {
933 	long x = 0;
934 	int cpu;
935 
936 	for_each_possible_cpu(cpu)
937 		x += per_cpu(memcg->vmstats_local->events[event], cpu);
938 	return x;
939 }
940 
941 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
942 					 struct page *page,
943 					 int nr_pages)
944 {
945 	/* pagein of a big page is an event. So, ignore page size */
946 	if (nr_pages > 0)
947 		__count_memcg_events(memcg, PGPGIN, 1);
948 	else {
949 		__count_memcg_events(memcg, PGPGOUT, 1);
950 		nr_pages = -nr_pages; /* for event */
951 	}
952 
953 	__this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages);
954 }
955 
956 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
957 				       enum mem_cgroup_events_target target)
958 {
959 	unsigned long val, next;
960 
961 	val = __this_cpu_read(memcg->vmstats_percpu->nr_page_events);
962 	next = __this_cpu_read(memcg->vmstats_percpu->targets[target]);
963 	/* from time_after() in jiffies.h */
964 	if ((long)(next - val) < 0) {
965 		switch (target) {
966 		case MEM_CGROUP_TARGET_THRESH:
967 			next = val + THRESHOLDS_EVENTS_TARGET;
968 			break;
969 		case MEM_CGROUP_TARGET_SOFTLIMIT:
970 			next = val + SOFTLIMIT_EVENTS_TARGET;
971 			break;
972 		default:
973 			break;
974 		}
975 		__this_cpu_write(memcg->vmstats_percpu->targets[target], next);
976 		return true;
977 	}
978 	return false;
979 }
980 
981 /*
982  * Check events in order.
983  *
984  */
985 static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
986 {
987 	/* threshold event is triggered in finer grain than soft limit */
988 	if (unlikely(mem_cgroup_event_ratelimit(memcg,
989 						MEM_CGROUP_TARGET_THRESH))) {
990 		bool do_softlimit;
991 
992 		do_softlimit = mem_cgroup_event_ratelimit(memcg,
993 						MEM_CGROUP_TARGET_SOFTLIMIT);
994 		mem_cgroup_threshold(memcg);
995 		if (unlikely(do_softlimit))
996 			mem_cgroup_update_tree(memcg, page);
997 	}
998 }
999 
1000 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
1001 {
1002 	/*
1003 	 * mm_update_next_owner() may clear mm->owner to NULL
1004 	 * if it races with swapoff, page migration, etc.
1005 	 * So this can be called with p == NULL.
1006 	 */
1007 	if (unlikely(!p))
1008 		return NULL;
1009 
1010 	return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
1011 }
1012 EXPORT_SYMBOL(mem_cgroup_from_task);
1013 
1014 /**
1015  * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg.
1016  * @mm: mm from which memcg should be extracted. It can be NULL.
1017  *
1018  * Obtain a reference on mm->memcg and returns it if successful. Otherwise
1019  * root_mem_cgroup is returned. However if mem_cgroup is disabled, NULL is
1020  * returned.
1021  */
1022 struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
1023 {
1024 	struct mem_cgroup *memcg;
1025 
1026 	if (mem_cgroup_disabled())
1027 		return NULL;
1028 
1029 	rcu_read_lock();
1030 	do {
1031 		/*
1032 		 * Page cache insertions can happen withou an
1033 		 * actual mm context, e.g. during disk probing
1034 		 * on boot, loopback IO, acct() writes etc.
1035 		 */
1036 		if (unlikely(!mm))
1037 			memcg = root_mem_cgroup;
1038 		else {
1039 			memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1040 			if (unlikely(!memcg))
1041 				memcg = root_mem_cgroup;
1042 		}
1043 	} while (!css_tryget(&memcg->css));
1044 	rcu_read_unlock();
1045 	return memcg;
1046 }
1047 EXPORT_SYMBOL(get_mem_cgroup_from_mm);
1048 
1049 /**
1050  * get_mem_cgroup_from_page: Obtain a reference on given page's memcg.
1051  * @page: page from which memcg should be extracted.
1052  *
1053  * Obtain a reference on page->memcg and returns it if successful. Otherwise
1054  * root_mem_cgroup is returned.
1055  */
1056 struct mem_cgroup *get_mem_cgroup_from_page(struct page *page)
1057 {
1058 	struct mem_cgroup *memcg = page->mem_cgroup;
1059 
1060 	if (mem_cgroup_disabled())
1061 		return NULL;
1062 
1063 	rcu_read_lock();
1064 	/* Page should not get uncharged and freed memcg under us. */
1065 	if (!memcg || WARN_ON_ONCE(!css_tryget(&memcg->css)))
1066 		memcg = root_mem_cgroup;
1067 	rcu_read_unlock();
1068 	return memcg;
1069 }
1070 EXPORT_SYMBOL(get_mem_cgroup_from_page);
1071 
1072 /**
1073  * If current->active_memcg is non-NULL, do not fallback to current->mm->memcg.
1074  */
1075 static __always_inline struct mem_cgroup *get_mem_cgroup_from_current(void)
1076 {
1077 	if (unlikely(current->active_memcg)) {
1078 		struct mem_cgroup *memcg;
1079 
1080 		rcu_read_lock();
1081 		/* current->active_memcg must hold a ref. */
1082 		if (WARN_ON_ONCE(!css_tryget(&current->active_memcg->css)))
1083 			memcg = root_mem_cgroup;
1084 		else
1085 			memcg = current->active_memcg;
1086 		rcu_read_unlock();
1087 		return memcg;
1088 	}
1089 	return get_mem_cgroup_from_mm(current->mm);
1090 }
1091 
1092 /**
1093  * mem_cgroup_iter - iterate over memory cgroup hierarchy
1094  * @root: hierarchy root
1095  * @prev: previously returned memcg, NULL on first invocation
1096  * @reclaim: cookie for shared reclaim walks, NULL for full walks
1097  *
1098  * Returns references to children of the hierarchy below @root, or
1099  * @root itself, or %NULL after a full round-trip.
1100  *
1101  * Caller must pass the return value in @prev on subsequent
1102  * invocations for reference counting, or use mem_cgroup_iter_break()
1103  * to cancel a hierarchy walk before the round-trip is complete.
1104  *
1105  * Reclaimers can specify a node and a priority level in @reclaim to
1106  * divide up the memcgs in the hierarchy among all concurrent
1107  * reclaimers operating on the same node and priority.
1108  */
1109 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
1110 				   struct mem_cgroup *prev,
1111 				   struct mem_cgroup_reclaim_cookie *reclaim)
1112 {
1113 	struct mem_cgroup_reclaim_iter *iter;
1114 	struct cgroup_subsys_state *css = NULL;
1115 	struct mem_cgroup *memcg = NULL;
1116 	struct mem_cgroup *pos = NULL;
1117 
1118 	if (mem_cgroup_disabled())
1119 		return NULL;
1120 
1121 	if (!root)
1122 		root = root_mem_cgroup;
1123 
1124 	if (prev && !reclaim)
1125 		pos = prev;
1126 
1127 	if (!root->use_hierarchy && root != root_mem_cgroup) {
1128 		if (prev)
1129 			goto out;
1130 		return root;
1131 	}
1132 
1133 	rcu_read_lock();
1134 
1135 	if (reclaim) {
1136 		struct mem_cgroup_per_node *mz;
1137 
1138 		mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id);
1139 		iter = &mz->iter;
1140 
1141 		if (prev && reclaim->generation != iter->generation)
1142 			goto out_unlock;
1143 
1144 		while (1) {
1145 			pos = READ_ONCE(iter->position);
1146 			if (!pos || css_tryget(&pos->css))
1147 				break;
1148 			/*
1149 			 * css reference reached zero, so iter->position will
1150 			 * be cleared by ->css_released. However, we should not
1151 			 * rely on this happening soon, because ->css_released
1152 			 * is called from a work queue, and by busy-waiting we
1153 			 * might block it. So we clear iter->position right
1154 			 * away.
1155 			 */
1156 			(void)cmpxchg(&iter->position, pos, NULL);
1157 		}
1158 	}
1159 
1160 	if (pos)
1161 		css = &pos->css;
1162 
1163 	for (;;) {
1164 		css = css_next_descendant_pre(css, &root->css);
1165 		if (!css) {
1166 			/*
1167 			 * Reclaimers share the hierarchy walk, and a
1168 			 * new one might jump in right at the end of
1169 			 * the hierarchy - make sure they see at least
1170 			 * one group and restart from the beginning.
1171 			 */
1172 			if (!prev)
1173 				continue;
1174 			break;
1175 		}
1176 
1177 		/*
1178 		 * Verify the css and acquire a reference.  The root
1179 		 * is provided by the caller, so we know it's alive
1180 		 * and kicking, and don't take an extra reference.
1181 		 */
1182 		memcg = mem_cgroup_from_css(css);
1183 
1184 		if (css == &root->css)
1185 			break;
1186 
1187 		if (css_tryget(css))
1188 			break;
1189 
1190 		memcg = NULL;
1191 	}
1192 
1193 	if (reclaim) {
1194 		/*
1195 		 * The position could have already been updated by a competing
1196 		 * thread, so check that the value hasn't changed since we read
1197 		 * it to avoid reclaiming from the same cgroup twice.
1198 		 */
1199 		(void)cmpxchg(&iter->position, pos, memcg);
1200 
1201 		if (pos)
1202 			css_put(&pos->css);
1203 
1204 		if (!memcg)
1205 			iter->generation++;
1206 		else if (!prev)
1207 			reclaim->generation = iter->generation;
1208 	}
1209 
1210 out_unlock:
1211 	rcu_read_unlock();
1212 out:
1213 	if (prev && prev != root)
1214 		css_put(&prev->css);
1215 
1216 	return memcg;
1217 }
1218 
1219 /**
1220  * mem_cgroup_iter_break - abort a hierarchy walk prematurely
1221  * @root: hierarchy root
1222  * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
1223  */
1224 void mem_cgroup_iter_break(struct mem_cgroup *root,
1225 			   struct mem_cgroup *prev)
1226 {
1227 	if (!root)
1228 		root = root_mem_cgroup;
1229 	if (prev && prev != root)
1230 		css_put(&prev->css);
1231 }
1232 
1233 static void __invalidate_reclaim_iterators(struct mem_cgroup *from,
1234 					struct mem_cgroup *dead_memcg)
1235 {
1236 	struct mem_cgroup_reclaim_iter *iter;
1237 	struct mem_cgroup_per_node *mz;
1238 	int nid;
1239 
1240 	for_each_node(nid) {
1241 		mz = mem_cgroup_nodeinfo(from, nid);
1242 		iter = &mz->iter;
1243 		cmpxchg(&iter->position, dead_memcg, NULL);
1244 	}
1245 }
1246 
1247 static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
1248 {
1249 	struct mem_cgroup *memcg = dead_memcg;
1250 	struct mem_cgroup *last;
1251 
1252 	do {
1253 		__invalidate_reclaim_iterators(memcg, dead_memcg);
1254 		last = memcg;
1255 	} while ((memcg = parent_mem_cgroup(memcg)));
1256 
1257 	/*
1258 	 * When cgruop1 non-hierarchy mode is used,
1259 	 * parent_mem_cgroup() does not walk all the way up to the
1260 	 * cgroup root (root_mem_cgroup). So we have to handle
1261 	 * dead_memcg from cgroup root separately.
1262 	 */
1263 	if (last != root_mem_cgroup)
1264 		__invalidate_reclaim_iterators(root_mem_cgroup,
1265 						dead_memcg);
1266 }
1267 
1268 /**
1269  * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
1270  * @memcg: hierarchy root
1271  * @fn: function to call for each task
1272  * @arg: argument passed to @fn
1273  *
1274  * This function iterates over tasks attached to @memcg or to any of its
1275  * descendants and calls @fn for each task. If @fn returns a non-zero
1276  * value, the function breaks the iteration loop and returns the value.
1277  * Otherwise, it will iterate over all tasks and return 0.
1278  *
1279  * This function must not be called for the root memory cgroup.
1280  */
1281 int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
1282 			  int (*fn)(struct task_struct *, void *), void *arg)
1283 {
1284 	struct mem_cgroup *iter;
1285 	int ret = 0;
1286 
1287 	BUG_ON(memcg == root_mem_cgroup);
1288 
1289 	for_each_mem_cgroup_tree(iter, memcg) {
1290 		struct css_task_iter it;
1291 		struct task_struct *task;
1292 
1293 		css_task_iter_start(&iter->css, CSS_TASK_ITER_PROCS, &it);
1294 		while (!ret && (task = css_task_iter_next(&it)))
1295 			ret = fn(task, arg);
1296 		css_task_iter_end(&it);
1297 		if (ret) {
1298 			mem_cgroup_iter_break(memcg, iter);
1299 			break;
1300 		}
1301 	}
1302 	return ret;
1303 }
1304 
1305 /**
1306  * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page
1307  * @page: the page
1308  * @pgdat: pgdat of the page
1309  *
1310  * This function relies on page->mem_cgroup being stable - see the
1311  * access rules in commit_charge().
1312  */
1313 struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgdat)
1314 {
1315 	struct mem_cgroup_per_node *mz;
1316 	struct mem_cgroup *memcg;
1317 	struct lruvec *lruvec;
1318 
1319 	if (mem_cgroup_disabled()) {
1320 		lruvec = &pgdat->__lruvec;
1321 		goto out;
1322 	}
1323 
1324 	memcg = page->mem_cgroup;
1325 	/*
1326 	 * Swapcache readahead pages are added to the LRU - and
1327 	 * possibly migrated - before they are charged.
1328 	 */
1329 	if (!memcg)
1330 		memcg = root_mem_cgroup;
1331 
1332 	mz = mem_cgroup_page_nodeinfo(memcg, page);
1333 	lruvec = &mz->lruvec;
1334 out:
1335 	/*
1336 	 * Since a node can be onlined after the mem_cgroup was created,
1337 	 * we have to be prepared to initialize lruvec->zone here;
1338 	 * and if offlined then reonlined, we need to reinitialize it.
1339 	 */
1340 	if (unlikely(lruvec->pgdat != pgdat))
1341 		lruvec->pgdat = pgdat;
1342 	return lruvec;
1343 }
1344 
1345 /**
1346  * mem_cgroup_update_lru_size - account for adding or removing an lru page
1347  * @lruvec: mem_cgroup per zone lru vector
1348  * @lru: index of lru list the page is sitting on
1349  * @zid: zone id of the accounted pages
1350  * @nr_pages: positive when adding or negative when removing
1351  *
1352  * This function must be called under lru_lock, just before a page is added
1353  * to or just after a page is removed from an lru list (that ordering being
1354  * so as to allow it to check that lru_size 0 is consistent with list_empty).
1355  */
1356 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
1357 				int zid, int nr_pages)
1358 {
1359 	struct mem_cgroup_per_node *mz;
1360 	unsigned long *lru_size;
1361 	long size;
1362 
1363 	if (mem_cgroup_disabled())
1364 		return;
1365 
1366 	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
1367 	lru_size = &mz->lru_zone_size[zid][lru];
1368 
1369 	if (nr_pages < 0)
1370 		*lru_size += nr_pages;
1371 
1372 	size = *lru_size;
1373 	if (WARN_ONCE(size < 0,
1374 		"%s(%p, %d, %d): lru_size %ld\n",
1375 		__func__, lruvec, lru, nr_pages, size)) {
1376 		VM_BUG_ON(1);
1377 		*lru_size = 0;
1378 	}
1379 
1380 	if (nr_pages > 0)
1381 		*lru_size += nr_pages;
1382 }
1383 
1384 /**
1385  * mem_cgroup_margin - calculate chargeable space of a memory cgroup
1386  * @memcg: the memory cgroup
1387  *
1388  * Returns the maximum amount of memory @mem can be charged with, in
1389  * pages.
1390  */
1391 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1392 {
1393 	unsigned long margin = 0;
1394 	unsigned long count;
1395 	unsigned long limit;
1396 
1397 	count = page_counter_read(&memcg->memory);
1398 	limit = READ_ONCE(memcg->memory.max);
1399 	if (count < limit)
1400 		margin = limit - count;
1401 
1402 	if (do_memsw_account()) {
1403 		count = page_counter_read(&memcg->memsw);
1404 		limit = READ_ONCE(memcg->memsw.max);
1405 		if (count < limit)
1406 			margin = min(margin, limit - count);
1407 		else
1408 			margin = 0;
1409 	}
1410 
1411 	return margin;
1412 }
1413 
1414 /*
1415  * A routine for checking "mem" is under move_account() or not.
1416  *
1417  * Checking a cgroup is mc.from or mc.to or under hierarchy of
1418  * moving cgroups. This is for waiting at high-memory pressure
1419  * caused by "move".
1420  */
1421 static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1422 {
1423 	struct mem_cgroup *from;
1424 	struct mem_cgroup *to;
1425 	bool ret = false;
1426 	/*
1427 	 * Unlike task_move routines, we access mc.to, mc.from not under
1428 	 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1429 	 */
1430 	spin_lock(&mc.lock);
1431 	from = mc.from;
1432 	to = mc.to;
1433 	if (!from)
1434 		goto unlock;
1435 
1436 	ret = mem_cgroup_is_descendant(from, memcg) ||
1437 		mem_cgroup_is_descendant(to, memcg);
1438 unlock:
1439 	spin_unlock(&mc.lock);
1440 	return ret;
1441 }
1442 
1443 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1444 {
1445 	if (mc.moving_task && current != mc.moving_task) {
1446 		if (mem_cgroup_under_move(memcg)) {
1447 			DEFINE_WAIT(wait);
1448 			prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1449 			/* moving charge context might have finished. */
1450 			if (mc.moving_task)
1451 				schedule();
1452 			finish_wait(&mc.waitq, &wait);
1453 			return true;
1454 		}
1455 	}
1456 	return false;
1457 }
1458 
1459 static char *memory_stat_format(struct mem_cgroup *memcg)
1460 {
1461 	struct seq_buf s;
1462 	int i;
1463 
1464 	seq_buf_init(&s, kmalloc(PAGE_SIZE, GFP_KERNEL), PAGE_SIZE);
1465 	if (!s.buffer)
1466 		return NULL;
1467 
1468 	/*
1469 	 * Provide statistics on the state of the memory subsystem as
1470 	 * well as cumulative event counters that show past behavior.
1471 	 *
1472 	 * This list is ordered following a combination of these gradients:
1473 	 * 1) generic big picture -> specifics and details
1474 	 * 2) reflecting userspace activity -> reflecting kernel heuristics
1475 	 *
1476 	 * Current memory state:
1477 	 */
1478 
1479 	seq_buf_printf(&s, "anon %llu\n",
1480 		       (u64)memcg_page_state(memcg, NR_ANON_MAPPED) *
1481 		       PAGE_SIZE);
1482 	seq_buf_printf(&s, "file %llu\n",
1483 		       (u64)memcg_page_state(memcg, NR_FILE_PAGES) *
1484 		       PAGE_SIZE);
1485 	seq_buf_printf(&s, "kernel_stack %llu\n",
1486 		       (u64)memcg_page_state(memcg, NR_KERNEL_STACK_KB) *
1487 		       1024);
1488 	seq_buf_printf(&s, "slab %llu\n",
1489 		       (u64)(memcg_page_state(memcg, NR_SLAB_RECLAIMABLE_B) +
1490 			     memcg_page_state(memcg, NR_SLAB_UNRECLAIMABLE_B)));
1491 	seq_buf_printf(&s, "percpu %llu\n",
1492 		       (u64)memcg_page_state(memcg, MEMCG_PERCPU_B));
1493 	seq_buf_printf(&s, "sock %llu\n",
1494 		       (u64)memcg_page_state(memcg, MEMCG_SOCK) *
1495 		       PAGE_SIZE);
1496 
1497 	seq_buf_printf(&s, "shmem %llu\n",
1498 		       (u64)memcg_page_state(memcg, NR_SHMEM) *
1499 		       PAGE_SIZE);
1500 	seq_buf_printf(&s, "file_mapped %llu\n",
1501 		       (u64)memcg_page_state(memcg, NR_FILE_MAPPED) *
1502 		       PAGE_SIZE);
1503 	seq_buf_printf(&s, "file_dirty %llu\n",
1504 		       (u64)memcg_page_state(memcg, NR_FILE_DIRTY) *
1505 		       PAGE_SIZE);
1506 	seq_buf_printf(&s, "file_writeback %llu\n",
1507 		       (u64)memcg_page_state(memcg, NR_WRITEBACK) *
1508 		       PAGE_SIZE);
1509 
1510 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1511 	seq_buf_printf(&s, "anon_thp %llu\n",
1512 		       (u64)memcg_page_state(memcg, NR_ANON_THPS) *
1513 		       HPAGE_PMD_SIZE);
1514 #endif
1515 
1516 	for (i = 0; i < NR_LRU_LISTS; i++)
1517 		seq_buf_printf(&s, "%s %llu\n", lru_list_name(i),
1518 			       (u64)memcg_page_state(memcg, NR_LRU_BASE + i) *
1519 			       PAGE_SIZE);
1520 
1521 	seq_buf_printf(&s, "slab_reclaimable %llu\n",
1522 		       (u64)memcg_page_state(memcg, NR_SLAB_RECLAIMABLE_B));
1523 	seq_buf_printf(&s, "slab_unreclaimable %llu\n",
1524 		       (u64)memcg_page_state(memcg, NR_SLAB_UNRECLAIMABLE_B));
1525 
1526 	/* Accumulated memory events */
1527 
1528 	seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGFAULT),
1529 		       memcg_events(memcg, PGFAULT));
1530 	seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGMAJFAULT),
1531 		       memcg_events(memcg, PGMAJFAULT));
1532 
1533 	seq_buf_printf(&s, "workingset_refault_anon %lu\n",
1534 		       memcg_page_state(memcg, WORKINGSET_REFAULT_ANON));
1535 	seq_buf_printf(&s, "workingset_refault_file %lu\n",
1536 		       memcg_page_state(memcg, WORKINGSET_REFAULT_FILE));
1537 	seq_buf_printf(&s, "workingset_activate_anon %lu\n",
1538 		       memcg_page_state(memcg, WORKINGSET_ACTIVATE_ANON));
1539 	seq_buf_printf(&s, "workingset_activate_file %lu\n",
1540 		       memcg_page_state(memcg, WORKINGSET_ACTIVATE_FILE));
1541 	seq_buf_printf(&s, "workingset_restore_anon %lu\n",
1542 		       memcg_page_state(memcg, WORKINGSET_RESTORE_ANON));
1543 	seq_buf_printf(&s, "workingset_restore_file %lu\n",
1544 		       memcg_page_state(memcg, WORKINGSET_RESTORE_FILE));
1545 	seq_buf_printf(&s, "workingset_nodereclaim %lu\n",
1546 		       memcg_page_state(memcg, WORKINGSET_NODERECLAIM));
1547 
1548 	seq_buf_printf(&s, "%s %lu\n",  vm_event_name(PGREFILL),
1549 		       memcg_events(memcg, PGREFILL));
1550 	seq_buf_printf(&s, "pgscan %lu\n",
1551 		       memcg_events(memcg, PGSCAN_KSWAPD) +
1552 		       memcg_events(memcg, PGSCAN_DIRECT));
1553 	seq_buf_printf(&s, "pgsteal %lu\n",
1554 		       memcg_events(memcg, PGSTEAL_KSWAPD) +
1555 		       memcg_events(memcg, PGSTEAL_DIRECT));
1556 	seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGACTIVATE),
1557 		       memcg_events(memcg, PGACTIVATE));
1558 	seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGDEACTIVATE),
1559 		       memcg_events(memcg, PGDEACTIVATE));
1560 	seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGLAZYFREE),
1561 		       memcg_events(memcg, PGLAZYFREE));
1562 	seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGLAZYFREED),
1563 		       memcg_events(memcg, PGLAZYFREED));
1564 
1565 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1566 	seq_buf_printf(&s, "%s %lu\n", vm_event_name(THP_FAULT_ALLOC),
1567 		       memcg_events(memcg, THP_FAULT_ALLOC));
1568 	seq_buf_printf(&s, "%s %lu\n", vm_event_name(THP_COLLAPSE_ALLOC),
1569 		       memcg_events(memcg, THP_COLLAPSE_ALLOC));
1570 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1571 
1572 	/* The above should easily fit into one page */
1573 	WARN_ON_ONCE(seq_buf_has_overflowed(&s));
1574 
1575 	return s.buffer;
1576 }
1577 
1578 #define K(x) ((x) << (PAGE_SHIFT-10))
1579 /**
1580  * mem_cgroup_print_oom_context: Print OOM information relevant to
1581  * memory controller.
1582  * @memcg: The memory cgroup that went over limit
1583  * @p: Task that is going to be killed
1584  *
1585  * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1586  * enabled
1587  */
1588 void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
1589 {
1590 	rcu_read_lock();
1591 
1592 	if (memcg) {
1593 		pr_cont(",oom_memcg=");
1594 		pr_cont_cgroup_path(memcg->css.cgroup);
1595 	} else
1596 		pr_cont(",global_oom");
1597 	if (p) {
1598 		pr_cont(",task_memcg=");
1599 		pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
1600 	}
1601 	rcu_read_unlock();
1602 }
1603 
1604 /**
1605  * mem_cgroup_print_oom_meminfo: Print OOM memory information relevant to
1606  * memory controller.
1607  * @memcg: The memory cgroup that went over limit
1608  */
1609 void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
1610 {
1611 	char *buf;
1612 
1613 	pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
1614 		K((u64)page_counter_read(&memcg->memory)),
1615 		K((u64)READ_ONCE(memcg->memory.max)), memcg->memory.failcnt);
1616 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
1617 		pr_info("swap: usage %llukB, limit %llukB, failcnt %lu\n",
1618 			K((u64)page_counter_read(&memcg->swap)),
1619 			K((u64)READ_ONCE(memcg->swap.max)), memcg->swap.failcnt);
1620 	else {
1621 		pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
1622 			K((u64)page_counter_read(&memcg->memsw)),
1623 			K((u64)memcg->memsw.max), memcg->memsw.failcnt);
1624 		pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
1625 			K((u64)page_counter_read(&memcg->kmem)),
1626 			K((u64)memcg->kmem.max), memcg->kmem.failcnt);
1627 	}
1628 
1629 	pr_info("Memory cgroup stats for ");
1630 	pr_cont_cgroup_path(memcg->css.cgroup);
1631 	pr_cont(":");
1632 	buf = memory_stat_format(memcg);
1633 	if (!buf)
1634 		return;
1635 	pr_info("%s", buf);
1636 	kfree(buf);
1637 }
1638 
1639 /*
1640  * Return the memory (and swap, if configured) limit for a memcg.
1641  */
1642 unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
1643 {
1644 	unsigned long max;
1645 
1646 	max = READ_ONCE(memcg->memory.max);
1647 	if (mem_cgroup_swappiness(memcg)) {
1648 		unsigned long memsw_max;
1649 		unsigned long swap_max;
1650 
1651 		memsw_max = memcg->memsw.max;
1652 		swap_max = READ_ONCE(memcg->swap.max);
1653 		swap_max = min(swap_max, (unsigned long)total_swap_pages);
1654 		max = min(max + swap_max, memsw_max);
1655 	}
1656 	return max;
1657 }
1658 
1659 unsigned long mem_cgroup_size(struct mem_cgroup *memcg)
1660 {
1661 	return page_counter_read(&memcg->memory);
1662 }
1663 
1664 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1665 				     int order)
1666 {
1667 	struct oom_control oc = {
1668 		.zonelist = NULL,
1669 		.nodemask = NULL,
1670 		.memcg = memcg,
1671 		.gfp_mask = gfp_mask,
1672 		.order = order,
1673 	};
1674 	bool ret = true;
1675 
1676 	if (mutex_lock_killable(&oom_lock))
1677 		return true;
1678 
1679 	if (mem_cgroup_margin(memcg) >= (1 << order))
1680 		goto unlock;
1681 
1682 	/*
1683 	 * A few threads which were not waiting at mutex_lock_killable() can
1684 	 * fail to bail out. Therefore, check again after holding oom_lock.
1685 	 */
1686 	ret = should_force_charge() || out_of_memory(&oc);
1687 
1688 unlock:
1689 	mutex_unlock(&oom_lock);
1690 	return ret;
1691 }
1692 
1693 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1694 				   pg_data_t *pgdat,
1695 				   gfp_t gfp_mask,
1696 				   unsigned long *total_scanned)
1697 {
1698 	struct mem_cgroup *victim = NULL;
1699 	int total = 0;
1700 	int loop = 0;
1701 	unsigned long excess;
1702 	unsigned long nr_scanned;
1703 	struct mem_cgroup_reclaim_cookie reclaim = {
1704 		.pgdat = pgdat,
1705 	};
1706 
1707 	excess = soft_limit_excess(root_memcg);
1708 
1709 	while (1) {
1710 		victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
1711 		if (!victim) {
1712 			loop++;
1713 			if (loop >= 2) {
1714 				/*
1715 				 * If we have not been able to reclaim
1716 				 * anything, it might because there are
1717 				 * no reclaimable pages under this hierarchy
1718 				 */
1719 				if (!total)
1720 					break;
1721 				/*
1722 				 * We want to do more targeted reclaim.
1723 				 * excess >> 2 is not to excessive so as to
1724 				 * reclaim too much, nor too less that we keep
1725 				 * coming back to reclaim from this cgroup
1726 				 */
1727 				if (total >= (excess >> 2) ||
1728 					(loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
1729 					break;
1730 			}
1731 			continue;
1732 		}
1733 		total += mem_cgroup_shrink_node(victim, gfp_mask, false,
1734 					pgdat, &nr_scanned);
1735 		*total_scanned += nr_scanned;
1736 		if (!soft_limit_excess(root_memcg))
1737 			break;
1738 	}
1739 	mem_cgroup_iter_break(root_memcg, victim);
1740 	return total;
1741 }
1742 
1743 #ifdef CONFIG_LOCKDEP
1744 static struct lockdep_map memcg_oom_lock_dep_map = {
1745 	.name = "memcg_oom_lock",
1746 };
1747 #endif
1748 
1749 static DEFINE_SPINLOCK(memcg_oom_lock);
1750 
1751 /*
1752  * Check OOM-Killer is already running under our hierarchy.
1753  * If someone is running, return false.
1754  */
1755 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
1756 {
1757 	struct mem_cgroup *iter, *failed = NULL;
1758 
1759 	spin_lock(&memcg_oom_lock);
1760 
1761 	for_each_mem_cgroup_tree(iter, memcg) {
1762 		if (iter->oom_lock) {
1763 			/*
1764 			 * this subtree of our hierarchy is already locked
1765 			 * so we cannot give a lock.
1766 			 */
1767 			failed = iter;
1768 			mem_cgroup_iter_break(memcg, iter);
1769 			break;
1770 		} else
1771 			iter->oom_lock = true;
1772 	}
1773 
1774 	if (failed) {
1775 		/*
1776 		 * OK, we failed to lock the whole subtree so we have
1777 		 * to clean up what we set up to the failing subtree
1778 		 */
1779 		for_each_mem_cgroup_tree(iter, memcg) {
1780 			if (iter == failed) {
1781 				mem_cgroup_iter_break(memcg, iter);
1782 				break;
1783 			}
1784 			iter->oom_lock = false;
1785 		}
1786 	} else
1787 		mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
1788 
1789 	spin_unlock(&memcg_oom_lock);
1790 
1791 	return !failed;
1792 }
1793 
1794 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1795 {
1796 	struct mem_cgroup *iter;
1797 
1798 	spin_lock(&memcg_oom_lock);
1799 	mutex_release(&memcg_oom_lock_dep_map, _RET_IP_);
1800 	for_each_mem_cgroup_tree(iter, memcg)
1801 		iter->oom_lock = false;
1802 	spin_unlock(&memcg_oom_lock);
1803 }
1804 
1805 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1806 {
1807 	struct mem_cgroup *iter;
1808 
1809 	spin_lock(&memcg_oom_lock);
1810 	for_each_mem_cgroup_tree(iter, memcg)
1811 		iter->under_oom++;
1812 	spin_unlock(&memcg_oom_lock);
1813 }
1814 
1815 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1816 {
1817 	struct mem_cgroup *iter;
1818 
1819 	/*
1820 	 * When a new child is created while the hierarchy is under oom,
1821 	 * mem_cgroup_oom_lock() may not be called. Watch for underflow.
1822 	 */
1823 	spin_lock(&memcg_oom_lock);
1824 	for_each_mem_cgroup_tree(iter, memcg)
1825 		if (iter->under_oom > 0)
1826 			iter->under_oom--;
1827 	spin_unlock(&memcg_oom_lock);
1828 }
1829 
1830 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1831 
1832 struct oom_wait_info {
1833 	struct mem_cgroup *memcg;
1834 	wait_queue_entry_t	wait;
1835 };
1836 
1837 static int memcg_oom_wake_function(wait_queue_entry_t *wait,
1838 	unsigned mode, int sync, void *arg)
1839 {
1840 	struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
1841 	struct mem_cgroup *oom_wait_memcg;
1842 	struct oom_wait_info *oom_wait_info;
1843 
1844 	oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1845 	oom_wait_memcg = oom_wait_info->memcg;
1846 
1847 	if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
1848 	    !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
1849 		return 0;
1850 	return autoremove_wake_function(wait, mode, sync, arg);
1851 }
1852 
1853 static void memcg_oom_recover(struct mem_cgroup *memcg)
1854 {
1855 	/*
1856 	 * For the following lockless ->under_oom test, the only required
1857 	 * guarantee is that it must see the state asserted by an OOM when
1858 	 * this function is called as a result of userland actions
1859 	 * triggered by the notification of the OOM.  This is trivially
1860 	 * achieved by invoking mem_cgroup_mark_under_oom() before
1861 	 * triggering notification.
1862 	 */
1863 	if (memcg && memcg->under_oom)
1864 		__wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
1865 }
1866 
1867 enum oom_status {
1868 	OOM_SUCCESS,
1869 	OOM_FAILED,
1870 	OOM_ASYNC,
1871 	OOM_SKIPPED
1872 };
1873 
1874 static enum oom_status mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
1875 {
1876 	enum oom_status ret;
1877 	bool locked;
1878 
1879 	if (order > PAGE_ALLOC_COSTLY_ORDER)
1880 		return OOM_SKIPPED;
1881 
1882 	memcg_memory_event(memcg, MEMCG_OOM);
1883 
1884 	/*
1885 	 * We are in the middle of the charge context here, so we
1886 	 * don't want to block when potentially sitting on a callstack
1887 	 * that holds all kinds of filesystem and mm locks.
1888 	 *
1889 	 * cgroup1 allows disabling the OOM killer and waiting for outside
1890 	 * handling until the charge can succeed; remember the context and put
1891 	 * the task to sleep at the end of the page fault when all locks are
1892 	 * released.
1893 	 *
1894 	 * On the other hand, in-kernel OOM killer allows for an async victim
1895 	 * memory reclaim (oom_reaper) and that means that we are not solely
1896 	 * relying on the oom victim to make a forward progress and we can
1897 	 * invoke the oom killer here.
1898 	 *
1899 	 * Please note that mem_cgroup_out_of_memory might fail to find a
1900 	 * victim and then we have to bail out from the charge path.
1901 	 */
1902 	if (memcg->oom_kill_disable) {
1903 		if (!current->in_user_fault)
1904 			return OOM_SKIPPED;
1905 		css_get(&memcg->css);
1906 		current->memcg_in_oom = memcg;
1907 		current->memcg_oom_gfp_mask = mask;
1908 		current->memcg_oom_order = order;
1909 
1910 		return OOM_ASYNC;
1911 	}
1912 
1913 	mem_cgroup_mark_under_oom(memcg);
1914 
1915 	locked = mem_cgroup_oom_trylock(memcg);
1916 
1917 	if (locked)
1918 		mem_cgroup_oom_notify(memcg);
1919 
1920 	mem_cgroup_unmark_under_oom(memcg);
1921 	if (mem_cgroup_out_of_memory(memcg, mask, order))
1922 		ret = OOM_SUCCESS;
1923 	else
1924 		ret = OOM_FAILED;
1925 
1926 	if (locked)
1927 		mem_cgroup_oom_unlock(memcg);
1928 
1929 	return ret;
1930 }
1931 
1932 /**
1933  * mem_cgroup_oom_synchronize - complete memcg OOM handling
1934  * @handle: actually kill/wait or just clean up the OOM state
1935  *
1936  * This has to be called at the end of a page fault if the memcg OOM
1937  * handler was enabled.
1938  *
1939  * Memcg supports userspace OOM handling where failed allocations must
1940  * sleep on a waitqueue until the userspace task resolves the
1941  * situation.  Sleeping directly in the charge context with all kinds
1942  * of locks held is not a good idea, instead we remember an OOM state
1943  * in the task and mem_cgroup_oom_synchronize() has to be called at
1944  * the end of the page fault to complete the OOM handling.
1945  *
1946  * Returns %true if an ongoing memcg OOM situation was detected and
1947  * completed, %false otherwise.
1948  */
1949 bool mem_cgroup_oom_synchronize(bool handle)
1950 {
1951 	struct mem_cgroup *memcg = current->memcg_in_oom;
1952 	struct oom_wait_info owait;
1953 	bool locked;
1954 
1955 	/* OOM is global, do not handle */
1956 	if (!memcg)
1957 		return false;
1958 
1959 	if (!handle)
1960 		goto cleanup;
1961 
1962 	owait.memcg = memcg;
1963 	owait.wait.flags = 0;
1964 	owait.wait.func = memcg_oom_wake_function;
1965 	owait.wait.private = current;
1966 	INIT_LIST_HEAD(&owait.wait.entry);
1967 
1968 	prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1969 	mem_cgroup_mark_under_oom(memcg);
1970 
1971 	locked = mem_cgroup_oom_trylock(memcg);
1972 
1973 	if (locked)
1974 		mem_cgroup_oom_notify(memcg);
1975 
1976 	if (locked && !memcg->oom_kill_disable) {
1977 		mem_cgroup_unmark_under_oom(memcg);
1978 		finish_wait(&memcg_oom_waitq, &owait.wait);
1979 		mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
1980 					 current->memcg_oom_order);
1981 	} else {
1982 		schedule();
1983 		mem_cgroup_unmark_under_oom(memcg);
1984 		finish_wait(&memcg_oom_waitq, &owait.wait);
1985 	}
1986 
1987 	if (locked) {
1988 		mem_cgroup_oom_unlock(memcg);
1989 		/*
1990 		 * There is no guarantee that an OOM-lock contender
1991 		 * sees the wakeups triggered by the OOM kill
1992 		 * uncharges.  Wake any sleepers explicitely.
1993 		 */
1994 		memcg_oom_recover(memcg);
1995 	}
1996 cleanup:
1997 	current->memcg_in_oom = NULL;
1998 	css_put(&memcg->css);
1999 	return true;
2000 }
2001 
2002 /**
2003  * mem_cgroup_get_oom_group - get a memory cgroup to clean up after OOM
2004  * @victim: task to be killed by the OOM killer
2005  * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM
2006  *
2007  * Returns a pointer to a memory cgroup, which has to be cleaned up
2008  * by killing all belonging OOM-killable tasks.
2009  *
2010  * Caller has to call mem_cgroup_put() on the returned non-NULL memcg.
2011  */
2012 struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
2013 					    struct mem_cgroup *oom_domain)
2014 {
2015 	struct mem_cgroup *oom_group = NULL;
2016 	struct mem_cgroup *memcg;
2017 
2018 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
2019 		return NULL;
2020 
2021 	if (!oom_domain)
2022 		oom_domain = root_mem_cgroup;
2023 
2024 	rcu_read_lock();
2025 
2026 	memcg = mem_cgroup_from_task(victim);
2027 	if (memcg == root_mem_cgroup)
2028 		goto out;
2029 
2030 	/*
2031 	 * If the victim task has been asynchronously moved to a different
2032 	 * memory cgroup, we might end up killing tasks outside oom_domain.
2033 	 * In this case it's better to ignore memory.group.oom.
2034 	 */
2035 	if (unlikely(!mem_cgroup_is_descendant(memcg, oom_domain)))
2036 		goto out;
2037 
2038 	/*
2039 	 * Traverse the memory cgroup hierarchy from the victim task's
2040 	 * cgroup up to the OOMing cgroup (or root) to find the
2041 	 * highest-level memory cgroup with oom.group set.
2042 	 */
2043 	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
2044 		if (memcg->oom_group)
2045 			oom_group = memcg;
2046 
2047 		if (memcg == oom_domain)
2048 			break;
2049 	}
2050 
2051 	if (oom_group)
2052 		css_get(&oom_group->css);
2053 out:
2054 	rcu_read_unlock();
2055 
2056 	return oom_group;
2057 }
2058 
2059 void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
2060 {
2061 	pr_info("Tasks in ");
2062 	pr_cont_cgroup_path(memcg->css.cgroup);
2063 	pr_cont(" are going to be killed due to memory.oom.group set\n");
2064 }
2065 
2066 /**
2067  * lock_page_memcg - lock a page->mem_cgroup binding
2068  * @page: the page
2069  *
2070  * This function protects unlocked LRU pages from being moved to
2071  * another cgroup.
2072  *
2073  * It ensures lifetime of the returned memcg. Caller is responsible
2074  * for the lifetime of the page; __unlock_page_memcg() is available
2075  * when @page might get freed inside the locked section.
2076  */
2077 struct mem_cgroup *lock_page_memcg(struct page *page)
2078 {
2079 	struct page *head = compound_head(page); /* rmap on tail pages */
2080 	struct mem_cgroup *memcg;
2081 	unsigned long flags;
2082 
2083 	/*
2084 	 * The RCU lock is held throughout the transaction.  The fast
2085 	 * path can get away without acquiring the memcg->move_lock
2086 	 * because page moving starts with an RCU grace period.
2087 	 *
2088 	 * The RCU lock also protects the memcg from being freed when
2089 	 * the page state that is going to change is the only thing
2090 	 * preventing the page itself from being freed. E.g. writeback
2091 	 * doesn't hold a page reference and relies on PG_writeback to
2092 	 * keep off truncation, migration and so forth.
2093          */
2094 	rcu_read_lock();
2095 
2096 	if (mem_cgroup_disabled())
2097 		return NULL;
2098 again:
2099 	memcg = head->mem_cgroup;
2100 	if (unlikely(!memcg))
2101 		return NULL;
2102 
2103 	if (atomic_read(&memcg->moving_account) <= 0)
2104 		return memcg;
2105 
2106 	spin_lock_irqsave(&memcg->move_lock, flags);
2107 	if (memcg != head->mem_cgroup) {
2108 		spin_unlock_irqrestore(&memcg->move_lock, flags);
2109 		goto again;
2110 	}
2111 
2112 	/*
2113 	 * When charge migration first begins, we can have locked and
2114 	 * unlocked page stat updates happening concurrently.  Track
2115 	 * the task who has the lock for unlock_page_memcg().
2116 	 */
2117 	memcg->move_lock_task = current;
2118 	memcg->move_lock_flags = flags;
2119 
2120 	return memcg;
2121 }
2122 EXPORT_SYMBOL(lock_page_memcg);
2123 
2124 /**
2125  * __unlock_page_memcg - unlock and unpin a memcg
2126  * @memcg: the memcg
2127  *
2128  * Unlock and unpin a memcg returned by lock_page_memcg().
2129  */
2130 void __unlock_page_memcg(struct mem_cgroup *memcg)
2131 {
2132 	if (memcg && memcg->move_lock_task == current) {
2133 		unsigned long flags = memcg->move_lock_flags;
2134 
2135 		memcg->move_lock_task = NULL;
2136 		memcg->move_lock_flags = 0;
2137 
2138 		spin_unlock_irqrestore(&memcg->move_lock, flags);
2139 	}
2140 
2141 	rcu_read_unlock();
2142 }
2143 
2144 /**
2145  * unlock_page_memcg - unlock a page->mem_cgroup binding
2146  * @page: the page
2147  */
2148 void unlock_page_memcg(struct page *page)
2149 {
2150 	struct page *head = compound_head(page);
2151 
2152 	__unlock_page_memcg(head->mem_cgroup);
2153 }
2154 EXPORT_SYMBOL(unlock_page_memcg);
2155 
2156 struct memcg_stock_pcp {
2157 	struct mem_cgroup *cached; /* this never be root cgroup */
2158 	unsigned int nr_pages;
2159 
2160 #ifdef CONFIG_MEMCG_KMEM
2161 	struct obj_cgroup *cached_objcg;
2162 	unsigned int nr_bytes;
2163 #endif
2164 
2165 	struct work_struct work;
2166 	unsigned long flags;
2167 #define FLUSHING_CACHED_CHARGE	0
2168 };
2169 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
2170 static DEFINE_MUTEX(percpu_charge_mutex);
2171 
2172 #ifdef CONFIG_MEMCG_KMEM
2173 static void drain_obj_stock(struct memcg_stock_pcp *stock);
2174 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2175 				     struct mem_cgroup *root_memcg);
2176 
2177 #else
2178 static inline void drain_obj_stock(struct memcg_stock_pcp *stock)
2179 {
2180 }
2181 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2182 				     struct mem_cgroup *root_memcg)
2183 {
2184 	return false;
2185 }
2186 #endif
2187 
2188 /**
2189  * consume_stock: Try to consume stocked charge on this cpu.
2190  * @memcg: memcg to consume from.
2191  * @nr_pages: how many pages to charge.
2192  *
2193  * The charges will only happen if @memcg matches the current cpu's memcg
2194  * stock, and at least @nr_pages are available in that stock.  Failure to
2195  * service an allocation will refill the stock.
2196  *
2197  * returns true if successful, false otherwise.
2198  */
2199 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2200 {
2201 	struct memcg_stock_pcp *stock;
2202 	unsigned long flags;
2203 	bool ret = false;
2204 
2205 	if (nr_pages > MEMCG_CHARGE_BATCH)
2206 		return ret;
2207 
2208 	local_irq_save(flags);
2209 
2210 	stock = this_cpu_ptr(&memcg_stock);
2211 	if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
2212 		stock->nr_pages -= nr_pages;
2213 		ret = true;
2214 	}
2215 
2216 	local_irq_restore(flags);
2217 
2218 	return ret;
2219 }
2220 
2221 /*
2222  * Returns stocks cached in percpu and reset cached information.
2223  */
2224 static void drain_stock(struct memcg_stock_pcp *stock)
2225 {
2226 	struct mem_cgroup *old = stock->cached;
2227 
2228 	if (!old)
2229 		return;
2230 
2231 	if (stock->nr_pages) {
2232 		page_counter_uncharge(&old->memory, stock->nr_pages);
2233 		if (do_memsw_account())
2234 			page_counter_uncharge(&old->memsw, stock->nr_pages);
2235 		stock->nr_pages = 0;
2236 	}
2237 
2238 	css_put(&old->css);
2239 	stock->cached = NULL;
2240 }
2241 
2242 static void drain_local_stock(struct work_struct *dummy)
2243 {
2244 	struct memcg_stock_pcp *stock;
2245 	unsigned long flags;
2246 
2247 	/*
2248 	 * The only protection from memory hotplug vs. drain_stock races is
2249 	 * that we always operate on local CPU stock here with IRQ disabled
2250 	 */
2251 	local_irq_save(flags);
2252 
2253 	stock = this_cpu_ptr(&memcg_stock);
2254 	drain_obj_stock(stock);
2255 	drain_stock(stock);
2256 	clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
2257 
2258 	local_irq_restore(flags);
2259 }
2260 
2261 /*
2262  * Cache charges(val) to local per_cpu area.
2263  * This will be consumed by consume_stock() function, later.
2264  */
2265 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2266 {
2267 	struct memcg_stock_pcp *stock;
2268 	unsigned long flags;
2269 
2270 	local_irq_save(flags);
2271 
2272 	stock = this_cpu_ptr(&memcg_stock);
2273 	if (stock->cached != memcg) { /* reset if necessary */
2274 		drain_stock(stock);
2275 		css_get(&memcg->css);
2276 		stock->cached = memcg;
2277 	}
2278 	stock->nr_pages += nr_pages;
2279 
2280 	if (stock->nr_pages > MEMCG_CHARGE_BATCH)
2281 		drain_stock(stock);
2282 
2283 	local_irq_restore(flags);
2284 }
2285 
2286 /*
2287  * Drains all per-CPU charge caches for given root_memcg resp. subtree
2288  * of the hierarchy under it.
2289  */
2290 static void drain_all_stock(struct mem_cgroup *root_memcg)
2291 {
2292 	int cpu, curcpu;
2293 
2294 	/* If someone's already draining, avoid adding running more workers. */
2295 	if (!mutex_trylock(&percpu_charge_mutex))
2296 		return;
2297 	/*
2298 	 * Notify other cpus that system-wide "drain" is running
2299 	 * We do not care about races with the cpu hotplug because cpu down
2300 	 * as well as workers from this path always operate on the local
2301 	 * per-cpu data. CPU up doesn't touch memcg_stock at all.
2302 	 */
2303 	curcpu = get_cpu();
2304 	for_each_online_cpu(cpu) {
2305 		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2306 		struct mem_cgroup *memcg;
2307 		bool flush = false;
2308 
2309 		rcu_read_lock();
2310 		memcg = stock->cached;
2311 		if (memcg && stock->nr_pages &&
2312 		    mem_cgroup_is_descendant(memcg, root_memcg))
2313 			flush = true;
2314 		if (obj_stock_flush_required(stock, root_memcg))
2315 			flush = true;
2316 		rcu_read_unlock();
2317 
2318 		if (flush &&
2319 		    !test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
2320 			if (cpu == curcpu)
2321 				drain_local_stock(&stock->work);
2322 			else
2323 				schedule_work_on(cpu, &stock->work);
2324 		}
2325 	}
2326 	put_cpu();
2327 	mutex_unlock(&percpu_charge_mutex);
2328 }
2329 
2330 static int memcg_hotplug_cpu_dead(unsigned int cpu)
2331 {
2332 	struct memcg_stock_pcp *stock;
2333 	struct mem_cgroup *memcg, *mi;
2334 
2335 	stock = &per_cpu(memcg_stock, cpu);
2336 	drain_stock(stock);
2337 
2338 	for_each_mem_cgroup(memcg) {
2339 		int i;
2340 
2341 		for (i = 0; i < MEMCG_NR_STAT; i++) {
2342 			int nid;
2343 			long x;
2344 
2345 			x = this_cpu_xchg(memcg->vmstats_percpu->stat[i], 0);
2346 			if (x)
2347 				for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
2348 					atomic_long_add(x, &memcg->vmstats[i]);
2349 
2350 			if (i >= NR_VM_NODE_STAT_ITEMS)
2351 				continue;
2352 
2353 			for_each_node(nid) {
2354 				struct mem_cgroup_per_node *pn;
2355 
2356 				pn = mem_cgroup_nodeinfo(memcg, nid);
2357 				x = this_cpu_xchg(pn->lruvec_stat_cpu->count[i], 0);
2358 				if (x)
2359 					do {
2360 						atomic_long_add(x, &pn->lruvec_stat[i]);
2361 					} while ((pn = parent_nodeinfo(pn, nid)));
2362 			}
2363 		}
2364 
2365 		for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
2366 			long x;
2367 
2368 			x = this_cpu_xchg(memcg->vmstats_percpu->events[i], 0);
2369 			if (x)
2370 				for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
2371 					atomic_long_add(x, &memcg->vmevents[i]);
2372 		}
2373 	}
2374 
2375 	return 0;
2376 }
2377 
2378 static unsigned long reclaim_high(struct mem_cgroup *memcg,
2379 				  unsigned int nr_pages,
2380 				  gfp_t gfp_mask)
2381 {
2382 	unsigned long nr_reclaimed = 0;
2383 
2384 	do {
2385 		unsigned long pflags;
2386 
2387 		if (page_counter_read(&memcg->memory) <=
2388 		    READ_ONCE(memcg->memory.high))
2389 			continue;
2390 
2391 		memcg_memory_event(memcg, MEMCG_HIGH);
2392 
2393 		psi_memstall_enter(&pflags);
2394 		nr_reclaimed += try_to_free_mem_cgroup_pages(memcg, nr_pages,
2395 							     gfp_mask, true);
2396 		psi_memstall_leave(&pflags);
2397 	} while ((memcg = parent_mem_cgroup(memcg)) &&
2398 		 !mem_cgroup_is_root(memcg));
2399 
2400 	return nr_reclaimed;
2401 }
2402 
2403 static void high_work_func(struct work_struct *work)
2404 {
2405 	struct mem_cgroup *memcg;
2406 
2407 	memcg = container_of(work, struct mem_cgroup, high_work);
2408 	reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);
2409 }
2410 
2411 /*
2412  * Clamp the maximum sleep time per allocation batch to 2 seconds. This is
2413  * enough to still cause a significant slowdown in most cases, while still
2414  * allowing diagnostics and tracing to proceed without becoming stuck.
2415  */
2416 #define MEMCG_MAX_HIGH_DELAY_JIFFIES (2UL*HZ)
2417 
2418 /*
2419  * When calculating the delay, we use these either side of the exponentiation to
2420  * maintain precision and scale to a reasonable number of jiffies (see the table
2421  * below.
2422  *
2423  * - MEMCG_DELAY_PRECISION_SHIFT: Extra precision bits while translating the
2424  *   overage ratio to a delay.
2425  * - MEMCG_DELAY_SCALING_SHIFT: The number of bits to scale down the
2426  *   proposed penalty in order to reduce to a reasonable number of jiffies, and
2427  *   to produce a reasonable delay curve.
2428  *
2429  * MEMCG_DELAY_SCALING_SHIFT just happens to be a number that produces a
2430  * reasonable delay curve compared to precision-adjusted overage, not
2431  * penalising heavily at first, but still making sure that growth beyond the
2432  * limit penalises misbehaviour cgroups by slowing them down exponentially. For
2433  * example, with a high of 100 megabytes:
2434  *
2435  *  +-------+------------------------+
2436  *  | usage | time to allocate in ms |
2437  *  +-------+------------------------+
2438  *  | 100M  |                      0 |
2439  *  | 101M  |                      6 |
2440  *  | 102M  |                     25 |
2441  *  | 103M  |                     57 |
2442  *  | 104M  |                    102 |
2443  *  | 105M  |                    159 |
2444  *  | 106M  |                    230 |
2445  *  | 107M  |                    313 |
2446  *  | 108M  |                    409 |
2447  *  | 109M  |                    518 |
2448  *  | 110M  |                    639 |
2449  *  | 111M  |                    774 |
2450  *  | 112M  |                    921 |
2451  *  | 113M  |                   1081 |
2452  *  | 114M  |                   1254 |
2453  *  | 115M  |                   1439 |
2454  *  | 116M  |                   1638 |
2455  *  | 117M  |                   1849 |
2456  *  | 118M  |                   2000 |
2457  *  | 119M  |                   2000 |
2458  *  | 120M  |                   2000 |
2459  *  +-------+------------------------+
2460  */
2461  #define MEMCG_DELAY_PRECISION_SHIFT 20
2462  #define MEMCG_DELAY_SCALING_SHIFT 14
2463 
2464 static u64 calculate_overage(unsigned long usage, unsigned long high)
2465 {
2466 	u64 overage;
2467 
2468 	if (usage <= high)
2469 		return 0;
2470 
2471 	/*
2472 	 * Prevent division by 0 in overage calculation by acting as if
2473 	 * it was a threshold of 1 page
2474 	 */
2475 	high = max(high, 1UL);
2476 
2477 	overage = usage - high;
2478 	overage <<= MEMCG_DELAY_PRECISION_SHIFT;
2479 	return div64_u64(overage, high);
2480 }
2481 
2482 static u64 mem_find_max_overage(struct mem_cgroup *memcg)
2483 {
2484 	u64 overage, max_overage = 0;
2485 
2486 	do {
2487 		overage = calculate_overage(page_counter_read(&memcg->memory),
2488 					    READ_ONCE(memcg->memory.high));
2489 		max_overage = max(overage, max_overage);
2490 	} while ((memcg = parent_mem_cgroup(memcg)) &&
2491 		 !mem_cgroup_is_root(memcg));
2492 
2493 	return max_overage;
2494 }
2495 
2496 static u64 swap_find_max_overage(struct mem_cgroup *memcg)
2497 {
2498 	u64 overage, max_overage = 0;
2499 
2500 	do {
2501 		overage = calculate_overage(page_counter_read(&memcg->swap),
2502 					    READ_ONCE(memcg->swap.high));
2503 		if (overage)
2504 			memcg_memory_event(memcg, MEMCG_SWAP_HIGH);
2505 		max_overage = max(overage, max_overage);
2506 	} while ((memcg = parent_mem_cgroup(memcg)) &&
2507 		 !mem_cgroup_is_root(memcg));
2508 
2509 	return max_overage;
2510 }
2511 
2512 /*
2513  * Get the number of jiffies that we should penalise a mischievous cgroup which
2514  * is exceeding its memory.high by checking both it and its ancestors.
2515  */
2516 static unsigned long calculate_high_delay(struct mem_cgroup *memcg,
2517 					  unsigned int nr_pages,
2518 					  u64 max_overage)
2519 {
2520 	unsigned long penalty_jiffies;
2521 
2522 	if (!max_overage)
2523 		return 0;
2524 
2525 	/*
2526 	 * We use overage compared to memory.high to calculate the number of
2527 	 * jiffies to sleep (penalty_jiffies). Ideally this value should be
2528 	 * fairly lenient on small overages, and increasingly harsh when the
2529 	 * memcg in question makes it clear that it has no intention of stopping
2530 	 * its crazy behaviour, so we exponentially increase the delay based on
2531 	 * overage amount.
2532 	 */
2533 	penalty_jiffies = max_overage * max_overage * HZ;
2534 	penalty_jiffies >>= MEMCG_DELAY_PRECISION_SHIFT;
2535 	penalty_jiffies >>= MEMCG_DELAY_SCALING_SHIFT;
2536 
2537 	/*
2538 	 * Factor in the task's own contribution to the overage, such that four
2539 	 * N-sized allocations are throttled approximately the same as one
2540 	 * 4N-sized allocation.
2541 	 *
2542 	 * MEMCG_CHARGE_BATCH pages is nominal, so work out how much smaller or
2543 	 * larger the current charge patch is than that.
2544 	 */
2545 	return penalty_jiffies * nr_pages / MEMCG_CHARGE_BATCH;
2546 }
2547 
2548 /*
2549  * Scheduled by try_charge() to be executed from the userland return path
2550  * and reclaims memory over the high limit.
2551  */
2552 void mem_cgroup_handle_over_high(void)
2553 {
2554 	unsigned long penalty_jiffies;
2555 	unsigned long pflags;
2556 	unsigned long nr_reclaimed;
2557 	unsigned int nr_pages = current->memcg_nr_pages_over_high;
2558 	int nr_retries = MAX_RECLAIM_RETRIES;
2559 	struct mem_cgroup *memcg;
2560 	bool in_retry = false;
2561 
2562 	if (likely(!nr_pages))
2563 		return;
2564 
2565 	memcg = get_mem_cgroup_from_mm(current->mm);
2566 	current->memcg_nr_pages_over_high = 0;
2567 
2568 retry_reclaim:
2569 	/*
2570 	 * The allocating task should reclaim at least the batch size, but for
2571 	 * subsequent retries we only want to do what's necessary to prevent oom
2572 	 * or breaching resource isolation.
2573 	 *
2574 	 * This is distinct from memory.max or page allocator behaviour because
2575 	 * memory.high is currently batched, whereas memory.max and the page
2576 	 * allocator run every time an allocation is made.
2577 	 */
2578 	nr_reclaimed = reclaim_high(memcg,
2579 				    in_retry ? SWAP_CLUSTER_MAX : nr_pages,
2580 				    GFP_KERNEL);
2581 
2582 	/*
2583 	 * memory.high is breached and reclaim is unable to keep up. Throttle
2584 	 * allocators proactively to slow down excessive growth.
2585 	 */
2586 	penalty_jiffies = calculate_high_delay(memcg, nr_pages,
2587 					       mem_find_max_overage(memcg));
2588 
2589 	penalty_jiffies += calculate_high_delay(memcg, nr_pages,
2590 						swap_find_max_overage(memcg));
2591 
2592 	/*
2593 	 * Clamp the max delay per usermode return so as to still keep the
2594 	 * application moving forwards and also permit diagnostics, albeit
2595 	 * extremely slowly.
2596 	 */
2597 	penalty_jiffies = min(penalty_jiffies, MEMCG_MAX_HIGH_DELAY_JIFFIES);
2598 
2599 	/*
2600 	 * Don't sleep if the amount of jiffies this memcg owes us is so low
2601 	 * that it's not even worth doing, in an attempt to be nice to those who
2602 	 * go only a small amount over their memory.high value and maybe haven't
2603 	 * been aggressively reclaimed enough yet.
2604 	 */
2605 	if (penalty_jiffies <= HZ / 100)
2606 		goto out;
2607 
2608 	/*
2609 	 * If reclaim is making forward progress but we're still over
2610 	 * memory.high, we want to encourage that rather than doing allocator
2611 	 * throttling.
2612 	 */
2613 	if (nr_reclaimed || nr_retries--) {
2614 		in_retry = true;
2615 		goto retry_reclaim;
2616 	}
2617 
2618 	/*
2619 	 * If we exit early, we're guaranteed to die (since
2620 	 * schedule_timeout_killable sets TASK_KILLABLE). This means we don't
2621 	 * need to account for any ill-begotten jiffies to pay them off later.
2622 	 */
2623 	psi_memstall_enter(&pflags);
2624 	schedule_timeout_killable(penalty_jiffies);
2625 	psi_memstall_leave(&pflags);
2626 
2627 out:
2628 	css_put(&memcg->css);
2629 }
2630 
2631 static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2632 		      unsigned int nr_pages)
2633 {
2634 	unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages);
2635 	int nr_retries = MAX_RECLAIM_RETRIES;
2636 	struct mem_cgroup *mem_over_limit;
2637 	struct page_counter *counter;
2638 	enum oom_status oom_status;
2639 	unsigned long nr_reclaimed;
2640 	bool may_swap = true;
2641 	bool drained = false;
2642 	unsigned long pflags;
2643 
2644 	if (mem_cgroup_is_root(memcg))
2645 		return 0;
2646 retry:
2647 	if (consume_stock(memcg, nr_pages))
2648 		return 0;
2649 
2650 	if (!do_memsw_account() ||
2651 	    page_counter_try_charge(&memcg->memsw, batch, &counter)) {
2652 		if (page_counter_try_charge(&memcg->memory, batch, &counter))
2653 			goto done_restock;
2654 		if (do_memsw_account())
2655 			page_counter_uncharge(&memcg->memsw, batch);
2656 		mem_over_limit = mem_cgroup_from_counter(counter, memory);
2657 	} else {
2658 		mem_over_limit = mem_cgroup_from_counter(counter, memsw);
2659 		may_swap = false;
2660 	}
2661 
2662 	if (batch > nr_pages) {
2663 		batch = nr_pages;
2664 		goto retry;
2665 	}
2666 
2667 	/*
2668 	 * Memcg doesn't have a dedicated reserve for atomic
2669 	 * allocations. But like the global atomic pool, we need to
2670 	 * put the burden of reclaim on regular allocation requests
2671 	 * and let these go through as privileged allocations.
2672 	 */
2673 	if (gfp_mask & __GFP_ATOMIC)
2674 		goto force;
2675 
2676 	/*
2677 	 * Unlike in global OOM situations, memcg is not in a physical
2678 	 * memory shortage.  Allow dying and OOM-killed tasks to
2679 	 * bypass the last charges so that they can exit quickly and
2680 	 * free their memory.
2681 	 */
2682 	if (unlikely(should_force_charge()))
2683 		goto force;
2684 
2685 	/*
2686 	 * Prevent unbounded recursion when reclaim operations need to
2687 	 * allocate memory. This might exceed the limits temporarily,
2688 	 * but we prefer facilitating memory reclaim and getting back
2689 	 * under the limit over triggering OOM kills in these cases.
2690 	 */
2691 	if (unlikely(current->flags & PF_MEMALLOC))
2692 		goto force;
2693 
2694 	if (unlikely(task_in_memcg_oom(current)))
2695 		goto nomem;
2696 
2697 	if (!gfpflags_allow_blocking(gfp_mask))
2698 		goto nomem;
2699 
2700 	memcg_memory_event(mem_over_limit, MEMCG_MAX);
2701 
2702 	psi_memstall_enter(&pflags);
2703 	nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
2704 						    gfp_mask, may_swap);
2705 	psi_memstall_leave(&pflags);
2706 
2707 	if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
2708 		goto retry;
2709 
2710 	if (!drained) {
2711 		drain_all_stock(mem_over_limit);
2712 		drained = true;
2713 		goto retry;
2714 	}
2715 
2716 	if (gfp_mask & __GFP_NORETRY)
2717 		goto nomem;
2718 	/*
2719 	 * Even though the limit is exceeded at this point, reclaim
2720 	 * may have been able to free some pages.  Retry the charge
2721 	 * before killing the task.
2722 	 *
2723 	 * Only for regular pages, though: huge pages are rather
2724 	 * unlikely to succeed so close to the limit, and we fall back
2725 	 * to regular pages anyway in case of failure.
2726 	 */
2727 	if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
2728 		goto retry;
2729 	/*
2730 	 * At task move, charge accounts can be doubly counted. So, it's
2731 	 * better to wait until the end of task_move if something is going on.
2732 	 */
2733 	if (mem_cgroup_wait_acct_move(mem_over_limit))
2734 		goto retry;
2735 
2736 	if (nr_retries--)
2737 		goto retry;
2738 
2739 	if (gfp_mask & __GFP_RETRY_MAYFAIL)
2740 		goto nomem;
2741 
2742 	if (gfp_mask & __GFP_NOFAIL)
2743 		goto force;
2744 
2745 	if (fatal_signal_pending(current))
2746 		goto force;
2747 
2748 	/*
2749 	 * keep retrying as long as the memcg oom killer is able to make
2750 	 * a forward progress or bypass the charge if the oom killer
2751 	 * couldn't make any progress.
2752 	 */
2753 	oom_status = mem_cgroup_oom(mem_over_limit, gfp_mask,
2754 		       get_order(nr_pages * PAGE_SIZE));
2755 	switch (oom_status) {
2756 	case OOM_SUCCESS:
2757 		nr_retries = MAX_RECLAIM_RETRIES;
2758 		goto retry;
2759 	case OOM_FAILED:
2760 		goto force;
2761 	default:
2762 		goto nomem;
2763 	}
2764 nomem:
2765 	if (!(gfp_mask & __GFP_NOFAIL))
2766 		return -ENOMEM;
2767 force:
2768 	/*
2769 	 * The allocation either can't fail or will lead to more memory
2770 	 * being freed very soon.  Allow memory usage go over the limit
2771 	 * temporarily by force charging it.
2772 	 */
2773 	page_counter_charge(&memcg->memory, nr_pages);
2774 	if (do_memsw_account())
2775 		page_counter_charge(&memcg->memsw, nr_pages);
2776 
2777 	return 0;
2778 
2779 done_restock:
2780 	if (batch > nr_pages)
2781 		refill_stock(memcg, batch - nr_pages);
2782 
2783 	/*
2784 	 * If the hierarchy is above the normal consumption range, schedule
2785 	 * reclaim on returning to userland.  We can perform reclaim here
2786 	 * if __GFP_RECLAIM but let's always punt for simplicity and so that
2787 	 * GFP_KERNEL can consistently be used during reclaim.  @memcg is
2788 	 * not recorded as it most likely matches current's and won't
2789 	 * change in the meantime.  As high limit is checked again before
2790 	 * reclaim, the cost of mismatch is negligible.
2791 	 */
2792 	do {
2793 		bool mem_high, swap_high;
2794 
2795 		mem_high = page_counter_read(&memcg->memory) >
2796 			READ_ONCE(memcg->memory.high);
2797 		swap_high = page_counter_read(&memcg->swap) >
2798 			READ_ONCE(memcg->swap.high);
2799 
2800 		/* Don't bother a random interrupted task */
2801 		if (in_interrupt()) {
2802 			if (mem_high) {
2803 				schedule_work(&memcg->high_work);
2804 				break;
2805 			}
2806 			continue;
2807 		}
2808 
2809 		if (mem_high || swap_high) {
2810 			/*
2811 			 * The allocating tasks in this cgroup will need to do
2812 			 * reclaim or be throttled to prevent further growth
2813 			 * of the memory or swap footprints.
2814 			 *
2815 			 * Target some best-effort fairness between the tasks,
2816 			 * and distribute reclaim work and delay penalties
2817 			 * based on how much each task is actually allocating.
2818 			 */
2819 			current->memcg_nr_pages_over_high += batch;
2820 			set_notify_resume(current);
2821 			break;
2822 		}
2823 	} while ((memcg = parent_mem_cgroup(memcg)));
2824 
2825 	return 0;
2826 }
2827 
2828 #if defined(CONFIG_MEMCG_KMEM) || defined(CONFIG_MMU)
2829 static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2830 {
2831 	if (mem_cgroup_is_root(memcg))
2832 		return;
2833 
2834 	page_counter_uncharge(&memcg->memory, nr_pages);
2835 	if (do_memsw_account())
2836 		page_counter_uncharge(&memcg->memsw, nr_pages);
2837 }
2838 #endif
2839 
2840 static void commit_charge(struct page *page, struct mem_cgroup *memcg)
2841 {
2842 	VM_BUG_ON_PAGE(page->mem_cgroup, page);
2843 	/*
2844 	 * Any of the following ensures page->mem_cgroup stability:
2845 	 *
2846 	 * - the page lock
2847 	 * - LRU isolation
2848 	 * - lock_page_memcg()
2849 	 * - exclusive reference
2850 	 */
2851 	page->mem_cgroup = memcg;
2852 }
2853 
2854 #ifdef CONFIG_MEMCG_KMEM
2855 int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
2856 				 gfp_t gfp)
2857 {
2858 	unsigned int objects = objs_per_slab_page(s, page);
2859 	void *vec;
2860 
2861 	vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp,
2862 			   page_to_nid(page));
2863 	if (!vec)
2864 		return -ENOMEM;
2865 
2866 	if (cmpxchg(&page->obj_cgroups, NULL,
2867 		    (struct obj_cgroup **) ((unsigned long)vec | 0x1UL)))
2868 		kfree(vec);
2869 	else
2870 		kmemleak_not_leak(vec);
2871 
2872 	return 0;
2873 }
2874 
2875 /*
2876  * Returns a pointer to the memory cgroup to which the kernel object is charged.
2877  *
2878  * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(),
2879  * cgroup_mutex, etc.
2880  */
2881 struct mem_cgroup *mem_cgroup_from_obj(void *p)
2882 {
2883 	struct page *page;
2884 
2885 	if (mem_cgroup_disabled())
2886 		return NULL;
2887 
2888 	page = virt_to_head_page(p);
2889 
2890 	/*
2891 	 * Slab objects are accounted individually, not per-page.
2892 	 * Memcg membership data for each individual object is saved in
2893 	 * the page->obj_cgroups.
2894 	 */
2895 	if (page_has_obj_cgroups(page)) {
2896 		struct obj_cgroup *objcg;
2897 		unsigned int off;
2898 
2899 		off = obj_to_index(page->slab_cache, page, p);
2900 		objcg = page_obj_cgroups(page)[off];
2901 		if (objcg)
2902 			return obj_cgroup_memcg(objcg);
2903 
2904 		return NULL;
2905 	}
2906 
2907 	/* All other pages use page->mem_cgroup */
2908 	return page->mem_cgroup;
2909 }
2910 
2911 __always_inline struct obj_cgroup *get_obj_cgroup_from_current(void)
2912 {
2913 	struct obj_cgroup *objcg = NULL;
2914 	struct mem_cgroup *memcg;
2915 
2916 	if (unlikely(!current->mm && !current->active_memcg))
2917 		return NULL;
2918 
2919 	rcu_read_lock();
2920 	if (unlikely(current->active_memcg))
2921 		memcg = rcu_dereference(current->active_memcg);
2922 	else
2923 		memcg = mem_cgroup_from_task(current);
2924 
2925 	for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) {
2926 		objcg = rcu_dereference(memcg->objcg);
2927 		if (objcg && obj_cgroup_tryget(objcg))
2928 			break;
2929 	}
2930 	rcu_read_unlock();
2931 
2932 	return objcg;
2933 }
2934 
2935 static int memcg_alloc_cache_id(void)
2936 {
2937 	int id, size;
2938 	int err;
2939 
2940 	id = ida_simple_get(&memcg_cache_ida,
2941 			    0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
2942 	if (id < 0)
2943 		return id;
2944 
2945 	if (id < memcg_nr_cache_ids)
2946 		return id;
2947 
2948 	/*
2949 	 * There's no space for the new id in memcg_caches arrays,
2950 	 * so we have to grow them.
2951 	 */
2952 	down_write(&memcg_cache_ids_sem);
2953 
2954 	size = 2 * (id + 1);
2955 	if (size < MEMCG_CACHES_MIN_SIZE)
2956 		size = MEMCG_CACHES_MIN_SIZE;
2957 	else if (size > MEMCG_CACHES_MAX_SIZE)
2958 		size = MEMCG_CACHES_MAX_SIZE;
2959 
2960 	err = memcg_update_all_list_lrus(size);
2961 	if (!err)
2962 		memcg_nr_cache_ids = size;
2963 
2964 	up_write(&memcg_cache_ids_sem);
2965 
2966 	if (err) {
2967 		ida_simple_remove(&memcg_cache_ida, id);
2968 		return err;
2969 	}
2970 	return id;
2971 }
2972 
2973 static void memcg_free_cache_id(int id)
2974 {
2975 	ida_simple_remove(&memcg_cache_ida, id);
2976 }
2977 
2978 /**
2979  * __memcg_kmem_charge: charge a number of kernel pages to a memcg
2980  * @memcg: memory cgroup to charge
2981  * @gfp: reclaim mode
2982  * @nr_pages: number of pages to charge
2983  *
2984  * Returns 0 on success, an error code on failure.
2985  */
2986 int __memcg_kmem_charge(struct mem_cgroup *memcg, gfp_t gfp,
2987 			unsigned int nr_pages)
2988 {
2989 	struct page_counter *counter;
2990 	int ret;
2991 
2992 	ret = try_charge(memcg, gfp, nr_pages);
2993 	if (ret)
2994 		return ret;
2995 
2996 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
2997 	    !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
2998 
2999 		/*
3000 		 * Enforce __GFP_NOFAIL allocation because callers are not
3001 		 * prepared to see failures and likely do not have any failure
3002 		 * handling code.
3003 		 */
3004 		if (gfp & __GFP_NOFAIL) {
3005 			page_counter_charge(&memcg->kmem, nr_pages);
3006 			return 0;
3007 		}
3008 		cancel_charge(memcg, nr_pages);
3009 		return -ENOMEM;
3010 	}
3011 	return 0;
3012 }
3013 
3014 /**
3015  * __memcg_kmem_uncharge: uncharge a number of kernel pages from a memcg
3016  * @memcg: memcg to uncharge
3017  * @nr_pages: number of pages to uncharge
3018  */
3019 void __memcg_kmem_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages)
3020 {
3021 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
3022 		page_counter_uncharge(&memcg->kmem, nr_pages);
3023 
3024 	page_counter_uncharge(&memcg->memory, nr_pages);
3025 	if (do_memsw_account())
3026 		page_counter_uncharge(&memcg->memsw, nr_pages);
3027 }
3028 
3029 /**
3030  * __memcg_kmem_charge_page: charge a kmem page to the current memory cgroup
3031  * @page: page to charge
3032  * @gfp: reclaim mode
3033  * @order: allocation order
3034  *
3035  * Returns 0 on success, an error code on failure.
3036  */
3037 int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
3038 {
3039 	struct mem_cgroup *memcg;
3040 	int ret = 0;
3041 
3042 	if (memcg_kmem_bypass())
3043 		return 0;
3044 
3045 	memcg = get_mem_cgroup_from_current();
3046 	if (!mem_cgroup_is_root(memcg)) {
3047 		ret = __memcg_kmem_charge(memcg, gfp, 1 << order);
3048 		if (!ret) {
3049 			page->mem_cgroup = memcg;
3050 			__SetPageKmemcg(page);
3051 			return 0;
3052 		}
3053 	}
3054 	css_put(&memcg->css);
3055 	return ret;
3056 }
3057 
3058 /**
3059  * __memcg_kmem_uncharge_page: uncharge a kmem page
3060  * @page: page to uncharge
3061  * @order: allocation order
3062  */
3063 void __memcg_kmem_uncharge_page(struct page *page, int order)
3064 {
3065 	struct mem_cgroup *memcg = page->mem_cgroup;
3066 	unsigned int nr_pages = 1 << order;
3067 
3068 	if (!memcg)
3069 		return;
3070 
3071 	VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
3072 	__memcg_kmem_uncharge(memcg, nr_pages);
3073 	page->mem_cgroup = NULL;
3074 	css_put(&memcg->css);
3075 
3076 	/* slab pages do not have PageKmemcg flag set */
3077 	if (PageKmemcg(page))
3078 		__ClearPageKmemcg(page);
3079 }
3080 
3081 static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes)
3082 {
3083 	struct memcg_stock_pcp *stock;
3084 	unsigned long flags;
3085 	bool ret = false;
3086 
3087 	local_irq_save(flags);
3088 
3089 	stock = this_cpu_ptr(&memcg_stock);
3090 	if (objcg == stock->cached_objcg && stock->nr_bytes >= nr_bytes) {
3091 		stock->nr_bytes -= nr_bytes;
3092 		ret = true;
3093 	}
3094 
3095 	local_irq_restore(flags);
3096 
3097 	return ret;
3098 }
3099 
3100 static void drain_obj_stock(struct memcg_stock_pcp *stock)
3101 {
3102 	struct obj_cgroup *old = stock->cached_objcg;
3103 
3104 	if (!old)
3105 		return;
3106 
3107 	if (stock->nr_bytes) {
3108 		unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT;
3109 		unsigned int nr_bytes = stock->nr_bytes & (PAGE_SIZE - 1);
3110 
3111 		if (nr_pages) {
3112 			rcu_read_lock();
3113 			__memcg_kmem_uncharge(obj_cgroup_memcg(old), nr_pages);
3114 			rcu_read_unlock();
3115 		}
3116 
3117 		/*
3118 		 * The leftover is flushed to the centralized per-memcg value.
3119 		 * On the next attempt to refill obj stock it will be moved
3120 		 * to a per-cpu stock (probably, on an other CPU), see
3121 		 * refill_obj_stock().
3122 		 *
3123 		 * How often it's flushed is a trade-off between the memory
3124 		 * limit enforcement accuracy and potential CPU contention,
3125 		 * so it might be changed in the future.
3126 		 */
3127 		atomic_add(nr_bytes, &old->nr_charged_bytes);
3128 		stock->nr_bytes = 0;
3129 	}
3130 
3131 	obj_cgroup_put(old);
3132 	stock->cached_objcg = NULL;
3133 }
3134 
3135 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
3136 				     struct mem_cgroup *root_memcg)
3137 {
3138 	struct mem_cgroup *memcg;
3139 
3140 	if (stock->cached_objcg) {
3141 		memcg = obj_cgroup_memcg(stock->cached_objcg);
3142 		if (memcg && mem_cgroup_is_descendant(memcg, root_memcg))
3143 			return true;
3144 	}
3145 
3146 	return false;
3147 }
3148 
3149 static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes)
3150 {
3151 	struct memcg_stock_pcp *stock;
3152 	unsigned long flags;
3153 
3154 	local_irq_save(flags);
3155 
3156 	stock = this_cpu_ptr(&memcg_stock);
3157 	if (stock->cached_objcg != objcg) { /* reset if necessary */
3158 		drain_obj_stock(stock);
3159 		obj_cgroup_get(objcg);
3160 		stock->cached_objcg = objcg;
3161 		stock->nr_bytes = atomic_xchg(&objcg->nr_charged_bytes, 0);
3162 	}
3163 	stock->nr_bytes += nr_bytes;
3164 
3165 	if (stock->nr_bytes > PAGE_SIZE)
3166 		drain_obj_stock(stock);
3167 
3168 	local_irq_restore(flags);
3169 }
3170 
3171 int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size)
3172 {
3173 	struct mem_cgroup *memcg;
3174 	unsigned int nr_pages, nr_bytes;
3175 	int ret;
3176 
3177 	if (consume_obj_stock(objcg, size))
3178 		return 0;
3179 
3180 	/*
3181 	 * In theory, memcg->nr_charged_bytes can have enough
3182 	 * pre-charged bytes to satisfy the allocation. However,
3183 	 * flushing memcg->nr_charged_bytes requires two atomic
3184 	 * operations, and memcg->nr_charged_bytes can't be big,
3185 	 * so it's better to ignore it and try grab some new pages.
3186 	 * memcg->nr_charged_bytes will be flushed in
3187 	 * refill_obj_stock(), called from this function or
3188 	 * independently later.
3189 	 */
3190 	rcu_read_lock();
3191 	memcg = obj_cgroup_memcg(objcg);
3192 	css_get(&memcg->css);
3193 	rcu_read_unlock();
3194 
3195 	nr_pages = size >> PAGE_SHIFT;
3196 	nr_bytes = size & (PAGE_SIZE - 1);
3197 
3198 	if (nr_bytes)
3199 		nr_pages += 1;
3200 
3201 	ret = __memcg_kmem_charge(memcg, gfp, nr_pages);
3202 	if (!ret && nr_bytes)
3203 		refill_obj_stock(objcg, PAGE_SIZE - nr_bytes);
3204 
3205 	css_put(&memcg->css);
3206 	return ret;
3207 }
3208 
3209 void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size)
3210 {
3211 	refill_obj_stock(objcg, size);
3212 }
3213 
3214 #endif /* CONFIG_MEMCG_KMEM */
3215 
3216 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3217 
3218 /*
3219  * Because tail pages are not marked as "used", set it. We're under
3220  * pgdat->lru_lock and migration entries setup in all page mappings.
3221  */
3222 void mem_cgroup_split_huge_fixup(struct page *head)
3223 {
3224 	struct mem_cgroup *memcg = head->mem_cgroup;
3225 	int i;
3226 
3227 	if (mem_cgroup_disabled())
3228 		return;
3229 
3230 	for (i = 1; i < HPAGE_PMD_NR; i++) {
3231 		css_get(&memcg->css);
3232 		head[i].mem_cgroup = memcg;
3233 	}
3234 }
3235 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
3236 
3237 #ifdef CONFIG_MEMCG_SWAP
3238 /**
3239  * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
3240  * @entry: swap entry to be moved
3241  * @from:  mem_cgroup which the entry is moved from
3242  * @to:  mem_cgroup which the entry is moved to
3243  *
3244  * It succeeds only when the swap_cgroup's record for this entry is the same
3245  * as the mem_cgroup's id of @from.
3246  *
3247  * Returns 0 on success, -EINVAL on failure.
3248  *
3249  * The caller must have charged to @to, IOW, called page_counter_charge() about
3250  * both res and memsw, and called css_get().
3251  */
3252 static int mem_cgroup_move_swap_account(swp_entry_t entry,
3253 				struct mem_cgroup *from, struct mem_cgroup *to)
3254 {
3255 	unsigned short old_id, new_id;
3256 
3257 	old_id = mem_cgroup_id(from);
3258 	new_id = mem_cgroup_id(to);
3259 
3260 	if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
3261 		mod_memcg_state(from, MEMCG_SWAP, -1);
3262 		mod_memcg_state(to, MEMCG_SWAP, 1);
3263 		return 0;
3264 	}
3265 	return -EINVAL;
3266 }
3267 #else
3268 static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
3269 				struct mem_cgroup *from, struct mem_cgroup *to)
3270 {
3271 	return -EINVAL;
3272 }
3273 #endif
3274 
3275 static DEFINE_MUTEX(memcg_max_mutex);
3276 
3277 static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
3278 				 unsigned long max, bool memsw)
3279 {
3280 	bool enlarge = false;
3281 	bool drained = false;
3282 	int ret;
3283 	bool limits_invariant;
3284 	struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory;
3285 
3286 	do {
3287 		if (signal_pending(current)) {
3288 			ret = -EINTR;
3289 			break;
3290 		}
3291 
3292 		mutex_lock(&memcg_max_mutex);
3293 		/*
3294 		 * Make sure that the new limit (memsw or memory limit) doesn't
3295 		 * break our basic invariant rule memory.max <= memsw.max.
3296 		 */
3297 		limits_invariant = memsw ? max >= READ_ONCE(memcg->memory.max) :
3298 					   max <= memcg->memsw.max;
3299 		if (!limits_invariant) {
3300 			mutex_unlock(&memcg_max_mutex);
3301 			ret = -EINVAL;
3302 			break;
3303 		}
3304 		if (max > counter->max)
3305 			enlarge = true;
3306 		ret = page_counter_set_max(counter, max);
3307 		mutex_unlock(&memcg_max_mutex);
3308 
3309 		if (!ret)
3310 			break;
3311 
3312 		if (!drained) {
3313 			drain_all_stock(memcg);
3314 			drained = true;
3315 			continue;
3316 		}
3317 
3318 		if (!try_to_free_mem_cgroup_pages(memcg, 1,
3319 					GFP_KERNEL, !memsw)) {
3320 			ret = -EBUSY;
3321 			break;
3322 		}
3323 	} while (true);
3324 
3325 	if (!ret && enlarge)
3326 		memcg_oom_recover(memcg);
3327 
3328 	return ret;
3329 }
3330 
3331 unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
3332 					    gfp_t gfp_mask,
3333 					    unsigned long *total_scanned)
3334 {
3335 	unsigned long nr_reclaimed = 0;
3336 	struct mem_cgroup_per_node *mz, *next_mz = NULL;
3337 	unsigned long reclaimed;
3338 	int loop = 0;
3339 	struct mem_cgroup_tree_per_node *mctz;
3340 	unsigned long excess;
3341 	unsigned long nr_scanned;
3342 
3343 	if (order > 0)
3344 		return 0;
3345 
3346 	mctz = soft_limit_tree_node(pgdat->node_id);
3347 
3348 	/*
3349 	 * Do not even bother to check the largest node if the root
3350 	 * is empty. Do it lockless to prevent lock bouncing. Races
3351 	 * are acceptable as soft limit is best effort anyway.
3352 	 */
3353 	if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
3354 		return 0;
3355 
3356 	/*
3357 	 * This loop can run a while, specially if mem_cgroup's continuously
3358 	 * keep exceeding their soft limit and putting the system under
3359 	 * pressure
3360 	 */
3361 	do {
3362 		if (next_mz)
3363 			mz = next_mz;
3364 		else
3365 			mz = mem_cgroup_largest_soft_limit_node(mctz);
3366 		if (!mz)
3367 			break;
3368 
3369 		nr_scanned = 0;
3370 		reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
3371 						    gfp_mask, &nr_scanned);
3372 		nr_reclaimed += reclaimed;
3373 		*total_scanned += nr_scanned;
3374 		spin_lock_irq(&mctz->lock);
3375 		__mem_cgroup_remove_exceeded(mz, mctz);
3376 
3377 		/*
3378 		 * If we failed to reclaim anything from this memory cgroup
3379 		 * it is time to move on to the next cgroup
3380 		 */
3381 		next_mz = NULL;
3382 		if (!reclaimed)
3383 			next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
3384 
3385 		excess = soft_limit_excess(mz->memcg);
3386 		/*
3387 		 * One school of thought says that we should not add
3388 		 * back the node to the tree if reclaim returns 0.
3389 		 * But our reclaim could return 0, simply because due
3390 		 * to priority we are exposing a smaller subset of
3391 		 * memory to reclaim from. Consider this as a longer
3392 		 * term TODO.
3393 		 */
3394 		/* If excess == 0, no tree ops */
3395 		__mem_cgroup_insert_exceeded(mz, mctz, excess);
3396 		spin_unlock_irq(&mctz->lock);
3397 		css_put(&mz->memcg->css);
3398 		loop++;
3399 		/*
3400 		 * Could not reclaim anything and there are no more
3401 		 * mem cgroups to try or we seem to be looping without
3402 		 * reclaiming anything.
3403 		 */
3404 		if (!nr_reclaimed &&
3405 			(next_mz == NULL ||
3406 			loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
3407 			break;
3408 	} while (!nr_reclaimed);
3409 	if (next_mz)
3410 		css_put(&next_mz->memcg->css);
3411 	return nr_reclaimed;
3412 }
3413 
3414 /*
3415  * Test whether @memcg has children, dead or alive.  Note that this
3416  * function doesn't care whether @memcg has use_hierarchy enabled and
3417  * returns %true if there are child csses according to the cgroup
3418  * hierarchy.  Testing use_hierarchy is the caller's responsibility.
3419  */
3420 static inline bool memcg_has_children(struct mem_cgroup *memcg)
3421 {
3422 	bool ret;
3423 
3424 	rcu_read_lock();
3425 	ret = css_next_child(NULL, &memcg->css);
3426 	rcu_read_unlock();
3427 	return ret;
3428 }
3429 
3430 /*
3431  * Reclaims as many pages from the given memcg as possible.
3432  *
3433  * Caller is responsible for holding css reference for memcg.
3434  */
3435 static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
3436 {
3437 	int nr_retries = MAX_RECLAIM_RETRIES;
3438 
3439 	/* we call try-to-free pages for make this cgroup empty */
3440 	lru_add_drain_all();
3441 
3442 	drain_all_stock(memcg);
3443 
3444 	/* try to free all pages in this cgroup */
3445 	while (nr_retries && page_counter_read(&memcg->memory)) {
3446 		int progress;
3447 
3448 		if (signal_pending(current))
3449 			return -EINTR;
3450 
3451 		progress = try_to_free_mem_cgroup_pages(memcg, 1,
3452 							GFP_KERNEL, true);
3453 		if (!progress) {
3454 			nr_retries--;
3455 			/* maybe some writeback is necessary */
3456 			congestion_wait(BLK_RW_ASYNC, HZ/10);
3457 		}
3458 
3459 	}
3460 
3461 	return 0;
3462 }
3463 
3464 static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
3465 					    char *buf, size_t nbytes,
3466 					    loff_t off)
3467 {
3468 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3469 
3470 	if (mem_cgroup_is_root(memcg))
3471 		return -EINVAL;
3472 	return mem_cgroup_force_empty(memcg) ?: nbytes;
3473 }
3474 
3475 static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
3476 				     struct cftype *cft)
3477 {
3478 	return mem_cgroup_from_css(css)->use_hierarchy;
3479 }
3480 
3481 static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
3482 				      struct cftype *cft, u64 val)
3483 {
3484 	int retval = 0;
3485 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3486 	struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);
3487 
3488 	if (memcg->use_hierarchy == val)
3489 		return 0;
3490 
3491 	/*
3492 	 * If parent's use_hierarchy is set, we can't make any modifications
3493 	 * in the child subtrees. If it is unset, then the change can
3494 	 * occur, provided the current cgroup has no children.
3495 	 *
3496 	 * For the root cgroup, parent_mem is NULL, we allow value to be
3497 	 * set if there are no children.
3498 	 */
3499 	if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
3500 				(val == 1 || val == 0)) {
3501 		if (!memcg_has_children(memcg))
3502 			memcg->use_hierarchy = val;
3503 		else
3504 			retval = -EBUSY;
3505 	} else
3506 		retval = -EINVAL;
3507 
3508 	return retval;
3509 }
3510 
3511 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
3512 {
3513 	unsigned long val;
3514 
3515 	if (mem_cgroup_is_root(memcg)) {
3516 		val = memcg_page_state(memcg, NR_FILE_PAGES) +
3517 			memcg_page_state(memcg, NR_ANON_MAPPED);
3518 		if (swap)
3519 			val += memcg_page_state(memcg, MEMCG_SWAP);
3520 	} else {
3521 		if (!swap)
3522 			val = page_counter_read(&memcg->memory);
3523 		else
3524 			val = page_counter_read(&memcg->memsw);
3525 	}
3526 	return val;
3527 }
3528 
3529 enum {
3530 	RES_USAGE,
3531 	RES_LIMIT,
3532 	RES_MAX_USAGE,
3533 	RES_FAILCNT,
3534 	RES_SOFT_LIMIT,
3535 };
3536 
3537 static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
3538 			       struct cftype *cft)
3539 {
3540 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3541 	struct page_counter *counter;
3542 
3543 	switch (MEMFILE_TYPE(cft->private)) {
3544 	case _MEM:
3545 		counter = &memcg->memory;
3546 		break;
3547 	case _MEMSWAP:
3548 		counter = &memcg->memsw;
3549 		break;
3550 	case _KMEM:
3551 		counter = &memcg->kmem;
3552 		break;
3553 	case _TCP:
3554 		counter = &memcg->tcpmem;
3555 		break;
3556 	default:
3557 		BUG();
3558 	}
3559 
3560 	switch (MEMFILE_ATTR(cft->private)) {
3561 	case RES_USAGE:
3562 		if (counter == &memcg->memory)
3563 			return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
3564 		if (counter == &memcg->memsw)
3565 			return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
3566 		return (u64)page_counter_read(counter) * PAGE_SIZE;
3567 	case RES_LIMIT:
3568 		return (u64)counter->max * PAGE_SIZE;
3569 	case RES_MAX_USAGE:
3570 		return (u64)counter->watermark * PAGE_SIZE;
3571 	case RES_FAILCNT:
3572 		return counter->failcnt;
3573 	case RES_SOFT_LIMIT:
3574 		return (u64)memcg->soft_limit * PAGE_SIZE;
3575 	default:
3576 		BUG();
3577 	}
3578 }
3579 
3580 static void memcg_flush_percpu_vmstats(struct mem_cgroup *memcg)
3581 {
3582 	unsigned long stat[MEMCG_NR_STAT] = {0};
3583 	struct mem_cgroup *mi;
3584 	int node, cpu, i;
3585 
3586 	for_each_online_cpu(cpu)
3587 		for (i = 0; i < MEMCG_NR_STAT; i++)
3588 			stat[i] += per_cpu(memcg->vmstats_percpu->stat[i], cpu);
3589 
3590 	for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
3591 		for (i = 0; i < MEMCG_NR_STAT; i++)
3592 			atomic_long_add(stat[i], &mi->vmstats[i]);
3593 
3594 	for_each_node(node) {
3595 		struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
3596 		struct mem_cgroup_per_node *pi;
3597 
3598 		for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
3599 			stat[i] = 0;
3600 
3601 		for_each_online_cpu(cpu)
3602 			for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
3603 				stat[i] += per_cpu(
3604 					pn->lruvec_stat_cpu->count[i], cpu);
3605 
3606 		for (pi = pn; pi; pi = parent_nodeinfo(pi, node))
3607 			for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
3608 				atomic_long_add(stat[i], &pi->lruvec_stat[i]);
3609 	}
3610 }
3611 
3612 static void memcg_flush_percpu_vmevents(struct mem_cgroup *memcg)
3613 {
3614 	unsigned long events[NR_VM_EVENT_ITEMS];
3615 	struct mem_cgroup *mi;
3616 	int cpu, i;
3617 
3618 	for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
3619 		events[i] = 0;
3620 
3621 	for_each_online_cpu(cpu)
3622 		for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
3623 			events[i] += per_cpu(memcg->vmstats_percpu->events[i],
3624 					     cpu);
3625 
3626 	for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
3627 		for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
3628 			atomic_long_add(events[i], &mi->vmevents[i]);
3629 }
3630 
3631 #ifdef CONFIG_MEMCG_KMEM
3632 static int memcg_online_kmem(struct mem_cgroup *memcg)
3633 {
3634 	struct obj_cgroup *objcg;
3635 	int memcg_id;
3636 
3637 	if (cgroup_memory_nokmem)
3638 		return 0;
3639 
3640 	BUG_ON(memcg->kmemcg_id >= 0);
3641 	BUG_ON(memcg->kmem_state);
3642 
3643 	memcg_id = memcg_alloc_cache_id();
3644 	if (memcg_id < 0)
3645 		return memcg_id;
3646 
3647 	objcg = obj_cgroup_alloc();
3648 	if (!objcg) {
3649 		memcg_free_cache_id(memcg_id);
3650 		return -ENOMEM;
3651 	}
3652 	objcg->memcg = memcg;
3653 	rcu_assign_pointer(memcg->objcg, objcg);
3654 
3655 	static_branch_enable(&memcg_kmem_enabled_key);
3656 
3657 	/*
3658 	 * A memory cgroup is considered kmem-online as soon as it gets
3659 	 * kmemcg_id. Setting the id after enabling static branching will
3660 	 * guarantee no one starts accounting before all call sites are
3661 	 * patched.
3662 	 */
3663 	memcg->kmemcg_id = memcg_id;
3664 	memcg->kmem_state = KMEM_ONLINE;
3665 
3666 	return 0;
3667 }
3668 
3669 static void memcg_offline_kmem(struct mem_cgroup *memcg)
3670 {
3671 	struct cgroup_subsys_state *css;
3672 	struct mem_cgroup *parent, *child;
3673 	int kmemcg_id;
3674 
3675 	if (memcg->kmem_state != KMEM_ONLINE)
3676 		return;
3677 
3678 	memcg->kmem_state = KMEM_ALLOCATED;
3679 
3680 	parent = parent_mem_cgroup(memcg);
3681 	if (!parent)
3682 		parent = root_mem_cgroup;
3683 
3684 	memcg_reparent_objcgs(memcg, parent);
3685 
3686 	kmemcg_id = memcg->kmemcg_id;
3687 	BUG_ON(kmemcg_id < 0);
3688 
3689 	/*
3690 	 * Change kmemcg_id of this cgroup and all its descendants to the
3691 	 * parent's id, and then move all entries from this cgroup's list_lrus
3692 	 * to ones of the parent. After we have finished, all list_lrus
3693 	 * corresponding to this cgroup are guaranteed to remain empty. The
3694 	 * ordering is imposed by list_lru_node->lock taken by
3695 	 * memcg_drain_all_list_lrus().
3696 	 */
3697 	rcu_read_lock(); /* can be called from css_free w/o cgroup_mutex */
3698 	css_for_each_descendant_pre(css, &memcg->css) {
3699 		child = mem_cgroup_from_css(css);
3700 		BUG_ON(child->kmemcg_id != kmemcg_id);
3701 		child->kmemcg_id = parent->kmemcg_id;
3702 		if (!memcg->use_hierarchy)
3703 			break;
3704 	}
3705 	rcu_read_unlock();
3706 
3707 	memcg_drain_all_list_lrus(kmemcg_id, parent);
3708 
3709 	memcg_free_cache_id(kmemcg_id);
3710 }
3711 
3712 static void memcg_free_kmem(struct mem_cgroup *memcg)
3713 {
3714 	/* css_alloc() failed, offlining didn't happen */
3715 	if (unlikely(memcg->kmem_state == KMEM_ONLINE))
3716 		memcg_offline_kmem(memcg);
3717 }
3718 #else
3719 static int memcg_online_kmem(struct mem_cgroup *memcg)
3720 {
3721 	return 0;
3722 }
3723 static void memcg_offline_kmem(struct mem_cgroup *memcg)
3724 {
3725 }
3726 static void memcg_free_kmem(struct mem_cgroup *memcg)
3727 {
3728 }
3729 #endif /* CONFIG_MEMCG_KMEM */
3730 
3731 static int memcg_update_kmem_max(struct mem_cgroup *memcg,
3732 				 unsigned long max)
3733 {
3734 	int ret;
3735 
3736 	mutex_lock(&memcg_max_mutex);
3737 	ret = page_counter_set_max(&memcg->kmem, max);
3738 	mutex_unlock(&memcg_max_mutex);
3739 	return ret;
3740 }
3741 
3742 static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max)
3743 {
3744 	int ret;
3745 
3746 	mutex_lock(&memcg_max_mutex);
3747 
3748 	ret = page_counter_set_max(&memcg->tcpmem, max);
3749 	if (ret)
3750 		goto out;
3751 
3752 	if (!memcg->tcpmem_active) {
3753 		/*
3754 		 * The active flag needs to be written after the static_key
3755 		 * update. This is what guarantees that the socket activation
3756 		 * function is the last one to run. See mem_cgroup_sk_alloc()
3757 		 * for details, and note that we don't mark any socket as
3758 		 * belonging to this memcg until that flag is up.
3759 		 *
3760 		 * We need to do this, because static_keys will span multiple
3761 		 * sites, but we can't control their order. If we mark a socket
3762 		 * as accounted, but the accounting functions are not patched in
3763 		 * yet, we'll lose accounting.
3764 		 *
3765 		 * We never race with the readers in mem_cgroup_sk_alloc(),
3766 		 * because when this value change, the code to process it is not
3767 		 * patched in yet.
3768 		 */
3769 		static_branch_inc(&memcg_sockets_enabled_key);
3770 		memcg->tcpmem_active = true;
3771 	}
3772 out:
3773 	mutex_unlock(&memcg_max_mutex);
3774 	return ret;
3775 }
3776 
3777 /*
3778  * The user of this function is...
3779  * RES_LIMIT.
3780  */
3781 static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
3782 				char *buf, size_t nbytes, loff_t off)
3783 {
3784 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3785 	unsigned long nr_pages;
3786 	int ret;
3787 
3788 	buf = strstrip(buf);
3789 	ret = page_counter_memparse(buf, "-1", &nr_pages);
3790 	if (ret)
3791 		return ret;
3792 
3793 	switch (MEMFILE_ATTR(of_cft(of)->private)) {
3794 	case RES_LIMIT:
3795 		if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
3796 			ret = -EINVAL;
3797 			break;
3798 		}
3799 		switch (MEMFILE_TYPE(of_cft(of)->private)) {
3800 		case _MEM:
3801 			ret = mem_cgroup_resize_max(memcg, nr_pages, false);
3802 			break;
3803 		case _MEMSWAP:
3804 			ret = mem_cgroup_resize_max(memcg, nr_pages, true);
3805 			break;
3806 		case _KMEM:
3807 			pr_warn_once("kmem.limit_in_bytes is deprecated and will be removed. "
3808 				     "Please report your usecase to linux-mm@kvack.org if you "
3809 				     "depend on this functionality.\n");
3810 			ret = memcg_update_kmem_max(memcg, nr_pages);
3811 			break;
3812 		case _TCP:
3813 			ret = memcg_update_tcp_max(memcg, nr_pages);
3814 			break;
3815 		}
3816 		break;
3817 	case RES_SOFT_LIMIT:
3818 		memcg->soft_limit = nr_pages;
3819 		ret = 0;
3820 		break;
3821 	}
3822 	return ret ?: nbytes;
3823 }
3824 
3825 static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
3826 				size_t nbytes, loff_t off)
3827 {
3828 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3829 	struct page_counter *counter;
3830 
3831 	switch (MEMFILE_TYPE(of_cft(of)->private)) {
3832 	case _MEM:
3833 		counter = &memcg->memory;
3834 		break;
3835 	case _MEMSWAP:
3836 		counter = &memcg->memsw;
3837 		break;
3838 	case _KMEM:
3839 		counter = &memcg->kmem;
3840 		break;
3841 	case _TCP:
3842 		counter = &memcg->tcpmem;
3843 		break;
3844 	default:
3845 		BUG();
3846 	}
3847 
3848 	switch (MEMFILE_ATTR(of_cft(of)->private)) {
3849 	case RES_MAX_USAGE:
3850 		page_counter_reset_watermark(counter);
3851 		break;
3852 	case RES_FAILCNT:
3853 		counter->failcnt = 0;
3854 		break;
3855 	default:
3856 		BUG();
3857 	}
3858 
3859 	return nbytes;
3860 }
3861 
3862 static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
3863 					struct cftype *cft)
3864 {
3865 	return mem_cgroup_from_css(css)->move_charge_at_immigrate;
3866 }
3867 
3868 #ifdef CONFIG_MMU
3869 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3870 					struct cftype *cft, u64 val)
3871 {
3872 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3873 
3874 	if (val & ~MOVE_MASK)
3875 		return -EINVAL;
3876 
3877 	/*
3878 	 * No kind of locking is needed in here, because ->can_attach() will
3879 	 * check this value once in the beginning of the process, and then carry
3880 	 * on with stale data. This means that changes to this value will only
3881 	 * affect task migrations starting after the change.
3882 	 */
3883 	memcg->move_charge_at_immigrate = val;
3884 	return 0;
3885 }
3886 #else
3887 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3888 					struct cftype *cft, u64 val)
3889 {
3890 	return -ENOSYS;
3891 }
3892 #endif
3893 
3894 #ifdef CONFIG_NUMA
3895 
3896 #define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))
3897 #define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))
3898 #define LRU_ALL	     ((1 << NR_LRU_LISTS) - 1)
3899 
3900 static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
3901 				int nid, unsigned int lru_mask, bool tree)
3902 {
3903 	struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
3904 	unsigned long nr = 0;
3905 	enum lru_list lru;
3906 
3907 	VM_BUG_ON((unsigned)nid >= nr_node_ids);
3908 
3909 	for_each_lru(lru) {
3910 		if (!(BIT(lru) & lru_mask))
3911 			continue;
3912 		if (tree)
3913 			nr += lruvec_page_state(lruvec, NR_LRU_BASE + lru);
3914 		else
3915 			nr += lruvec_page_state_local(lruvec, NR_LRU_BASE + lru);
3916 	}
3917 	return nr;
3918 }
3919 
3920 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
3921 					     unsigned int lru_mask,
3922 					     bool tree)
3923 {
3924 	unsigned long nr = 0;
3925 	enum lru_list lru;
3926 
3927 	for_each_lru(lru) {
3928 		if (!(BIT(lru) & lru_mask))
3929 			continue;
3930 		if (tree)
3931 			nr += memcg_page_state(memcg, NR_LRU_BASE + lru);
3932 		else
3933 			nr += memcg_page_state_local(memcg, NR_LRU_BASE + lru);
3934 	}
3935 	return nr;
3936 }
3937 
3938 static int memcg_numa_stat_show(struct seq_file *m, void *v)
3939 {
3940 	struct numa_stat {
3941 		const char *name;
3942 		unsigned int lru_mask;
3943 	};
3944 
3945 	static const struct numa_stat stats[] = {
3946 		{ "total", LRU_ALL },
3947 		{ "file", LRU_ALL_FILE },
3948 		{ "anon", LRU_ALL_ANON },
3949 		{ "unevictable", BIT(LRU_UNEVICTABLE) },
3950 	};
3951 	const struct numa_stat *stat;
3952 	int nid;
3953 	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
3954 
3955 	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3956 		seq_printf(m, "%s=%lu", stat->name,
3957 			   mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
3958 						   false));
3959 		for_each_node_state(nid, N_MEMORY)
3960 			seq_printf(m, " N%d=%lu", nid,
3961 				   mem_cgroup_node_nr_lru_pages(memcg, nid,
3962 							stat->lru_mask, false));
3963 		seq_putc(m, '\n');
3964 	}
3965 
3966 	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3967 
3968 		seq_printf(m, "hierarchical_%s=%lu", stat->name,
3969 			   mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
3970 						   true));
3971 		for_each_node_state(nid, N_MEMORY)
3972 			seq_printf(m, " N%d=%lu", nid,
3973 				   mem_cgroup_node_nr_lru_pages(memcg, nid,
3974 							stat->lru_mask, true));
3975 		seq_putc(m, '\n');
3976 	}
3977 
3978 	return 0;
3979 }
3980 #endif /* CONFIG_NUMA */
3981 
3982 static const unsigned int memcg1_stats[] = {
3983 	NR_FILE_PAGES,
3984 	NR_ANON_MAPPED,
3985 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3986 	NR_ANON_THPS,
3987 #endif
3988 	NR_SHMEM,
3989 	NR_FILE_MAPPED,
3990 	NR_FILE_DIRTY,
3991 	NR_WRITEBACK,
3992 	MEMCG_SWAP,
3993 };
3994 
3995 static const char *const memcg1_stat_names[] = {
3996 	"cache",
3997 	"rss",
3998 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3999 	"rss_huge",
4000 #endif
4001 	"shmem",
4002 	"mapped_file",
4003 	"dirty",
4004 	"writeback",
4005 	"swap",
4006 };
4007 
4008 /* Universal VM events cgroup1 shows, original sort order */
4009 static const unsigned int memcg1_events[] = {
4010 	PGPGIN,
4011 	PGPGOUT,
4012 	PGFAULT,
4013 	PGMAJFAULT,
4014 };
4015 
4016 static int memcg_stat_show(struct seq_file *m, void *v)
4017 {
4018 	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
4019 	unsigned long memory, memsw;
4020 	struct mem_cgroup *mi;
4021 	unsigned int i;
4022 
4023 	BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats));
4024 
4025 	for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
4026 		unsigned long nr;
4027 
4028 		if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
4029 			continue;
4030 		nr = memcg_page_state_local(memcg, memcg1_stats[i]);
4031 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4032 		if (memcg1_stats[i] == NR_ANON_THPS)
4033 			nr *= HPAGE_PMD_NR;
4034 #endif
4035 		seq_printf(m, "%s %lu\n", memcg1_stat_names[i], nr * PAGE_SIZE);
4036 	}
4037 
4038 	for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
4039 		seq_printf(m, "%s %lu\n", vm_event_name(memcg1_events[i]),
4040 			   memcg_events_local(memcg, memcg1_events[i]));
4041 
4042 	for (i = 0; i < NR_LRU_LISTS; i++)
4043 		seq_printf(m, "%s %lu\n", lru_list_name(i),
4044 			   memcg_page_state_local(memcg, NR_LRU_BASE + i) *
4045 			   PAGE_SIZE);
4046 
4047 	/* Hierarchical information */
4048 	memory = memsw = PAGE_COUNTER_MAX;
4049 	for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
4050 		memory = min(memory, READ_ONCE(mi->memory.max));
4051 		memsw = min(memsw, READ_ONCE(mi->memsw.max));
4052 	}
4053 	seq_printf(m, "hierarchical_memory_limit %llu\n",
4054 		   (u64)memory * PAGE_SIZE);
4055 	if (do_memsw_account())
4056 		seq_printf(m, "hierarchical_memsw_limit %llu\n",
4057 			   (u64)memsw * PAGE_SIZE);
4058 
4059 	for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
4060 		if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
4061 			continue;
4062 		seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i],
4063 			   (u64)memcg_page_state(memcg, memcg1_stats[i]) *
4064 			   PAGE_SIZE);
4065 	}
4066 
4067 	for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
4068 		seq_printf(m, "total_%s %llu\n",
4069 			   vm_event_name(memcg1_events[i]),
4070 			   (u64)memcg_events(memcg, memcg1_events[i]));
4071 
4072 	for (i = 0; i < NR_LRU_LISTS; i++)
4073 		seq_printf(m, "total_%s %llu\n", lru_list_name(i),
4074 			   (u64)memcg_page_state(memcg, NR_LRU_BASE + i) *
4075 			   PAGE_SIZE);
4076 
4077 #ifdef CONFIG_DEBUG_VM
4078 	{
4079 		pg_data_t *pgdat;
4080 		struct mem_cgroup_per_node *mz;
4081 		unsigned long anon_cost = 0;
4082 		unsigned long file_cost = 0;
4083 
4084 		for_each_online_pgdat(pgdat) {
4085 			mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
4086 
4087 			anon_cost += mz->lruvec.anon_cost;
4088 			file_cost += mz->lruvec.file_cost;
4089 		}
4090 		seq_printf(m, "anon_cost %lu\n", anon_cost);
4091 		seq_printf(m, "file_cost %lu\n", file_cost);
4092 	}
4093 #endif
4094 
4095 	return 0;
4096 }
4097 
4098 static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
4099 				      struct cftype *cft)
4100 {
4101 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4102 
4103 	return mem_cgroup_swappiness(memcg);
4104 }
4105 
4106 static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
4107 				       struct cftype *cft, u64 val)
4108 {
4109 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4110 
4111 	if (val > 100)
4112 		return -EINVAL;
4113 
4114 	if (css->parent)
4115 		memcg->swappiness = val;
4116 	else
4117 		vm_swappiness = val;
4118 
4119 	return 0;
4120 }
4121 
4122 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
4123 {
4124 	struct mem_cgroup_threshold_ary *t;
4125 	unsigned long usage;
4126 	int i;
4127 
4128 	rcu_read_lock();
4129 	if (!swap)
4130 		t = rcu_dereference(memcg->thresholds.primary);
4131 	else
4132 		t = rcu_dereference(memcg->memsw_thresholds.primary);
4133 
4134 	if (!t)
4135 		goto unlock;
4136 
4137 	usage = mem_cgroup_usage(memcg, swap);
4138 
4139 	/*
4140 	 * current_threshold points to threshold just below or equal to usage.
4141 	 * If it's not true, a threshold was crossed after last
4142 	 * call of __mem_cgroup_threshold().
4143 	 */
4144 	i = t->current_threshold;
4145 
4146 	/*
4147 	 * Iterate backward over array of thresholds starting from
4148 	 * current_threshold and check if a threshold is crossed.
4149 	 * If none of thresholds below usage is crossed, we read
4150 	 * only one element of the array here.
4151 	 */
4152 	for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
4153 		eventfd_signal(t->entries[i].eventfd, 1);
4154 
4155 	/* i = current_threshold + 1 */
4156 	i++;
4157 
4158 	/*
4159 	 * Iterate forward over array of thresholds starting from
4160 	 * current_threshold+1 and check if a threshold is crossed.
4161 	 * If none of thresholds above usage is crossed, we read
4162 	 * only one element of the array here.
4163 	 */
4164 	for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
4165 		eventfd_signal(t->entries[i].eventfd, 1);
4166 
4167 	/* Update current_threshold */
4168 	t->current_threshold = i - 1;
4169 unlock:
4170 	rcu_read_unlock();
4171 }
4172 
4173 static void mem_cgroup_threshold(struct mem_cgroup *memcg)
4174 {
4175 	while (memcg) {
4176 		__mem_cgroup_threshold(memcg, false);
4177 		if (do_memsw_account())
4178 			__mem_cgroup_threshold(memcg, true);
4179 
4180 		memcg = parent_mem_cgroup(memcg);
4181 	}
4182 }
4183 
4184 static int compare_thresholds(const void *a, const void *b)
4185 {
4186 	const struct mem_cgroup_threshold *_a = a;
4187 	const struct mem_cgroup_threshold *_b = b;
4188 
4189 	if (_a->threshold > _b->threshold)
4190 		return 1;
4191 
4192 	if (_a->threshold < _b->threshold)
4193 		return -1;
4194 
4195 	return 0;
4196 }
4197 
4198 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
4199 {
4200 	struct mem_cgroup_eventfd_list *ev;
4201 
4202 	spin_lock(&memcg_oom_lock);
4203 
4204 	list_for_each_entry(ev, &memcg->oom_notify, list)
4205 		eventfd_signal(ev->eventfd, 1);
4206 
4207 	spin_unlock(&memcg_oom_lock);
4208 	return 0;
4209 }
4210 
4211 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
4212 {
4213 	struct mem_cgroup *iter;
4214 
4215 	for_each_mem_cgroup_tree(iter, memcg)
4216 		mem_cgroup_oom_notify_cb(iter);
4217 }
4218 
4219 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
4220 	struct eventfd_ctx *eventfd, const char *args, enum res_type type)
4221 {
4222 	struct mem_cgroup_thresholds *thresholds;
4223 	struct mem_cgroup_threshold_ary *new;
4224 	unsigned long threshold;
4225 	unsigned long usage;
4226 	int i, size, ret;
4227 
4228 	ret = page_counter_memparse(args, "-1", &threshold);
4229 	if (ret)
4230 		return ret;
4231 
4232 	mutex_lock(&memcg->thresholds_lock);
4233 
4234 	if (type == _MEM) {
4235 		thresholds = &memcg->thresholds;
4236 		usage = mem_cgroup_usage(memcg, false);
4237 	} else if (type == _MEMSWAP) {
4238 		thresholds = &memcg->memsw_thresholds;
4239 		usage = mem_cgroup_usage(memcg, true);
4240 	} else
4241 		BUG();
4242 
4243 	/* Check if a threshold crossed before adding a new one */
4244 	if (thresholds->primary)
4245 		__mem_cgroup_threshold(memcg, type == _MEMSWAP);
4246 
4247 	size = thresholds->primary ? thresholds->primary->size + 1 : 1;
4248 
4249 	/* Allocate memory for new array of thresholds */
4250 	new = kmalloc(struct_size(new, entries, size), GFP_KERNEL);
4251 	if (!new) {
4252 		ret = -ENOMEM;
4253 		goto unlock;
4254 	}
4255 	new->size = size;
4256 
4257 	/* Copy thresholds (if any) to new array */
4258 	if (thresholds->primary) {
4259 		memcpy(new->entries, thresholds->primary->entries, (size - 1) *
4260 				sizeof(struct mem_cgroup_threshold));
4261 	}
4262 
4263 	/* Add new threshold */
4264 	new->entries[size - 1].eventfd = eventfd;
4265 	new->entries[size - 1].threshold = threshold;
4266 
4267 	/* Sort thresholds. Registering of new threshold isn't time-critical */
4268 	sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
4269 			compare_thresholds, NULL);
4270 
4271 	/* Find current threshold */
4272 	new->current_threshold = -1;
4273 	for (i = 0; i < size; i++) {
4274 		if (new->entries[i].threshold <= usage) {
4275 			/*
4276 			 * new->current_threshold will not be used until
4277 			 * rcu_assign_pointer(), so it's safe to increment
4278 			 * it here.
4279 			 */
4280 			++new->current_threshold;
4281 		} else
4282 			break;
4283 	}
4284 
4285 	/* Free old spare buffer and save old primary buffer as spare */
4286 	kfree(thresholds->spare);
4287 	thresholds->spare = thresholds->primary;
4288 
4289 	rcu_assign_pointer(thresholds->primary, new);
4290 
4291 	/* To be sure that nobody uses thresholds */
4292 	synchronize_rcu();
4293 
4294 unlock:
4295 	mutex_unlock(&memcg->thresholds_lock);
4296 
4297 	return ret;
4298 }
4299 
4300 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
4301 	struct eventfd_ctx *eventfd, const char *args)
4302 {
4303 	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
4304 }
4305 
4306 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
4307 	struct eventfd_ctx *eventfd, const char *args)
4308 {
4309 	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
4310 }
4311 
4312 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4313 	struct eventfd_ctx *eventfd, enum res_type type)
4314 {
4315 	struct mem_cgroup_thresholds *thresholds;
4316 	struct mem_cgroup_threshold_ary *new;
4317 	unsigned long usage;
4318 	int i, j, size, entries;
4319 
4320 	mutex_lock(&memcg->thresholds_lock);
4321 
4322 	if (type == _MEM) {
4323 		thresholds = &memcg->thresholds;
4324 		usage = mem_cgroup_usage(memcg, false);
4325 	} else if (type == _MEMSWAP) {
4326 		thresholds = &memcg->memsw_thresholds;
4327 		usage = mem_cgroup_usage(memcg, true);
4328 	} else
4329 		BUG();
4330 
4331 	if (!thresholds->primary)
4332 		goto unlock;
4333 
4334 	/* Check if a threshold crossed before removing */
4335 	__mem_cgroup_threshold(memcg, type == _MEMSWAP);
4336 
4337 	/* Calculate new number of threshold */
4338 	size = entries = 0;
4339 	for (i = 0; i < thresholds->primary->size; i++) {
4340 		if (thresholds->primary->entries[i].eventfd != eventfd)
4341 			size++;
4342 		else
4343 			entries++;
4344 	}
4345 
4346 	new = thresholds->spare;
4347 
4348 	/* If no items related to eventfd have been cleared, nothing to do */
4349 	if (!entries)
4350 		goto unlock;
4351 
4352 	/* Set thresholds array to NULL if we don't have thresholds */
4353 	if (!size) {
4354 		kfree(new);
4355 		new = NULL;
4356 		goto swap_buffers;
4357 	}
4358 
4359 	new->size = size;
4360 
4361 	/* Copy thresholds and find current threshold */
4362 	new->current_threshold = -1;
4363 	for (i = 0, j = 0; i < thresholds->primary->size; i++) {
4364 		if (thresholds->primary->entries[i].eventfd == eventfd)
4365 			continue;
4366 
4367 		new->entries[j] = thresholds->primary->entries[i];
4368 		if (new->entries[j].threshold <= usage) {
4369 			/*
4370 			 * new->current_threshold will not be used
4371 			 * until rcu_assign_pointer(), so it's safe to increment
4372 			 * it here.
4373 			 */
4374 			++new->current_threshold;
4375 		}
4376 		j++;
4377 	}
4378 
4379 swap_buffers:
4380 	/* Swap primary and spare array */
4381 	thresholds->spare = thresholds->primary;
4382 
4383 	rcu_assign_pointer(thresholds->primary, new);
4384 
4385 	/* To be sure that nobody uses thresholds */
4386 	synchronize_rcu();
4387 
4388 	/* If all events are unregistered, free the spare array */
4389 	if (!new) {
4390 		kfree(thresholds->spare);
4391 		thresholds->spare = NULL;
4392 	}
4393 unlock:
4394 	mutex_unlock(&memcg->thresholds_lock);
4395 }
4396 
4397 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4398 	struct eventfd_ctx *eventfd)
4399 {
4400 	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
4401 }
4402 
4403 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4404 	struct eventfd_ctx *eventfd)
4405 {
4406 	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
4407 }
4408 
4409 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
4410 	struct eventfd_ctx *eventfd, const char *args)
4411 {
4412 	struct mem_cgroup_eventfd_list *event;
4413 
4414 	event = kmalloc(sizeof(*event),	GFP_KERNEL);
4415 	if (!event)
4416 		return -ENOMEM;
4417 
4418 	spin_lock(&memcg_oom_lock);
4419 
4420 	event->eventfd = eventfd;
4421 	list_add(&event->list, &memcg->oom_notify);
4422 
4423 	/* already in OOM ? */
4424 	if (memcg->under_oom)
4425 		eventfd_signal(eventfd, 1);
4426 	spin_unlock(&memcg_oom_lock);
4427 
4428 	return 0;
4429 }
4430 
4431 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
4432 	struct eventfd_ctx *eventfd)
4433 {
4434 	struct mem_cgroup_eventfd_list *ev, *tmp;
4435 
4436 	spin_lock(&memcg_oom_lock);
4437 
4438 	list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
4439 		if (ev->eventfd == eventfd) {
4440 			list_del(&ev->list);
4441 			kfree(ev);
4442 		}
4443 	}
4444 
4445 	spin_unlock(&memcg_oom_lock);
4446 }
4447 
4448 static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
4449 {
4450 	struct mem_cgroup *memcg = mem_cgroup_from_seq(sf);
4451 
4452 	seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
4453 	seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
4454 	seq_printf(sf, "oom_kill %lu\n",
4455 		   atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL]));
4456 	return 0;
4457 }
4458 
4459 static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
4460 	struct cftype *cft, u64 val)
4461 {
4462 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4463 
4464 	/* cannot set to root cgroup and only 0 and 1 are allowed */
4465 	if (!css->parent || !((val == 0) || (val == 1)))
4466 		return -EINVAL;
4467 
4468 	memcg->oom_kill_disable = val;
4469 	if (!val)
4470 		memcg_oom_recover(memcg);
4471 
4472 	return 0;
4473 }
4474 
4475 #ifdef CONFIG_CGROUP_WRITEBACK
4476 
4477 #include <trace/events/writeback.h>
4478 
4479 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
4480 {
4481 	return wb_domain_init(&memcg->cgwb_domain, gfp);
4482 }
4483 
4484 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
4485 {
4486 	wb_domain_exit(&memcg->cgwb_domain);
4487 }
4488 
4489 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
4490 {
4491 	wb_domain_size_changed(&memcg->cgwb_domain);
4492 }
4493 
4494 struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
4495 {
4496 	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4497 
4498 	if (!memcg->css.parent)
4499 		return NULL;
4500 
4501 	return &memcg->cgwb_domain;
4502 }
4503 
4504 /*
4505  * idx can be of type enum memcg_stat_item or node_stat_item.
4506  * Keep in sync with memcg_exact_page().
4507  */
4508 static unsigned long memcg_exact_page_state(struct mem_cgroup *memcg, int idx)
4509 {
4510 	long x = atomic_long_read(&memcg->vmstats[idx]);
4511 	int cpu;
4512 
4513 	for_each_online_cpu(cpu)
4514 		x += per_cpu_ptr(memcg->vmstats_percpu, cpu)->stat[idx];
4515 	if (x < 0)
4516 		x = 0;
4517 	return x;
4518 }
4519 
4520 /**
4521  * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
4522  * @wb: bdi_writeback in question
4523  * @pfilepages: out parameter for number of file pages
4524  * @pheadroom: out parameter for number of allocatable pages according to memcg
4525  * @pdirty: out parameter for number of dirty pages
4526  * @pwriteback: out parameter for number of pages under writeback
4527  *
4528  * Determine the numbers of file, headroom, dirty, and writeback pages in
4529  * @wb's memcg.  File, dirty and writeback are self-explanatory.  Headroom
4530  * is a bit more involved.
4531  *
4532  * A memcg's headroom is "min(max, high) - used".  In the hierarchy, the
4533  * headroom is calculated as the lowest headroom of itself and the
4534  * ancestors.  Note that this doesn't consider the actual amount of
4535  * available memory in the system.  The caller should further cap
4536  * *@pheadroom accordingly.
4537  */
4538 void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
4539 			 unsigned long *pheadroom, unsigned long *pdirty,
4540 			 unsigned long *pwriteback)
4541 {
4542 	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4543 	struct mem_cgroup *parent;
4544 
4545 	*pdirty = memcg_exact_page_state(memcg, NR_FILE_DIRTY);
4546 
4547 	*pwriteback = memcg_exact_page_state(memcg, NR_WRITEBACK);
4548 	*pfilepages = memcg_exact_page_state(memcg, NR_INACTIVE_FILE) +
4549 			memcg_exact_page_state(memcg, NR_ACTIVE_FILE);
4550 	*pheadroom = PAGE_COUNTER_MAX;
4551 
4552 	while ((parent = parent_mem_cgroup(memcg))) {
4553 		unsigned long ceiling = min(READ_ONCE(memcg->memory.max),
4554 					    READ_ONCE(memcg->memory.high));
4555 		unsigned long used = page_counter_read(&memcg->memory);
4556 
4557 		*pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
4558 		memcg = parent;
4559 	}
4560 }
4561 
4562 /*
4563  * Foreign dirty flushing
4564  *
4565  * There's an inherent mismatch between memcg and writeback.  The former
4566  * trackes ownership per-page while the latter per-inode.  This was a
4567  * deliberate design decision because honoring per-page ownership in the
4568  * writeback path is complicated, may lead to higher CPU and IO overheads
4569  * and deemed unnecessary given that write-sharing an inode across
4570  * different cgroups isn't a common use-case.
4571  *
4572  * Combined with inode majority-writer ownership switching, this works well
4573  * enough in most cases but there are some pathological cases.  For
4574  * example, let's say there are two cgroups A and B which keep writing to
4575  * different but confined parts of the same inode.  B owns the inode and
4576  * A's memory is limited far below B's.  A's dirty ratio can rise enough to
4577  * trigger balance_dirty_pages() sleeps but B's can be low enough to avoid
4578  * triggering background writeback.  A will be slowed down without a way to
4579  * make writeback of the dirty pages happen.
4580  *
4581  * Conditions like the above can lead to a cgroup getting repatedly and
4582  * severely throttled after making some progress after each
4583  * dirty_expire_interval while the underyling IO device is almost
4584  * completely idle.
4585  *
4586  * Solving this problem completely requires matching the ownership tracking
4587  * granularities between memcg and writeback in either direction.  However,
4588  * the more egregious behaviors can be avoided by simply remembering the
4589  * most recent foreign dirtying events and initiating remote flushes on
4590  * them when local writeback isn't enough to keep the memory clean enough.
4591  *
4592  * The following two functions implement such mechanism.  When a foreign
4593  * page - a page whose memcg and writeback ownerships don't match - is
4594  * dirtied, mem_cgroup_track_foreign_dirty() records the inode owning
4595  * bdi_writeback on the page owning memcg.  When balance_dirty_pages()
4596  * decides that the memcg needs to sleep due to high dirty ratio, it calls
4597  * mem_cgroup_flush_foreign() which queues writeback on the recorded
4598  * foreign bdi_writebacks which haven't expired.  Both the numbers of
4599  * recorded bdi_writebacks and concurrent in-flight foreign writebacks are
4600  * limited to MEMCG_CGWB_FRN_CNT.
4601  *
4602  * The mechanism only remembers IDs and doesn't hold any object references.
4603  * As being wrong occasionally doesn't matter, updates and accesses to the
4604  * records are lockless and racy.
4605  */
4606 void mem_cgroup_track_foreign_dirty_slowpath(struct page *page,
4607 					     struct bdi_writeback *wb)
4608 {
4609 	struct mem_cgroup *memcg = page->mem_cgroup;
4610 	struct memcg_cgwb_frn *frn;
4611 	u64 now = get_jiffies_64();
4612 	u64 oldest_at = now;
4613 	int oldest = -1;
4614 	int i;
4615 
4616 	trace_track_foreign_dirty(page, wb);
4617 
4618 	/*
4619 	 * Pick the slot to use.  If there is already a slot for @wb, keep
4620 	 * using it.  If not replace the oldest one which isn't being
4621 	 * written out.
4622 	 */
4623 	for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
4624 		frn = &memcg->cgwb_frn[i];
4625 		if (frn->bdi_id == wb->bdi->id &&
4626 		    frn->memcg_id == wb->memcg_css->id)
4627 			break;
4628 		if (time_before64(frn->at, oldest_at) &&
4629 		    atomic_read(&frn->done.cnt) == 1) {
4630 			oldest = i;
4631 			oldest_at = frn->at;
4632 		}
4633 	}
4634 
4635 	if (i < MEMCG_CGWB_FRN_CNT) {
4636 		/*
4637 		 * Re-using an existing one.  Update timestamp lazily to
4638 		 * avoid making the cacheline hot.  We want them to be
4639 		 * reasonably up-to-date and significantly shorter than
4640 		 * dirty_expire_interval as that's what expires the record.
4641 		 * Use the shorter of 1s and dirty_expire_interval / 8.
4642 		 */
4643 		unsigned long update_intv =
4644 			min_t(unsigned long, HZ,
4645 			      msecs_to_jiffies(dirty_expire_interval * 10) / 8);
4646 
4647 		if (time_before64(frn->at, now - update_intv))
4648 			frn->at = now;
4649 	} else if (oldest >= 0) {
4650 		/* replace the oldest free one */
4651 		frn = &memcg->cgwb_frn[oldest];
4652 		frn->bdi_id = wb->bdi->id;
4653 		frn->memcg_id = wb->memcg_css->id;
4654 		frn->at = now;
4655 	}
4656 }
4657 
4658 /* issue foreign writeback flushes for recorded foreign dirtying events */
4659 void mem_cgroup_flush_foreign(struct bdi_writeback *wb)
4660 {
4661 	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4662 	unsigned long intv = msecs_to_jiffies(dirty_expire_interval * 10);
4663 	u64 now = jiffies_64;
4664 	int i;
4665 
4666 	for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
4667 		struct memcg_cgwb_frn *frn = &memcg->cgwb_frn[i];
4668 
4669 		/*
4670 		 * If the record is older than dirty_expire_interval,
4671 		 * writeback on it has already started.  No need to kick it
4672 		 * off again.  Also, don't start a new one if there's
4673 		 * already one in flight.
4674 		 */
4675 		if (time_after64(frn->at, now - intv) &&
4676 		    atomic_read(&frn->done.cnt) == 1) {
4677 			frn->at = 0;
4678 			trace_flush_foreign(wb, frn->bdi_id, frn->memcg_id);
4679 			cgroup_writeback_by_id(frn->bdi_id, frn->memcg_id, 0,
4680 					       WB_REASON_FOREIGN_FLUSH,
4681 					       &frn->done);
4682 		}
4683 	}
4684 }
4685 
4686 #else	/* CONFIG_CGROUP_WRITEBACK */
4687 
4688 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
4689 {
4690 	return 0;
4691 }
4692 
4693 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
4694 {
4695 }
4696 
4697 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
4698 {
4699 }
4700 
4701 #endif	/* CONFIG_CGROUP_WRITEBACK */
4702 
4703 /*
4704  * DO NOT USE IN NEW FILES.
4705  *
4706  * "cgroup.event_control" implementation.
4707  *
4708  * This is way over-engineered.  It tries to support fully configurable
4709  * events for each user.  Such level of flexibility is completely
4710  * unnecessary especially in the light of the planned unified hierarchy.
4711  *
4712  * Please deprecate this and replace with something simpler if at all
4713  * possible.
4714  */
4715 
4716 /*
4717  * Unregister event and free resources.
4718  *
4719  * Gets called from workqueue.
4720  */
4721 static void memcg_event_remove(struct work_struct *work)
4722 {
4723 	struct mem_cgroup_event *event =
4724 		container_of(work, struct mem_cgroup_event, remove);
4725 	struct mem_cgroup *memcg = event->memcg;
4726 
4727 	remove_wait_queue(event->wqh, &event->wait);
4728 
4729 	event->unregister_event(memcg, event->eventfd);
4730 
4731 	/* Notify userspace the event is going away. */
4732 	eventfd_signal(event->eventfd, 1);
4733 
4734 	eventfd_ctx_put(event->eventfd);
4735 	kfree(event);
4736 	css_put(&memcg->css);
4737 }
4738 
4739 /*
4740  * Gets called on EPOLLHUP on eventfd when user closes it.
4741  *
4742  * Called with wqh->lock held and interrupts disabled.
4743  */
4744 static int memcg_event_wake(wait_queue_entry_t *wait, unsigned mode,
4745 			    int sync, void *key)
4746 {
4747 	struct mem_cgroup_event *event =
4748 		container_of(wait, struct mem_cgroup_event, wait);
4749 	struct mem_cgroup *memcg = event->memcg;
4750 	__poll_t flags = key_to_poll(key);
4751 
4752 	if (flags & EPOLLHUP) {
4753 		/*
4754 		 * If the event has been detached at cgroup removal, we
4755 		 * can simply return knowing the other side will cleanup
4756 		 * for us.
4757 		 *
4758 		 * We can't race against event freeing since the other
4759 		 * side will require wqh->lock via remove_wait_queue(),
4760 		 * which we hold.
4761 		 */
4762 		spin_lock(&memcg->event_list_lock);
4763 		if (!list_empty(&event->list)) {
4764 			list_del_init(&event->list);
4765 			/*
4766 			 * We are in atomic context, but cgroup_event_remove()
4767 			 * may sleep, so we have to call it in workqueue.
4768 			 */
4769 			schedule_work(&event->remove);
4770 		}
4771 		spin_unlock(&memcg->event_list_lock);
4772 	}
4773 
4774 	return 0;
4775 }
4776 
4777 static void memcg_event_ptable_queue_proc(struct file *file,
4778 		wait_queue_head_t *wqh, poll_table *pt)
4779 {
4780 	struct mem_cgroup_event *event =
4781 		container_of(pt, struct mem_cgroup_event, pt);
4782 
4783 	event->wqh = wqh;
4784 	add_wait_queue(wqh, &event->wait);
4785 }
4786 
4787 /*
4788  * DO NOT USE IN NEW FILES.
4789  *
4790  * Parse input and register new cgroup event handler.
4791  *
4792  * Input must be in format '<event_fd> <control_fd> <args>'.
4793  * Interpretation of args is defined by control file implementation.
4794  */
4795 static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
4796 					 char *buf, size_t nbytes, loff_t off)
4797 {
4798 	struct cgroup_subsys_state *css = of_css(of);
4799 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4800 	struct mem_cgroup_event *event;
4801 	struct cgroup_subsys_state *cfile_css;
4802 	unsigned int efd, cfd;
4803 	struct fd efile;
4804 	struct fd cfile;
4805 	const char *name;
4806 	char *endp;
4807 	int ret;
4808 
4809 	buf = strstrip(buf);
4810 
4811 	efd = simple_strtoul(buf, &endp, 10);
4812 	if (*endp != ' ')
4813 		return -EINVAL;
4814 	buf = endp + 1;
4815 
4816 	cfd = simple_strtoul(buf, &endp, 10);
4817 	if ((*endp != ' ') && (*endp != '\0'))
4818 		return -EINVAL;
4819 	buf = endp + 1;
4820 
4821 	event = kzalloc(sizeof(*event), GFP_KERNEL);
4822 	if (!event)
4823 		return -ENOMEM;
4824 
4825 	event->memcg = memcg;
4826 	INIT_LIST_HEAD(&event->list);
4827 	init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
4828 	init_waitqueue_func_entry(&event->wait, memcg_event_wake);
4829 	INIT_WORK(&event->remove, memcg_event_remove);
4830 
4831 	efile = fdget(efd);
4832 	if (!efile.file) {
4833 		ret = -EBADF;
4834 		goto out_kfree;
4835 	}
4836 
4837 	event->eventfd = eventfd_ctx_fileget(efile.file);
4838 	if (IS_ERR(event->eventfd)) {
4839 		ret = PTR_ERR(event->eventfd);
4840 		goto out_put_efile;
4841 	}
4842 
4843 	cfile = fdget(cfd);
4844 	if (!cfile.file) {
4845 		ret = -EBADF;
4846 		goto out_put_eventfd;
4847 	}
4848 
4849 	/* the process need read permission on control file */
4850 	/* AV: shouldn't we check that it's been opened for read instead? */
4851 	ret = inode_permission(file_inode(cfile.file), MAY_READ);
4852 	if (ret < 0)
4853 		goto out_put_cfile;
4854 
4855 	/*
4856 	 * Determine the event callbacks and set them in @event.  This used
4857 	 * to be done via struct cftype but cgroup core no longer knows
4858 	 * about these events.  The following is crude but the whole thing
4859 	 * is for compatibility anyway.
4860 	 *
4861 	 * DO NOT ADD NEW FILES.
4862 	 */
4863 	name = cfile.file->f_path.dentry->d_name.name;
4864 
4865 	if (!strcmp(name, "memory.usage_in_bytes")) {
4866 		event->register_event = mem_cgroup_usage_register_event;
4867 		event->unregister_event = mem_cgroup_usage_unregister_event;
4868 	} else if (!strcmp(name, "memory.oom_control")) {
4869 		event->register_event = mem_cgroup_oom_register_event;
4870 		event->unregister_event = mem_cgroup_oom_unregister_event;
4871 	} else if (!strcmp(name, "memory.pressure_level")) {
4872 		event->register_event = vmpressure_register_event;
4873 		event->unregister_event = vmpressure_unregister_event;
4874 	} else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
4875 		event->register_event = memsw_cgroup_usage_register_event;
4876 		event->unregister_event = memsw_cgroup_usage_unregister_event;
4877 	} else {
4878 		ret = -EINVAL;
4879 		goto out_put_cfile;
4880 	}
4881 
4882 	/*
4883 	 * Verify @cfile should belong to @css.  Also, remaining events are
4884 	 * automatically removed on cgroup destruction but the removal is
4885 	 * asynchronous, so take an extra ref on @css.
4886 	 */
4887 	cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,
4888 					       &memory_cgrp_subsys);
4889 	ret = -EINVAL;
4890 	if (IS_ERR(cfile_css))
4891 		goto out_put_cfile;
4892 	if (cfile_css != css) {
4893 		css_put(cfile_css);
4894 		goto out_put_cfile;
4895 	}
4896 
4897 	ret = event->register_event(memcg, event->eventfd, buf);
4898 	if (ret)
4899 		goto out_put_css;
4900 
4901 	vfs_poll(efile.file, &event->pt);
4902 
4903 	spin_lock(&memcg->event_list_lock);
4904 	list_add(&event->list, &memcg->event_list);
4905 	spin_unlock(&memcg->event_list_lock);
4906 
4907 	fdput(cfile);
4908 	fdput(efile);
4909 
4910 	return nbytes;
4911 
4912 out_put_css:
4913 	css_put(css);
4914 out_put_cfile:
4915 	fdput(cfile);
4916 out_put_eventfd:
4917 	eventfd_ctx_put(event->eventfd);
4918 out_put_efile:
4919 	fdput(efile);
4920 out_kfree:
4921 	kfree(event);
4922 
4923 	return ret;
4924 }
4925 
4926 static struct cftype mem_cgroup_legacy_files[] = {
4927 	{
4928 		.name = "usage_in_bytes",
4929 		.private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
4930 		.read_u64 = mem_cgroup_read_u64,
4931 	},
4932 	{
4933 		.name = "max_usage_in_bytes",
4934 		.private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
4935 		.write = mem_cgroup_reset,
4936 		.read_u64 = mem_cgroup_read_u64,
4937 	},
4938 	{
4939 		.name = "limit_in_bytes",
4940 		.private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
4941 		.write = mem_cgroup_write,
4942 		.read_u64 = mem_cgroup_read_u64,
4943 	},
4944 	{
4945 		.name = "soft_limit_in_bytes",
4946 		.private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
4947 		.write = mem_cgroup_write,
4948 		.read_u64 = mem_cgroup_read_u64,
4949 	},
4950 	{
4951 		.name = "failcnt",
4952 		.private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
4953 		.write = mem_cgroup_reset,
4954 		.read_u64 = mem_cgroup_read_u64,
4955 	},
4956 	{
4957 		.name = "stat",
4958 		.seq_show = memcg_stat_show,
4959 	},
4960 	{
4961 		.name = "force_empty",
4962 		.write = mem_cgroup_force_empty_write,
4963 	},
4964 	{
4965 		.name = "use_hierarchy",
4966 		.write_u64 = mem_cgroup_hierarchy_write,
4967 		.read_u64 = mem_cgroup_hierarchy_read,
4968 	},
4969 	{
4970 		.name = "cgroup.event_control",		/* XXX: for compat */
4971 		.write = memcg_write_event_control,
4972 		.flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
4973 	},
4974 	{
4975 		.name = "swappiness",
4976 		.read_u64 = mem_cgroup_swappiness_read,
4977 		.write_u64 = mem_cgroup_swappiness_write,
4978 	},
4979 	{
4980 		.name = "move_charge_at_immigrate",
4981 		.read_u64 = mem_cgroup_move_charge_read,
4982 		.write_u64 = mem_cgroup_move_charge_write,
4983 	},
4984 	{
4985 		.name = "oom_control",
4986 		.seq_show = mem_cgroup_oom_control_read,
4987 		.write_u64 = mem_cgroup_oom_control_write,
4988 		.private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
4989 	},
4990 	{
4991 		.name = "pressure_level",
4992 	},
4993 #ifdef CONFIG_NUMA
4994 	{
4995 		.name = "numa_stat",
4996 		.seq_show = memcg_numa_stat_show,
4997 	},
4998 #endif
4999 	{
5000 		.name = "kmem.limit_in_bytes",
5001 		.private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
5002 		.write = mem_cgroup_write,
5003 		.read_u64 = mem_cgroup_read_u64,
5004 	},
5005 	{
5006 		.name = "kmem.usage_in_bytes",
5007 		.private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
5008 		.read_u64 = mem_cgroup_read_u64,
5009 	},
5010 	{
5011 		.name = "kmem.failcnt",
5012 		.private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
5013 		.write = mem_cgroup_reset,
5014 		.read_u64 = mem_cgroup_read_u64,
5015 	},
5016 	{
5017 		.name = "kmem.max_usage_in_bytes",
5018 		.private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
5019 		.write = mem_cgroup_reset,
5020 		.read_u64 = mem_cgroup_read_u64,
5021 	},
5022 #if defined(CONFIG_MEMCG_KMEM) && \
5023 	(defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG))
5024 	{
5025 		.name = "kmem.slabinfo",
5026 		.seq_show = memcg_slab_show,
5027 	},
5028 #endif
5029 	{
5030 		.name = "kmem.tcp.limit_in_bytes",
5031 		.private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
5032 		.write = mem_cgroup_write,
5033 		.read_u64 = mem_cgroup_read_u64,
5034 	},
5035 	{
5036 		.name = "kmem.tcp.usage_in_bytes",
5037 		.private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
5038 		.read_u64 = mem_cgroup_read_u64,
5039 	},
5040 	{
5041 		.name = "kmem.tcp.failcnt",
5042 		.private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
5043 		.write = mem_cgroup_reset,
5044 		.read_u64 = mem_cgroup_read_u64,
5045 	},
5046 	{
5047 		.name = "kmem.tcp.max_usage_in_bytes",
5048 		.private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
5049 		.write = mem_cgroup_reset,
5050 		.read_u64 = mem_cgroup_read_u64,
5051 	},
5052 	{ },	/* terminate */
5053 };
5054 
5055 /*
5056  * Private memory cgroup IDR
5057  *
5058  * Swap-out records and page cache shadow entries need to store memcg
5059  * references in constrained space, so we maintain an ID space that is
5060  * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of
5061  * memory-controlled cgroups to 64k.
5062  *
5063  * However, there usually are many references to the offline CSS after
5064  * the cgroup has been destroyed, such as page cache or reclaimable
5065  * slab objects, that don't need to hang on to the ID. We want to keep
5066  * those dead CSS from occupying IDs, or we might quickly exhaust the
5067  * relatively small ID space and prevent the creation of new cgroups
5068  * even when there are much fewer than 64k cgroups - possibly none.
5069  *
5070  * Maintain a private 16-bit ID space for memcg, and allow the ID to
5071  * be freed and recycled when it's no longer needed, which is usually
5072  * when the CSS is offlined.
5073  *
5074  * The only exception to that are records of swapped out tmpfs/shmem
5075  * pages that need to be attributed to live ancestors on swapin. But
5076  * those references are manageable from userspace.
5077  */
5078 
5079 static DEFINE_IDR(mem_cgroup_idr);
5080 
5081 static void mem_cgroup_id_remove(struct mem_cgroup *memcg)
5082 {
5083 	if (memcg->id.id > 0) {
5084 		idr_remove(&mem_cgroup_idr, memcg->id.id);
5085 		memcg->id.id = 0;
5086 	}
5087 }
5088 
5089 static void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg,
5090 						  unsigned int n)
5091 {
5092 	refcount_add(n, &memcg->id.ref);
5093 }
5094 
5095 static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
5096 {
5097 	if (refcount_sub_and_test(n, &memcg->id.ref)) {
5098 		mem_cgroup_id_remove(memcg);
5099 
5100 		/* Memcg ID pins CSS */
5101 		css_put(&memcg->css);
5102 	}
5103 }
5104 
5105 static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
5106 {
5107 	mem_cgroup_id_put_many(memcg, 1);
5108 }
5109 
5110 /**
5111  * mem_cgroup_from_id - look up a memcg from a memcg id
5112  * @id: the memcg id to look up
5113  *
5114  * Caller must hold rcu_read_lock().
5115  */
5116 struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
5117 {
5118 	WARN_ON_ONCE(!rcu_read_lock_held());
5119 	return idr_find(&mem_cgroup_idr, id);
5120 }
5121 
5122 static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
5123 {
5124 	struct mem_cgroup_per_node *pn;
5125 	int tmp = node;
5126 	/*
5127 	 * This routine is called against possible nodes.
5128 	 * But it's BUG to call kmalloc() against offline node.
5129 	 *
5130 	 * TODO: this routine can waste much memory for nodes which will
5131 	 *       never be onlined. It's better to use memory hotplug callback
5132 	 *       function.
5133 	 */
5134 	if (!node_state(node, N_NORMAL_MEMORY))
5135 		tmp = -1;
5136 	pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
5137 	if (!pn)
5138 		return 1;
5139 
5140 	pn->lruvec_stat_local = alloc_percpu_gfp(struct lruvec_stat,
5141 						 GFP_KERNEL_ACCOUNT);
5142 	if (!pn->lruvec_stat_local) {
5143 		kfree(pn);
5144 		return 1;
5145 	}
5146 
5147 	pn->lruvec_stat_cpu = alloc_percpu_gfp(struct lruvec_stat,
5148 					       GFP_KERNEL_ACCOUNT);
5149 	if (!pn->lruvec_stat_cpu) {
5150 		free_percpu(pn->lruvec_stat_local);
5151 		kfree(pn);
5152 		return 1;
5153 	}
5154 
5155 	lruvec_init(&pn->lruvec);
5156 	pn->usage_in_excess = 0;
5157 	pn->on_tree = false;
5158 	pn->memcg = memcg;
5159 
5160 	memcg->nodeinfo[node] = pn;
5161 	return 0;
5162 }
5163 
5164 static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
5165 {
5166 	struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
5167 
5168 	if (!pn)
5169 		return;
5170 
5171 	free_percpu(pn->lruvec_stat_cpu);
5172 	free_percpu(pn->lruvec_stat_local);
5173 	kfree(pn);
5174 }
5175 
5176 static void __mem_cgroup_free(struct mem_cgroup *memcg)
5177 {
5178 	int node;
5179 
5180 	for_each_node(node)
5181 		free_mem_cgroup_per_node_info(memcg, node);
5182 	free_percpu(memcg->vmstats_percpu);
5183 	free_percpu(memcg->vmstats_local);
5184 	kfree(memcg);
5185 }
5186 
5187 static void mem_cgroup_free(struct mem_cgroup *memcg)
5188 {
5189 	memcg_wb_domain_exit(memcg);
5190 	/*
5191 	 * Flush percpu vmstats and vmevents to guarantee the value correctness
5192 	 * on parent's and all ancestor levels.
5193 	 */
5194 	memcg_flush_percpu_vmstats(memcg);
5195 	memcg_flush_percpu_vmevents(memcg);
5196 	__mem_cgroup_free(memcg);
5197 }
5198 
5199 static struct mem_cgroup *mem_cgroup_alloc(void)
5200 {
5201 	struct mem_cgroup *memcg;
5202 	unsigned int size;
5203 	int node;
5204 	int __maybe_unused i;
5205 	long error = -ENOMEM;
5206 
5207 	size = sizeof(struct mem_cgroup);
5208 	size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);
5209 
5210 	memcg = kzalloc(size, GFP_KERNEL);
5211 	if (!memcg)
5212 		return ERR_PTR(error);
5213 
5214 	memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
5215 				 1, MEM_CGROUP_ID_MAX,
5216 				 GFP_KERNEL);
5217 	if (memcg->id.id < 0) {
5218 		error = memcg->id.id;
5219 		goto fail;
5220 	}
5221 
5222 	memcg->vmstats_local = alloc_percpu_gfp(struct memcg_vmstats_percpu,
5223 						GFP_KERNEL_ACCOUNT);
5224 	if (!memcg->vmstats_local)
5225 		goto fail;
5226 
5227 	memcg->vmstats_percpu = alloc_percpu_gfp(struct memcg_vmstats_percpu,
5228 						 GFP_KERNEL_ACCOUNT);
5229 	if (!memcg->vmstats_percpu)
5230 		goto fail;
5231 
5232 	for_each_node(node)
5233 		if (alloc_mem_cgroup_per_node_info(memcg, node))
5234 			goto fail;
5235 
5236 	if (memcg_wb_domain_init(memcg, GFP_KERNEL))
5237 		goto fail;
5238 
5239 	INIT_WORK(&memcg->high_work, high_work_func);
5240 	INIT_LIST_HEAD(&memcg->oom_notify);
5241 	mutex_init(&memcg->thresholds_lock);
5242 	spin_lock_init(&memcg->move_lock);
5243 	vmpressure_init(&memcg->vmpressure);
5244 	INIT_LIST_HEAD(&memcg->event_list);
5245 	spin_lock_init(&memcg->event_list_lock);
5246 	memcg->socket_pressure = jiffies;
5247 #ifdef CONFIG_MEMCG_KMEM
5248 	memcg->kmemcg_id = -1;
5249 	INIT_LIST_HEAD(&memcg->objcg_list);
5250 #endif
5251 #ifdef CONFIG_CGROUP_WRITEBACK
5252 	INIT_LIST_HEAD(&memcg->cgwb_list);
5253 	for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
5254 		memcg->cgwb_frn[i].done =
5255 			__WB_COMPLETION_INIT(&memcg_cgwb_frn_waitq);
5256 #endif
5257 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
5258 	spin_lock_init(&memcg->deferred_split_queue.split_queue_lock);
5259 	INIT_LIST_HEAD(&memcg->deferred_split_queue.split_queue);
5260 	memcg->deferred_split_queue.split_queue_len = 0;
5261 #endif
5262 	idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
5263 	return memcg;
5264 fail:
5265 	mem_cgroup_id_remove(memcg);
5266 	__mem_cgroup_free(memcg);
5267 	return ERR_PTR(error);
5268 }
5269 
5270 static struct cgroup_subsys_state * __ref
5271 mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
5272 {
5273 	struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
5274 	struct mem_cgroup *memcg;
5275 	long error = -ENOMEM;
5276 
5277 	memalloc_use_memcg(parent);
5278 	memcg = mem_cgroup_alloc();
5279 	memalloc_unuse_memcg();
5280 	if (IS_ERR(memcg))
5281 		return ERR_CAST(memcg);
5282 
5283 	page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
5284 	memcg->soft_limit = PAGE_COUNTER_MAX;
5285 	page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
5286 	if (parent) {
5287 		memcg->swappiness = mem_cgroup_swappiness(parent);
5288 		memcg->oom_kill_disable = parent->oom_kill_disable;
5289 	}
5290 	if (parent && parent->use_hierarchy) {
5291 		memcg->use_hierarchy = true;
5292 		page_counter_init(&memcg->memory, &parent->memory);
5293 		page_counter_init(&memcg->swap, &parent->swap);
5294 		page_counter_init(&memcg->memsw, &parent->memsw);
5295 		page_counter_init(&memcg->kmem, &parent->kmem);
5296 		page_counter_init(&memcg->tcpmem, &parent->tcpmem);
5297 	} else {
5298 		page_counter_init(&memcg->memory, NULL);
5299 		page_counter_init(&memcg->swap, NULL);
5300 		page_counter_init(&memcg->memsw, NULL);
5301 		page_counter_init(&memcg->kmem, NULL);
5302 		page_counter_init(&memcg->tcpmem, NULL);
5303 		/*
5304 		 * Deeper hierachy with use_hierarchy == false doesn't make
5305 		 * much sense so let cgroup subsystem know about this
5306 		 * unfortunate state in our controller.
5307 		 */
5308 		if (parent != root_mem_cgroup)
5309 			memory_cgrp_subsys.broken_hierarchy = true;
5310 	}
5311 
5312 	/* The following stuff does not apply to the root */
5313 	if (!parent) {
5314 		root_mem_cgroup = memcg;
5315 		return &memcg->css;
5316 	}
5317 
5318 	error = memcg_online_kmem(memcg);
5319 	if (error)
5320 		goto fail;
5321 
5322 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
5323 		static_branch_inc(&memcg_sockets_enabled_key);
5324 
5325 	return &memcg->css;
5326 fail:
5327 	mem_cgroup_id_remove(memcg);
5328 	mem_cgroup_free(memcg);
5329 	return ERR_PTR(error);
5330 }
5331 
5332 static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
5333 {
5334 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5335 
5336 	/*
5337 	 * A memcg must be visible for memcg_expand_shrinker_maps()
5338 	 * by the time the maps are allocated. So, we allocate maps
5339 	 * here, when for_each_mem_cgroup() can't skip it.
5340 	 */
5341 	if (memcg_alloc_shrinker_maps(memcg)) {
5342 		mem_cgroup_id_remove(memcg);
5343 		return -ENOMEM;
5344 	}
5345 
5346 	/* Online state pins memcg ID, memcg ID pins CSS */
5347 	refcount_set(&memcg->id.ref, 1);
5348 	css_get(css);
5349 	return 0;
5350 }
5351 
5352 static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
5353 {
5354 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5355 	struct mem_cgroup_event *event, *tmp;
5356 
5357 	/*
5358 	 * Unregister events and notify userspace.
5359 	 * Notify userspace about cgroup removing only after rmdir of cgroup
5360 	 * directory to avoid race between userspace and kernelspace.
5361 	 */
5362 	spin_lock(&memcg->event_list_lock);
5363 	list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
5364 		list_del_init(&event->list);
5365 		schedule_work(&event->remove);
5366 	}
5367 	spin_unlock(&memcg->event_list_lock);
5368 
5369 	page_counter_set_min(&memcg->memory, 0);
5370 	page_counter_set_low(&memcg->memory, 0);
5371 
5372 	memcg_offline_kmem(memcg);
5373 	wb_memcg_offline(memcg);
5374 
5375 	drain_all_stock(memcg);
5376 
5377 	mem_cgroup_id_put(memcg);
5378 }
5379 
5380 static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
5381 {
5382 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5383 
5384 	invalidate_reclaim_iterators(memcg);
5385 }
5386 
5387 static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
5388 {
5389 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5390 	int __maybe_unused i;
5391 
5392 #ifdef CONFIG_CGROUP_WRITEBACK
5393 	for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
5394 		wb_wait_for_completion(&memcg->cgwb_frn[i].done);
5395 #endif
5396 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
5397 		static_branch_dec(&memcg_sockets_enabled_key);
5398 
5399 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
5400 		static_branch_dec(&memcg_sockets_enabled_key);
5401 
5402 	vmpressure_cleanup(&memcg->vmpressure);
5403 	cancel_work_sync(&memcg->high_work);
5404 	mem_cgroup_remove_from_trees(memcg);
5405 	memcg_free_shrinker_maps(memcg);
5406 	memcg_free_kmem(memcg);
5407 	mem_cgroup_free(memcg);
5408 }
5409 
5410 /**
5411  * mem_cgroup_css_reset - reset the states of a mem_cgroup
5412  * @css: the target css
5413  *
5414  * Reset the states of the mem_cgroup associated with @css.  This is
5415  * invoked when the userland requests disabling on the default hierarchy
5416  * but the memcg is pinned through dependency.  The memcg should stop
5417  * applying policies and should revert to the vanilla state as it may be
5418  * made visible again.
5419  *
5420  * The current implementation only resets the essential configurations.
5421  * This needs to be expanded to cover all the visible parts.
5422  */
5423 static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
5424 {
5425 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5426 
5427 	page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX);
5428 	page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX);
5429 	page_counter_set_max(&memcg->memsw, PAGE_COUNTER_MAX);
5430 	page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX);
5431 	page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX);
5432 	page_counter_set_min(&memcg->memory, 0);
5433 	page_counter_set_low(&memcg->memory, 0);
5434 	page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
5435 	memcg->soft_limit = PAGE_COUNTER_MAX;
5436 	page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
5437 	memcg_wb_domain_size_changed(memcg);
5438 }
5439 
5440 #ifdef CONFIG_MMU
5441 /* Handlers for move charge at task migration. */
5442 static int mem_cgroup_do_precharge(unsigned long count)
5443 {
5444 	int ret;
5445 
5446 	/* Try a single bulk charge without reclaim first, kswapd may wake */
5447 	ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
5448 	if (!ret) {
5449 		mc.precharge += count;
5450 		return ret;
5451 	}
5452 
5453 	/* Try charges one by one with reclaim, but do not retry */
5454 	while (count--) {
5455 		ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
5456 		if (ret)
5457 			return ret;
5458 		mc.precharge++;
5459 		cond_resched();
5460 	}
5461 	return 0;
5462 }
5463 
5464 union mc_target {
5465 	struct page	*page;
5466 	swp_entry_t	ent;
5467 };
5468 
5469 enum mc_target_type {
5470 	MC_TARGET_NONE = 0,
5471 	MC_TARGET_PAGE,
5472 	MC_TARGET_SWAP,
5473 	MC_TARGET_DEVICE,
5474 };
5475 
5476 static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
5477 						unsigned long addr, pte_t ptent)
5478 {
5479 	struct page *page = vm_normal_page(vma, addr, ptent);
5480 
5481 	if (!page || !page_mapped(page))
5482 		return NULL;
5483 	if (PageAnon(page)) {
5484 		if (!(mc.flags & MOVE_ANON))
5485 			return NULL;
5486 	} else {
5487 		if (!(mc.flags & MOVE_FILE))
5488 			return NULL;
5489 	}
5490 	if (!get_page_unless_zero(page))
5491 		return NULL;
5492 
5493 	return page;
5494 }
5495 
5496 #if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE)
5497 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
5498 			pte_t ptent, swp_entry_t *entry)
5499 {
5500 	struct page *page = NULL;
5501 	swp_entry_t ent = pte_to_swp_entry(ptent);
5502 
5503 	if (!(mc.flags & MOVE_ANON) || non_swap_entry(ent))
5504 		return NULL;
5505 
5506 	/*
5507 	 * Handle MEMORY_DEVICE_PRIVATE which are ZONE_DEVICE page belonging to
5508 	 * a device and because they are not accessible by CPU they are store
5509 	 * as special swap entry in the CPU page table.
5510 	 */
5511 	if (is_device_private_entry(ent)) {
5512 		page = device_private_entry_to_page(ent);
5513 		/*
5514 		 * MEMORY_DEVICE_PRIVATE means ZONE_DEVICE page and which have
5515 		 * a refcount of 1 when free (unlike normal page)
5516 		 */
5517 		if (!page_ref_add_unless(page, 1, 1))
5518 			return NULL;
5519 		return page;
5520 	}
5521 
5522 	/*
5523 	 * Because lookup_swap_cache() updates some statistics counter,
5524 	 * we call find_get_page() with swapper_space directly.
5525 	 */
5526 	page = find_get_page(swap_address_space(ent), swp_offset(ent));
5527 	entry->val = ent.val;
5528 
5529 	return page;
5530 }
5531 #else
5532 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
5533 			pte_t ptent, swp_entry_t *entry)
5534 {
5535 	return NULL;
5536 }
5537 #endif
5538 
5539 static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
5540 			unsigned long addr, pte_t ptent, swp_entry_t *entry)
5541 {
5542 	struct page *page = NULL;
5543 	struct address_space *mapping;
5544 	pgoff_t pgoff;
5545 
5546 	if (!vma->vm_file) /* anonymous vma */
5547 		return NULL;
5548 	if (!(mc.flags & MOVE_FILE))
5549 		return NULL;
5550 
5551 	mapping = vma->vm_file->f_mapping;
5552 	pgoff = linear_page_index(vma, addr);
5553 
5554 	/* page is moved even if it's not RSS of this task(page-faulted). */
5555 #ifdef CONFIG_SWAP
5556 	/* shmem/tmpfs may report page out on swap: account for that too. */
5557 	if (shmem_mapping(mapping)) {
5558 		page = find_get_entry(mapping, pgoff);
5559 		if (xa_is_value(page)) {
5560 			swp_entry_t swp = radix_to_swp_entry(page);
5561 			*entry = swp;
5562 			page = find_get_page(swap_address_space(swp),
5563 					     swp_offset(swp));
5564 		}
5565 	} else
5566 		page = find_get_page(mapping, pgoff);
5567 #else
5568 	page = find_get_page(mapping, pgoff);
5569 #endif
5570 	return page;
5571 }
5572 
5573 /**
5574  * mem_cgroup_move_account - move account of the page
5575  * @page: the page
5576  * @compound: charge the page as compound or small page
5577  * @from: mem_cgroup which the page is moved from.
5578  * @to:	mem_cgroup which the page is moved to. @from != @to.
5579  *
5580  * The caller must make sure the page is not on LRU (isolate_page() is useful.)
5581  *
5582  * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
5583  * from old cgroup.
5584  */
5585 static int mem_cgroup_move_account(struct page *page,
5586 				   bool compound,
5587 				   struct mem_cgroup *from,
5588 				   struct mem_cgroup *to)
5589 {
5590 	struct lruvec *from_vec, *to_vec;
5591 	struct pglist_data *pgdat;
5592 	unsigned int nr_pages = compound ? thp_nr_pages(page) : 1;
5593 	int ret;
5594 
5595 	VM_BUG_ON(from == to);
5596 	VM_BUG_ON_PAGE(PageLRU(page), page);
5597 	VM_BUG_ON(compound && !PageTransHuge(page));
5598 
5599 	/*
5600 	 * Prevent mem_cgroup_migrate() from looking at
5601 	 * page->mem_cgroup of its source page while we change it.
5602 	 */
5603 	ret = -EBUSY;
5604 	if (!trylock_page(page))
5605 		goto out;
5606 
5607 	ret = -EINVAL;
5608 	if (page->mem_cgroup != from)
5609 		goto out_unlock;
5610 
5611 	pgdat = page_pgdat(page);
5612 	from_vec = mem_cgroup_lruvec(from, pgdat);
5613 	to_vec = mem_cgroup_lruvec(to, pgdat);
5614 
5615 	lock_page_memcg(page);
5616 
5617 	if (PageAnon(page)) {
5618 		if (page_mapped(page)) {
5619 			__mod_lruvec_state(from_vec, NR_ANON_MAPPED, -nr_pages);
5620 			__mod_lruvec_state(to_vec, NR_ANON_MAPPED, nr_pages);
5621 			if (PageTransHuge(page)) {
5622 				__mod_lruvec_state(from_vec, NR_ANON_THPS,
5623 						   -nr_pages);
5624 				__mod_lruvec_state(to_vec, NR_ANON_THPS,
5625 						   nr_pages);
5626 			}
5627 
5628 		}
5629 	} else {
5630 		__mod_lruvec_state(from_vec, NR_FILE_PAGES, -nr_pages);
5631 		__mod_lruvec_state(to_vec, NR_FILE_PAGES, nr_pages);
5632 
5633 		if (PageSwapBacked(page)) {
5634 			__mod_lruvec_state(from_vec, NR_SHMEM, -nr_pages);
5635 			__mod_lruvec_state(to_vec, NR_SHMEM, nr_pages);
5636 		}
5637 
5638 		if (page_mapped(page)) {
5639 			__mod_lruvec_state(from_vec, NR_FILE_MAPPED, -nr_pages);
5640 			__mod_lruvec_state(to_vec, NR_FILE_MAPPED, nr_pages);
5641 		}
5642 
5643 		if (PageDirty(page)) {
5644 			struct address_space *mapping = page_mapping(page);
5645 
5646 			if (mapping_cap_account_dirty(mapping)) {
5647 				__mod_lruvec_state(from_vec, NR_FILE_DIRTY,
5648 						   -nr_pages);
5649 				__mod_lruvec_state(to_vec, NR_FILE_DIRTY,
5650 						   nr_pages);
5651 			}
5652 		}
5653 	}
5654 
5655 	if (PageWriteback(page)) {
5656 		__mod_lruvec_state(from_vec, NR_WRITEBACK, -nr_pages);
5657 		__mod_lruvec_state(to_vec, NR_WRITEBACK, nr_pages);
5658 	}
5659 
5660 	/*
5661 	 * All state has been migrated, let's switch to the new memcg.
5662 	 *
5663 	 * It is safe to change page->mem_cgroup here because the page
5664 	 * is referenced, charged, isolated, and locked: we can't race
5665 	 * with (un)charging, migration, LRU putback, or anything else
5666 	 * that would rely on a stable page->mem_cgroup.
5667 	 *
5668 	 * Note that lock_page_memcg is a memcg lock, not a page lock,
5669 	 * to save space. As soon as we switch page->mem_cgroup to a
5670 	 * new memcg that isn't locked, the above state can change
5671 	 * concurrently again. Make sure we're truly done with it.
5672 	 */
5673 	smp_mb();
5674 
5675 	css_get(&to->css);
5676 	css_put(&from->css);
5677 
5678 	page->mem_cgroup = to;
5679 
5680 	__unlock_page_memcg(from);
5681 
5682 	ret = 0;
5683 
5684 	local_irq_disable();
5685 	mem_cgroup_charge_statistics(to, page, nr_pages);
5686 	memcg_check_events(to, page);
5687 	mem_cgroup_charge_statistics(from, page, -nr_pages);
5688 	memcg_check_events(from, page);
5689 	local_irq_enable();
5690 out_unlock:
5691 	unlock_page(page);
5692 out:
5693 	return ret;
5694 }
5695 
5696 /**
5697  * get_mctgt_type - get target type of moving charge
5698  * @vma: the vma the pte to be checked belongs
5699  * @addr: the address corresponding to the pte to be checked
5700  * @ptent: the pte to be checked
5701  * @target: the pointer the target page or swap ent will be stored(can be NULL)
5702  *
5703  * Returns
5704  *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
5705  *   1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
5706  *     move charge. if @target is not NULL, the page is stored in target->page
5707  *     with extra refcnt got(Callers should handle it).
5708  *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
5709  *     target for charge migration. if @target is not NULL, the entry is stored
5710  *     in target->ent.
5711  *   3(MC_TARGET_DEVICE): like MC_TARGET_PAGE  but page is MEMORY_DEVICE_PRIVATE
5712  *     (so ZONE_DEVICE page and thus not on the lru).
5713  *     For now we such page is charge like a regular page would be as for all
5714  *     intent and purposes it is just special memory taking the place of a
5715  *     regular page.
5716  *
5717  *     See Documentations/vm/hmm.txt and include/linux/hmm.h
5718  *
5719  * Called with pte lock held.
5720  */
5721 
5722 static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
5723 		unsigned long addr, pte_t ptent, union mc_target *target)
5724 {
5725 	struct page *page = NULL;
5726 	enum mc_target_type ret = MC_TARGET_NONE;
5727 	swp_entry_t ent = { .val = 0 };
5728 
5729 	if (pte_present(ptent))
5730 		page = mc_handle_present_pte(vma, addr, ptent);
5731 	else if (is_swap_pte(ptent))
5732 		page = mc_handle_swap_pte(vma, ptent, &ent);
5733 	else if (pte_none(ptent))
5734 		page = mc_handle_file_pte(vma, addr, ptent, &ent);
5735 
5736 	if (!page && !ent.val)
5737 		return ret;
5738 	if (page) {
5739 		/*
5740 		 * Do only loose check w/o serialization.
5741 		 * mem_cgroup_move_account() checks the page is valid or
5742 		 * not under LRU exclusion.
5743 		 */
5744 		if (page->mem_cgroup == mc.from) {
5745 			ret = MC_TARGET_PAGE;
5746 			if (is_device_private_page(page))
5747 				ret = MC_TARGET_DEVICE;
5748 			if (target)
5749 				target->page = page;
5750 		}
5751 		if (!ret || !target)
5752 			put_page(page);
5753 	}
5754 	/*
5755 	 * There is a swap entry and a page doesn't exist or isn't charged.
5756 	 * But we cannot move a tail-page in a THP.
5757 	 */
5758 	if (ent.val && !ret && (!page || !PageTransCompound(page)) &&
5759 	    mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
5760 		ret = MC_TARGET_SWAP;
5761 		if (target)
5762 			target->ent = ent;
5763 	}
5764 	return ret;
5765 }
5766 
5767 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
5768 /*
5769  * We don't consider PMD mapped swapping or file mapped pages because THP does
5770  * not support them for now.
5771  * Caller should make sure that pmd_trans_huge(pmd) is true.
5772  */
5773 static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
5774 		unsigned long addr, pmd_t pmd, union mc_target *target)
5775 {
5776 	struct page *page = NULL;
5777 	enum mc_target_type ret = MC_TARGET_NONE;
5778 
5779 	if (unlikely(is_swap_pmd(pmd))) {
5780 		VM_BUG_ON(thp_migration_supported() &&
5781 				  !is_pmd_migration_entry(pmd));
5782 		return ret;
5783 	}
5784 	page = pmd_page(pmd);
5785 	VM_BUG_ON_PAGE(!page || !PageHead(page), page);
5786 	if (!(mc.flags & MOVE_ANON))
5787 		return ret;
5788 	if (page->mem_cgroup == mc.from) {
5789 		ret = MC_TARGET_PAGE;
5790 		if (target) {
5791 			get_page(page);
5792 			target->page = page;
5793 		}
5794 	}
5795 	return ret;
5796 }
5797 #else
5798 static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
5799 		unsigned long addr, pmd_t pmd, union mc_target *target)
5800 {
5801 	return MC_TARGET_NONE;
5802 }
5803 #endif
5804 
5805 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
5806 					unsigned long addr, unsigned long end,
5807 					struct mm_walk *walk)
5808 {
5809 	struct vm_area_struct *vma = walk->vma;
5810 	pte_t *pte;
5811 	spinlock_t *ptl;
5812 
5813 	ptl = pmd_trans_huge_lock(pmd, vma);
5814 	if (ptl) {
5815 		/*
5816 		 * Note their can not be MC_TARGET_DEVICE for now as we do not
5817 		 * support transparent huge page with MEMORY_DEVICE_PRIVATE but
5818 		 * this might change.
5819 		 */
5820 		if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
5821 			mc.precharge += HPAGE_PMD_NR;
5822 		spin_unlock(ptl);
5823 		return 0;
5824 	}
5825 
5826 	if (pmd_trans_unstable(pmd))
5827 		return 0;
5828 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
5829 	for (; addr != end; pte++, addr += PAGE_SIZE)
5830 		if (get_mctgt_type(vma, addr, *pte, NULL))
5831 			mc.precharge++;	/* increment precharge temporarily */
5832 	pte_unmap_unlock(pte - 1, ptl);
5833 	cond_resched();
5834 
5835 	return 0;
5836 }
5837 
5838 static const struct mm_walk_ops precharge_walk_ops = {
5839 	.pmd_entry	= mem_cgroup_count_precharge_pte_range,
5840 };
5841 
5842 static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
5843 {
5844 	unsigned long precharge;
5845 
5846 	mmap_read_lock(mm);
5847 	walk_page_range(mm, 0, mm->highest_vm_end, &precharge_walk_ops, NULL);
5848 	mmap_read_unlock(mm);
5849 
5850 	precharge = mc.precharge;
5851 	mc.precharge = 0;
5852 
5853 	return precharge;
5854 }
5855 
5856 static int mem_cgroup_precharge_mc(struct mm_struct *mm)
5857 {
5858 	unsigned long precharge = mem_cgroup_count_precharge(mm);
5859 
5860 	VM_BUG_ON(mc.moving_task);
5861 	mc.moving_task = current;
5862 	return mem_cgroup_do_precharge(precharge);
5863 }
5864 
5865 /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
5866 static void __mem_cgroup_clear_mc(void)
5867 {
5868 	struct mem_cgroup *from = mc.from;
5869 	struct mem_cgroup *to = mc.to;
5870 
5871 	/* we must uncharge all the leftover precharges from mc.to */
5872 	if (mc.precharge) {
5873 		cancel_charge(mc.to, mc.precharge);
5874 		mc.precharge = 0;
5875 	}
5876 	/*
5877 	 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
5878 	 * we must uncharge here.
5879 	 */
5880 	if (mc.moved_charge) {
5881 		cancel_charge(mc.from, mc.moved_charge);
5882 		mc.moved_charge = 0;
5883 	}
5884 	/* we must fixup refcnts and charges */
5885 	if (mc.moved_swap) {
5886 		/* uncharge swap account from the old cgroup */
5887 		if (!mem_cgroup_is_root(mc.from))
5888 			page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
5889 
5890 		mem_cgroup_id_put_many(mc.from, mc.moved_swap);
5891 
5892 		/*
5893 		 * we charged both to->memory and to->memsw, so we
5894 		 * should uncharge to->memory.
5895 		 */
5896 		if (!mem_cgroup_is_root(mc.to))
5897 			page_counter_uncharge(&mc.to->memory, mc.moved_swap);
5898 
5899 		mc.moved_swap = 0;
5900 	}
5901 	memcg_oom_recover(from);
5902 	memcg_oom_recover(to);
5903 	wake_up_all(&mc.waitq);
5904 }
5905 
5906 static void mem_cgroup_clear_mc(void)
5907 {
5908 	struct mm_struct *mm = mc.mm;
5909 
5910 	/*
5911 	 * we must clear moving_task before waking up waiters at the end of
5912 	 * task migration.
5913 	 */
5914 	mc.moving_task = NULL;
5915 	__mem_cgroup_clear_mc();
5916 	spin_lock(&mc.lock);
5917 	mc.from = NULL;
5918 	mc.to = NULL;
5919 	mc.mm = NULL;
5920 	spin_unlock(&mc.lock);
5921 
5922 	mmput(mm);
5923 }
5924 
5925 static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
5926 {
5927 	struct cgroup_subsys_state *css;
5928 	struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
5929 	struct mem_cgroup *from;
5930 	struct task_struct *leader, *p;
5931 	struct mm_struct *mm;
5932 	unsigned long move_flags;
5933 	int ret = 0;
5934 
5935 	/* charge immigration isn't supported on the default hierarchy */
5936 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
5937 		return 0;
5938 
5939 	/*
5940 	 * Multi-process migrations only happen on the default hierarchy
5941 	 * where charge immigration is not used.  Perform charge
5942 	 * immigration if @tset contains a leader and whine if there are
5943 	 * multiple.
5944 	 */
5945 	p = NULL;
5946 	cgroup_taskset_for_each_leader(leader, css, tset) {
5947 		WARN_ON_ONCE(p);
5948 		p = leader;
5949 		memcg = mem_cgroup_from_css(css);
5950 	}
5951 	if (!p)
5952 		return 0;
5953 
5954 	/*
5955 	 * We are now commited to this value whatever it is. Changes in this
5956 	 * tunable will only affect upcoming migrations, not the current one.
5957 	 * So we need to save it, and keep it going.
5958 	 */
5959 	move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
5960 	if (!move_flags)
5961 		return 0;
5962 
5963 	from = mem_cgroup_from_task(p);
5964 
5965 	VM_BUG_ON(from == memcg);
5966 
5967 	mm = get_task_mm(p);
5968 	if (!mm)
5969 		return 0;
5970 	/* We move charges only when we move a owner of the mm */
5971 	if (mm->owner == p) {
5972 		VM_BUG_ON(mc.from);
5973 		VM_BUG_ON(mc.to);
5974 		VM_BUG_ON(mc.precharge);
5975 		VM_BUG_ON(mc.moved_charge);
5976 		VM_BUG_ON(mc.moved_swap);
5977 
5978 		spin_lock(&mc.lock);
5979 		mc.mm = mm;
5980 		mc.from = from;
5981 		mc.to = memcg;
5982 		mc.flags = move_flags;
5983 		spin_unlock(&mc.lock);
5984 		/* We set mc.moving_task later */
5985 
5986 		ret = mem_cgroup_precharge_mc(mm);
5987 		if (ret)
5988 			mem_cgroup_clear_mc();
5989 	} else {
5990 		mmput(mm);
5991 	}
5992 	return ret;
5993 }
5994 
5995 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
5996 {
5997 	if (mc.to)
5998 		mem_cgroup_clear_mc();
5999 }
6000 
6001 static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
6002 				unsigned long addr, unsigned long end,
6003 				struct mm_walk *walk)
6004 {
6005 	int ret = 0;
6006 	struct vm_area_struct *vma = walk->vma;
6007 	pte_t *pte;
6008 	spinlock_t *ptl;
6009 	enum mc_target_type target_type;
6010 	union mc_target target;
6011 	struct page *page;
6012 
6013 	ptl = pmd_trans_huge_lock(pmd, vma);
6014 	if (ptl) {
6015 		if (mc.precharge < HPAGE_PMD_NR) {
6016 			spin_unlock(ptl);
6017 			return 0;
6018 		}
6019 		target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
6020 		if (target_type == MC_TARGET_PAGE) {
6021 			page = target.page;
6022 			if (!isolate_lru_page(page)) {
6023 				if (!mem_cgroup_move_account(page, true,
6024 							     mc.from, mc.to)) {
6025 					mc.precharge -= HPAGE_PMD_NR;
6026 					mc.moved_charge += HPAGE_PMD_NR;
6027 				}
6028 				putback_lru_page(page);
6029 			}
6030 			put_page(page);
6031 		} else if (target_type == MC_TARGET_DEVICE) {
6032 			page = target.page;
6033 			if (!mem_cgroup_move_account(page, true,
6034 						     mc.from, mc.to)) {
6035 				mc.precharge -= HPAGE_PMD_NR;
6036 				mc.moved_charge += HPAGE_PMD_NR;
6037 			}
6038 			put_page(page);
6039 		}
6040 		spin_unlock(ptl);
6041 		return 0;
6042 	}
6043 
6044 	if (pmd_trans_unstable(pmd))
6045 		return 0;
6046 retry:
6047 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
6048 	for (; addr != end; addr += PAGE_SIZE) {
6049 		pte_t ptent = *(pte++);
6050 		bool device = false;
6051 		swp_entry_t ent;
6052 
6053 		if (!mc.precharge)
6054 			break;
6055 
6056 		switch (get_mctgt_type(vma, addr, ptent, &target)) {
6057 		case MC_TARGET_DEVICE:
6058 			device = true;
6059 			fallthrough;
6060 		case MC_TARGET_PAGE:
6061 			page = target.page;
6062 			/*
6063 			 * We can have a part of the split pmd here. Moving it
6064 			 * can be done but it would be too convoluted so simply
6065 			 * ignore such a partial THP and keep it in original
6066 			 * memcg. There should be somebody mapping the head.
6067 			 */
6068 			if (PageTransCompound(page))
6069 				goto put;
6070 			if (!device && isolate_lru_page(page))
6071 				goto put;
6072 			if (!mem_cgroup_move_account(page, false,
6073 						mc.from, mc.to)) {
6074 				mc.precharge--;
6075 				/* we uncharge from mc.from later. */
6076 				mc.moved_charge++;
6077 			}
6078 			if (!device)
6079 				putback_lru_page(page);
6080 put:			/* get_mctgt_type() gets the page */
6081 			put_page(page);
6082 			break;
6083 		case MC_TARGET_SWAP:
6084 			ent = target.ent;
6085 			if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
6086 				mc.precharge--;
6087 				mem_cgroup_id_get_many(mc.to, 1);
6088 				/* we fixup other refcnts and charges later. */
6089 				mc.moved_swap++;
6090 			}
6091 			break;
6092 		default:
6093 			break;
6094 		}
6095 	}
6096 	pte_unmap_unlock(pte - 1, ptl);
6097 	cond_resched();
6098 
6099 	if (addr != end) {
6100 		/*
6101 		 * We have consumed all precharges we got in can_attach().
6102 		 * We try charge one by one, but don't do any additional
6103 		 * charges to mc.to if we have failed in charge once in attach()
6104 		 * phase.
6105 		 */
6106 		ret = mem_cgroup_do_precharge(1);
6107 		if (!ret)
6108 			goto retry;
6109 	}
6110 
6111 	return ret;
6112 }
6113 
6114 static const struct mm_walk_ops charge_walk_ops = {
6115 	.pmd_entry	= mem_cgroup_move_charge_pte_range,
6116 };
6117 
6118 static void mem_cgroup_move_charge(void)
6119 {
6120 	lru_add_drain_all();
6121 	/*
6122 	 * Signal lock_page_memcg() to take the memcg's move_lock
6123 	 * while we're moving its pages to another memcg. Then wait
6124 	 * for already started RCU-only updates to finish.
6125 	 */
6126 	atomic_inc(&mc.from->moving_account);
6127 	synchronize_rcu();
6128 retry:
6129 	if (unlikely(!mmap_read_trylock(mc.mm))) {
6130 		/*
6131 		 * Someone who are holding the mmap_lock might be waiting in
6132 		 * waitq. So we cancel all extra charges, wake up all waiters,
6133 		 * and retry. Because we cancel precharges, we might not be able
6134 		 * to move enough charges, but moving charge is a best-effort
6135 		 * feature anyway, so it wouldn't be a big problem.
6136 		 */
6137 		__mem_cgroup_clear_mc();
6138 		cond_resched();
6139 		goto retry;
6140 	}
6141 	/*
6142 	 * When we have consumed all precharges and failed in doing
6143 	 * additional charge, the page walk just aborts.
6144 	 */
6145 	walk_page_range(mc.mm, 0, mc.mm->highest_vm_end, &charge_walk_ops,
6146 			NULL);
6147 
6148 	mmap_read_unlock(mc.mm);
6149 	atomic_dec(&mc.from->moving_account);
6150 }
6151 
6152 static void mem_cgroup_move_task(void)
6153 {
6154 	if (mc.to) {
6155 		mem_cgroup_move_charge();
6156 		mem_cgroup_clear_mc();
6157 	}
6158 }
6159 #else	/* !CONFIG_MMU */
6160 static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
6161 {
6162 	return 0;
6163 }
6164 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
6165 {
6166 }
6167 static void mem_cgroup_move_task(void)
6168 {
6169 }
6170 #endif
6171 
6172 /*
6173  * Cgroup retains root cgroups across [un]mount cycles making it necessary
6174  * to verify whether we're attached to the default hierarchy on each mount
6175  * attempt.
6176  */
6177 static void mem_cgroup_bind(struct cgroup_subsys_state *root_css)
6178 {
6179 	/*
6180 	 * use_hierarchy is forced on the default hierarchy.  cgroup core
6181 	 * guarantees that @root doesn't have any children, so turning it
6182 	 * on for the root memcg is enough.
6183 	 */
6184 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
6185 		root_mem_cgroup->use_hierarchy = true;
6186 	else
6187 		root_mem_cgroup->use_hierarchy = false;
6188 }
6189 
6190 static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value)
6191 {
6192 	if (value == PAGE_COUNTER_MAX)
6193 		seq_puts(m, "max\n");
6194 	else
6195 		seq_printf(m, "%llu\n", (u64)value * PAGE_SIZE);
6196 
6197 	return 0;
6198 }
6199 
6200 static u64 memory_current_read(struct cgroup_subsys_state *css,
6201 			       struct cftype *cft)
6202 {
6203 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6204 
6205 	return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
6206 }
6207 
6208 static int memory_min_show(struct seq_file *m, void *v)
6209 {
6210 	return seq_puts_memcg_tunable(m,
6211 		READ_ONCE(mem_cgroup_from_seq(m)->memory.min));
6212 }
6213 
6214 static ssize_t memory_min_write(struct kernfs_open_file *of,
6215 				char *buf, size_t nbytes, loff_t off)
6216 {
6217 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6218 	unsigned long min;
6219 	int err;
6220 
6221 	buf = strstrip(buf);
6222 	err = page_counter_memparse(buf, "max", &min);
6223 	if (err)
6224 		return err;
6225 
6226 	page_counter_set_min(&memcg->memory, min);
6227 
6228 	return nbytes;
6229 }
6230 
6231 static int memory_low_show(struct seq_file *m, void *v)
6232 {
6233 	return seq_puts_memcg_tunable(m,
6234 		READ_ONCE(mem_cgroup_from_seq(m)->memory.low));
6235 }
6236 
6237 static ssize_t memory_low_write(struct kernfs_open_file *of,
6238 				char *buf, size_t nbytes, loff_t off)
6239 {
6240 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6241 	unsigned long low;
6242 	int err;
6243 
6244 	buf = strstrip(buf);
6245 	err = page_counter_memparse(buf, "max", &low);
6246 	if (err)
6247 		return err;
6248 
6249 	page_counter_set_low(&memcg->memory, low);
6250 
6251 	return nbytes;
6252 }
6253 
6254 static int memory_high_show(struct seq_file *m, void *v)
6255 {
6256 	return seq_puts_memcg_tunable(m,
6257 		READ_ONCE(mem_cgroup_from_seq(m)->memory.high));
6258 }
6259 
6260 static ssize_t memory_high_write(struct kernfs_open_file *of,
6261 				 char *buf, size_t nbytes, loff_t off)
6262 {
6263 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6264 	unsigned int nr_retries = MAX_RECLAIM_RETRIES;
6265 	bool drained = false;
6266 	unsigned long high;
6267 	int err;
6268 
6269 	buf = strstrip(buf);
6270 	err = page_counter_memparse(buf, "max", &high);
6271 	if (err)
6272 		return err;
6273 
6274 	for (;;) {
6275 		unsigned long nr_pages = page_counter_read(&memcg->memory);
6276 		unsigned long reclaimed;
6277 
6278 		if (nr_pages <= high)
6279 			break;
6280 
6281 		if (signal_pending(current))
6282 			break;
6283 
6284 		if (!drained) {
6285 			drain_all_stock(memcg);
6286 			drained = true;
6287 			continue;
6288 		}
6289 
6290 		reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
6291 							 GFP_KERNEL, true);
6292 
6293 		if (!reclaimed && !nr_retries--)
6294 			break;
6295 	}
6296 
6297 	page_counter_set_high(&memcg->memory, high);
6298 
6299 	memcg_wb_domain_size_changed(memcg);
6300 
6301 	return nbytes;
6302 }
6303 
6304 static int memory_max_show(struct seq_file *m, void *v)
6305 {
6306 	return seq_puts_memcg_tunable(m,
6307 		READ_ONCE(mem_cgroup_from_seq(m)->memory.max));
6308 }
6309 
6310 static ssize_t memory_max_write(struct kernfs_open_file *of,
6311 				char *buf, size_t nbytes, loff_t off)
6312 {
6313 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6314 	unsigned int nr_reclaims = MAX_RECLAIM_RETRIES;
6315 	bool drained = false;
6316 	unsigned long max;
6317 	int err;
6318 
6319 	buf = strstrip(buf);
6320 	err = page_counter_memparse(buf, "max", &max);
6321 	if (err)
6322 		return err;
6323 
6324 	xchg(&memcg->memory.max, max);
6325 
6326 	for (;;) {
6327 		unsigned long nr_pages = page_counter_read(&memcg->memory);
6328 
6329 		if (nr_pages <= max)
6330 			break;
6331 
6332 		if (signal_pending(current))
6333 			break;
6334 
6335 		if (!drained) {
6336 			drain_all_stock(memcg);
6337 			drained = true;
6338 			continue;
6339 		}
6340 
6341 		if (nr_reclaims) {
6342 			if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
6343 							  GFP_KERNEL, true))
6344 				nr_reclaims--;
6345 			continue;
6346 		}
6347 
6348 		memcg_memory_event(memcg, MEMCG_OOM);
6349 		if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
6350 			break;
6351 	}
6352 
6353 	memcg_wb_domain_size_changed(memcg);
6354 	return nbytes;
6355 }
6356 
6357 static void __memory_events_show(struct seq_file *m, atomic_long_t *events)
6358 {
6359 	seq_printf(m, "low %lu\n", atomic_long_read(&events[MEMCG_LOW]));
6360 	seq_printf(m, "high %lu\n", atomic_long_read(&events[MEMCG_HIGH]));
6361 	seq_printf(m, "max %lu\n", atomic_long_read(&events[MEMCG_MAX]));
6362 	seq_printf(m, "oom %lu\n", atomic_long_read(&events[MEMCG_OOM]));
6363 	seq_printf(m, "oom_kill %lu\n",
6364 		   atomic_long_read(&events[MEMCG_OOM_KILL]));
6365 }
6366 
6367 static int memory_events_show(struct seq_file *m, void *v)
6368 {
6369 	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6370 
6371 	__memory_events_show(m, memcg->memory_events);
6372 	return 0;
6373 }
6374 
6375 static int memory_events_local_show(struct seq_file *m, void *v)
6376 {
6377 	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6378 
6379 	__memory_events_show(m, memcg->memory_events_local);
6380 	return 0;
6381 }
6382 
6383 static int memory_stat_show(struct seq_file *m, void *v)
6384 {
6385 	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6386 	char *buf;
6387 
6388 	buf = memory_stat_format(memcg);
6389 	if (!buf)
6390 		return -ENOMEM;
6391 	seq_puts(m, buf);
6392 	kfree(buf);
6393 	return 0;
6394 }
6395 
6396 static int memory_oom_group_show(struct seq_file *m, void *v)
6397 {
6398 	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6399 
6400 	seq_printf(m, "%d\n", memcg->oom_group);
6401 
6402 	return 0;
6403 }
6404 
6405 static ssize_t memory_oom_group_write(struct kernfs_open_file *of,
6406 				      char *buf, size_t nbytes, loff_t off)
6407 {
6408 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6409 	int ret, oom_group;
6410 
6411 	buf = strstrip(buf);
6412 	if (!buf)
6413 		return -EINVAL;
6414 
6415 	ret = kstrtoint(buf, 0, &oom_group);
6416 	if (ret)
6417 		return ret;
6418 
6419 	if (oom_group != 0 && oom_group != 1)
6420 		return -EINVAL;
6421 
6422 	memcg->oom_group = oom_group;
6423 
6424 	return nbytes;
6425 }
6426 
6427 static struct cftype memory_files[] = {
6428 	{
6429 		.name = "current",
6430 		.flags = CFTYPE_NOT_ON_ROOT,
6431 		.read_u64 = memory_current_read,
6432 	},
6433 	{
6434 		.name = "min",
6435 		.flags = CFTYPE_NOT_ON_ROOT,
6436 		.seq_show = memory_min_show,
6437 		.write = memory_min_write,
6438 	},
6439 	{
6440 		.name = "low",
6441 		.flags = CFTYPE_NOT_ON_ROOT,
6442 		.seq_show = memory_low_show,
6443 		.write = memory_low_write,
6444 	},
6445 	{
6446 		.name = "high",
6447 		.flags = CFTYPE_NOT_ON_ROOT,
6448 		.seq_show = memory_high_show,
6449 		.write = memory_high_write,
6450 	},
6451 	{
6452 		.name = "max",
6453 		.flags = CFTYPE_NOT_ON_ROOT,
6454 		.seq_show = memory_max_show,
6455 		.write = memory_max_write,
6456 	},
6457 	{
6458 		.name = "events",
6459 		.flags = CFTYPE_NOT_ON_ROOT,
6460 		.file_offset = offsetof(struct mem_cgroup, events_file),
6461 		.seq_show = memory_events_show,
6462 	},
6463 	{
6464 		.name = "events.local",
6465 		.flags = CFTYPE_NOT_ON_ROOT,
6466 		.file_offset = offsetof(struct mem_cgroup, events_local_file),
6467 		.seq_show = memory_events_local_show,
6468 	},
6469 	{
6470 		.name = "stat",
6471 		.seq_show = memory_stat_show,
6472 	},
6473 	{
6474 		.name = "oom.group",
6475 		.flags = CFTYPE_NOT_ON_ROOT | CFTYPE_NS_DELEGATABLE,
6476 		.seq_show = memory_oom_group_show,
6477 		.write = memory_oom_group_write,
6478 	},
6479 	{ }	/* terminate */
6480 };
6481 
6482 struct cgroup_subsys memory_cgrp_subsys = {
6483 	.css_alloc = mem_cgroup_css_alloc,
6484 	.css_online = mem_cgroup_css_online,
6485 	.css_offline = mem_cgroup_css_offline,
6486 	.css_released = mem_cgroup_css_released,
6487 	.css_free = mem_cgroup_css_free,
6488 	.css_reset = mem_cgroup_css_reset,
6489 	.can_attach = mem_cgroup_can_attach,
6490 	.cancel_attach = mem_cgroup_cancel_attach,
6491 	.post_attach = mem_cgroup_move_task,
6492 	.bind = mem_cgroup_bind,
6493 	.dfl_cftypes = memory_files,
6494 	.legacy_cftypes = mem_cgroup_legacy_files,
6495 	.early_init = 0,
6496 };
6497 
6498 /*
6499  * This function calculates an individual cgroup's effective
6500  * protection which is derived from its own memory.min/low, its
6501  * parent's and siblings' settings, as well as the actual memory
6502  * distribution in the tree.
6503  *
6504  * The following rules apply to the effective protection values:
6505  *
6506  * 1. At the first level of reclaim, effective protection is equal to
6507  *    the declared protection in memory.min and memory.low.
6508  *
6509  * 2. To enable safe delegation of the protection configuration, at
6510  *    subsequent levels the effective protection is capped to the
6511  *    parent's effective protection.
6512  *
6513  * 3. To make complex and dynamic subtrees easier to configure, the
6514  *    user is allowed to overcommit the declared protection at a given
6515  *    level. If that is the case, the parent's effective protection is
6516  *    distributed to the children in proportion to how much protection
6517  *    they have declared and how much of it they are utilizing.
6518  *
6519  *    This makes distribution proportional, but also work-conserving:
6520  *    if one cgroup claims much more protection than it uses memory,
6521  *    the unused remainder is available to its siblings.
6522  *
6523  * 4. Conversely, when the declared protection is undercommitted at a
6524  *    given level, the distribution of the larger parental protection
6525  *    budget is NOT proportional. A cgroup's protection from a sibling
6526  *    is capped to its own memory.min/low setting.
6527  *
6528  * 5. However, to allow protecting recursive subtrees from each other
6529  *    without having to declare each individual cgroup's fixed share
6530  *    of the ancestor's claim to protection, any unutilized -
6531  *    "floating" - protection from up the tree is distributed in
6532  *    proportion to each cgroup's *usage*. This makes the protection
6533  *    neutral wrt sibling cgroups and lets them compete freely over
6534  *    the shared parental protection budget, but it protects the
6535  *    subtree as a whole from neighboring subtrees.
6536  *
6537  * Note that 4. and 5. are not in conflict: 4. is about protecting
6538  * against immediate siblings whereas 5. is about protecting against
6539  * neighboring subtrees.
6540  */
6541 static unsigned long effective_protection(unsigned long usage,
6542 					  unsigned long parent_usage,
6543 					  unsigned long setting,
6544 					  unsigned long parent_effective,
6545 					  unsigned long siblings_protected)
6546 {
6547 	unsigned long protected;
6548 	unsigned long ep;
6549 
6550 	protected = min(usage, setting);
6551 	/*
6552 	 * If all cgroups at this level combined claim and use more
6553 	 * protection then what the parent affords them, distribute
6554 	 * shares in proportion to utilization.
6555 	 *
6556 	 * We are using actual utilization rather than the statically
6557 	 * claimed protection in order to be work-conserving: claimed
6558 	 * but unused protection is available to siblings that would
6559 	 * otherwise get a smaller chunk than what they claimed.
6560 	 */
6561 	if (siblings_protected > parent_effective)
6562 		return protected * parent_effective / siblings_protected;
6563 
6564 	/*
6565 	 * Ok, utilized protection of all children is within what the
6566 	 * parent affords them, so we know whatever this child claims
6567 	 * and utilizes is effectively protected.
6568 	 *
6569 	 * If there is unprotected usage beyond this value, reclaim
6570 	 * will apply pressure in proportion to that amount.
6571 	 *
6572 	 * If there is unutilized protection, the cgroup will be fully
6573 	 * shielded from reclaim, but we do return a smaller value for
6574 	 * protection than what the group could enjoy in theory. This
6575 	 * is okay. With the overcommit distribution above, effective
6576 	 * protection is always dependent on how memory is actually
6577 	 * consumed among the siblings anyway.
6578 	 */
6579 	ep = protected;
6580 
6581 	/*
6582 	 * If the children aren't claiming (all of) the protection
6583 	 * afforded to them by the parent, distribute the remainder in
6584 	 * proportion to the (unprotected) memory of each cgroup. That
6585 	 * way, cgroups that aren't explicitly prioritized wrt each
6586 	 * other compete freely over the allowance, but they are
6587 	 * collectively protected from neighboring trees.
6588 	 *
6589 	 * We're using unprotected memory for the weight so that if
6590 	 * some cgroups DO claim explicit protection, we don't protect
6591 	 * the same bytes twice.
6592 	 *
6593 	 * Check both usage and parent_usage against the respective
6594 	 * protected values. One should imply the other, but they
6595 	 * aren't read atomically - make sure the division is sane.
6596 	 */
6597 	if (!(cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT))
6598 		return ep;
6599 	if (parent_effective > siblings_protected &&
6600 	    parent_usage > siblings_protected &&
6601 	    usage > protected) {
6602 		unsigned long unclaimed;
6603 
6604 		unclaimed = parent_effective - siblings_protected;
6605 		unclaimed *= usage - protected;
6606 		unclaimed /= parent_usage - siblings_protected;
6607 
6608 		ep += unclaimed;
6609 	}
6610 
6611 	return ep;
6612 }
6613 
6614 /**
6615  * mem_cgroup_protected - check if memory consumption is in the normal range
6616  * @root: the top ancestor of the sub-tree being checked
6617  * @memcg: the memory cgroup to check
6618  *
6619  * WARNING: This function is not stateless! It can only be used as part
6620  *          of a top-down tree iteration, not for isolated queries.
6621  */
6622 void mem_cgroup_calculate_protection(struct mem_cgroup *root,
6623 				     struct mem_cgroup *memcg)
6624 {
6625 	unsigned long usage, parent_usage;
6626 	struct mem_cgroup *parent;
6627 
6628 	if (mem_cgroup_disabled())
6629 		return;
6630 
6631 	if (!root)
6632 		root = root_mem_cgroup;
6633 
6634 	/*
6635 	 * Effective values of the reclaim targets are ignored so they
6636 	 * can be stale. Have a look at mem_cgroup_protection for more
6637 	 * details.
6638 	 * TODO: calculation should be more robust so that we do not need
6639 	 * that special casing.
6640 	 */
6641 	if (memcg == root)
6642 		return;
6643 
6644 	usage = page_counter_read(&memcg->memory);
6645 	if (!usage)
6646 		return;
6647 
6648 	parent = parent_mem_cgroup(memcg);
6649 	/* No parent means a non-hierarchical mode on v1 memcg */
6650 	if (!parent)
6651 		return;
6652 
6653 	if (parent == root) {
6654 		memcg->memory.emin = READ_ONCE(memcg->memory.min);
6655 		memcg->memory.elow = READ_ONCE(memcg->memory.low);
6656 		return;
6657 	}
6658 
6659 	parent_usage = page_counter_read(&parent->memory);
6660 
6661 	WRITE_ONCE(memcg->memory.emin, effective_protection(usage, parent_usage,
6662 			READ_ONCE(memcg->memory.min),
6663 			READ_ONCE(parent->memory.emin),
6664 			atomic_long_read(&parent->memory.children_min_usage)));
6665 
6666 	WRITE_ONCE(memcg->memory.elow, effective_protection(usage, parent_usage,
6667 			READ_ONCE(memcg->memory.low),
6668 			READ_ONCE(parent->memory.elow),
6669 			atomic_long_read(&parent->memory.children_low_usage)));
6670 }
6671 
6672 /**
6673  * mem_cgroup_charge - charge a newly allocated page to a cgroup
6674  * @page: page to charge
6675  * @mm: mm context of the victim
6676  * @gfp_mask: reclaim mode
6677  *
6678  * Try to charge @page to the memcg that @mm belongs to, reclaiming
6679  * pages according to @gfp_mask if necessary.
6680  *
6681  * Returns 0 on success. Otherwise, an error code is returned.
6682  */
6683 int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
6684 {
6685 	unsigned int nr_pages = thp_nr_pages(page);
6686 	struct mem_cgroup *memcg = NULL;
6687 	int ret = 0;
6688 
6689 	if (mem_cgroup_disabled())
6690 		goto out;
6691 
6692 	if (PageSwapCache(page)) {
6693 		swp_entry_t ent = { .val = page_private(page), };
6694 		unsigned short id;
6695 
6696 		/*
6697 		 * Every swap fault against a single page tries to charge the
6698 		 * page, bail as early as possible.  shmem_unuse() encounters
6699 		 * already charged pages, too.  page->mem_cgroup is protected
6700 		 * by the page lock, which serializes swap cache removal, which
6701 		 * in turn serializes uncharging.
6702 		 */
6703 		VM_BUG_ON_PAGE(!PageLocked(page), page);
6704 		if (compound_head(page)->mem_cgroup)
6705 			goto out;
6706 
6707 		id = lookup_swap_cgroup_id(ent);
6708 		rcu_read_lock();
6709 		memcg = mem_cgroup_from_id(id);
6710 		if (memcg && !css_tryget_online(&memcg->css))
6711 			memcg = NULL;
6712 		rcu_read_unlock();
6713 	}
6714 
6715 	if (!memcg)
6716 		memcg = get_mem_cgroup_from_mm(mm);
6717 
6718 	ret = try_charge(memcg, gfp_mask, nr_pages);
6719 	if (ret)
6720 		goto out_put;
6721 
6722 	css_get(&memcg->css);
6723 	commit_charge(page, memcg);
6724 
6725 	local_irq_disable();
6726 	mem_cgroup_charge_statistics(memcg, page, nr_pages);
6727 	memcg_check_events(memcg, page);
6728 	local_irq_enable();
6729 
6730 	if (PageSwapCache(page)) {
6731 		swp_entry_t entry = { .val = page_private(page) };
6732 		/*
6733 		 * The swap entry might not get freed for a long time,
6734 		 * let's not wait for it.  The page already received a
6735 		 * memory+swap charge, drop the swap entry duplicate.
6736 		 */
6737 		mem_cgroup_uncharge_swap(entry, nr_pages);
6738 	}
6739 
6740 out_put:
6741 	css_put(&memcg->css);
6742 out:
6743 	return ret;
6744 }
6745 
6746 struct uncharge_gather {
6747 	struct mem_cgroup *memcg;
6748 	unsigned long nr_pages;
6749 	unsigned long pgpgout;
6750 	unsigned long nr_kmem;
6751 	struct page *dummy_page;
6752 };
6753 
6754 static inline void uncharge_gather_clear(struct uncharge_gather *ug)
6755 {
6756 	memset(ug, 0, sizeof(*ug));
6757 }
6758 
6759 static void uncharge_batch(const struct uncharge_gather *ug)
6760 {
6761 	unsigned long flags;
6762 
6763 	if (!mem_cgroup_is_root(ug->memcg)) {
6764 		page_counter_uncharge(&ug->memcg->memory, ug->nr_pages);
6765 		if (do_memsw_account())
6766 			page_counter_uncharge(&ug->memcg->memsw, ug->nr_pages);
6767 		if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && ug->nr_kmem)
6768 			page_counter_uncharge(&ug->memcg->kmem, ug->nr_kmem);
6769 		memcg_oom_recover(ug->memcg);
6770 	}
6771 
6772 	local_irq_save(flags);
6773 	__count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);
6774 	__this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_pages);
6775 	memcg_check_events(ug->memcg, ug->dummy_page);
6776 	local_irq_restore(flags);
6777 
6778 	/* drop reference from uncharge_page */
6779 	css_put(&ug->memcg->css);
6780 }
6781 
6782 static void uncharge_page(struct page *page, struct uncharge_gather *ug)
6783 {
6784 	unsigned long nr_pages;
6785 
6786 	VM_BUG_ON_PAGE(PageLRU(page), page);
6787 
6788 	if (!page->mem_cgroup)
6789 		return;
6790 
6791 	/*
6792 	 * Nobody should be changing or seriously looking at
6793 	 * page->mem_cgroup at this point, we have fully
6794 	 * exclusive access to the page.
6795 	 */
6796 
6797 	if (ug->memcg != page->mem_cgroup) {
6798 		if (ug->memcg) {
6799 			uncharge_batch(ug);
6800 			uncharge_gather_clear(ug);
6801 		}
6802 		ug->memcg = page->mem_cgroup;
6803 
6804 		/* pairs with css_put in uncharge_batch */
6805 		css_get(&ug->memcg->css);
6806 	}
6807 
6808 	nr_pages = compound_nr(page);
6809 	ug->nr_pages += nr_pages;
6810 
6811 	if (!PageKmemcg(page)) {
6812 		ug->pgpgout++;
6813 	} else {
6814 		ug->nr_kmem += nr_pages;
6815 		__ClearPageKmemcg(page);
6816 	}
6817 
6818 	ug->dummy_page = page;
6819 	page->mem_cgroup = NULL;
6820 	css_put(&ug->memcg->css);
6821 }
6822 
6823 static void uncharge_list(struct list_head *page_list)
6824 {
6825 	struct uncharge_gather ug;
6826 	struct list_head *next;
6827 
6828 	uncharge_gather_clear(&ug);
6829 
6830 	/*
6831 	 * Note that the list can be a single page->lru; hence the
6832 	 * do-while loop instead of a simple list_for_each_entry().
6833 	 */
6834 	next = page_list->next;
6835 	do {
6836 		struct page *page;
6837 
6838 		page = list_entry(next, struct page, lru);
6839 		next = page->lru.next;
6840 
6841 		uncharge_page(page, &ug);
6842 	} while (next != page_list);
6843 
6844 	if (ug.memcg)
6845 		uncharge_batch(&ug);
6846 }
6847 
6848 /**
6849  * mem_cgroup_uncharge - uncharge a page
6850  * @page: page to uncharge
6851  *
6852  * Uncharge a page previously charged with mem_cgroup_charge().
6853  */
6854 void mem_cgroup_uncharge(struct page *page)
6855 {
6856 	struct uncharge_gather ug;
6857 
6858 	if (mem_cgroup_disabled())
6859 		return;
6860 
6861 	/* Don't touch page->lru of any random page, pre-check: */
6862 	if (!page->mem_cgroup)
6863 		return;
6864 
6865 	uncharge_gather_clear(&ug);
6866 	uncharge_page(page, &ug);
6867 	uncharge_batch(&ug);
6868 }
6869 
6870 /**
6871  * mem_cgroup_uncharge_list - uncharge a list of page
6872  * @page_list: list of pages to uncharge
6873  *
6874  * Uncharge a list of pages previously charged with
6875  * mem_cgroup_charge().
6876  */
6877 void mem_cgroup_uncharge_list(struct list_head *page_list)
6878 {
6879 	if (mem_cgroup_disabled())
6880 		return;
6881 
6882 	if (!list_empty(page_list))
6883 		uncharge_list(page_list);
6884 }
6885 
6886 /**
6887  * mem_cgroup_migrate - charge a page's replacement
6888  * @oldpage: currently circulating page
6889  * @newpage: replacement page
6890  *
6891  * Charge @newpage as a replacement page for @oldpage. @oldpage will
6892  * be uncharged upon free.
6893  *
6894  * Both pages must be locked, @newpage->mapping must be set up.
6895  */
6896 void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
6897 {
6898 	struct mem_cgroup *memcg;
6899 	unsigned int nr_pages;
6900 	unsigned long flags;
6901 
6902 	VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
6903 	VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
6904 	VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage);
6905 	VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage),
6906 		       newpage);
6907 
6908 	if (mem_cgroup_disabled())
6909 		return;
6910 
6911 	/* Page cache replacement: new page already charged? */
6912 	if (newpage->mem_cgroup)
6913 		return;
6914 
6915 	/* Swapcache readahead pages can get replaced before being charged */
6916 	memcg = oldpage->mem_cgroup;
6917 	if (!memcg)
6918 		return;
6919 
6920 	/* Force-charge the new page. The old one will be freed soon */
6921 	nr_pages = thp_nr_pages(newpage);
6922 
6923 	page_counter_charge(&memcg->memory, nr_pages);
6924 	if (do_memsw_account())
6925 		page_counter_charge(&memcg->memsw, nr_pages);
6926 
6927 	css_get(&memcg->css);
6928 	commit_charge(newpage, memcg);
6929 
6930 	local_irq_save(flags);
6931 	mem_cgroup_charge_statistics(memcg, newpage, nr_pages);
6932 	memcg_check_events(memcg, newpage);
6933 	local_irq_restore(flags);
6934 }
6935 
6936 DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
6937 EXPORT_SYMBOL(memcg_sockets_enabled_key);
6938 
6939 void mem_cgroup_sk_alloc(struct sock *sk)
6940 {
6941 	struct mem_cgroup *memcg;
6942 
6943 	if (!mem_cgroup_sockets_enabled)
6944 		return;
6945 
6946 	/* Do not associate the sock with unrelated interrupted task's memcg. */
6947 	if (in_interrupt())
6948 		return;
6949 
6950 	rcu_read_lock();
6951 	memcg = mem_cgroup_from_task(current);
6952 	if (memcg == root_mem_cgroup)
6953 		goto out;
6954 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
6955 		goto out;
6956 	if (css_tryget(&memcg->css))
6957 		sk->sk_memcg = memcg;
6958 out:
6959 	rcu_read_unlock();
6960 }
6961 
6962 void mem_cgroup_sk_free(struct sock *sk)
6963 {
6964 	if (sk->sk_memcg)
6965 		css_put(&sk->sk_memcg->css);
6966 }
6967 
6968 /**
6969  * mem_cgroup_charge_skmem - charge socket memory
6970  * @memcg: memcg to charge
6971  * @nr_pages: number of pages to charge
6972  *
6973  * Charges @nr_pages to @memcg. Returns %true if the charge fit within
6974  * @memcg's configured limit, %false if the charge had to be forced.
6975  */
6976 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
6977 {
6978 	gfp_t gfp_mask = GFP_KERNEL;
6979 
6980 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
6981 		struct page_counter *fail;
6982 
6983 		if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
6984 			memcg->tcpmem_pressure = 0;
6985 			return true;
6986 		}
6987 		page_counter_charge(&memcg->tcpmem, nr_pages);
6988 		memcg->tcpmem_pressure = 1;
6989 		return false;
6990 	}
6991 
6992 	/* Don't block in the packet receive path */
6993 	if (in_softirq())
6994 		gfp_mask = GFP_NOWAIT;
6995 
6996 	mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);
6997 
6998 	if (try_charge(memcg, gfp_mask, nr_pages) == 0)
6999 		return true;
7000 
7001 	try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages);
7002 	return false;
7003 }
7004 
7005 /**
7006  * mem_cgroup_uncharge_skmem - uncharge socket memory
7007  * @memcg: memcg to uncharge
7008  * @nr_pages: number of pages to uncharge
7009  */
7010 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
7011 {
7012 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
7013 		page_counter_uncharge(&memcg->tcpmem, nr_pages);
7014 		return;
7015 	}
7016 
7017 	mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
7018 
7019 	refill_stock(memcg, nr_pages);
7020 }
7021 
7022 static int __init cgroup_memory(char *s)
7023 {
7024 	char *token;
7025 
7026 	while ((token = strsep(&s, ",")) != NULL) {
7027 		if (!*token)
7028 			continue;
7029 		if (!strcmp(token, "nosocket"))
7030 			cgroup_memory_nosocket = true;
7031 		if (!strcmp(token, "nokmem"))
7032 			cgroup_memory_nokmem = true;
7033 	}
7034 	return 0;
7035 }
7036 __setup("cgroup.memory=", cgroup_memory);
7037 
7038 /*
7039  * subsys_initcall() for memory controller.
7040  *
7041  * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this
7042  * context because of lock dependencies (cgroup_lock -> cpu hotplug) but
7043  * basically everything that doesn't depend on a specific mem_cgroup structure
7044  * should be initialized from here.
7045  */
7046 static int __init mem_cgroup_init(void)
7047 {
7048 	int cpu, node;
7049 
7050 	cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
7051 				  memcg_hotplug_cpu_dead);
7052 
7053 	for_each_possible_cpu(cpu)
7054 		INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
7055 			  drain_local_stock);
7056 
7057 	for_each_node(node) {
7058 		struct mem_cgroup_tree_per_node *rtpn;
7059 
7060 		rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
7061 				    node_online(node) ? node : NUMA_NO_NODE);
7062 
7063 		rtpn->rb_root = RB_ROOT;
7064 		rtpn->rb_rightmost = NULL;
7065 		spin_lock_init(&rtpn->lock);
7066 		soft_limit_tree.rb_tree_per_node[node] = rtpn;
7067 	}
7068 
7069 	return 0;
7070 }
7071 subsys_initcall(mem_cgroup_init);
7072 
7073 #ifdef CONFIG_MEMCG_SWAP
7074 static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
7075 {
7076 	while (!refcount_inc_not_zero(&memcg->id.ref)) {
7077 		/*
7078 		 * The root cgroup cannot be destroyed, so it's refcount must
7079 		 * always be >= 1.
7080 		 */
7081 		if (WARN_ON_ONCE(memcg == root_mem_cgroup)) {
7082 			VM_BUG_ON(1);
7083 			break;
7084 		}
7085 		memcg = parent_mem_cgroup(memcg);
7086 		if (!memcg)
7087 			memcg = root_mem_cgroup;
7088 	}
7089 	return memcg;
7090 }
7091 
7092 /**
7093  * mem_cgroup_swapout - transfer a memsw charge to swap
7094  * @page: page whose memsw charge to transfer
7095  * @entry: swap entry to move the charge to
7096  *
7097  * Transfer the memsw charge of @page to @entry.
7098  */
7099 void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
7100 {
7101 	struct mem_cgroup *memcg, *swap_memcg;
7102 	unsigned int nr_entries;
7103 	unsigned short oldid;
7104 
7105 	VM_BUG_ON_PAGE(PageLRU(page), page);
7106 	VM_BUG_ON_PAGE(page_count(page), page);
7107 
7108 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
7109 		return;
7110 
7111 	memcg = page->mem_cgroup;
7112 
7113 	/* Readahead page, never charged */
7114 	if (!memcg)
7115 		return;
7116 
7117 	/*
7118 	 * In case the memcg owning these pages has been offlined and doesn't
7119 	 * have an ID allocated to it anymore, charge the closest online
7120 	 * ancestor for the swap instead and transfer the memory+swap charge.
7121 	 */
7122 	swap_memcg = mem_cgroup_id_get_online(memcg);
7123 	nr_entries = thp_nr_pages(page);
7124 	/* Get references for the tail pages, too */
7125 	if (nr_entries > 1)
7126 		mem_cgroup_id_get_many(swap_memcg, nr_entries - 1);
7127 	oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg),
7128 				   nr_entries);
7129 	VM_BUG_ON_PAGE(oldid, page);
7130 	mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);
7131 
7132 	page->mem_cgroup = NULL;
7133 
7134 	if (!mem_cgroup_is_root(memcg))
7135 		page_counter_uncharge(&memcg->memory, nr_entries);
7136 
7137 	if (!cgroup_memory_noswap && memcg != swap_memcg) {
7138 		if (!mem_cgroup_is_root(swap_memcg))
7139 			page_counter_charge(&swap_memcg->memsw, nr_entries);
7140 		page_counter_uncharge(&memcg->memsw, nr_entries);
7141 	}
7142 
7143 	/*
7144 	 * Interrupts should be disabled here because the caller holds the
7145 	 * i_pages lock which is taken with interrupts-off. It is
7146 	 * important here to have the interrupts disabled because it is the
7147 	 * only synchronisation we have for updating the per-CPU variables.
7148 	 */
7149 	VM_BUG_ON(!irqs_disabled());
7150 	mem_cgroup_charge_statistics(memcg, page, -nr_entries);
7151 	memcg_check_events(memcg, page);
7152 
7153 	css_put(&memcg->css);
7154 }
7155 
7156 /**
7157  * mem_cgroup_try_charge_swap - try charging swap space for a page
7158  * @page: page being added to swap
7159  * @entry: swap entry to charge
7160  *
7161  * Try to charge @page's memcg for the swap space at @entry.
7162  *
7163  * Returns 0 on success, -ENOMEM on failure.
7164  */
7165 int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
7166 {
7167 	unsigned int nr_pages = thp_nr_pages(page);
7168 	struct page_counter *counter;
7169 	struct mem_cgroup *memcg;
7170 	unsigned short oldid;
7171 
7172 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
7173 		return 0;
7174 
7175 	memcg = page->mem_cgroup;
7176 
7177 	/* Readahead page, never charged */
7178 	if (!memcg)
7179 		return 0;
7180 
7181 	if (!entry.val) {
7182 		memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
7183 		return 0;
7184 	}
7185 
7186 	memcg = mem_cgroup_id_get_online(memcg);
7187 
7188 	if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg) &&
7189 	    !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
7190 		memcg_memory_event(memcg, MEMCG_SWAP_MAX);
7191 		memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
7192 		mem_cgroup_id_put(memcg);
7193 		return -ENOMEM;
7194 	}
7195 
7196 	/* Get references for the tail pages, too */
7197 	if (nr_pages > 1)
7198 		mem_cgroup_id_get_many(memcg, nr_pages - 1);
7199 	oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages);
7200 	VM_BUG_ON_PAGE(oldid, page);
7201 	mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
7202 
7203 	return 0;
7204 }
7205 
7206 /**
7207  * mem_cgroup_uncharge_swap - uncharge swap space
7208  * @entry: swap entry to uncharge
7209  * @nr_pages: the amount of swap space to uncharge
7210  */
7211 void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
7212 {
7213 	struct mem_cgroup *memcg;
7214 	unsigned short id;
7215 
7216 	id = swap_cgroup_record(entry, 0, nr_pages);
7217 	rcu_read_lock();
7218 	memcg = mem_cgroup_from_id(id);
7219 	if (memcg) {
7220 		if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg)) {
7221 			if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
7222 				page_counter_uncharge(&memcg->swap, nr_pages);
7223 			else
7224 				page_counter_uncharge(&memcg->memsw, nr_pages);
7225 		}
7226 		mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
7227 		mem_cgroup_id_put_many(memcg, nr_pages);
7228 	}
7229 	rcu_read_unlock();
7230 }
7231 
7232 long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
7233 {
7234 	long nr_swap_pages = get_nr_swap_pages();
7235 
7236 	if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
7237 		return nr_swap_pages;
7238 	for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
7239 		nr_swap_pages = min_t(long, nr_swap_pages,
7240 				      READ_ONCE(memcg->swap.max) -
7241 				      page_counter_read(&memcg->swap));
7242 	return nr_swap_pages;
7243 }
7244 
7245 bool mem_cgroup_swap_full(struct page *page)
7246 {
7247 	struct mem_cgroup *memcg;
7248 
7249 	VM_BUG_ON_PAGE(!PageLocked(page), page);
7250 
7251 	if (vm_swap_full())
7252 		return true;
7253 	if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
7254 		return false;
7255 
7256 	memcg = page->mem_cgroup;
7257 	if (!memcg)
7258 		return false;
7259 
7260 	for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) {
7261 		unsigned long usage = page_counter_read(&memcg->swap);
7262 
7263 		if (usage * 2 >= READ_ONCE(memcg->swap.high) ||
7264 		    usage * 2 >= READ_ONCE(memcg->swap.max))
7265 			return true;
7266 	}
7267 
7268 	return false;
7269 }
7270 
7271 static int __init setup_swap_account(char *s)
7272 {
7273 	if (!strcmp(s, "1"))
7274 		cgroup_memory_noswap = 0;
7275 	else if (!strcmp(s, "0"))
7276 		cgroup_memory_noswap = 1;
7277 	return 1;
7278 }
7279 __setup("swapaccount=", setup_swap_account);
7280 
7281 static u64 swap_current_read(struct cgroup_subsys_state *css,
7282 			     struct cftype *cft)
7283 {
7284 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
7285 
7286 	return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
7287 }
7288 
7289 static int swap_high_show(struct seq_file *m, void *v)
7290 {
7291 	return seq_puts_memcg_tunable(m,
7292 		READ_ONCE(mem_cgroup_from_seq(m)->swap.high));
7293 }
7294 
7295 static ssize_t swap_high_write(struct kernfs_open_file *of,
7296 			       char *buf, size_t nbytes, loff_t off)
7297 {
7298 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7299 	unsigned long high;
7300 	int err;
7301 
7302 	buf = strstrip(buf);
7303 	err = page_counter_memparse(buf, "max", &high);
7304 	if (err)
7305 		return err;
7306 
7307 	page_counter_set_high(&memcg->swap, high);
7308 
7309 	return nbytes;
7310 }
7311 
7312 static int swap_max_show(struct seq_file *m, void *v)
7313 {
7314 	return seq_puts_memcg_tunable(m,
7315 		READ_ONCE(mem_cgroup_from_seq(m)->swap.max));
7316 }
7317 
7318 static ssize_t swap_max_write(struct kernfs_open_file *of,
7319 			      char *buf, size_t nbytes, loff_t off)
7320 {
7321 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7322 	unsigned long max;
7323 	int err;
7324 
7325 	buf = strstrip(buf);
7326 	err = page_counter_memparse(buf, "max", &max);
7327 	if (err)
7328 		return err;
7329 
7330 	xchg(&memcg->swap.max, max);
7331 
7332 	return nbytes;
7333 }
7334 
7335 static int swap_events_show(struct seq_file *m, void *v)
7336 {
7337 	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
7338 
7339 	seq_printf(m, "high %lu\n",
7340 		   atomic_long_read(&memcg->memory_events[MEMCG_SWAP_HIGH]));
7341 	seq_printf(m, "max %lu\n",
7342 		   atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]));
7343 	seq_printf(m, "fail %lu\n",
7344 		   atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL]));
7345 
7346 	return 0;
7347 }
7348 
7349 static struct cftype swap_files[] = {
7350 	{
7351 		.name = "swap.current",
7352 		.flags = CFTYPE_NOT_ON_ROOT,
7353 		.read_u64 = swap_current_read,
7354 	},
7355 	{
7356 		.name = "swap.high",
7357 		.flags = CFTYPE_NOT_ON_ROOT,
7358 		.seq_show = swap_high_show,
7359 		.write = swap_high_write,
7360 	},
7361 	{
7362 		.name = "swap.max",
7363 		.flags = CFTYPE_NOT_ON_ROOT,
7364 		.seq_show = swap_max_show,
7365 		.write = swap_max_write,
7366 	},
7367 	{
7368 		.name = "swap.events",
7369 		.flags = CFTYPE_NOT_ON_ROOT,
7370 		.file_offset = offsetof(struct mem_cgroup, swap_events_file),
7371 		.seq_show = swap_events_show,
7372 	},
7373 	{ }	/* terminate */
7374 };
7375 
7376 static struct cftype memsw_files[] = {
7377 	{
7378 		.name = "memsw.usage_in_bytes",
7379 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
7380 		.read_u64 = mem_cgroup_read_u64,
7381 	},
7382 	{
7383 		.name = "memsw.max_usage_in_bytes",
7384 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
7385 		.write = mem_cgroup_reset,
7386 		.read_u64 = mem_cgroup_read_u64,
7387 	},
7388 	{
7389 		.name = "memsw.limit_in_bytes",
7390 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
7391 		.write = mem_cgroup_write,
7392 		.read_u64 = mem_cgroup_read_u64,
7393 	},
7394 	{
7395 		.name = "memsw.failcnt",
7396 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
7397 		.write = mem_cgroup_reset,
7398 		.read_u64 = mem_cgroup_read_u64,
7399 	},
7400 	{ },	/* terminate */
7401 };
7402 
7403 /*
7404  * If mem_cgroup_swap_init() is implemented as a subsys_initcall()
7405  * instead of a core_initcall(), this could mean cgroup_memory_noswap still
7406  * remains set to false even when memcg is disabled via "cgroup_disable=memory"
7407  * boot parameter. This may result in premature OOPS inside
7408  * mem_cgroup_get_nr_swap_pages() function in corner cases.
7409  */
7410 static int __init mem_cgroup_swap_init(void)
7411 {
7412 	/* No memory control -> no swap control */
7413 	if (mem_cgroup_disabled())
7414 		cgroup_memory_noswap = true;
7415 
7416 	if (cgroup_memory_noswap)
7417 		return 0;
7418 
7419 	WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, swap_files));
7420 	WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys, memsw_files));
7421 
7422 	return 0;
7423 }
7424 core_initcall(mem_cgroup_swap_init);
7425 
7426 #endif /* CONFIG_MEMCG_SWAP */
7427