xref: /openbmc/linux/mm/memcontrol.c (revision 711aab1d)
1 /* memcontrol.c - Memory Controller
2  *
3  * Copyright IBM Corporation, 2007
4  * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5  *
6  * Copyright 2007 OpenVZ SWsoft Inc
7  * Author: Pavel Emelianov <xemul@openvz.org>
8  *
9  * Memory thresholds
10  * Copyright (C) 2009 Nokia Corporation
11  * Author: Kirill A. Shutemov
12  *
13  * Kernel Memory Controller
14  * Copyright (C) 2012 Parallels Inc. and Google Inc.
15  * Authors: Glauber Costa and Suleiman Souhlal
16  *
17  * Native page reclaim
18  * Charge lifetime sanitation
19  * Lockless page tracking & accounting
20  * Unified hierarchy configuration model
21  * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
22  *
23  * This program is free software; you can redistribute it and/or modify
24  * it under the terms of the GNU General Public License as published by
25  * the Free Software Foundation; either version 2 of the License, or
26  * (at your option) any later version.
27  *
28  * This program is distributed in the hope that it will be useful,
29  * but WITHOUT ANY WARRANTY; without even the implied warranty of
30  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
31  * GNU General Public License for more details.
32  */
33 
34 #include <linux/page_counter.h>
35 #include <linux/memcontrol.h>
36 #include <linux/cgroup.h>
37 #include <linux/mm.h>
38 #include <linux/sched/mm.h>
39 #include <linux/shmem_fs.h>
40 #include <linux/hugetlb.h>
41 #include <linux/pagemap.h>
42 #include <linux/smp.h>
43 #include <linux/page-flags.h>
44 #include <linux/backing-dev.h>
45 #include <linux/bit_spinlock.h>
46 #include <linux/rcupdate.h>
47 #include <linux/limits.h>
48 #include <linux/export.h>
49 #include <linux/mutex.h>
50 #include <linux/rbtree.h>
51 #include <linux/slab.h>
52 #include <linux/swap.h>
53 #include <linux/swapops.h>
54 #include <linux/spinlock.h>
55 #include <linux/eventfd.h>
56 #include <linux/poll.h>
57 #include <linux/sort.h>
58 #include <linux/fs.h>
59 #include <linux/seq_file.h>
60 #include <linux/vmpressure.h>
61 #include <linux/mm_inline.h>
62 #include <linux/swap_cgroup.h>
63 #include <linux/cpu.h>
64 #include <linux/oom.h>
65 #include <linux/lockdep.h>
66 #include <linux/file.h>
67 #include <linux/tracehook.h>
68 #include "internal.h"
69 #include <net/sock.h>
70 #include <net/ip.h>
71 #include "slab.h"
72 
73 #include <linux/uaccess.h>
74 
75 #include <trace/events/vmscan.h>
76 
77 struct cgroup_subsys memory_cgrp_subsys __read_mostly;
78 EXPORT_SYMBOL(memory_cgrp_subsys);
79 
80 struct mem_cgroup *root_mem_cgroup __read_mostly;
81 
82 #define MEM_CGROUP_RECLAIM_RETRIES	5
83 
84 /* Socket memory accounting disabled? */
85 static bool cgroup_memory_nosocket;
86 
87 /* Kernel memory accounting disabled? */
88 static bool cgroup_memory_nokmem;
89 
90 /* Whether the swap controller is active */
91 #ifdef CONFIG_MEMCG_SWAP
92 int do_swap_account __read_mostly;
93 #else
94 #define do_swap_account		0
95 #endif
96 
97 /* Whether legacy memory+swap accounting is active */
98 static bool do_memsw_account(void)
99 {
100 	return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && do_swap_account;
101 }
102 
103 static const char *const mem_cgroup_lru_names[] = {
104 	"inactive_anon",
105 	"active_anon",
106 	"inactive_file",
107 	"active_file",
108 	"unevictable",
109 };
110 
111 #define THRESHOLDS_EVENTS_TARGET 128
112 #define SOFTLIMIT_EVENTS_TARGET 1024
113 #define NUMAINFO_EVENTS_TARGET	1024
114 
115 /*
116  * Cgroups above their limits are maintained in a RB-Tree, independent of
117  * their hierarchy representation
118  */
119 
120 struct mem_cgroup_tree_per_node {
121 	struct rb_root rb_root;
122 	struct rb_node *rb_rightmost;
123 	spinlock_t lock;
124 };
125 
126 struct mem_cgroup_tree {
127 	struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
128 };
129 
130 static struct mem_cgroup_tree soft_limit_tree __read_mostly;
131 
132 /* for OOM */
133 struct mem_cgroup_eventfd_list {
134 	struct list_head list;
135 	struct eventfd_ctx *eventfd;
136 };
137 
138 /*
139  * cgroup_event represents events which userspace want to receive.
140  */
141 struct mem_cgroup_event {
142 	/*
143 	 * memcg which the event belongs to.
144 	 */
145 	struct mem_cgroup *memcg;
146 	/*
147 	 * eventfd to signal userspace about the event.
148 	 */
149 	struct eventfd_ctx *eventfd;
150 	/*
151 	 * Each of these stored in a list by the cgroup.
152 	 */
153 	struct list_head list;
154 	/*
155 	 * register_event() callback will be used to add new userspace
156 	 * waiter for changes related to this event.  Use eventfd_signal()
157 	 * on eventfd to send notification to userspace.
158 	 */
159 	int (*register_event)(struct mem_cgroup *memcg,
160 			      struct eventfd_ctx *eventfd, const char *args);
161 	/*
162 	 * unregister_event() callback will be called when userspace closes
163 	 * the eventfd or on cgroup removing.  This callback must be set,
164 	 * if you want provide notification functionality.
165 	 */
166 	void (*unregister_event)(struct mem_cgroup *memcg,
167 				 struct eventfd_ctx *eventfd);
168 	/*
169 	 * All fields below needed to unregister event when
170 	 * userspace closes eventfd.
171 	 */
172 	poll_table pt;
173 	wait_queue_head_t *wqh;
174 	wait_queue_entry_t wait;
175 	struct work_struct remove;
176 };
177 
178 static void mem_cgroup_threshold(struct mem_cgroup *memcg);
179 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
180 
181 /* Stuffs for move charges at task migration. */
182 /*
183  * Types of charges to be moved.
184  */
185 #define MOVE_ANON	0x1U
186 #define MOVE_FILE	0x2U
187 #define MOVE_MASK	(MOVE_ANON | MOVE_FILE)
188 
189 /* "mc" and its members are protected by cgroup_mutex */
190 static struct move_charge_struct {
191 	spinlock_t	  lock; /* for from, to */
192 	struct mm_struct  *mm;
193 	struct mem_cgroup *from;
194 	struct mem_cgroup *to;
195 	unsigned long flags;
196 	unsigned long precharge;
197 	unsigned long moved_charge;
198 	unsigned long moved_swap;
199 	struct task_struct *moving_task;	/* a task moving charges */
200 	wait_queue_head_t waitq;		/* a waitq for other context */
201 } mc = {
202 	.lock = __SPIN_LOCK_UNLOCKED(mc.lock),
203 	.waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
204 };
205 
206 /*
207  * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
208  * limit reclaim to prevent infinite loops, if they ever occur.
209  */
210 #define	MEM_CGROUP_MAX_RECLAIM_LOOPS		100
211 #define	MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS	2
212 
213 enum charge_type {
214 	MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
215 	MEM_CGROUP_CHARGE_TYPE_ANON,
216 	MEM_CGROUP_CHARGE_TYPE_SWAPOUT,	/* for accounting swapcache */
217 	MEM_CGROUP_CHARGE_TYPE_DROP,	/* a page was unused swap cache */
218 	NR_CHARGE_TYPE,
219 };
220 
221 /* for encoding cft->private value on file */
222 enum res_type {
223 	_MEM,
224 	_MEMSWAP,
225 	_OOM_TYPE,
226 	_KMEM,
227 	_TCP,
228 };
229 
230 #define MEMFILE_PRIVATE(x, val)	((x) << 16 | (val))
231 #define MEMFILE_TYPE(val)	((val) >> 16 & 0xffff)
232 #define MEMFILE_ATTR(val)	((val) & 0xffff)
233 /* Used for OOM nofiier */
234 #define OOM_CONTROL		(0)
235 
236 /* Some nice accessors for the vmpressure. */
237 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
238 {
239 	if (!memcg)
240 		memcg = root_mem_cgroup;
241 	return &memcg->vmpressure;
242 }
243 
244 struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
245 {
246 	return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
247 }
248 
249 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
250 {
251 	return (memcg == root_mem_cgroup);
252 }
253 
254 #ifndef CONFIG_SLOB
255 /*
256  * This will be the memcg's index in each cache's ->memcg_params.memcg_caches.
257  * The main reason for not using cgroup id for this:
258  *  this works better in sparse environments, where we have a lot of memcgs,
259  *  but only a few kmem-limited. Or also, if we have, for instance, 200
260  *  memcgs, and none but the 200th is kmem-limited, we'd have to have a
261  *  200 entry array for that.
262  *
263  * The current size of the caches array is stored in memcg_nr_cache_ids. It
264  * will double each time we have to increase it.
265  */
266 static DEFINE_IDA(memcg_cache_ida);
267 int memcg_nr_cache_ids;
268 
269 /* Protects memcg_nr_cache_ids */
270 static DECLARE_RWSEM(memcg_cache_ids_sem);
271 
272 void memcg_get_cache_ids(void)
273 {
274 	down_read(&memcg_cache_ids_sem);
275 }
276 
277 void memcg_put_cache_ids(void)
278 {
279 	up_read(&memcg_cache_ids_sem);
280 }
281 
282 /*
283  * MIN_SIZE is different than 1, because we would like to avoid going through
284  * the alloc/free process all the time. In a small machine, 4 kmem-limited
285  * cgroups is a reasonable guess. In the future, it could be a parameter or
286  * tunable, but that is strictly not necessary.
287  *
288  * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get
289  * this constant directly from cgroup, but it is understandable that this is
290  * better kept as an internal representation in cgroup.c. In any case, the
291  * cgrp_id space is not getting any smaller, and we don't have to necessarily
292  * increase ours as well if it increases.
293  */
294 #define MEMCG_CACHES_MIN_SIZE 4
295 #define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
296 
297 /*
298  * A lot of the calls to the cache allocation functions are expected to be
299  * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
300  * conditional to this static branch, we'll have to allow modules that does
301  * kmem_cache_alloc and the such to see this symbol as well
302  */
303 DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);
304 EXPORT_SYMBOL(memcg_kmem_enabled_key);
305 
306 struct workqueue_struct *memcg_kmem_cache_wq;
307 
308 #endif /* !CONFIG_SLOB */
309 
310 /**
311  * mem_cgroup_css_from_page - css of the memcg associated with a page
312  * @page: page of interest
313  *
314  * If memcg is bound to the default hierarchy, css of the memcg associated
315  * with @page is returned.  The returned css remains associated with @page
316  * until it is released.
317  *
318  * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
319  * is returned.
320  */
321 struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
322 {
323 	struct mem_cgroup *memcg;
324 
325 	memcg = page->mem_cgroup;
326 
327 	if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
328 		memcg = root_mem_cgroup;
329 
330 	return &memcg->css;
331 }
332 
333 /**
334  * page_cgroup_ino - return inode number of the memcg a page is charged to
335  * @page: the page
336  *
337  * Look up the closest online ancestor of the memory cgroup @page is charged to
338  * and return its inode number or 0 if @page is not charged to any cgroup. It
339  * is safe to call this function without holding a reference to @page.
340  *
341  * Note, this function is inherently racy, because there is nothing to prevent
342  * the cgroup inode from getting torn down and potentially reallocated a moment
343  * after page_cgroup_ino() returns, so it only should be used by callers that
344  * do not care (such as procfs interfaces).
345  */
346 ino_t page_cgroup_ino(struct page *page)
347 {
348 	struct mem_cgroup *memcg;
349 	unsigned long ino = 0;
350 
351 	rcu_read_lock();
352 	memcg = READ_ONCE(page->mem_cgroup);
353 	while (memcg && !(memcg->css.flags & CSS_ONLINE))
354 		memcg = parent_mem_cgroup(memcg);
355 	if (memcg)
356 		ino = cgroup_ino(memcg->css.cgroup);
357 	rcu_read_unlock();
358 	return ino;
359 }
360 
361 static struct mem_cgroup_per_node *
362 mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page)
363 {
364 	int nid = page_to_nid(page);
365 
366 	return memcg->nodeinfo[nid];
367 }
368 
369 static struct mem_cgroup_tree_per_node *
370 soft_limit_tree_node(int nid)
371 {
372 	return soft_limit_tree.rb_tree_per_node[nid];
373 }
374 
375 static struct mem_cgroup_tree_per_node *
376 soft_limit_tree_from_page(struct page *page)
377 {
378 	int nid = page_to_nid(page);
379 
380 	return soft_limit_tree.rb_tree_per_node[nid];
381 }
382 
383 static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
384 					 struct mem_cgroup_tree_per_node *mctz,
385 					 unsigned long new_usage_in_excess)
386 {
387 	struct rb_node **p = &mctz->rb_root.rb_node;
388 	struct rb_node *parent = NULL;
389 	struct mem_cgroup_per_node *mz_node;
390 	bool rightmost = true;
391 
392 	if (mz->on_tree)
393 		return;
394 
395 	mz->usage_in_excess = new_usage_in_excess;
396 	if (!mz->usage_in_excess)
397 		return;
398 	while (*p) {
399 		parent = *p;
400 		mz_node = rb_entry(parent, struct mem_cgroup_per_node,
401 					tree_node);
402 		if (mz->usage_in_excess < mz_node->usage_in_excess) {
403 			p = &(*p)->rb_left;
404 			rightmost = false;
405 		}
406 
407 		/*
408 		 * We can't avoid mem cgroups that are over their soft
409 		 * limit by the same amount
410 		 */
411 		else if (mz->usage_in_excess >= mz_node->usage_in_excess)
412 			p = &(*p)->rb_right;
413 	}
414 
415 	if (rightmost)
416 		mctz->rb_rightmost = &mz->tree_node;
417 
418 	rb_link_node(&mz->tree_node, parent, p);
419 	rb_insert_color(&mz->tree_node, &mctz->rb_root);
420 	mz->on_tree = true;
421 }
422 
423 static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
424 					 struct mem_cgroup_tree_per_node *mctz)
425 {
426 	if (!mz->on_tree)
427 		return;
428 
429 	if (&mz->tree_node == mctz->rb_rightmost)
430 		mctz->rb_rightmost = rb_prev(&mz->tree_node);
431 
432 	rb_erase(&mz->tree_node, &mctz->rb_root);
433 	mz->on_tree = false;
434 }
435 
436 static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
437 				       struct mem_cgroup_tree_per_node *mctz)
438 {
439 	unsigned long flags;
440 
441 	spin_lock_irqsave(&mctz->lock, flags);
442 	__mem_cgroup_remove_exceeded(mz, mctz);
443 	spin_unlock_irqrestore(&mctz->lock, flags);
444 }
445 
446 static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
447 {
448 	unsigned long nr_pages = page_counter_read(&memcg->memory);
449 	unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
450 	unsigned long excess = 0;
451 
452 	if (nr_pages > soft_limit)
453 		excess = nr_pages - soft_limit;
454 
455 	return excess;
456 }
457 
458 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
459 {
460 	unsigned long excess;
461 	struct mem_cgroup_per_node *mz;
462 	struct mem_cgroup_tree_per_node *mctz;
463 
464 	mctz = soft_limit_tree_from_page(page);
465 	if (!mctz)
466 		return;
467 	/*
468 	 * Necessary to update all ancestors when hierarchy is used.
469 	 * because their event counter is not touched.
470 	 */
471 	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
472 		mz = mem_cgroup_page_nodeinfo(memcg, page);
473 		excess = soft_limit_excess(memcg);
474 		/*
475 		 * We have to update the tree if mz is on RB-tree or
476 		 * mem is over its softlimit.
477 		 */
478 		if (excess || mz->on_tree) {
479 			unsigned long flags;
480 
481 			spin_lock_irqsave(&mctz->lock, flags);
482 			/* if on-tree, remove it */
483 			if (mz->on_tree)
484 				__mem_cgroup_remove_exceeded(mz, mctz);
485 			/*
486 			 * Insert again. mz->usage_in_excess will be updated.
487 			 * If excess is 0, no tree ops.
488 			 */
489 			__mem_cgroup_insert_exceeded(mz, mctz, excess);
490 			spin_unlock_irqrestore(&mctz->lock, flags);
491 		}
492 	}
493 }
494 
495 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
496 {
497 	struct mem_cgroup_tree_per_node *mctz;
498 	struct mem_cgroup_per_node *mz;
499 	int nid;
500 
501 	for_each_node(nid) {
502 		mz = mem_cgroup_nodeinfo(memcg, nid);
503 		mctz = soft_limit_tree_node(nid);
504 		if (mctz)
505 			mem_cgroup_remove_exceeded(mz, mctz);
506 	}
507 }
508 
509 static struct mem_cgroup_per_node *
510 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
511 {
512 	struct mem_cgroup_per_node *mz;
513 
514 retry:
515 	mz = NULL;
516 	if (!mctz->rb_rightmost)
517 		goto done;		/* Nothing to reclaim from */
518 
519 	mz = rb_entry(mctz->rb_rightmost,
520 		      struct mem_cgroup_per_node, tree_node);
521 	/*
522 	 * Remove the node now but someone else can add it back,
523 	 * we will to add it back at the end of reclaim to its correct
524 	 * position in the tree.
525 	 */
526 	__mem_cgroup_remove_exceeded(mz, mctz);
527 	if (!soft_limit_excess(mz->memcg) ||
528 	    !css_tryget_online(&mz->memcg->css))
529 		goto retry;
530 done:
531 	return mz;
532 }
533 
534 static struct mem_cgroup_per_node *
535 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
536 {
537 	struct mem_cgroup_per_node *mz;
538 
539 	spin_lock_irq(&mctz->lock);
540 	mz = __mem_cgroup_largest_soft_limit_node(mctz);
541 	spin_unlock_irq(&mctz->lock);
542 	return mz;
543 }
544 
545 /*
546  * Return page count for single (non recursive) @memcg.
547  *
548  * Implementation Note: reading percpu statistics for memcg.
549  *
550  * Both of vmstat[] and percpu_counter has threshold and do periodic
551  * synchronization to implement "quick" read. There are trade-off between
552  * reading cost and precision of value. Then, we may have a chance to implement
553  * a periodic synchronization of counter in memcg's counter.
554  *
555  * But this _read() function is used for user interface now. The user accounts
556  * memory usage by memory cgroup and he _always_ requires exact value because
557  * he accounts memory. Even if we provide quick-and-fuzzy read, we always
558  * have to visit all online cpus and make sum. So, for now, unnecessary
559  * synchronization is not implemented. (just implemented for cpu hotplug)
560  *
561  * If there are kernel internal actions which can make use of some not-exact
562  * value, and reading all cpu value can be performance bottleneck in some
563  * common workload, threshold and synchronization as vmstat[] should be
564  * implemented.
565  *
566  * The parameter idx can be of type enum memcg_event_item or vm_event_item.
567  */
568 
569 static unsigned long memcg_sum_events(struct mem_cgroup *memcg,
570 				      int event)
571 {
572 	unsigned long val = 0;
573 	int cpu;
574 
575 	for_each_possible_cpu(cpu)
576 		val += per_cpu(memcg->stat->events[event], cpu);
577 	return val;
578 }
579 
580 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
581 					 struct page *page,
582 					 bool compound, int nr_pages)
583 {
584 	/*
585 	 * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
586 	 * counted as CACHE even if it's on ANON LRU.
587 	 */
588 	if (PageAnon(page))
589 		__this_cpu_add(memcg->stat->count[MEMCG_RSS], nr_pages);
590 	else {
591 		__this_cpu_add(memcg->stat->count[MEMCG_CACHE], nr_pages);
592 		if (PageSwapBacked(page))
593 			__this_cpu_add(memcg->stat->count[NR_SHMEM], nr_pages);
594 	}
595 
596 	if (compound) {
597 		VM_BUG_ON_PAGE(!PageTransHuge(page), page);
598 		__this_cpu_add(memcg->stat->count[MEMCG_RSS_HUGE], nr_pages);
599 	}
600 
601 	/* pagein of a big page is an event. So, ignore page size */
602 	if (nr_pages > 0)
603 		__this_cpu_inc(memcg->stat->events[PGPGIN]);
604 	else {
605 		__this_cpu_inc(memcg->stat->events[PGPGOUT]);
606 		nr_pages = -nr_pages; /* for event */
607 	}
608 
609 	__this_cpu_add(memcg->stat->nr_page_events, nr_pages);
610 }
611 
612 unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
613 					   int nid, unsigned int lru_mask)
614 {
615 	struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg);
616 	unsigned long nr = 0;
617 	enum lru_list lru;
618 
619 	VM_BUG_ON((unsigned)nid >= nr_node_ids);
620 
621 	for_each_lru(lru) {
622 		if (!(BIT(lru) & lru_mask))
623 			continue;
624 		nr += mem_cgroup_get_lru_size(lruvec, lru);
625 	}
626 	return nr;
627 }
628 
629 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
630 			unsigned int lru_mask)
631 {
632 	unsigned long nr = 0;
633 	int nid;
634 
635 	for_each_node_state(nid, N_MEMORY)
636 		nr += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
637 	return nr;
638 }
639 
640 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
641 				       enum mem_cgroup_events_target target)
642 {
643 	unsigned long val, next;
644 
645 	val = __this_cpu_read(memcg->stat->nr_page_events);
646 	next = __this_cpu_read(memcg->stat->targets[target]);
647 	/* from time_after() in jiffies.h */
648 	if ((long)(next - val) < 0) {
649 		switch (target) {
650 		case MEM_CGROUP_TARGET_THRESH:
651 			next = val + THRESHOLDS_EVENTS_TARGET;
652 			break;
653 		case MEM_CGROUP_TARGET_SOFTLIMIT:
654 			next = val + SOFTLIMIT_EVENTS_TARGET;
655 			break;
656 		case MEM_CGROUP_TARGET_NUMAINFO:
657 			next = val + NUMAINFO_EVENTS_TARGET;
658 			break;
659 		default:
660 			break;
661 		}
662 		__this_cpu_write(memcg->stat->targets[target], next);
663 		return true;
664 	}
665 	return false;
666 }
667 
668 /*
669  * Check events in order.
670  *
671  */
672 static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
673 {
674 	/* threshold event is triggered in finer grain than soft limit */
675 	if (unlikely(mem_cgroup_event_ratelimit(memcg,
676 						MEM_CGROUP_TARGET_THRESH))) {
677 		bool do_softlimit;
678 		bool do_numainfo __maybe_unused;
679 
680 		do_softlimit = mem_cgroup_event_ratelimit(memcg,
681 						MEM_CGROUP_TARGET_SOFTLIMIT);
682 #if MAX_NUMNODES > 1
683 		do_numainfo = mem_cgroup_event_ratelimit(memcg,
684 						MEM_CGROUP_TARGET_NUMAINFO);
685 #endif
686 		mem_cgroup_threshold(memcg);
687 		if (unlikely(do_softlimit))
688 			mem_cgroup_update_tree(memcg, page);
689 #if MAX_NUMNODES > 1
690 		if (unlikely(do_numainfo))
691 			atomic_inc(&memcg->numainfo_events);
692 #endif
693 	}
694 }
695 
696 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
697 {
698 	/*
699 	 * mm_update_next_owner() may clear mm->owner to NULL
700 	 * if it races with swapoff, page migration, etc.
701 	 * So this can be called with p == NULL.
702 	 */
703 	if (unlikely(!p))
704 		return NULL;
705 
706 	return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
707 }
708 EXPORT_SYMBOL(mem_cgroup_from_task);
709 
710 static struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
711 {
712 	struct mem_cgroup *memcg = NULL;
713 
714 	rcu_read_lock();
715 	do {
716 		/*
717 		 * Page cache insertions can happen withou an
718 		 * actual mm context, e.g. during disk probing
719 		 * on boot, loopback IO, acct() writes etc.
720 		 */
721 		if (unlikely(!mm))
722 			memcg = root_mem_cgroup;
723 		else {
724 			memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
725 			if (unlikely(!memcg))
726 				memcg = root_mem_cgroup;
727 		}
728 	} while (!css_tryget_online(&memcg->css));
729 	rcu_read_unlock();
730 	return memcg;
731 }
732 
733 /**
734  * mem_cgroup_iter - iterate over memory cgroup hierarchy
735  * @root: hierarchy root
736  * @prev: previously returned memcg, NULL on first invocation
737  * @reclaim: cookie for shared reclaim walks, NULL for full walks
738  *
739  * Returns references to children of the hierarchy below @root, or
740  * @root itself, or %NULL after a full round-trip.
741  *
742  * Caller must pass the return value in @prev on subsequent
743  * invocations for reference counting, or use mem_cgroup_iter_break()
744  * to cancel a hierarchy walk before the round-trip is complete.
745  *
746  * Reclaimers can specify a zone and a priority level in @reclaim to
747  * divide up the memcgs in the hierarchy among all concurrent
748  * reclaimers operating on the same zone and priority.
749  */
750 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
751 				   struct mem_cgroup *prev,
752 				   struct mem_cgroup_reclaim_cookie *reclaim)
753 {
754 	struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
755 	struct cgroup_subsys_state *css = NULL;
756 	struct mem_cgroup *memcg = NULL;
757 	struct mem_cgroup *pos = NULL;
758 
759 	if (mem_cgroup_disabled())
760 		return NULL;
761 
762 	if (!root)
763 		root = root_mem_cgroup;
764 
765 	if (prev && !reclaim)
766 		pos = prev;
767 
768 	if (!root->use_hierarchy && root != root_mem_cgroup) {
769 		if (prev)
770 			goto out;
771 		return root;
772 	}
773 
774 	rcu_read_lock();
775 
776 	if (reclaim) {
777 		struct mem_cgroup_per_node *mz;
778 
779 		mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id);
780 		iter = &mz->iter[reclaim->priority];
781 
782 		if (prev && reclaim->generation != iter->generation)
783 			goto out_unlock;
784 
785 		while (1) {
786 			pos = READ_ONCE(iter->position);
787 			if (!pos || css_tryget(&pos->css))
788 				break;
789 			/*
790 			 * css reference reached zero, so iter->position will
791 			 * be cleared by ->css_released. However, we should not
792 			 * rely on this happening soon, because ->css_released
793 			 * is called from a work queue, and by busy-waiting we
794 			 * might block it. So we clear iter->position right
795 			 * away.
796 			 */
797 			(void)cmpxchg(&iter->position, pos, NULL);
798 		}
799 	}
800 
801 	if (pos)
802 		css = &pos->css;
803 
804 	for (;;) {
805 		css = css_next_descendant_pre(css, &root->css);
806 		if (!css) {
807 			/*
808 			 * Reclaimers share the hierarchy walk, and a
809 			 * new one might jump in right at the end of
810 			 * the hierarchy - make sure they see at least
811 			 * one group and restart from the beginning.
812 			 */
813 			if (!prev)
814 				continue;
815 			break;
816 		}
817 
818 		/*
819 		 * Verify the css and acquire a reference.  The root
820 		 * is provided by the caller, so we know it's alive
821 		 * and kicking, and don't take an extra reference.
822 		 */
823 		memcg = mem_cgroup_from_css(css);
824 
825 		if (css == &root->css)
826 			break;
827 
828 		if (css_tryget(css))
829 			break;
830 
831 		memcg = NULL;
832 	}
833 
834 	if (reclaim) {
835 		/*
836 		 * The position could have already been updated by a competing
837 		 * thread, so check that the value hasn't changed since we read
838 		 * it to avoid reclaiming from the same cgroup twice.
839 		 */
840 		(void)cmpxchg(&iter->position, pos, memcg);
841 
842 		if (pos)
843 			css_put(&pos->css);
844 
845 		if (!memcg)
846 			iter->generation++;
847 		else if (!prev)
848 			reclaim->generation = iter->generation;
849 	}
850 
851 out_unlock:
852 	rcu_read_unlock();
853 out:
854 	if (prev && prev != root)
855 		css_put(&prev->css);
856 
857 	return memcg;
858 }
859 
860 /**
861  * mem_cgroup_iter_break - abort a hierarchy walk prematurely
862  * @root: hierarchy root
863  * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
864  */
865 void mem_cgroup_iter_break(struct mem_cgroup *root,
866 			   struct mem_cgroup *prev)
867 {
868 	if (!root)
869 		root = root_mem_cgroup;
870 	if (prev && prev != root)
871 		css_put(&prev->css);
872 }
873 
874 static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
875 {
876 	struct mem_cgroup *memcg = dead_memcg;
877 	struct mem_cgroup_reclaim_iter *iter;
878 	struct mem_cgroup_per_node *mz;
879 	int nid;
880 	int i;
881 
882 	while ((memcg = parent_mem_cgroup(memcg))) {
883 		for_each_node(nid) {
884 			mz = mem_cgroup_nodeinfo(memcg, nid);
885 			for (i = 0; i <= DEF_PRIORITY; i++) {
886 				iter = &mz->iter[i];
887 				cmpxchg(&iter->position,
888 					dead_memcg, NULL);
889 			}
890 		}
891 	}
892 }
893 
894 /*
895  * Iteration constructs for visiting all cgroups (under a tree).  If
896  * loops are exited prematurely (break), mem_cgroup_iter_break() must
897  * be used for reference counting.
898  */
899 #define for_each_mem_cgroup_tree(iter, root)		\
900 	for (iter = mem_cgroup_iter(root, NULL, NULL);	\
901 	     iter != NULL;				\
902 	     iter = mem_cgroup_iter(root, iter, NULL))
903 
904 #define for_each_mem_cgroup(iter)			\
905 	for (iter = mem_cgroup_iter(NULL, NULL, NULL);	\
906 	     iter != NULL;				\
907 	     iter = mem_cgroup_iter(NULL, iter, NULL))
908 
909 /**
910  * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
911  * @memcg: hierarchy root
912  * @fn: function to call for each task
913  * @arg: argument passed to @fn
914  *
915  * This function iterates over tasks attached to @memcg or to any of its
916  * descendants and calls @fn for each task. If @fn returns a non-zero
917  * value, the function breaks the iteration loop and returns the value.
918  * Otherwise, it will iterate over all tasks and return 0.
919  *
920  * This function must not be called for the root memory cgroup.
921  */
922 int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
923 			  int (*fn)(struct task_struct *, void *), void *arg)
924 {
925 	struct mem_cgroup *iter;
926 	int ret = 0;
927 
928 	BUG_ON(memcg == root_mem_cgroup);
929 
930 	for_each_mem_cgroup_tree(iter, memcg) {
931 		struct css_task_iter it;
932 		struct task_struct *task;
933 
934 		css_task_iter_start(&iter->css, 0, &it);
935 		while (!ret && (task = css_task_iter_next(&it)))
936 			ret = fn(task, arg);
937 		css_task_iter_end(&it);
938 		if (ret) {
939 			mem_cgroup_iter_break(memcg, iter);
940 			break;
941 		}
942 	}
943 	return ret;
944 }
945 
946 /**
947  * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page
948  * @page: the page
949  * @zone: zone of the page
950  *
951  * This function is only safe when following the LRU page isolation
952  * and putback protocol: the LRU lock must be held, and the page must
953  * either be PageLRU() or the caller must have isolated/allocated it.
954  */
955 struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgdat)
956 {
957 	struct mem_cgroup_per_node *mz;
958 	struct mem_cgroup *memcg;
959 	struct lruvec *lruvec;
960 
961 	if (mem_cgroup_disabled()) {
962 		lruvec = &pgdat->lruvec;
963 		goto out;
964 	}
965 
966 	memcg = page->mem_cgroup;
967 	/*
968 	 * Swapcache readahead pages are added to the LRU - and
969 	 * possibly migrated - before they are charged.
970 	 */
971 	if (!memcg)
972 		memcg = root_mem_cgroup;
973 
974 	mz = mem_cgroup_page_nodeinfo(memcg, page);
975 	lruvec = &mz->lruvec;
976 out:
977 	/*
978 	 * Since a node can be onlined after the mem_cgroup was created,
979 	 * we have to be prepared to initialize lruvec->zone here;
980 	 * and if offlined then reonlined, we need to reinitialize it.
981 	 */
982 	if (unlikely(lruvec->pgdat != pgdat))
983 		lruvec->pgdat = pgdat;
984 	return lruvec;
985 }
986 
987 /**
988  * mem_cgroup_update_lru_size - account for adding or removing an lru page
989  * @lruvec: mem_cgroup per zone lru vector
990  * @lru: index of lru list the page is sitting on
991  * @zid: zone id of the accounted pages
992  * @nr_pages: positive when adding or negative when removing
993  *
994  * This function must be called under lru_lock, just before a page is added
995  * to or just after a page is removed from an lru list (that ordering being
996  * so as to allow it to check that lru_size 0 is consistent with list_empty).
997  */
998 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
999 				int zid, int nr_pages)
1000 {
1001 	struct mem_cgroup_per_node *mz;
1002 	unsigned long *lru_size;
1003 	long size;
1004 
1005 	if (mem_cgroup_disabled())
1006 		return;
1007 
1008 	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
1009 	lru_size = &mz->lru_zone_size[zid][lru];
1010 
1011 	if (nr_pages < 0)
1012 		*lru_size += nr_pages;
1013 
1014 	size = *lru_size;
1015 	if (WARN_ONCE(size < 0,
1016 		"%s(%p, %d, %d): lru_size %ld\n",
1017 		__func__, lruvec, lru, nr_pages, size)) {
1018 		VM_BUG_ON(1);
1019 		*lru_size = 0;
1020 	}
1021 
1022 	if (nr_pages > 0)
1023 		*lru_size += nr_pages;
1024 }
1025 
1026 bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg)
1027 {
1028 	struct mem_cgroup *task_memcg;
1029 	struct task_struct *p;
1030 	bool ret;
1031 
1032 	p = find_lock_task_mm(task);
1033 	if (p) {
1034 		task_memcg = get_mem_cgroup_from_mm(p->mm);
1035 		task_unlock(p);
1036 	} else {
1037 		/*
1038 		 * All threads may have already detached their mm's, but the oom
1039 		 * killer still needs to detect if they have already been oom
1040 		 * killed to prevent needlessly killing additional tasks.
1041 		 */
1042 		rcu_read_lock();
1043 		task_memcg = mem_cgroup_from_task(task);
1044 		css_get(&task_memcg->css);
1045 		rcu_read_unlock();
1046 	}
1047 	ret = mem_cgroup_is_descendant(task_memcg, memcg);
1048 	css_put(&task_memcg->css);
1049 	return ret;
1050 }
1051 
1052 /**
1053  * mem_cgroup_margin - calculate chargeable space of a memory cgroup
1054  * @memcg: the memory cgroup
1055  *
1056  * Returns the maximum amount of memory @mem can be charged with, in
1057  * pages.
1058  */
1059 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1060 {
1061 	unsigned long margin = 0;
1062 	unsigned long count;
1063 	unsigned long limit;
1064 
1065 	count = page_counter_read(&memcg->memory);
1066 	limit = READ_ONCE(memcg->memory.limit);
1067 	if (count < limit)
1068 		margin = limit - count;
1069 
1070 	if (do_memsw_account()) {
1071 		count = page_counter_read(&memcg->memsw);
1072 		limit = READ_ONCE(memcg->memsw.limit);
1073 		if (count <= limit)
1074 			margin = min(margin, limit - count);
1075 		else
1076 			margin = 0;
1077 	}
1078 
1079 	return margin;
1080 }
1081 
1082 /*
1083  * A routine for checking "mem" is under move_account() or not.
1084  *
1085  * Checking a cgroup is mc.from or mc.to or under hierarchy of
1086  * moving cgroups. This is for waiting at high-memory pressure
1087  * caused by "move".
1088  */
1089 static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1090 {
1091 	struct mem_cgroup *from;
1092 	struct mem_cgroup *to;
1093 	bool ret = false;
1094 	/*
1095 	 * Unlike task_move routines, we access mc.to, mc.from not under
1096 	 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1097 	 */
1098 	spin_lock(&mc.lock);
1099 	from = mc.from;
1100 	to = mc.to;
1101 	if (!from)
1102 		goto unlock;
1103 
1104 	ret = mem_cgroup_is_descendant(from, memcg) ||
1105 		mem_cgroup_is_descendant(to, memcg);
1106 unlock:
1107 	spin_unlock(&mc.lock);
1108 	return ret;
1109 }
1110 
1111 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1112 {
1113 	if (mc.moving_task && current != mc.moving_task) {
1114 		if (mem_cgroup_under_move(memcg)) {
1115 			DEFINE_WAIT(wait);
1116 			prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1117 			/* moving charge context might have finished. */
1118 			if (mc.moving_task)
1119 				schedule();
1120 			finish_wait(&mc.waitq, &wait);
1121 			return true;
1122 		}
1123 	}
1124 	return false;
1125 }
1126 
1127 unsigned int memcg1_stats[] = {
1128 	MEMCG_CACHE,
1129 	MEMCG_RSS,
1130 	MEMCG_RSS_HUGE,
1131 	NR_SHMEM,
1132 	NR_FILE_MAPPED,
1133 	NR_FILE_DIRTY,
1134 	NR_WRITEBACK,
1135 	MEMCG_SWAP,
1136 };
1137 
1138 static const char *const memcg1_stat_names[] = {
1139 	"cache",
1140 	"rss",
1141 	"rss_huge",
1142 	"shmem",
1143 	"mapped_file",
1144 	"dirty",
1145 	"writeback",
1146 	"swap",
1147 };
1148 
1149 #define K(x) ((x) << (PAGE_SHIFT-10))
1150 /**
1151  * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller.
1152  * @memcg: The memory cgroup that went over limit
1153  * @p: Task that is going to be killed
1154  *
1155  * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1156  * enabled
1157  */
1158 void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
1159 {
1160 	struct mem_cgroup *iter;
1161 	unsigned int i;
1162 
1163 	rcu_read_lock();
1164 
1165 	if (p) {
1166 		pr_info("Task in ");
1167 		pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
1168 		pr_cont(" killed as a result of limit of ");
1169 	} else {
1170 		pr_info("Memory limit reached of cgroup ");
1171 	}
1172 
1173 	pr_cont_cgroup_path(memcg->css.cgroup);
1174 	pr_cont("\n");
1175 
1176 	rcu_read_unlock();
1177 
1178 	pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
1179 		K((u64)page_counter_read(&memcg->memory)),
1180 		K((u64)memcg->memory.limit), memcg->memory.failcnt);
1181 	pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
1182 		K((u64)page_counter_read(&memcg->memsw)),
1183 		K((u64)memcg->memsw.limit), memcg->memsw.failcnt);
1184 	pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
1185 		K((u64)page_counter_read(&memcg->kmem)),
1186 		K((u64)memcg->kmem.limit), memcg->kmem.failcnt);
1187 
1188 	for_each_mem_cgroup_tree(iter, memcg) {
1189 		pr_info("Memory cgroup stats for ");
1190 		pr_cont_cgroup_path(iter->css.cgroup);
1191 		pr_cont(":");
1192 
1193 		for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
1194 			if (memcg1_stats[i] == MEMCG_SWAP && !do_swap_account)
1195 				continue;
1196 			pr_cont(" %s:%luKB", memcg1_stat_names[i],
1197 				K(memcg_page_state(iter, memcg1_stats[i])));
1198 		}
1199 
1200 		for (i = 0; i < NR_LRU_LISTS; i++)
1201 			pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
1202 				K(mem_cgroup_nr_lru_pages(iter, BIT(i))));
1203 
1204 		pr_cont("\n");
1205 	}
1206 }
1207 
1208 /*
1209  * This function returns the number of memcg under hierarchy tree. Returns
1210  * 1(self count) if no children.
1211  */
1212 static int mem_cgroup_count_children(struct mem_cgroup *memcg)
1213 {
1214 	int num = 0;
1215 	struct mem_cgroup *iter;
1216 
1217 	for_each_mem_cgroup_tree(iter, memcg)
1218 		num++;
1219 	return num;
1220 }
1221 
1222 /*
1223  * Return the memory (and swap, if configured) limit for a memcg.
1224  */
1225 unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg)
1226 {
1227 	unsigned long limit;
1228 
1229 	limit = memcg->memory.limit;
1230 	if (mem_cgroup_swappiness(memcg)) {
1231 		unsigned long memsw_limit;
1232 		unsigned long swap_limit;
1233 
1234 		memsw_limit = memcg->memsw.limit;
1235 		swap_limit = memcg->swap.limit;
1236 		swap_limit = min(swap_limit, (unsigned long)total_swap_pages);
1237 		limit = min(limit + swap_limit, memsw_limit);
1238 	}
1239 	return limit;
1240 }
1241 
1242 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1243 				     int order)
1244 {
1245 	struct oom_control oc = {
1246 		.zonelist = NULL,
1247 		.nodemask = NULL,
1248 		.memcg = memcg,
1249 		.gfp_mask = gfp_mask,
1250 		.order = order,
1251 	};
1252 	bool ret;
1253 
1254 	mutex_lock(&oom_lock);
1255 	ret = out_of_memory(&oc);
1256 	mutex_unlock(&oom_lock);
1257 	return ret;
1258 }
1259 
1260 #if MAX_NUMNODES > 1
1261 
1262 /**
1263  * test_mem_cgroup_node_reclaimable
1264  * @memcg: the target memcg
1265  * @nid: the node ID to be checked.
1266  * @noswap : specify true here if the user wants flle only information.
1267  *
1268  * This function returns whether the specified memcg contains any
1269  * reclaimable pages on a node. Returns true if there are any reclaimable
1270  * pages in the node.
1271  */
1272 static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1273 		int nid, bool noswap)
1274 {
1275 	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
1276 		return true;
1277 	if (noswap || !total_swap_pages)
1278 		return false;
1279 	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
1280 		return true;
1281 	return false;
1282 
1283 }
1284 
1285 /*
1286  * Always updating the nodemask is not very good - even if we have an empty
1287  * list or the wrong list here, we can start from some node and traverse all
1288  * nodes based on the zonelist. So update the list loosely once per 10 secs.
1289  *
1290  */
1291 static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1292 {
1293 	int nid;
1294 	/*
1295 	 * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
1296 	 * pagein/pageout changes since the last update.
1297 	 */
1298 	if (!atomic_read(&memcg->numainfo_events))
1299 		return;
1300 	if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1301 		return;
1302 
1303 	/* make a nodemask where this memcg uses memory from */
1304 	memcg->scan_nodes = node_states[N_MEMORY];
1305 
1306 	for_each_node_mask(nid, node_states[N_MEMORY]) {
1307 
1308 		if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
1309 			node_clear(nid, memcg->scan_nodes);
1310 	}
1311 
1312 	atomic_set(&memcg->numainfo_events, 0);
1313 	atomic_set(&memcg->numainfo_updating, 0);
1314 }
1315 
1316 /*
1317  * Selecting a node where we start reclaim from. Because what we need is just
1318  * reducing usage counter, start from anywhere is O,K. Considering
1319  * memory reclaim from current node, there are pros. and cons.
1320  *
1321  * Freeing memory from current node means freeing memory from a node which
1322  * we'll use or we've used. So, it may make LRU bad. And if several threads
1323  * hit limits, it will see a contention on a node. But freeing from remote
1324  * node means more costs for memory reclaim because of memory latency.
1325  *
1326  * Now, we use round-robin. Better algorithm is welcomed.
1327  */
1328 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1329 {
1330 	int node;
1331 
1332 	mem_cgroup_may_update_nodemask(memcg);
1333 	node = memcg->last_scanned_node;
1334 
1335 	node = next_node_in(node, memcg->scan_nodes);
1336 	/*
1337 	 * mem_cgroup_may_update_nodemask might have seen no reclaimmable pages
1338 	 * last time it really checked all the LRUs due to rate limiting.
1339 	 * Fallback to the current node in that case for simplicity.
1340 	 */
1341 	if (unlikely(node == MAX_NUMNODES))
1342 		node = numa_node_id();
1343 
1344 	memcg->last_scanned_node = node;
1345 	return node;
1346 }
1347 #else
1348 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1349 {
1350 	return 0;
1351 }
1352 #endif
1353 
1354 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1355 				   pg_data_t *pgdat,
1356 				   gfp_t gfp_mask,
1357 				   unsigned long *total_scanned)
1358 {
1359 	struct mem_cgroup *victim = NULL;
1360 	int total = 0;
1361 	int loop = 0;
1362 	unsigned long excess;
1363 	unsigned long nr_scanned;
1364 	struct mem_cgroup_reclaim_cookie reclaim = {
1365 		.pgdat = pgdat,
1366 		.priority = 0,
1367 	};
1368 
1369 	excess = soft_limit_excess(root_memcg);
1370 
1371 	while (1) {
1372 		victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
1373 		if (!victim) {
1374 			loop++;
1375 			if (loop >= 2) {
1376 				/*
1377 				 * If we have not been able to reclaim
1378 				 * anything, it might because there are
1379 				 * no reclaimable pages under this hierarchy
1380 				 */
1381 				if (!total)
1382 					break;
1383 				/*
1384 				 * We want to do more targeted reclaim.
1385 				 * excess >> 2 is not to excessive so as to
1386 				 * reclaim too much, nor too less that we keep
1387 				 * coming back to reclaim from this cgroup
1388 				 */
1389 				if (total >= (excess >> 2) ||
1390 					(loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
1391 					break;
1392 			}
1393 			continue;
1394 		}
1395 		total += mem_cgroup_shrink_node(victim, gfp_mask, false,
1396 					pgdat, &nr_scanned);
1397 		*total_scanned += nr_scanned;
1398 		if (!soft_limit_excess(root_memcg))
1399 			break;
1400 	}
1401 	mem_cgroup_iter_break(root_memcg, victim);
1402 	return total;
1403 }
1404 
1405 #ifdef CONFIG_LOCKDEP
1406 static struct lockdep_map memcg_oom_lock_dep_map = {
1407 	.name = "memcg_oom_lock",
1408 };
1409 #endif
1410 
1411 static DEFINE_SPINLOCK(memcg_oom_lock);
1412 
1413 /*
1414  * Check OOM-Killer is already running under our hierarchy.
1415  * If someone is running, return false.
1416  */
1417 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
1418 {
1419 	struct mem_cgroup *iter, *failed = NULL;
1420 
1421 	spin_lock(&memcg_oom_lock);
1422 
1423 	for_each_mem_cgroup_tree(iter, memcg) {
1424 		if (iter->oom_lock) {
1425 			/*
1426 			 * this subtree of our hierarchy is already locked
1427 			 * so we cannot give a lock.
1428 			 */
1429 			failed = iter;
1430 			mem_cgroup_iter_break(memcg, iter);
1431 			break;
1432 		} else
1433 			iter->oom_lock = true;
1434 	}
1435 
1436 	if (failed) {
1437 		/*
1438 		 * OK, we failed to lock the whole subtree so we have
1439 		 * to clean up what we set up to the failing subtree
1440 		 */
1441 		for_each_mem_cgroup_tree(iter, memcg) {
1442 			if (iter == failed) {
1443 				mem_cgroup_iter_break(memcg, iter);
1444 				break;
1445 			}
1446 			iter->oom_lock = false;
1447 		}
1448 	} else
1449 		mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
1450 
1451 	spin_unlock(&memcg_oom_lock);
1452 
1453 	return !failed;
1454 }
1455 
1456 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1457 {
1458 	struct mem_cgroup *iter;
1459 
1460 	spin_lock(&memcg_oom_lock);
1461 	mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_);
1462 	for_each_mem_cgroup_tree(iter, memcg)
1463 		iter->oom_lock = false;
1464 	spin_unlock(&memcg_oom_lock);
1465 }
1466 
1467 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1468 {
1469 	struct mem_cgroup *iter;
1470 
1471 	spin_lock(&memcg_oom_lock);
1472 	for_each_mem_cgroup_tree(iter, memcg)
1473 		iter->under_oom++;
1474 	spin_unlock(&memcg_oom_lock);
1475 }
1476 
1477 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1478 {
1479 	struct mem_cgroup *iter;
1480 
1481 	/*
1482 	 * When a new child is created while the hierarchy is under oom,
1483 	 * mem_cgroup_oom_lock() may not be called. Watch for underflow.
1484 	 */
1485 	spin_lock(&memcg_oom_lock);
1486 	for_each_mem_cgroup_tree(iter, memcg)
1487 		if (iter->under_oom > 0)
1488 			iter->under_oom--;
1489 	spin_unlock(&memcg_oom_lock);
1490 }
1491 
1492 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1493 
1494 struct oom_wait_info {
1495 	struct mem_cgroup *memcg;
1496 	wait_queue_entry_t	wait;
1497 };
1498 
1499 static int memcg_oom_wake_function(wait_queue_entry_t *wait,
1500 	unsigned mode, int sync, void *arg)
1501 {
1502 	struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
1503 	struct mem_cgroup *oom_wait_memcg;
1504 	struct oom_wait_info *oom_wait_info;
1505 
1506 	oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1507 	oom_wait_memcg = oom_wait_info->memcg;
1508 
1509 	if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
1510 	    !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
1511 		return 0;
1512 	return autoremove_wake_function(wait, mode, sync, arg);
1513 }
1514 
1515 static void memcg_oom_recover(struct mem_cgroup *memcg)
1516 {
1517 	/*
1518 	 * For the following lockless ->under_oom test, the only required
1519 	 * guarantee is that it must see the state asserted by an OOM when
1520 	 * this function is called as a result of userland actions
1521 	 * triggered by the notification of the OOM.  This is trivially
1522 	 * achieved by invoking mem_cgroup_mark_under_oom() before
1523 	 * triggering notification.
1524 	 */
1525 	if (memcg && memcg->under_oom)
1526 		__wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
1527 }
1528 
1529 static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
1530 {
1531 	if (!current->memcg_may_oom)
1532 		return;
1533 	/*
1534 	 * We are in the middle of the charge context here, so we
1535 	 * don't want to block when potentially sitting on a callstack
1536 	 * that holds all kinds of filesystem and mm locks.
1537 	 *
1538 	 * Also, the caller may handle a failed allocation gracefully
1539 	 * (like optional page cache readahead) and so an OOM killer
1540 	 * invocation might not even be necessary.
1541 	 *
1542 	 * That's why we don't do anything here except remember the
1543 	 * OOM context and then deal with it at the end of the page
1544 	 * fault when the stack is unwound, the locks are released,
1545 	 * and when we know whether the fault was overall successful.
1546 	 */
1547 	css_get(&memcg->css);
1548 	current->memcg_in_oom = memcg;
1549 	current->memcg_oom_gfp_mask = mask;
1550 	current->memcg_oom_order = order;
1551 }
1552 
1553 /**
1554  * mem_cgroup_oom_synchronize - complete memcg OOM handling
1555  * @handle: actually kill/wait or just clean up the OOM state
1556  *
1557  * This has to be called at the end of a page fault if the memcg OOM
1558  * handler was enabled.
1559  *
1560  * Memcg supports userspace OOM handling where failed allocations must
1561  * sleep on a waitqueue until the userspace task resolves the
1562  * situation.  Sleeping directly in the charge context with all kinds
1563  * of locks held is not a good idea, instead we remember an OOM state
1564  * in the task and mem_cgroup_oom_synchronize() has to be called at
1565  * the end of the page fault to complete the OOM handling.
1566  *
1567  * Returns %true if an ongoing memcg OOM situation was detected and
1568  * completed, %false otherwise.
1569  */
1570 bool mem_cgroup_oom_synchronize(bool handle)
1571 {
1572 	struct mem_cgroup *memcg = current->memcg_in_oom;
1573 	struct oom_wait_info owait;
1574 	bool locked;
1575 
1576 	/* OOM is global, do not handle */
1577 	if (!memcg)
1578 		return false;
1579 
1580 	if (!handle)
1581 		goto cleanup;
1582 
1583 	owait.memcg = memcg;
1584 	owait.wait.flags = 0;
1585 	owait.wait.func = memcg_oom_wake_function;
1586 	owait.wait.private = current;
1587 	INIT_LIST_HEAD(&owait.wait.entry);
1588 
1589 	prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1590 	mem_cgroup_mark_under_oom(memcg);
1591 
1592 	locked = mem_cgroup_oom_trylock(memcg);
1593 
1594 	if (locked)
1595 		mem_cgroup_oom_notify(memcg);
1596 
1597 	if (locked && !memcg->oom_kill_disable) {
1598 		mem_cgroup_unmark_under_oom(memcg);
1599 		finish_wait(&memcg_oom_waitq, &owait.wait);
1600 		mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
1601 					 current->memcg_oom_order);
1602 	} else {
1603 		schedule();
1604 		mem_cgroup_unmark_under_oom(memcg);
1605 		finish_wait(&memcg_oom_waitq, &owait.wait);
1606 	}
1607 
1608 	if (locked) {
1609 		mem_cgroup_oom_unlock(memcg);
1610 		/*
1611 		 * There is no guarantee that an OOM-lock contender
1612 		 * sees the wakeups triggered by the OOM kill
1613 		 * uncharges.  Wake any sleepers explicitely.
1614 		 */
1615 		memcg_oom_recover(memcg);
1616 	}
1617 cleanup:
1618 	current->memcg_in_oom = NULL;
1619 	css_put(&memcg->css);
1620 	return true;
1621 }
1622 
1623 /**
1624  * lock_page_memcg - lock a page->mem_cgroup binding
1625  * @page: the page
1626  *
1627  * This function protects unlocked LRU pages from being moved to
1628  * another cgroup.
1629  *
1630  * It ensures lifetime of the returned memcg. Caller is responsible
1631  * for the lifetime of the page; __unlock_page_memcg() is available
1632  * when @page might get freed inside the locked section.
1633  */
1634 struct mem_cgroup *lock_page_memcg(struct page *page)
1635 {
1636 	struct mem_cgroup *memcg;
1637 	unsigned long flags;
1638 
1639 	/*
1640 	 * The RCU lock is held throughout the transaction.  The fast
1641 	 * path can get away without acquiring the memcg->move_lock
1642 	 * because page moving starts with an RCU grace period.
1643 	 *
1644 	 * The RCU lock also protects the memcg from being freed when
1645 	 * the page state that is going to change is the only thing
1646 	 * preventing the page itself from being freed. E.g. writeback
1647 	 * doesn't hold a page reference and relies on PG_writeback to
1648 	 * keep off truncation, migration and so forth.
1649          */
1650 	rcu_read_lock();
1651 
1652 	if (mem_cgroup_disabled())
1653 		return NULL;
1654 again:
1655 	memcg = page->mem_cgroup;
1656 	if (unlikely(!memcg))
1657 		return NULL;
1658 
1659 	if (atomic_read(&memcg->moving_account) <= 0)
1660 		return memcg;
1661 
1662 	spin_lock_irqsave(&memcg->move_lock, flags);
1663 	if (memcg != page->mem_cgroup) {
1664 		spin_unlock_irqrestore(&memcg->move_lock, flags);
1665 		goto again;
1666 	}
1667 
1668 	/*
1669 	 * When charge migration first begins, we can have locked and
1670 	 * unlocked page stat updates happening concurrently.  Track
1671 	 * the task who has the lock for unlock_page_memcg().
1672 	 */
1673 	memcg->move_lock_task = current;
1674 	memcg->move_lock_flags = flags;
1675 
1676 	return memcg;
1677 }
1678 EXPORT_SYMBOL(lock_page_memcg);
1679 
1680 /**
1681  * __unlock_page_memcg - unlock and unpin a memcg
1682  * @memcg: the memcg
1683  *
1684  * Unlock and unpin a memcg returned by lock_page_memcg().
1685  */
1686 void __unlock_page_memcg(struct mem_cgroup *memcg)
1687 {
1688 	if (memcg && memcg->move_lock_task == current) {
1689 		unsigned long flags = memcg->move_lock_flags;
1690 
1691 		memcg->move_lock_task = NULL;
1692 		memcg->move_lock_flags = 0;
1693 
1694 		spin_unlock_irqrestore(&memcg->move_lock, flags);
1695 	}
1696 
1697 	rcu_read_unlock();
1698 }
1699 
1700 /**
1701  * unlock_page_memcg - unlock a page->mem_cgroup binding
1702  * @page: the page
1703  */
1704 void unlock_page_memcg(struct page *page)
1705 {
1706 	__unlock_page_memcg(page->mem_cgroup);
1707 }
1708 EXPORT_SYMBOL(unlock_page_memcg);
1709 
1710 /*
1711  * size of first charge trial. "32" comes from vmscan.c's magic value.
1712  * TODO: maybe necessary to use big numbers in big irons.
1713  */
1714 #define CHARGE_BATCH	32U
1715 struct memcg_stock_pcp {
1716 	struct mem_cgroup *cached; /* this never be root cgroup */
1717 	unsigned int nr_pages;
1718 	struct work_struct work;
1719 	unsigned long flags;
1720 #define FLUSHING_CACHED_CHARGE	0
1721 };
1722 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
1723 static DEFINE_MUTEX(percpu_charge_mutex);
1724 
1725 /**
1726  * consume_stock: Try to consume stocked charge on this cpu.
1727  * @memcg: memcg to consume from.
1728  * @nr_pages: how many pages to charge.
1729  *
1730  * The charges will only happen if @memcg matches the current cpu's memcg
1731  * stock, and at least @nr_pages are available in that stock.  Failure to
1732  * service an allocation will refill the stock.
1733  *
1734  * returns true if successful, false otherwise.
1735  */
1736 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
1737 {
1738 	struct memcg_stock_pcp *stock;
1739 	unsigned long flags;
1740 	bool ret = false;
1741 
1742 	if (nr_pages > CHARGE_BATCH)
1743 		return ret;
1744 
1745 	local_irq_save(flags);
1746 
1747 	stock = this_cpu_ptr(&memcg_stock);
1748 	if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
1749 		stock->nr_pages -= nr_pages;
1750 		ret = true;
1751 	}
1752 
1753 	local_irq_restore(flags);
1754 
1755 	return ret;
1756 }
1757 
1758 /*
1759  * Returns stocks cached in percpu and reset cached information.
1760  */
1761 static void drain_stock(struct memcg_stock_pcp *stock)
1762 {
1763 	struct mem_cgroup *old = stock->cached;
1764 
1765 	if (stock->nr_pages) {
1766 		page_counter_uncharge(&old->memory, stock->nr_pages);
1767 		if (do_memsw_account())
1768 			page_counter_uncharge(&old->memsw, stock->nr_pages);
1769 		css_put_many(&old->css, stock->nr_pages);
1770 		stock->nr_pages = 0;
1771 	}
1772 	stock->cached = NULL;
1773 }
1774 
1775 static void drain_local_stock(struct work_struct *dummy)
1776 {
1777 	struct memcg_stock_pcp *stock;
1778 	unsigned long flags;
1779 
1780 	local_irq_save(flags);
1781 
1782 	stock = this_cpu_ptr(&memcg_stock);
1783 	drain_stock(stock);
1784 	clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
1785 
1786 	local_irq_restore(flags);
1787 }
1788 
1789 /*
1790  * Cache charges(val) to local per_cpu area.
1791  * This will be consumed by consume_stock() function, later.
1792  */
1793 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
1794 {
1795 	struct memcg_stock_pcp *stock;
1796 	unsigned long flags;
1797 
1798 	local_irq_save(flags);
1799 
1800 	stock = this_cpu_ptr(&memcg_stock);
1801 	if (stock->cached != memcg) { /* reset if necessary */
1802 		drain_stock(stock);
1803 		stock->cached = memcg;
1804 	}
1805 	stock->nr_pages += nr_pages;
1806 
1807 	if (stock->nr_pages > CHARGE_BATCH)
1808 		drain_stock(stock);
1809 
1810 	local_irq_restore(flags);
1811 }
1812 
1813 /*
1814  * Drains all per-CPU charge caches for given root_memcg resp. subtree
1815  * of the hierarchy under it.
1816  */
1817 static void drain_all_stock(struct mem_cgroup *root_memcg)
1818 {
1819 	int cpu, curcpu;
1820 
1821 	/* If someone's already draining, avoid adding running more workers. */
1822 	if (!mutex_trylock(&percpu_charge_mutex))
1823 		return;
1824 	/* Notify other cpus that system-wide "drain" is running */
1825 	get_online_cpus();
1826 	curcpu = get_cpu();
1827 	for_each_online_cpu(cpu) {
1828 		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
1829 		struct mem_cgroup *memcg;
1830 
1831 		memcg = stock->cached;
1832 		if (!memcg || !stock->nr_pages)
1833 			continue;
1834 		if (!mem_cgroup_is_descendant(memcg, root_memcg))
1835 			continue;
1836 		if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
1837 			if (cpu == curcpu)
1838 				drain_local_stock(&stock->work);
1839 			else
1840 				schedule_work_on(cpu, &stock->work);
1841 		}
1842 	}
1843 	put_cpu();
1844 	put_online_cpus();
1845 	mutex_unlock(&percpu_charge_mutex);
1846 }
1847 
1848 static int memcg_hotplug_cpu_dead(unsigned int cpu)
1849 {
1850 	struct memcg_stock_pcp *stock;
1851 
1852 	stock = &per_cpu(memcg_stock, cpu);
1853 	drain_stock(stock);
1854 	return 0;
1855 }
1856 
1857 static void reclaim_high(struct mem_cgroup *memcg,
1858 			 unsigned int nr_pages,
1859 			 gfp_t gfp_mask)
1860 {
1861 	do {
1862 		if (page_counter_read(&memcg->memory) <= memcg->high)
1863 			continue;
1864 		mem_cgroup_event(memcg, MEMCG_HIGH);
1865 		try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true);
1866 	} while ((memcg = parent_mem_cgroup(memcg)));
1867 }
1868 
1869 static void high_work_func(struct work_struct *work)
1870 {
1871 	struct mem_cgroup *memcg;
1872 
1873 	memcg = container_of(work, struct mem_cgroup, high_work);
1874 	reclaim_high(memcg, CHARGE_BATCH, GFP_KERNEL);
1875 }
1876 
1877 /*
1878  * Scheduled by try_charge() to be executed from the userland return path
1879  * and reclaims memory over the high limit.
1880  */
1881 void mem_cgroup_handle_over_high(void)
1882 {
1883 	unsigned int nr_pages = current->memcg_nr_pages_over_high;
1884 	struct mem_cgroup *memcg;
1885 
1886 	if (likely(!nr_pages))
1887 		return;
1888 
1889 	memcg = get_mem_cgroup_from_mm(current->mm);
1890 	reclaim_high(memcg, nr_pages, GFP_KERNEL);
1891 	css_put(&memcg->css);
1892 	current->memcg_nr_pages_over_high = 0;
1893 }
1894 
1895 static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
1896 		      unsigned int nr_pages)
1897 {
1898 	unsigned int batch = max(CHARGE_BATCH, nr_pages);
1899 	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
1900 	struct mem_cgroup *mem_over_limit;
1901 	struct page_counter *counter;
1902 	unsigned long nr_reclaimed;
1903 	bool may_swap = true;
1904 	bool drained = false;
1905 
1906 	if (mem_cgroup_is_root(memcg))
1907 		return 0;
1908 retry:
1909 	if (consume_stock(memcg, nr_pages))
1910 		return 0;
1911 
1912 	if (!do_memsw_account() ||
1913 	    page_counter_try_charge(&memcg->memsw, batch, &counter)) {
1914 		if (page_counter_try_charge(&memcg->memory, batch, &counter))
1915 			goto done_restock;
1916 		if (do_memsw_account())
1917 			page_counter_uncharge(&memcg->memsw, batch);
1918 		mem_over_limit = mem_cgroup_from_counter(counter, memory);
1919 	} else {
1920 		mem_over_limit = mem_cgroup_from_counter(counter, memsw);
1921 		may_swap = false;
1922 	}
1923 
1924 	if (batch > nr_pages) {
1925 		batch = nr_pages;
1926 		goto retry;
1927 	}
1928 
1929 	/*
1930 	 * Unlike in global OOM situations, memcg is not in a physical
1931 	 * memory shortage.  Allow dying and OOM-killed tasks to
1932 	 * bypass the last charges so that they can exit quickly and
1933 	 * free their memory.
1934 	 */
1935 	if (unlikely(tsk_is_oom_victim(current) ||
1936 		     fatal_signal_pending(current) ||
1937 		     current->flags & PF_EXITING))
1938 		goto force;
1939 
1940 	/*
1941 	 * Prevent unbounded recursion when reclaim operations need to
1942 	 * allocate memory. This might exceed the limits temporarily,
1943 	 * but we prefer facilitating memory reclaim and getting back
1944 	 * under the limit over triggering OOM kills in these cases.
1945 	 */
1946 	if (unlikely(current->flags & PF_MEMALLOC))
1947 		goto force;
1948 
1949 	if (unlikely(task_in_memcg_oom(current)))
1950 		goto nomem;
1951 
1952 	if (!gfpflags_allow_blocking(gfp_mask))
1953 		goto nomem;
1954 
1955 	mem_cgroup_event(mem_over_limit, MEMCG_MAX);
1956 
1957 	nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
1958 						    gfp_mask, may_swap);
1959 
1960 	if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
1961 		goto retry;
1962 
1963 	if (!drained) {
1964 		drain_all_stock(mem_over_limit);
1965 		drained = true;
1966 		goto retry;
1967 	}
1968 
1969 	if (gfp_mask & __GFP_NORETRY)
1970 		goto nomem;
1971 	/*
1972 	 * Even though the limit is exceeded at this point, reclaim
1973 	 * may have been able to free some pages.  Retry the charge
1974 	 * before killing the task.
1975 	 *
1976 	 * Only for regular pages, though: huge pages are rather
1977 	 * unlikely to succeed so close to the limit, and we fall back
1978 	 * to regular pages anyway in case of failure.
1979 	 */
1980 	if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
1981 		goto retry;
1982 	/*
1983 	 * At task move, charge accounts can be doubly counted. So, it's
1984 	 * better to wait until the end of task_move if something is going on.
1985 	 */
1986 	if (mem_cgroup_wait_acct_move(mem_over_limit))
1987 		goto retry;
1988 
1989 	if (nr_retries--)
1990 		goto retry;
1991 
1992 	if (gfp_mask & __GFP_NOFAIL)
1993 		goto force;
1994 
1995 	if (fatal_signal_pending(current))
1996 		goto force;
1997 
1998 	mem_cgroup_event(mem_over_limit, MEMCG_OOM);
1999 
2000 	mem_cgroup_oom(mem_over_limit, gfp_mask,
2001 		       get_order(nr_pages * PAGE_SIZE));
2002 nomem:
2003 	if (!(gfp_mask & __GFP_NOFAIL))
2004 		return -ENOMEM;
2005 force:
2006 	/*
2007 	 * The allocation either can't fail or will lead to more memory
2008 	 * being freed very soon.  Allow memory usage go over the limit
2009 	 * temporarily by force charging it.
2010 	 */
2011 	page_counter_charge(&memcg->memory, nr_pages);
2012 	if (do_memsw_account())
2013 		page_counter_charge(&memcg->memsw, nr_pages);
2014 	css_get_many(&memcg->css, nr_pages);
2015 
2016 	return 0;
2017 
2018 done_restock:
2019 	css_get_many(&memcg->css, batch);
2020 	if (batch > nr_pages)
2021 		refill_stock(memcg, batch - nr_pages);
2022 
2023 	/*
2024 	 * If the hierarchy is above the normal consumption range, schedule
2025 	 * reclaim on returning to userland.  We can perform reclaim here
2026 	 * if __GFP_RECLAIM but let's always punt for simplicity and so that
2027 	 * GFP_KERNEL can consistently be used during reclaim.  @memcg is
2028 	 * not recorded as it most likely matches current's and won't
2029 	 * change in the meantime.  As high limit is checked again before
2030 	 * reclaim, the cost of mismatch is negligible.
2031 	 */
2032 	do {
2033 		if (page_counter_read(&memcg->memory) > memcg->high) {
2034 			/* Don't bother a random interrupted task */
2035 			if (in_interrupt()) {
2036 				schedule_work(&memcg->high_work);
2037 				break;
2038 			}
2039 			current->memcg_nr_pages_over_high += batch;
2040 			set_notify_resume(current);
2041 			break;
2042 		}
2043 	} while ((memcg = parent_mem_cgroup(memcg)));
2044 
2045 	return 0;
2046 }
2047 
2048 static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2049 {
2050 	if (mem_cgroup_is_root(memcg))
2051 		return;
2052 
2053 	page_counter_uncharge(&memcg->memory, nr_pages);
2054 	if (do_memsw_account())
2055 		page_counter_uncharge(&memcg->memsw, nr_pages);
2056 
2057 	css_put_many(&memcg->css, nr_pages);
2058 }
2059 
2060 static void lock_page_lru(struct page *page, int *isolated)
2061 {
2062 	struct zone *zone = page_zone(page);
2063 
2064 	spin_lock_irq(zone_lru_lock(zone));
2065 	if (PageLRU(page)) {
2066 		struct lruvec *lruvec;
2067 
2068 		lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
2069 		ClearPageLRU(page);
2070 		del_page_from_lru_list(page, lruvec, page_lru(page));
2071 		*isolated = 1;
2072 	} else
2073 		*isolated = 0;
2074 }
2075 
2076 static void unlock_page_lru(struct page *page, int isolated)
2077 {
2078 	struct zone *zone = page_zone(page);
2079 
2080 	if (isolated) {
2081 		struct lruvec *lruvec;
2082 
2083 		lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
2084 		VM_BUG_ON_PAGE(PageLRU(page), page);
2085 		SetPageLRU(page);
2086 		add_page_to_lru_list(page, lruvec, page_lru(page));
2087 	}
2088 	spin_unlock_irq(zone_lru_lock(zone));
2089 }
2090 
2091 static void commit_charge(struct page *page, struct mem_cgroup *memcg,
2092 			  bool lrucare)
2093 {
2094 	int isolated;
2095 
2096 	VM_BUG_ON_PAGE(page->mem_cgroup, page);
2097 
2098 	/*
2099 	 * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
2100 	 * may already be on some other mem_cgroup's LRU.  Take care of it.
2101 	 */
2102 	if (lrucare)
2103 		lock_page_lru(page, &isolated);
2104 
2105 	/*
2106 	 * Nobody should be changing or seriously looking at
2107 	 * page->mem_cgroup at this point:
2108 	 *
2109 	 * - the page is uncharged
2110 	 *
2111 	 * - the page is off-LRU
2112 	 *
2113 	 * - an anonymous fault has exclusive page access, except for
2114 	 *   a locked page table
2115 	 *
2116 	 * - a page cache insertion, a swapin fault, or a migration
2117 	 *   have the page locked
2118 	 */
2119 	page->mem_cgroup = memcg;
2120 
2121 	if (lrucare)
2122 		unlock_page_lru(page, isolated);
2123 }
2124 
2125 #ifndef CONFIG_SLOB
2126 static int memcg_alloc_cache_id(void)
2127 {
2128 	int id, size;
2129 	int err;
2130 
2131 	id = ida_simple_get(&memcg_cache_ida,
2132 			    0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
2133 	if (id < 0)
2134 		return id;
2135 
2136 	if (id < memcg_nr_cache_ids)
2137 		return id;
2138 
2139 	/*
2140 	 * There's no space for the new id in memcg_caches arrays,
2141 	 * so we have to grow them.
2142 	 */
2143 	down_write(&memcg_cache_ids_sem);
2144 
2145 	size = 2 * (id + 1);
2146 	if (size < MEMCG_CACHES_MIN_SIZE)
2147 		size = MEMCG_CACHES_MIN_SIZE;
2148 	else if (size > MEMCG_CACHES_MAX_SIZE)
2149 		size = MEMCG_CACHES_MAX_SIZE;
2150 
2151 	err = memcg_update_all_caches(size);
2152 	if (!err)
2153 		err = memcg_update_all_list_lrus(size);
2154 	if (!err)
2155 		memcg_nr_cache_ids = size;
2156 
2157 	up_write(&memcg_cache_ids_sem);
2158 
2159 	if (err) {
2160 		ida_simple_remove(&memcg_cache_ida, id);
2161 		return err;
2162 	}
2163 	return id;
2164 }
2165 
2166 static void memcg_free_cache_id(int id)
2167 {
2168 	ida_simple_remove(&memcg_cache_ida, id);
2169 }
2170 
2171 struct memcg_kmem_cache_create_work {
2172 	struct mem_cgroup *memcg;
2173 	struct kmem_cache *cachep;
2174 	struct work_struct work;
2175 };
2176 
2177 static void memcg_kmem_cache_create_func(struct work_struct *w)
2178 {
2179 	struct memcg_kmem_cache_create_work *cw =
2180 		container_of(w, struct memcg_kmem_cache_create_work, work);
2181 	struct mem_cgroup *memcg = cw->memcg;
2182 	struct kmem_cache *cachep = cw->cachep;
2183 
2184 	memcg_create_kmem_cache(memcg, cachep);
2185 
2186 	css_put(&memcg->css);
2187 	kfree(cw);
2188 }
2189 
2190 /*
2191  * Enqueue the creation of a per-memcg kmem_cache.
2192  */
2193 static void __memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
2194 					       struct kmem_cache *cachep)
2195 {
2196 	struct memcg_kmem_cache_create_work *cw;
2197 
2198 	cw = kmalloc(sizeof(*cw), GFP_NOWAIT);
2199 	if (!cw)
2200 		return;
2201 
2202 	css_get(&memcg->css);
2203 
2204 	cw->memcg = memcg;
2205 	cw->cachep = cachep;
2206 	INIT_WORK(&cw->work, memcg_kmem_cache_create_func);
2207 
2208 	queue_work(memcg_kmem_cache_wq, &cw->work);
2209 }
2210 
2211 static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
2212 					     struct kmem_cache *cachep)
2213 {
2214 	/*
2215 	 * We need to stop accounting when we kmalloc, because if the
2216 	 * corresponding kmalloc cache is not yet created, the first allocation
2217 	 * in __memcg_schedule_kmem_cache_create will recurse.
2218 	 *
2219 	 * However, it is better to enclose the whole function. Depending on
2220 	 * the debugging options enabled, INIT_WORK(), for instance, can
2221 	 * trigger an allocation. This too, will make us recurse. Because at
2222 	 * this point we can't allow ourselves back into memcg_kmem_get_cache,
2223 	 * the safest choice is to do it like this, wrapping the whole function.
2224 	 */
2225 	current->memcg_kmem_skip_account = 1;
2226 	__memcg_schedule_kmem_cache_create(memcg, cachep);
2227 	current->memcg_kmem_skip_account = 0;
2228 }
2229 
2230 static inline bool memcg_kmem_bypass(void)
2231 {
2232 	if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))
2233 		return true;
2234 	return false;
2235 }
2236 
2237 /**
2238  * memcg_kmem_get_cache: select the correct per-memcg cache for allocation
2239  * @cachep: the original global kmem cache
2240  *
2241  * Return the kmem_cache we're supposed to use for a slab allocation.
2242  * We try to use the current memcg's version of the cache.
2243  *
2244  * If the cache does not exist yet, if we are the first user of it, we
2245  * create it asynchronously in a workqueue and let the current allocation
2246  * go through with the original cache.
2247  *
2248  * This function takes a reference to the cache it returns to assure it
2249  * won't get destroyed while we are working with it. Once the caller is
2250  * done with it, memcg_kmem_put_cache() must be called to release the
2251  * reference.
2252  */
2253 struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep)
2254 {
2255 	struct mem_cgroup *memcg;
2256 	struct kmem_cache *memcg_cachep;
2257 	int kmemcg_id;
2258 
2259 	VM_BUG_ON(!is_root_cache(cachep));
2260 
2261 	if (memcg_kmem_bypass())
2262 		return cachep;
2263 
2264 	if (current->memcg_kmem_skip_account)
2265 		return cachep;
2266 
2267 	memcg = get_mem_cgroup_from_mm(current->mm);
2268 	kmemcg_id = READ_ONCE(memcg->kmemcg_id);
2269 	if (kmemcg_id < 0)
2270 		goto out;
2271 
2272 	memcg_cachep = cache_from_memcg_idx(cachep, kmemcg_id);
2273 	if (likely(memcg_cachep))
2274 		return memcg_cachep;
2275 
2276 	/*
2277 	 * If we are in a safe context (can wait, and not in interrupt
2278 	 * context), we could be be predictable and return right away.
2279 	 * This would guarantee that the allocation being performed
2280 	 * already belongs in the new cache.
2281 	 *
2282 	 * However, there are some clashes that can arrive from locking.
2283 	 * For instance, because we acquire the slab_mutex while doing
2284 	 * memcg_create_kmem_cache, this means no further allocation
2285 	 * could happen with the slab_mutex held. So it's better to
2286 	 * defer everything.
2287 	 */
2288 	memcg_schedule_kmem_cache_create(memcg, cachep);
2289 out:
2290 	css_put(&memcg->css);
2291 	return cachep;
2292 }
2293 
2294 /**
2295  * memcg_kmem_put_cache: drop reference taken by memcg_kmem_get_cache
2296  * @cachep: the cache returned by memcg_kmem_get_cache
2297  */
2298 void memcg_kmem_put_cache(struct kmem_cache *cachep)
2299 {
2300 	if (!is_root_cache(cachep))
2301 		css_put(&cachep->memcg_params.memcg->css);
2302 }
2303 
2304 /**
2305  * memcg_kmem_charge: charge a kmem page
2306  * @page: page to charge
2307  * @gfp: reclaim mode
2308  * @order: allocation order
2309  * @memcg: memory cgroup to charge
2310  *
2311  * Returns 0 on success, an error code on failure.
2312  */
2313 int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
2314 			    struct mem_cgroup *memcg)
2315 {
2316 	unsigned int nr_pages = 1 << order;
2317 	struct page_counter *counter;
2318 	int ret;
2319 
2320 	ret = try_charge(memcg, gfp, nr_pages);
2321 	if (ret)
2322 		return ret;
2323 
2324 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
2325 	    !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
2326 		cancel_charge(memcg, nr_pages);
2327 		return -ENOMEM;
2328 	}
2329 
2330 	page->mem_cgroup = memcg;
2331 
2332 	return 0;
2333 }
2334 
2335 /**
2336  * memcg_kmem_charge: charge a kmem page to the current memory cgroup
2337  * @page: page to charge
2338  * @gfp: reclaim mode
2339  * @order: allocation order
2340  *
2341  * Returns 0 on success, an error code on failure.
2342  */
2343 int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
2344 {
2345 	struct mem_cgroup *memcg;
2346 	int ret = 0;
2347 
2348 	if (memcg_kmem_bypass())
2349 		return 0;
2350 
2351 	memcg = get_mem_cgroup_from_mm(current->mm);
2352 	if (!mem_cgroup_is_root(memcg)) {
2353 		ret = memcg_kmem_charge_memcg(page, gfp, order, memcg);
2354 		if (!ret)
2355 			__SetPageKmemcg(page);
2356 	}
2357 	css_put(&memcg->css);
2358 	return ret;
2359 }
2360 /**
2361  * memcg_kmem_uncharge: uncharge a kmem page
2362  * @page: page to uncharge
2363  * @order: allocation order
2364  */
2365 void memcg_kmem_uncharge(struct page *page, int order)
2366 {
2367 	struct mem_cgroup *memcg = page->mem_cgroup;
2368 	unsigned int nr_pages = 1 << order;
2369 
2370 	if (!memcg)
2371 		return;
2372 
2373 	VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
2374 
2375 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
2376 		page_counter_uncharge(&memcg->kmem, nr_pages);
2377 
2378 	page_counter_uncharge(&memcg->memory, nr_pages);
2379 	if (do_memsw_account())
2380 		page_counter_uncharge(&memcg->memsw, nr_pages);
2381 
2382 	page->mem_cgroup = NULL;
2383 
2384 	/* slab pages do not have PageKmemcg flag set */
2385 	if (PageKmemcg(page))
2386 		__ClearPageKmemcg(page);
2387 
2388 	css_put_many(&memcg->css, nr_pages);
2389 }
2390 #endif /* !CONFIG_SLOB */
2391 
2392 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2393 
2394 /*
2395  * Because tail pages are not marked as "used", set it. We're under
2396  * zone_lru_lock and migration entries setup in all page mappings.
2397  */
2398 void mem_cgroup_split_huge_fixup(struct page *head)
2399 {
2400 	int i;
2401 
2402 	if (mem_cgroup_disabled())
2403 		return;
2404 
2405 	for (i = 1; i < HPAGE_PMD_NR; i++)
2406 		head[i].mem_cgroup = head->mem_cgroup;
2407 
2408 	__this_cpu_sub(head->mem_cgroup->stat->count[MEMCG_RSS_HUGE],
2409 		       HPAGE_PMD_NR);
2410 }
2411 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
2412 
2413 #ifdef CONFIG_MEMCG_SWAP
2414 static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
2415 				       int nr_entries)
2416 {
2417 	this_cpu_add(memcg->stat->count[MEMCG_SWAP], nr_entries);
2418 }
2419 
2420 /**
2421  * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
2422  * @entry: swap entry to be moved
2423  * @from:  mem_cgroup which the entry is moved from
2424  * @to:  mem_cgroup which the entry is moved to
2425  *
2426  * It succeeds only when the swap_cgroup's record for this entry is the same
2427  * as the mem_cgroup's id of @from.
2428  *
2429  * Returns 0 on success, -EINVAL on failure.
2430  *
2431  * The caller must have charged to @to, IOW, called page_counter_charge() about
2432  * both res and memsw, and called css_get().
2433  */
2434 static int mem_cgroup_move_swap_account(swp_entry_t entry,
2435 				struct mem_cgroup *from, struct mem_cgroup *to)
2436 {
2437 	unsigned short old_id, new_id;
2438 
2439 	old_id = mem_cgroup_id(from);
2440 	new_id = mem_cgroup_id(to);
2441 
2442 	if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
2443 		mem_cgroup_swap_statistics(from, -1);
2444 		mem_cgroup_swap_statistics(to, 1);
2445 		return 0;
2446 	}
2447 	return -EINVAL;
2448 }
2449 #else
2450 static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
2451 				struct mem_cgroup *from, struct mem_cgroup *to)
2452 {
2453 	return -EINVAL;
2454 }
2455 #endif
2456 
2457 static DEFINE_MUTEX(memcg_limit_mutex);
2458 
2459 static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
2460 				   unsigned long limit)
2461 {
2462 	unsigned long curusage;
2463 	unsigned long oldusage;
2464 	bool enlarge = false;
2465 	int retry_count;
2466 	int ret;
2467 
2468 	/*
2469 	 * For keeping hierarchical_reclaim simple, how long we should retry
2470 	 * is depends on callers. We set our retry-count to be function
2471 	 * of # of children which we should visit in this loop.
2472 	 */
2473 	retry_count = MEM_CGROUP_RECLAIM_RETRIES *
2474 		      mem_cgroup_count_children(memcg);
2475 
2476 	oldusage = page_counter_read(&memcg->memory);
2477 
2478 	do {
2479 		if (signal_pending(current)) {
2480 			ret = -EINTR;
2481 			break;
2482 		}
2483 
2484 		mutex_lock(&memcg_limit_mutex);
2485 		if (limit > memcg->memsw.limit) {
2486 			mutex_unlock(&memcg_limit_mutex);
2487 			ret = -EINVAL;
2488 			break;
2489 		}
2490 		if (limit > memcg->memory.limit)
2491 			enlarge = true;
2492 		ret = page_counter_limit(&memcg->memory, limit);
2493 		mutex_unlock(&memcg_limit_mutex);
2494 
2495 		if (!ret)
2496 			break;
2497 
2498 		try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, true);
2499 
2500 		curusage = page_counter_read(&memcg->memory);
2501 		/* Usage is reduced ? */
2502 		if (curusage >= oldusage)
2503 			retry_count--;
2504 		else
2505 			oldusage = curusage;
2506 	} while (retry_count);
2507 
2508 	if (!ret && enlarge)
2509 		memcg_oom_recover(memcg);
2510 
2511 	return ret;
2512 }
2513 
2514 static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
2515 					 unsigned long limit)
2516 {
2517 	unsigned long curusage;
2518 	unsigned long oldusage;
2519 	bool enlarge = false;
2520 	int retry_count;
2521 	int ret;
2522 
2523 	/* see mem_cgroup_resize_res_limit */
2524 	retry_count = MEM_CGROUP_RECLAIM_RETRIES *
2525 		      mem_cgroup_count_children(memcg);
2526 
2527 	oldusage = page_counter_read(&memcg->memsw);
2528 
2529 	do {
2530 		if (signal_pending(current)) {
2531 			ret = -EINTR;
2532 			break;
2533 		}
2534 
2535 		mutex_lock(&memcg_limit_mutex);
2536 		if (limit < memcg->memory.limit) {
2537 			mutex_unlock(&memcg_limit_mutex);
2538 			ret = -EINVAL;
2539 			break;
2540 		}
2541 		if (limit > memcg->memsw.limit)
2542 			enlarge = true;
2543 		ret = page_counter_limit(&memcg->memsw, limit);
2544 		mutex_unlock(&memcg_limit_mutex);
2545 
2546 		if (!ret)
2547 			break;
2548 
2549 		try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, false);
2550 
2551 		curusage = page_counter_read(&memcg->memsw);
2552 		/* Usage is reduced ? */
2553 		if (curusage >= oldusage)
2554 			retry_count--;
2555 		else
2556 			oldusage = curusage;
2557 	} while (retry_count);
2558 
2559 	if (!ret && enlarge)
2560 		memcg_oom_recover(memcg);
2561 
2562 	return ret;
2563 }
2564 
2565 unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
2566 					    gfp_t gfp_mask,
2567 					    unsigned long *total_scanned)
2568 {
2569 	unsigned long nr_reclaimed = 0;
2570 	struct mem_cgroup_per_node *mz, *next_mz = NULL;
2571 	unsigned long reclaimed;
2572 	int loop = 0;
2573 	struct mem_cgroup_tree_per_node *mctz;
2574 	unsigned long excess;
2575 	unsigned long nr_scanned;
2576 
2577 	if (order > 0)
2578 		return 0;
2579 
2580 	mctz = soft_limit_tree_node(pgdat->node_id);
2581 
2582 	/*
2583 	 * Do not even bother to check the largest node if the root
2584 	 * is empty. Do it lockless to prevent lock bouncing. Races
2585 	 * are acceptable as soft limit is best effort anyway.
2586 	 */
2587 	if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
2588 		return 0;
2589 
2590 	/*
2591 	 * This loop can run a while, specially if mem_cgroup's continuously
2592 	 * keep exceeding their soft limit and putting the system under
2593 	 * pressure
2594 	 */
2595 	do {
2596 		if (next_mz)
2597 			mz = next_mz;
2598 		else
2599 			mz = mem_cgroup_largest_soft_limit_node(mctz);
2600 		if (!mz)
2601 			break;
2602 
2603 		nr_scanned = 0;
2604 		reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
2605 						    gfp_mask, &nr_scanned);
2606 		nr_reclaimed += reclaimed;
2607 		*total_scanned += nr_scanned;
2608 		spin_lock_irq(&mctz->lock);
2609 		__mem_cgroup_remove_exceeded(mz, mctz);
2610 
2611 		/*
2612 		 * If we failed to reclaim anything from this memory cgroup
2613 		 * it is time to move on to the next cgroup
2614 		 */
2615 		next_mz = NULL;
2616 		if (!reclaimed)
2617 			next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
2618 
2619 		excess = soft_limit_excess(mz->memcg);
2620 		/*
2621 		 * One school of thought says that we should not add
2622 		 * back the node to the tree if reclaim returns 0.
2623 		 * But our reclaim could return 0, simply because due
2624 		 * to priority we are exposing a smaller subset of
2625 		 * memory to reclaim from. Consider this as a longer
2626 		 * term TODO.
2627 		 */
2628 		/* If excess == 0, no tree ops */
2629 		__mem_cgroup_insert_exceeded(mz, mctz, excess);
2630 		spin_unlock_irq(&mctz->lock);
2631 		css_put(&mz->memcg->css);
2632 		loop++;
2633 		/*
2634 		 * Could not reclaim anything and there are no more
2635 		 * mem cgroups to try or we seem to be looping without
2636 		 * reclaiming anything.
2637 		 */
2638 		if (!nr_reclaimed &&
2639 			(next_mz == NULL ||
2640 			loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
2641 			break;
2642 	} while (!nr_reclaimed);
2643 	if (next_mz)
2644 		css_put(&next_mz->memcg->css);
2645 	return nr_reclaimed;
2646 }
2647 
2648 /*
2649  * Test whether @memcg has children, dead or alive.  Note that this
2650  * function doesn't care whether @memcg has use_hierarchy enabled and
2651  * returns %true if there are child csses according to the cgroup
2652  * hierarchy.  Testing use_hierarchy is the caller's responsiblity.
2653  */
2654 static inline bool memcg_has_children(struct mem_cgroup *memcg)
2655 {
2656 	bool ret;
2657 
2658 	rcu_read_lock();
2659 	ret = css_next_child(NULL, &memcg->css);
2660 	rcu_read_unlock();
2661 	return ret;
2662 }
2663 
2664 /*
2665  * Reclaims as many pages from the given memcg as possible.
2666  *
2667  * Caller is responsible for holding css reference for memcg.
2668  */
2669 static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
2670 {
2671 	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
2672 
2673 	/* we call try-to-free pages for make this cgroup empty */
2674 	lru_add_drain_all();
2675 	/* try to free all pages in this cgroup */
2676 	while (nr_retries && page_counter_read(&memcg->memory)) {
2677 		int progress;
2678 
2679 		if (signal_pending(current))
2680 			return -EINTR;
2681 
2682 		progress = try_to_free_mem_cgroup_pages(memcg, 1,
2683 							GFP_KERNEL, true);
2684 		if (!progress) {
2685 			nr_retries--;
2686 			/* maybe some writeback is necessary */
2687 			congestion_wait(BLK_RW_ASYNC, HZ/10);
2688 		}
2689 
2690 	}
2691 
2692 	return 0;
2693 }
2694 
2695 static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
2696 					    char *buf, size_t nbytes,
2697 					    loff_t off)
2698 {
2699 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2700 
2701 	if (mem_cgroup_is_root(memcg))
2702 		return -EINVAL;
2703 	return mem_cgroup_force_empty(memcg) ?: nbytes;
2704 }
2705 
2706 static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
2707 				     struct cftype *cft)
2708 {
2709 	return mem_cgroup_from_css(css)->use_hierarchy;
2710 }
2711 
2712 static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
2713 				      struct cftype *cft, u64 val)
2714 {
2715 	int retval = 0;
2716 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2717 	struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);
2718 
2719 	if (memcg->use_hierarchy == val)
2720 		return 0;
2721 
2722 	/*
2723 	 * If parent's use_hierarchy is set, we can't make any modifications
2724 	 * in the child subtrees. If it is unset, then the change can
2725 	 * occur, provided the current cgroup has no children.
2726 	 *
2727 	 * For the root cgroup, parent_mem is NULL, we allow value to be
2728 	 * set if there are no children.
2729 	 */
2730 	if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
2731 				(val == 1 || val == 0)) {
2732 		if (!memcg_has_children(memcg))
2733 			memcg->use_hierarchy = val;
2734 		else
2735 			retval = -EBUSY;
2736 	} else
2737 		retval = -EINVAL;
2738 
2739 	return retval;
2740 }
2741 
2742 static void tree_stat(struct mem_cgroup *memcg, unsigned long *stat)
2743 {
2744 	struct mem_cgroup *iter;
2745 	int i;
2746 
2747 	memset(stat, 0, sizeof(*stat) * MEMCG_NR_STAT);
2748 
2749 	for_each_mem_cgroup_tree(iter, memcg) {
2750 		for (i = 0; i < MEMCG_NR_STAT; i++)
2751 			stat[i] += memcg_page_state(iter, i);
2752 	}
2753 }
2754 
2755 static void tree_events(struct mem_cgroup *memcg, unsigned long *events)
2756 {
2757 	struct mem_cgroup *iter;
2758 	int i;
2759 
2760 	memset(events, 0, sizeof(*events) * MEMCG_NR_EVENTS);
2761 
2762 	for_each_mem_cgroup_tree(iter, memcg) {
2763 		for (i = 0; i < MEMCG_NR_EVENTS; i++)
2764 			events[i] += memcg_sum_events(iter, i);
2765 	}
2766 }
2767 
2768 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
2769 {
2770 	unsigned long val = 0;
2771 
2772 	if (mem_cgroup_is_root(memcg)) {
2773 		struct mem_cgroup *iter;
2774 
2775 		for_each_mem_cgroup_tree(iter, memcg) {
2776 			val += memcg_page_state(iter, MEMCG_CACHE);
2777 			val += memcg_page_state(iter, MEMCG_RSS);
2778 			if (swap)
2779 				val += memcg_page_state(iter, MEMCG_SWAP);
2780 		}
2781 	} else {
2782 		if (!swap)
2783 			val = page_counter_read(&memcg->memory);
2784 		else
2785 			val = page_counter_read(&memcg->memsw);
2786 	}
2787 	return val;
2788 }
2789 
2790 enum {
2791 	RES_USAGE,
2792 	RES_LIMIT,
2793 	RES_MAX_USAGE,
2794 	RES_FAILCNT,
2795 	RES_SOFT_LIMIT,
2796 };
2797 
2798 static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
2799 			       struct cftype *cft)
2800 {
2801 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2802 	struct page_counter *counter;
2803 
2804 	switch (MEMFILE_TYPE(cft->private)) {
2805 	case _MEM:
2806 		counter = &memcg->memory;
2807 		break;
2808 	case _MEMSWAP:
2809 		counter = &memcg->memsw;
2810 		break;
2811 	case _KMEM:
2812 		counter = &memcg->kmem;
2813 		break;
2814 	case _TCP:
2815 		counter = &memcg->tcpmem;
2816 		break;
2817 	default:
2818 		BUG();
2819 	}
2820 
2821 	switch (MEMFILE_ATTR(cft->private)) {
2822 	case RES_USAGE:
2823 		if (counter == &memcg->memory)
2824 			return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
2825 		if (counter == &memcg->memsw)
2826 			return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
2827 		return (u64)page_counter_read(counter) * PAGE_SIZE;
2828 	case RES_LIMIT:
2829 		return (u64)counter->limit * PAGE_SIZE;
2830 	case RES_MAX_USAGE:
2831 		return (u64)counter->watermark * PAGE_SIZE;
2832 	case RES_FAILCNT:
2833 		return counter->failcnt;
2834 	case RES_SOFT_LIMIT:
2835 		return (u64)memcg->soft_limit * PAGE_SIZE;
2836 	default:
2837 		BUG();
2838 	}
2839 }
2840 
2841 #ifndef CONFIG_SLOB
2842 static int memcg_online_kmem(struct mem_cgroup *memcg)
2843 {
2844 	int memcg_id;
2845 
2846 	if (cgroup_memory_nokmem)
2847 		return 0;
2848 
2849 	BUG_ON(memcg->kmemcg_id >= 0);
2850 	BUG_ON(memcg->kmem_state);
2851 
2852 	memcg_id = memcg_alloc_cache_id();
2853 	if (memcg_id < 0)
2854 		return memcg_id;
2855 
2856 	static_branch_inc(&memcg_kmem_enabled_key);
2857 	/*
2858 	 * A memory cgroup is considered kmem-online as soon as it gets
2859 	 * kmemcg_id. Setting the id after enabling static branching will
2860 	 * guarantee no one starts accounting before all call sites are
2861 	 * patched.
2862 	 */
2863 	memcg->kmemcg_id = memcg_id;
2864 	memcg->kmem_state = KMEM_ONLINE;
2865 	INIT_LIST_HEAD(&memcg->kmem_caches);
2866 
2867 	return 0;
2868 }
2869 
2870 static void memcg_offline_kmem(struct mem_cgroup *memcg)
2871 {
2872 	struct cgroup_subsys_state *css;
2873 	struct mem_cgroup *parent, *child;
2874 	int kmemcg_id;
2875 
2876 	if (memcg->kmem_state != KMEM_ONLINE)
2877 		return;
2878 	/*
2879 	 * Clear the online state before clearing memcg_caches array
2880 	 * entries. The slab_mutex in memcg_deactivate_kmem_caches()
2881 	 * guarantees that no cache will be created for this cgroup
2882 	 * after we are done (see memcg_create_kmem_cache()).
2883 	 */
2884 	memcg->kmem_state = KMEM_ALLOCATED;
2885 
2886 	memcg_deactivate_kmem_caches(memcg);
2887 
2888 	kmemcg_id = memcg->kmemcg_id;
2889 	BUG_ON(kmemcg_id < 0);
2890 
2891 	parent = parent_mem_cgroup(memcg);
2892 	if (!parent)
2893 		parent = root_mem_cgroup;
2894 
2895 	/*
2896 	 * Change kmemcg_id of this cgroup and all its descendants to the
2897 	 * parent's id, and then move all entries from this cgroup's list_lrus
2898 	 * to ones of the parent. After we have finished, all list_lrus
2899 	 * corresponding to this cgroup are guaranteed to remain empty. The
2900 	 * ordering is imposed by list_lru_node->lock taken by
2901 	 * memcg_drain_all_list_lrus().
2902 	 */
2903 	rcu_read_lock(); /* can be called from css_free w/o cgroup_mutex */
2904 	css_for_each_descendant_pre(css, &memcg->css) {
2905 		child = mem_cgroup_from_css(css);
2906 		BUG_ON(child->kmemcg_id != kmemcg_id);
2907 		child->kmemcg_id = parent->kmemcg_id;
2908 		if (!memcg->use_hierarchy)
2909 			break;
2910 	}
2911 	rcu_read_unlock();
2912 
2913 	memcg_drain_all_list_lrus(kmemcg_id, parent->kmemcg_id);
2914 
2915 	memcg_free_cache_id(kmemcg_id);
2916 }
2917 
2918 static void memcg_free_kmem(struct mem_cgroup *memcg)
2919 {
2920 	/* css_alloc() failed, offlining didn't happen */
2921 	if (unlikely(memcg->kmem_state == KMEM_ONLINE))
2922 		memcg_offline_kmem(memcg);
2923 
2924 	if (memcg->kmem_state == KMEM_ALLOCATED) {
2925 		memcg_destroy_kmem_caches(memcg);
2926 		static_branch_dec(&memcg_kmem_enabled_key);
2927 		WARN_ON(page_counter_read(&memcg->kmem));
2928 	}
2929 }
2930 #else
2931 static int memcg_online_kmem(struct mem_cgroup *memcg)
2932 {
2933 	return 0;
2934 }
2935 static void memcg_offline_kmem(struct mem_cgroup *memcg)
2936 {
2937 }
2938 static void memcg_free_kmem(struct mem_cgroup *memcg)
2939 {
2940 }
2941 #endif /* !CONFIG_SLOB */
2942 
2943 static int memcg_update_kmem_limit(struct mem_cgroup *memcg,
2944 				   unsigned long limit)
2945 {
2946 	int ret;
2947 
2948 	mutex_lock(&memcg_limit_mutex);
2949 	ret = page_counter_limit(&memcg->kmem, limit);
2950 	mutex_unlock(&memcg_limit_mutex);
2951 	return ret;
2952 }
2953 
2954 static int memcg_update_tcp_limit(struct mem_cgroup *memcg, unsigned long limit)
2955 {
2956 	int ret;
2957 
2958 	mutex_lock(&memcg_limit_mutex);
2959 
2960 	ret = page_counter_limit(&memcg->tcpmem, limit);
2961 	if (ret)
2962 		goto out;
2963 
2964 	if (!memcg->tcpmem_active) {
2965 		/*
2966 		 * The active flag needs to be written after the static_key
2967 		 * update. This is what guarantees that the socket activation
2968 		 * function is the last one to run. See mem_cgroup_sk_alloc()
2969 		 * for details, and note that we don't mark any socket as
2970 		 * belonging to this memcg until that flag is up.
2971 		 *
2972 		 * We need to do this, because static_keys will span multiple
2973 		 * sites, but we can't control their order. If we mark a socket
2974 		 * as accounted, but the accounting functions are not patched in
2975 		 * yet, we'll lose accounting.
2976 		 *
2977 		 * We never race with the readers in mem_cgroup_sk_alloc(),
2978 		 * because when this value change, the code to process it is not
2979 		 * patched in yet.
2980 		 */
2981 		static_branch_inc(&memcg_sockets_enabled_key);
2982 		memcg->tcpmem_active = true;
2983 	}
2984 out:
2985 	mutex_unlock(&memcg_limit_mutex);
2986 	return ret;
2987 }
2988 
2989 /*
2990  * The user of this function is...
2991  * RES_LIMIT.
2992  */
2993 static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
2994 				char *buf, size_t nbytes, loff_t off)
2995 {
2996 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2997 	unsigned long nr_pages;
2998 	int ret;
2999 
3000 	buf = strstrip(buf);
3001 	ret = page_counter_memparse(buf, "-1", &nr_pages);
3002 	if (ret)
3003 		return ret;
3004 
3005 	switch (MEMFILE_ATTR(of_cft(of)->private)) {
3006 	case RES_LIMIT:
3007 		if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
3008 			ret = -EINVAL;
3009 			break;
3010 		}
3011 		switch (MEMFILE_TYPE(of_cft(of)->private)) {
3012 		case _MEM:
3013 			ret = mem_cgroup_resize_limit(memcg, nr_pages);
3014 			break;
3015 		case _MEMSWAP:
3016 			ret = mem_cgroup_resize_memsw_limit(memcg, nr_pages);
3017 			break;
3018 		case _KMEM:
3019 			ret = memcg_update_kmem_limit(memcg, nr_pages);
3020 			break;
3021 		case _TCP:
3022 			ret = memcg_update_tcp_limit(memcg, nr_pages);
3023 			break;
3024 		}
3025 		break;
3026 	case RES_SOFT_LIMIT:
3027 		memcg->soft_limit = nr_pages;
3028 		ret = 0;
3029 		break;
3030 	}
3031 	return ret ?: nbytes;
3032 }
3033 
3034 static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
3035 				size_t nbytes, loff_t off)
3036 {
3037 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3038 	struct page_counter *counter;
3039 
3040 	switch (MEMFILE_TYPE(of_cft(of)->private)) {
3041 	case _MEM:
3042 		counter = &memcg->memory;
3043 		break;
3044 	case _MEMSWAP:
3045 		counter = &memcg->memsw;
3046 		break;
3047 	case _KMEM:
3048 		counter = &memcg->kmem;
3049 		break;
3050 	case _TCP:
3051 		counter = &memcg->tcpmem;
3052 		break;
3053 	default:
3054 		BUG();
3055 	}
3056 
3057 	switch (MEMFILE_ATTR(of_cft(of)->private)) {
3058 	case RES_MAX_USAGE:
3059 		page_counter_reset_watermark(counter);
3060 		break;
3061 	case RES_FAILCNT:
3062 		counter->failcnt = 0;
3063 		break;
3064 	default:
3065 		BUG();
3066 	}
3067 
3068 	return nbytes;
3069 }
3070 
3071 static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
3072 					struct cftype *cft)
3073 {
3074 	return mem_cgroup_from_css(css)->move_charge_at_immigrate;
3075 }
3076 
3077 #ifdef CONFIG_MMU
3078 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3079 					struct cftype *cft, u64 val)
3080 {
3081 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3082 
3083 	if (val & ~MOVE_MASK)
3084 		return -EINVAL;
3085 
3086 	/*
3087 	 * No kind of locking is needed in here, because ->can_attach() will
3088 	 * check this value once in the beginning of the process, and then carry
3089 	 * on with stale data. This means that changes to this value will only
3090 	 * affect task migrations starting after the change.
3091 	 */
3092 	memcg->move_charge_at_immigrate = val;
3093 	return 0;
3094 }
3095 #else
3096 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3097 					struct cftype *cft, u64 val)
3098 {
3099 	return -ENOSYS;
3100 }
3101 #endif
3102 
3103 #ifdef CONFIG_NUMA
3104 static int memcg_numa_stat_show(struct seq_file *m, void *v)
3105 {
3106 	struct numa_stat {
3107 		const char *name;
3108 		unsigned int lru_mask;
3109 	};
3110 
3111 	static const struct numa_stat stats[] = {
3112 		{ "total", LRU_ALL },
3113 		{ "file", LRU_ALL_FILE },
3114 		{ "anon", LRU_ALL_ANON },
3115 		{ "unevictable", BIT(LRU_UNEVICTABLE) },
3116 	};
3117 	const struct numa_stat *stat;
3118 	int nid;
3119 	unsigned long nr;
3120 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3121 
3122 	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3123 		nr = mem_cgroup_nr_lru_pages(memcg, stat->lru_mask);
3124 		seq_printf(m, "%s=%lu", stat->name, nr);
3125 		for_each_node_state(nid, N_MEMORY) {
3126 			nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
3127 							  stat->lru_mask);
3128 			seq_printf(m, " N%d=%lu", nid, nr);
3129 		}
3130 		seq_putc(m, '\n');
3131 	}
3132 
3133 	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3134 		struct mem_cgroup *iter;
3135 
3136 		nr = 0;
3137 		for_each_mem_cgroup_tree(iter, memcg)
3138 			nr += mem_cgroup_nr_lru_pages(iter, stat->lru_mask);
3139 		seq_printf(m, "hierarchical_%s=%lu", stat->name, nr);
3140 		for_each_node_state(nid, N_MEMORY) {
3141 			nr = 0;
3142 			for_each_mem_cgroup_tree(iter, memcg)
3143 				nr += mem_cgroup_node_nr_lru_pages(
3144 					iter, nid, stat->lru_mask);
3145 			seq_printf(m, " N%d=%lu", nid, nr);
3146 		}
3147 		seq_putc(m, '\n');
3148 	}
3149 
3150 	return 0;
3151 }
3152 #endif /* CONFIG_NUMA */
3153 
3154 /* Universal VM events cgroup1 shows, original sort order */
3155 unsigned int memcg1_events[] = {
3156 	PGPGIN,
3157 	PGPGOUT,
3158 	PGFAULT,
3159 	PGMAJFAULT,
3160 };
3161 
3162 static const char *const memcg1_event_names[] = {
3163 	"pgpgin",
3164 	"pgpgout",
3165 	"pgfault",
3166 	"pgmajfault",
3167 };
3168 
3169 static int memcg_stat_show(struct seq_file *m, void *v)
3170 {
3171 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3172 	unsigned long memory, memsw;
3173 	struct mem_cgroup *mi;
3174 	unsigned int i;
3175 
3176 	BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats));
3177 	BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
3178 
3179 	for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
3180 		if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
3181 			continue;
3182 		seq_printf(m, "%s %lu\n", memcg1_stat_names[i],
3183 			   memcg_page_state(memcg, memcg1_stats[i]) *
3184 			   PAGE_SIZE);
3185 	}
3186 
3187 	for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
3188 		seq_printf(m, "%s %lu\n", memcg1_event_names[i],
3189 			   memcg_sum_events(memcg, memcg1_events[i]));
3190 
3191 	for (i = 0; i < NR_LRU_LISTS; i++)
3192 		seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
3193 			   mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);
3194 
3195 	/* Hierarchical information */
3196 	memory = memsw = PAGE_COUNTER_MAX;
3197 	for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
3198 		memory = min(memory, mi->memory.limit);
3199 		memsw = min(memsw, mi->memsw.limit);
3200 	}
3201 	seq_printf(m, "hierarchical_memory_limit %llu\n",
3202 		   (u64)memory * PAGE_SIZE);
3203 	if (do_memsw_account())
3204 		seq_printf(m, "hierarchical_memsw_limit %llu\n",
3205 			   (u64)memsw * PAGE_SIZE);
3206 
3207 	for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
3208 		unsigned long long val = 0;
3209 
3210 		if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
3211 			continue;
3212 		for_each_mem_cgroup_tree(mi, memcg)
3213 			val += memcg_page_state(mi, memcg1_stats[i]) *
3214 			PAGE_SIZE;
3215 		seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i], val);
3216 	}
3217 
3218 	for (i = 0; i < ARRAY_SIZE(memcg1_events); i++) {
3219 		unsigned long long val = 0;
3220 
3221 		for_each_mem_cgroup_tree(mi, memcg)
3222 			val += memcg_sum_events(mi, memcg1_events[i]);
3223 		seq_printf(m, "total_%s %llu\n", memcg1_event_names[i], val);
3224 	}
3225 
3226 	for (i = 0; i < NR_LRU_LISTS; i++) {
3227 		unsigned long long val = 0;
3228 
3229 		for_each_mem_cgroup_tree(mi, memcg)
3230 			val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE;
3231 		seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val);
3232 	}
3233 
3234 #ifdef CONFIG_DEBUG_VM
3235 	{
3236 		pg_data_t *pgdat;
3237 		struct mem_cgroup_per_node *mz;
3238 		struct zone_reclaim_stat *rstat;
3239 		unsigned long recent_rotated[2] = {0, 0};
3240 		unsigned long recent_scanned[2] = {0, 0};
3241 
3242 		for_each_online_pgdat(pgdat) {
3243 			mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
3244 			rstat = &mz->lruvec.reclaim_stat;
3245 
3246 			recent_rotated[0] += rstat->recent_rotated[0];
3247 			recent_rotated[1] += rstat->recent_rotated[1];
3248 			recent_scanned[0] += rstat->recent_scanned[0];
3249 			recent_scanned[1] += rstat->recent_scanned[1];
3250 		}
3251 		seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
3252 		seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
3253 		seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
3254 		seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
3255 	}
3256 #endif
3257 
3258 	return 0;
3259 }
3260 
3261 static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
3262 				      struct cftype *cft)
3263 {
3264 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3265 
3266 	return mem_cgroup_swappiness(memcg);
3267 }
3268 
3269 static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
3270 				       struct cftype *cft, u64 val)
3271 {
3272 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3273 
3274 	if (val > 100)
3275 		return -EINVAL;
3276 
3277 	if (css->parent)
3278 		memcg->swappiness = val;
3279 	else
3280 		vm_swappiness = val;
3281 
3282 	return 0;
3283 }
3284 
3285 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
3286 {
3287 	struct mem_cgroup_threshold_ary *t;
3288 	unsigned long usage;
3289 	int i;
3290 
3291 	rcu_read_lock();
3292 	if (!swap)
3293 		t = rcu_dereference(memcg->thresholds.primary);
3294 	else
3295 		t = rcu_dereference(memcg->memsw_thresholds.primary);
3296 
3297 	if (!t)
3298 		goto unlock;
3299 
3300 	usage = mem_cgroup_usage(memcg, swap);
3301 
3302 	/*
3303 	 * current_threshold points to threshold just below or equal to usage.
3304 	 * If it's not true, a threshold was crossed after last
3305 	 * call of __mem_cgroup_threshold().
3306 	 */
3307 	i = t->current_threshold;
3308 
3309 	/*
3310 	 * Iterate backward over array of thresholds starting from
3311 	 * current_threshold and check if a threshold is crossed.
3312 	 * If none of thresholds below usage is crossed, we read
3313 	 * only one element of the array here.
3314 	 */
3315 	for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
3316 		eventfd_signal(t->entries[i].eventfd, 1);
3317 
3318 	/* i = current_threshold + 1 */
3319 	i++;
3320 
3321 	/*
3322 	 * Iterate forward over array of thresholds starting from
3323 	 * current_threshold+1 and check if a threshold is crossed.
3324 	 * If none of thresholds above usage is crossed, we read
3325 	 * only one element of the array here.
3326 	 */
3327 	for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
3328 		eventfd_signal(t->entries[i].eventfd, 1);
3329 
3330 	/* Update current_threshold */
3331 	t->current_threshold = i - 1;
3332 unlock:
3333 	rcu_read_unlock();
3334 }
3335 
3336 static void mem_cgroup_threshold(struct mem_cgroup *memcg)
3337 {
3338 	while (memcg) {
3339 		__mem_cgroup_threshold(memcg, false);
3340 		if (do_memsw_account())
3341 			__mem_cgroup_threshold(memcg, true);
3342 
3343 		memcg = parent_mem_cgroup(memcg);
3344 	}
3345 }
3346 
3347 static int compare_thresholds(const void *a, const void *b)
3348 {
3349 	const struct mem_cgroup_threshold *_a = a;
3350 	const struct mem_cgroup_threshold *_b = b;
3351 
3352 	if (_a->threshold > _b->threshold)
3353 		return 1;
3354 
3355 	if (_a->threshold < _b->threshold)
3356 		return -1;
3357 
3358 	return 0;
3359 }
3360 
3361 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
3362 {
3363 	struct mem_cgroup_eventfd_list *ev;
3364 
3365 	spin_lock(&memcg_oom_lock);
3366 
3367 	list_for_each_entry(ev, &memcg->oom_notify, list)
3368 		eventfd_signal(ev->eventfd, 1);
3369 
3370 	spin_unlock(&memcg_oom_lock);
3371 	return 0;
3372 }
3373 
3374 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
3375 {
3376 	struct mem_cgroup *iter;
3377 
3378 	for_each_mem_cgroup_tree(iter, memcg)
3379 		mem_cgroup_oom_notify_cb(iter);
3380 }
3381 
3382 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
3383 	struct eventfd_ctx *eventfd, const char *args, enum res_type type)
3384 {
3385 	struct mem_cgroup_thresholds *thresholds;
3386 	struct mem_cgroup_threshold_ary *new;
3387 	unsigned long threshold;
3388 	unsigned long usage;
3389 	int i, size, ret;
3390 
3391 	ret = page_counter_memparse(args, "-1", &threshold);
3392 	if (ret)
3393 		return ret;
3394 
3395 	mutex_lock(&memcg->thresholds_lock);
3396 
3397 	if (type == _MEM) {
3398 		thresholds = &memcg->thresholds;
3399 		usage = mem_cgroup_usage(memcg, false);
3400 	} else if (type == _MEMSWAP) {
3401 		thresholds = &memcg->memsw_thresholds;
3402 		usage = mem_cgroup_usage(memcg, true);
3403 	} else
3404 		BUG();
3405 
3406 	/* Check if a threshold crossed before adding a new one */
3407 	if (thresholds->primary)
3408 		__mem_cgroup_threshold(memcg, type == _MEMSWAP);
3409 
3410 	size = thresholds->primary ? thresholds->primary->size + 1 : 1;
3411 
3412 	/* Allocate memory for new array of thresholds */
3413 	new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
3414 			GFP_KERNEL);
3415 	if (!new) {
3416 		ret = -ENOMEM;
3417 		goto unlock;
3418 	}
3419 	new->size = size;
3420 
3421 	/* Copy thresholds (if any) to new array */
3422 	if (thresholds->primary) {
3423 		memcpy(new->entries, thresholds->primary->entries, (size - 1) *
3424 				sizeof(struct mem_cgroup_threshold));
3425 	}
3426 
3427 	/* Add new threshold */
3428 	new->entries[size - 1].eventfd = eventfd;
3429 	new->entries[size - 1].threshold = threshold;
3430 
3431 	/* Sort thresholds. Registering of new threshold isn't time-critical */
3432 	sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
3433 			compare_thresholds, NULL);
3434 
3435 	/* Find current threshold */
3436 	new->current_threshold = -1;
3437 	for (i = 0; i < size; i++) {
3438 		if (new->entries[i].threshold <= usage) {
3439 			/*
3440 			 * new->current_threshold will not be used until
3441 			 * rcu_assign_pointer(), so it's safe to increment
3442 			 * it here.
3443 			 */
3444 			++new->current_threshold;
3445 		} else
3446 			break;
3447 	}
3448 
3449 	/* Free old spare buffer and save old primary buffer as spare */
3450 	kfree(thresholds->spare);
3451 	thresholds->spare = thresholds->primary;
3452 
3453 	rcu_assign_pointer(thresholds->primary, new);
3454 
3455 	/* To be sure that nobody uses thresholds */
3456 	synchronize_rcu();
3457 
3458 unlock:
3459 	mutex_unlock(&memcg->thresholds_lock);
3460 
3461 	return ret;
3462 }
3463 
3464 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
3465 	struct eventfd_ctx *eventfd, const char *args)
3466 {
3467 	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
3468 }
3469 
3470 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
3471 	struct eventfd_ctx *eventfd, const char *args)
3472 {
3473 	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
3474 }
3475 
3476 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3477 	struct eventfd_ctx *eventfd, enum res_type type)
3478 {
3479 	struct mem_cgroup_thresholds *thresholds;
3480 	struct mem_cgroup_threshold_ary *new;
3481 	unsigned long usage;
3482 	int i, j, size;
3483 
3484 	mutex_lock(&memcg->thresholds_lock);
3485 
3486 	if (type == _MEM) {
3487 		thresholds = &memcg->thresholds;
3488 		usage = mem_cgroup_usage(memcg, false);
3489 	} else if (type == _MEMSWAP) {
3490 		thresholds = &memcg->memsw_thresholds;
3491 		usage = mem_cgroup_usage(memcg, true);
3492 	} else
3493 		BUG();
3494 
3495 	if (!thresholds->primary)
3496 		goto unlock;
3497 
3498 	/* Check if a threshold crossed before removing */
3499 	__mem_cgroup_threshold(memcg, type == _MEMSWAP);
3500 
3501 	/* Calculate new number of threshold */
3502 	size = 0;
3503 	for (i = 0; i < thresholds->primary->size; i++) {
3504 		if (thresholds->primary->entries[i].eventfd != eventfd)
3505 			size++;
3506 	}
3507 
3508 	new = thresholds->spare;
3509 
3510 	/* Set thresholds array to NULL if we don't have thresholds */
3511 	if (!size) {
3512 		kfree(new);
3513 		new = NULL;
3514 		goto swap_buffers;
3515 	}
3516 
3517 	new->size = size;
3518 
3519 	/* Copy thresholds and find current threshold */
3520 	new->current_threshold = -1;
3521 	for (i = 0, j = 0; i < thresholds->primary->size; i++) {
3522 		if (thresholds->primary->entries[i].eventfd == eventfd)
3523 			continue;
3524 
3525 		new->entries[j] = thresholds->primary->entries[i];
3526 		if (new->entries[j].threshold <= usage) {
3527 			/*
3528 			 * new->current_threshold will not be used
3529 			 * until rcu_assign_pointer(), so it's safe to increment
3530 			 * it here.
3531 			 */
3532 			++new->current_threshold;
3533 		}
3534 		j++;
3535 	}
3536 
3537 swap_buffers:
3538 	/* Swap primary and spare array */
3539 	thresholds->spare = thresholds->primary;
3540 
3541 	rcu_assign_pointer(thresholds->primary, new);
3542 
3543 	/* To be sure that nobody uses thresholds */
3544 	synchronize_rcu();
3545 
3546 	/* If all events are unregistered, free the spare array */
3547 	if (!new) {
3548 		kfree(thresholds->spare);
3549 		thresholds->spare = NULL;
3550 	}
3551 unlock:
3552 	mutex_unlock(&memcg->thresholds_lock);
3553 }
3554 
3555 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3556 	struct eventfd_ctx *eventfd)
3557 {
3558 	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
3559 }
3560 
3561 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3562 	struct eventfd_ctx *eventfd)
3563 {
3564 	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
3565 }
3566 
3567 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
3568 	struct eventfd_ctx *eventfd, const char *args)
3569 {
3570 	struct mem_cgroup_eventfd_list *event;
3571 
3572 	event = kmalloc(sizeof(*event),	GFP_KERNEL);
3573 	if (!event)
3574 		return -ENOMEM;
3575 
3576 	spin_lock(&memcg_oom_lock);
3577 
3578 	event->eventfd = eventfd;
3579 	list_add(&event->list, &memcg->oom_notify);
3580 
3581 	/* already in OOM ? */
3582 	if (memcg->under_oom)
3583 		eventfd_signal(eventfd, 1);
3584 	spin_unlock(&memcg_oom_lock);
3585 
3586 	return 0;
3587 }
3588 
3589 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
3590 	struct eventfd_ctx *eventfd)
3591 {
3592 	struct mem_cgroup_eventfd_list *ev, *tmp;
3593 
3594 	spin_lock(&memcg_oom_lock);
3595 
3596 	list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
3597 		if (ev->eventfd == eventfd) {
3598 			list_del(&ev->list);
3599 			kfree(ev);
3600 		}
3601 	}
3602 
3603 	spin_unlock(&memcg_oom_lock);
3604 }
3605 
3606 static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
3607 {
3608 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf));
3609 
3610 	seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
3611 	seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
3612 	seq_printf(sf, "oom_kill %lu\n", memcg_sum_events(memcg, OOM_KILL));
3613 	return 0;
3614 }
3615 
3616 static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
3617 	struct cftype *cft, u64 val)
3618 {
3619 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3620 
3621 	/* cannot set to root cgroup and only 0 and 1 are allowed */
3622 	if (!css->parent || !((val == 0) || (val == 1)))
3623 		return -EINVAL;
3624 
3625 	memcg->oom_kill_disable = val;
3626 	if (!val)
3627 		memcg_oom_recover(memcg);
3628 
3629 	return 0;
3630 }
3631 
3632 #ifdef CONFIG_CGROUP_WRITEBACK
3633 
3634 struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg)
3635 {
3636 	return &memcg->cgwb_list;
3637 }
3638 
3639 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
3640 {
3641 	return wb_domain_init(&memcg->cgwb_domain, gfp);
3642 }
3643 
3644 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
3645 {
3646 	wb_domain_exit(&memcg->cgwb_domain);
3647 }
3648 
3649 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
3650 {
3651 	wb_domain_size_changed(&memcg->cgwb_domain);
3652 }
3653 
3654 struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
3655 {
3656 	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
3657 
3658 	if (!memcg->css.parent)
3659 		return NULL;
3660 
3661 	return &memcg->cgwb_domain;
3662 }
3663 
3664 /**
3665  * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
3666  * @wb: bdi_writeback in question
3667  * @pfilepages: out parameter for number of file pages
3668  * @pheadroom: out parameter for number of allocatable pages according to memcg
3669  * @pdirty: out parameter for number of dirty pages
3670  * @pwriteback: out parameter for number of pages under writeback
3671  *
3672  * Determine the numbers of file, headroom, dirty, and writeback pages in
3673  * @wb's memcg.  File, dirty and writeback are self-explanatory.  Headroom
3674  * is a bit more involved.
3675  *
3676  * A memcg's headroom is "min(max, high) - used".  In the hierarchy, the
3677  * headroom is calculated as the lowest headroom of itself and the
3678  * ancestors.  Note that this doesn't consider the actual amount of
3679  * available memory in the system.  The caller should further cap
3680  * *@pheadroom accordingly.
3681  */
3682 void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
3683 			 unsigned long *pheadroom, unsigned long *pdirty,
3684 			 unsigned long *pwriteback)
3685 {
3686 	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
3687 	struct mem_cgroup *parent;
3688 
3689 	*pdirty = memcg_page_state(memcg, NR_FILE_DIRTY);
3690 
3691 	/* this should eventually include NR_UNSTABLE_NFS */
3692 	*pwriteback = memcg_page_state(memcg, NR_WRITEBACK);
3693 	*pfilepages = mem_cgroup_nr_lru_pages(memcg, (1 << LRU_INACTIVE_FILE) |
3694 						     (1 << LRU_ACTIVE_FILE));
3695 	*pheadroom = PAGE_COUNTER_MAX;
3696 
3697 	while ((parent = parent_mem_cgroup(memcg))) {
3698 		unsigned long ceiling = min(memcg->memory.limit, memcg->high);
3699 		unsigned long used = page_counter_read(&memcg->memory);
3700 
3701 		*pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
3702 		memcg = parent;
3703 	}
3704 }
3705 
3706 #else	/* CONFIG_CGROUP_WRITEBACK */
3707 
3708 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
3709 {
3710 	return 0;
3711 }
3712 
3713 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
3714 {
3715 }
3716 
3717 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
3718 {
3719 }
3720 
3721 #endif	/* CONFIG_CGROUP_WRITEBACK */
3722 
3723 /*
3724  * DO NOT USE IN NEW FILES.
3725  *
3726  * "cgroup.event_control" implementation.
3727  *
3728  * This is way over-engineered.  It tries to support fully configurable
3729  * events for each user.  Such level of flexibility is completely
3730  * unnecessary especially in the light of the planned unified hierarchy.
3731  *
3732  * Please deprecate this and replace with something simpler if at all
3733  * possible.
3734  */
3735 
3736 /*
3737  * Unregister event and free resources.
3738  *
3739  * Gets called from workqueue.
3740  */
3741 static void memcg_event_remove(struct work_struct *work)
3742 {
3743 	struct mem_cgroup_event *event =
3744 		container_of(work, struct mem_cgroup_event, remove);
3745 	struct mem_cgroup *memcg = event->memcg;
3746 
3747 	remove_wait_queue(event->wqh, &event->wait);
3748 
3749 	event->unregister_event(memcg, event->eventfd);
3750 
3751 	/* Notify userspace the event is going away. */
3752 	eventfd_signal(event->eventfd, 1);
3753 
3754 	eventfd_ctx_put(event->eventfd);
3755 	kfree(event);
3756 	css_put(&memcg->css);
3757 }
3758 
3759 /*
3760  * Gets called on POLLHUP on eventfd when user closes it.
3761  *
3762  * Called with wqh->lock held and interrupts disabled.
3763  */
3764 static int memcg_event_wake(wait_queue_entry_t *wait, unsigned mode,
3765 			    int sync, void *key)
3766 {
3767 	struct mem_cgroup_event *event =
3768 		container_of(wait, struct mem_cgroup_event, wait);
3769 	struct mem_cgroup *memcg = event->memcg;
3770 	unsigned long flags = (unsigned long)key;
3771 
3772 	if (flags & POLLHUP) {
3773 		/*
3774 		 * If the event has been detached at cgroup removal, we
3775 		 * can simply return knowing the other side will cleanup
3776 		 * for us.
3777 		 *
3778 		 * We can't race against event freeing since the other
3779 		 * side will require wqh->lock via remove_wait_queue(),
3780 		 * which we hold.
3781 		 */
3782 		spin_lock(&memcg->event_list_lock);
3783 		if (!list_empty(&event->list)) {
3784 			list_del_init(&event->list);
3785 			/*
3786 			 * We are in atomic context, but cgroup_event_remove()
3787 			 * may sleep, so we have to call it in workqueue.
3788 			 */
3789 			schedule_work(&event->remove);
3790 		}
3791 		spin_unlock(&memcg->event_list_lock);
3792 	}
3793 
3794 	return 0;
3795 }
3796 
3797 static void memcg_event_ptable_queue_proc(struct file *file,
3798 		wait_queue_head_t *wqh, poll_table *pt)
3799 {
3800 	struct mem_cgroup_event *event =
3801 		container_of(pt, struct mem_cgroup_event, pt);
3802 
3803 	event->wqh = wqh;
3804 	add_wait_queue(wqh, &event->wait);
3805 }
3806 
3807 /*
3808  * DO NOT USE IN NEW FILES.
3809  *
3810  * Parse input and register new cgroup event handler.
3811  *
3812  * Input must be in format '<event_fd> <control_fd> <args>'.
3813  * Interpretation of args is defined by control file implementation.
3814  */
3815 static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
3816 					 char *buf, size_t nbytes, loff_t off)
3817 {
3818 	struct cgroup_subsys_state *css = of_css(of);
3819 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3820 	struct mem_cgroup_event *event;
3821 	struct cgroup_subsys_state *cfile_css;
3822 	unsigned int efd, cfd;
3823 	struct fd efile;
3824 	struct fd cfile;
3825 	const char *name;
3826 	char *endp;
3827 	int ret;
3828 
3829 	buf = strstrip(buf);
3830 
3831 	efd = simple_strtoul(buf, &endp, 10);
3832 	if (*endp != ' ')
3833 		return -EINVAL;
3834 	buf = endp + 1;
3835 
3836 	cfd = simple_strtoul(buf, &endp, 10);
3837 	if ((*endp != ' ') && (*endp != '\0'))
3838 		return -EINVAL;
3839 	buf = endp + 1;
3840 
3841 	event = kzalloc(sizeof(*event), GFP_KERNEL);
3842 	if (!event)
3843 		return -ENOMEM;
3844 
3845 	event->memcg = memcg;
3846 	INIT_LIST_HEAD(&event->list);
3847 	init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
3848 	init_waitqueue_func_entry(&event->wait, memcg_event_wake);
3849 	INIT_WORK(&event->remove, memcg_event_remove);
3850 
3851 	efile = fdget(efd);
3852 	if (!efile.file) {
3853 		ret = -EBADF;
3854 		goto out_kfree;
3855 	}
3856 
3857 	event->eventfd = eventfd_ctx_fileget(efile.file);
3858 	if (IS_ERR(event->eventfd)) {
3859 		ret = PTR_ERR(event->eventfd);
3860 		goto out_put_efile;
3861 	}
3862 
3863 	cfile = fdget(cfd);
3864 	if (!cfile.file) {
3865 		ret = -EBADF;
3866 		goto out_put_eventfd;
3867 	}
3868 
3869 	/* the process need read permission on control file */
3870 	/* AV: shouldn't we check that it's been opened for read instead? */
3871 	ret = inode_permission(file_inode(cfile.file), MAY_READ);
3872 	if (ret < 0)
3873 		goto out_put_cfile;
3874 
3875 	/*
3876 	 * Determine the event callbacks and set them in @event.  This used
3877 	 * to be done via struct cftype but cgroup core no longer knows
3878 	 * about these events.  The following is crude but the whole thing
3879 	 * is for compatibility anyway.
3880 	 *
3881 	 * DO NOT ADD NEW FILES.
3882 	 */
3883 	name = cfile.file->f_path.dentry->d_name.name;
3884 
3885 	if (!strcmp(name, "memory.usage_in_bytes")) {
3886 		event->register_event = mem_cgroup_usage_register_event;
3887 		event->unregister_event = mem_cgroup_usage_unregister_event;
3888 	} else if (!strcmp(name, "memory.oom_control")) {
3889 		event->register_event = mem_cgroup_oom_register_event;
3890 		event->unregister_event = mem_cgroup_oom_unregister_event;
3891 	} else if (!strcmp(name, "memory.pressure_level")) {
3892 		event->register_event = vmpressure_register_event;
3893 		event->unregister_event = vmpressure_unregister_event;
3894 	} else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
3895 		event->register_event = memsw_cgroup_usage_register_event;
3896 		event->unregister_event = memsw_cgroup_usage_unregister_event;
3897 	} else {
3898 		ret = -EINVAL;
3899 		goto out_put_cfile;
3900 	}
3901 
3902 	/*
3903 	 * Verify @cfile should belong to @css.  Also, remaining events are
3904 	 * automatically removed on cgroup destruction but the removal is
3905 	 * asynchronous, so take an extra ref on @css.
3906 	 */
3907 	cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,
3908 					       &memory_cgrp_subsys);
3909 	ret = -EINVAL;
3910 	if (IS_ERR(cfile_css))
3911 		goto out_put_cfile;
3912 	if (cfile_css != css) {
3913 		css_put(cfile_css);
3914 		goto out_put_cfile;
3915 	}
3916 
3917 	ret = event->register_event(memcg, event->eventfd, buf);
3918 	if (ret)
3919 		goto out_put_css;
3920 
3921 	efile.file->f_op->poll(efile.file, &event->pt);
3922 
3923 	spin_lock(&memcg->event_list_lock);
3924 	list_add(&event->list, &memcg->event_list);
3925 	spin_unlock(&memcg->event_list_lock);
3926 
3927 	fdput(cfile);
3928 	fdput(efile);
3929 
3930 	return nbytes;
3931 
3932 out_put_css:
3933 	css_put(css);
3934 out_put_cfile:
3935 	fdput(cfile);
3936 out_put_eventfd:
3937 	eventfd_ctx_put(event->eventfd);
3938 out_put_efile:
3939 	fdput(efile);
3940 out_kfree:
3941 	kfree(event);
3942 
3943 	return ret;
3944 }
3945 
3946 static struct cftype mem_cgroup_legacy_files[] = {
3947 	{
3948 		.name = "usage_in_bytes",
3949 		.private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
3950 		.read_u64 = mem_cgroup_read_u64,
3951 	},
3952 	{
3953 		.name = "max_usage_in_bytes",
3954 		.private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
3955 		.write = mem_cgroup_reset,
3956 		.read_u64 = mem_cgroup_read_u64,
3957 	},
3958 	{
3959 		.name = "limit_in_bytes",
3960 		.private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
3961 		.write = mem_cgroup_write,
3962 		.read_u64 = mem_cgroup_read_u64,
3963 	},
3964 	{
3965 		.name = "soft_limit_in_bytes",
3966 		.private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
3967 		.write = mem_cgroup_write,
3968 		.read_u64 = mem_cgroup_read_u64,
3969 	},
3970 	{
3971 		.name = "failcnt",
3972 		.private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
3973 		.write = mem_cgroup_reset,
3974 		.read_u64 = mem_cgroup_read_u64,
3975 	},
3976 	{
3977 		.name = "stat",
3978 		.seq_show = memcg_stat_show,
3979 	},
3980 	{
3981 		.name = "force_empty",
3982 		.write = mem_cgroup_force_empty_write,
3983 	},
3984 	{
3985 		.name = "use_hierarchy",
3986 		.write_u64 = mem_cgroup_hierarchy_write,
3987 		.read_u64 = mem_cgroup_hierarchy_read,
3988 	},
3989 	{
3990 		.name = "cgroup.event_control",		/* XXX: for compat */
3991 		.write = memcg_write_event_control,
3992 		.flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
3993 	},
3994 	{
3995 		.name = "swappiness",
3996 		.read_u64 = mem_cgroup_swappiness_read,
3997 		.write_u64 = mem_cgroup_swappiness_write,
3998 	},
3999 	{
4000 		.name = "move_charge_at_immigrate",
4001 		.read_u64 = mem_cgroup_move_charge_read,
4002 		.write_u64 = mem_cgroup_move_charge_write,
4003 	},
4004 	{
4005 		.name = "oom_control",
4006 		.seq_show = mem_cgroup_oom_control_read,
4007 		.write_u64 = mem_cgroup_oom_control_write,
4008 		.private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
4009 	},
4010 	{
4011 		.name = "pressure_level",
4012 	},
4013 #ifdef CONFIG_NUMA
4014 	{
4015 		.name = "numa_stat",
4016 		.seq_show = memcg_numa_stat_show,
4017 	},
4018 #endif
4019 	{
4020 		.name = "kmem.limit_in_bytes",
4021 		.private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
4022 		.write = mem_cgroup_write,
4023 		.read_u64 = mem_cgroup_read_u64,
4024 	},
4025 	{
4026 		.name = "kmem.usage_in_bytes",
4027 		.private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
4028 		.read_u64 = mem_cgroup_read_u64,
4029 	},
4030 	{
4031 		.name = "kmem.failcnt",
4032 		.private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
4033 		.write = mem_cgroup_reset,
4034 		.read_u64 = mem_cgroup_read_u64,
4035 	},
4036 	{
4037 		.name = "kmem.max_usage_in_bytes",
4038 		.private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
4039 		.write = mem_cgroup_reset,
4040 		.read_u64 = mem_cgroup_read_u64,
4041 	},
4042 #ifdef CONFIG_SLABINFO
4043 	{
4044 		.name = "kmem.slabinfo",
4045 		.seq_start = memcg_slab_start,
4046 		.seq_next = memcg_slab_next,
4047 		.seq_stop = memcg_slab_stop,
4048 		.seq_show = memcg_slab_show,
4049 	},
4050 #endif
4051 	{
4052 		.name = "kmem.tcp.limit_in_bytes",
4053 		.private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
4054 		.write = mem_cgroup_write,
4055 		.read_u64 = mem_cgroup_read_u64,
4056 	},
4057 	{
4058 		.name = "kmem.tcp.usage_in_bytes",
4059 		.private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
4060 		.read_u64 = mem_cgroup_read_u64,
4061 	},
4062 	{
4063 		.name = "kmem.tcp.failcnt",
4064 		.private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
4065 		.write = mem_cgroup_reset,
4066 		.read_u64 = mem_cgroup_read_u64,
4067 	},
4068 	{
4069 		.name = "kmem.tcp.max_usage_in_bytes",
4070 		.private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
4071 		.write = mem_cgroup_reset,
4072 		.read_u64 = mem_cgroup_read_u64,
4073 	},
4074 	{ },	/* terminate */
4075 };
4076 
4077 /*
4078  * Private memory cgroup IDR
4079  *
4080  * Swap-out records and page cache shadow entries need to store memcg
4081  * references in constrained space, so we maintain an ID space that is
4082  * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of
4083  * memory-controlled cgroups to 64k.
4084  *
4085  * However, there usually are many references to the oflline CSS after
4086  * the cgroup has been destroyed, such as page cache or reclaimable
4087  * slab objects, that don't need to hang on to the ID. We want to keep
4088  * those dead CSS from occupying IDs, or we might quickly exhaust the
4089  * relatively small ID space and prevent the creation of new cgroups
4090  * even when there are much fewer than 64k cgroups - possibly none.
4091  *
4092  * Maintain a private 16-bit ID space for memcg, and allow the ID to
4093  * be freed and recycled when it's no longer needed, which is usually
4094  * when the CSS is offlined.
4095  *
4096  * The only exception to that are records of swapped out tmpfs/shmem
4097  * pages that need to be attributed to live ancestors on swapin. But
4098  * those references are manageable from userspace.
4099  */
4100 
4101 static DEFINE_IDR(mem_cgroup_idr);
4102 
4103 static void mem_cgroup_id_get_many(struct mem_cgroup *memcg, unsigned int n)
4104 {
4105 	VM_BUG_ON(atomic_read(&memcg->id.ref) <= 0);
4106 	atomic_add(n, &memcg->id.ref);
4107 }
4108 
4109 static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
4110 {
4111 	VM_BUG_ON(atomic_read(&memcg->id.ref) < n);
4112 	if (atomic_sub_and_test(n, &memcg->id.ref)) {
4113 		idr_remove(&mem_cgroup_idr, memcg->id.id);
4114 		memcg->id.id = 0;
4115 
4116 		/* Memcg ID pins CSS */
4117 		css_put(&memcg->css);
4118 	}
4119 }
4120 
4121 static inline void mem_cgroup_id_get(struct mem_cgroup *memcg)
4122 {
4123 	mem_cgroup_id_get_many(memcg, 1);
4124 }
4125 
4126 static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
4127 {
4128 	mem_cgroup_id_put_many(memcg, 1);
4129 }
4130 
4131 /**
4132  * mem_cgroup_from_id - look up a memcg from a memcg id
4133  * @id: the memcg id to look up
4134  *
4135  * Caller must hold rcu_read_lock().
4136  */
4137 struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
4138 {
4139 	WARN_ON_ONCE(!rcu_read_lock_held());
4140 	return idr_find(&mem_cgroup_idr, id);
4141 }
4142 
4143 static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
4144 {
4145 	struct mem_cgroup_per_node *pn;
4146 	int tmp = node;
4147 	/*
4148 	 * This routine is called against possible nodes.
4149 	 * But it's BUG to call kmalloc() against offline node.
4150 	 *
4151 	 * TODO: this routine can waste much memory for nodes which will
4152 	 *       never be onlined. It's better to use memory hotplug callback
4153 	 *       function.
4154 	 */
4155 	if (!node_state(node, N_NORMAL_MEMORY))
4156 		tmp = -1;
4157 	pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
4158 	if (!pn)
4159 		return 1;
4160 
4161 	pn->lruvec_stat = alloc_percpu(struct lruvec_stat);
4162 	if (!pn->lruvec_stat) {
4163 		kfree(pn);
4164 		return 1;
4165 	}
4166 
4167 	lruvec_init(&pn->lruvec);
4168 	pn->usage_in_excess = 0;
4169 	pn->on_tree = false;
4170 	pn->memcg = memcg;
4171 
4172 	memcg->nodeinfo[node] = pn;
4173 	return 0;
4174 }
4175 
4176 static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
4177 {
4178 	struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
4179 
4180 	free_percpu(pn->lruvec_stat);
4181 	kfree(pn);
4182 }
4183 
4184 static void __mem_cgroup_free(struct mem_cgroup *memcg)
4185 {
4186 	int node;
4187 
4188 	for_each_node(node)
4189 		free_mem_cgroup_per_node_info(memcg, node);
4190 	free_percpu(memcg->stat);
4191 	kfree(memcg);
4192 }
4193 
4194 static void mem_cgroup_free(struct mem_cgroup *memcg)
4195 {
4196 	memcg_wb_domain_exit(memcg);
4197 	__mem_cgroup_free(memcg);
4198 }
4199 
4200 static struct mem_cgroup *mem_cgroup_alloc(void)
4201 {
4202 	struct mem_cgroup *memcg;
4203 	size_t size;
4204 	int node;
4205 
4206 	size = sizeof(struct mem_cgroup);
4207 	size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);
4208 
4209 	memcg = kzalloc(size, GFP_KERNEL);
4210 	if (!memcg)
4211 		return NULL;
4212 
4213 	memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
4214 				 1, MEM_CGROUP_ID_MAX,
4215 				 GFP_KERNEL);
4216 	if (memcg->id.id < 0)
4217 		goto fail;
4218 
4219 	memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
4220 	if (!memcg->stat)
4221 		goto fail;
4222 
4223 	for_each_node(node)
4224 		if (alloc_mem_cgroup_per_node_info(memcg, node))
4225 			goto fail;
4226 
4227 	if (memcg_wb_domain_init(memcg, GFP_KERNEL))
4228 		goto fail;
4229 
4230 	INIT_WORK(&memcg->high_work, high_work_func);
4231 	memcg->last_scanned_node = MAX_NUMNODES;
4232 	INIT_LIST_HEAD(&memcg->oom_notify);
4233 	mutex_init(&memcg->thresholds_lock);
4234 	spin_lock_init(&memcg->move_lock);
4235 	vmpressure_init(&memcg->vmpressure);
4236 	INIT_LIST_HEAD(&memcg->event_list);
4237 	spin_lock_init(&memcg->event_list_lock);
4238 	memcg->socket_pressure = jiffies;
4239 #ifndef CONFIG_SLOB
4240 	memcg->kmemcg_id = -1;
4241 #endif
4242 #ifdef CONFIG_CGROUP_WRITEBACK
4243 	INIT_LIST_HEAD(&memcg->cgwb_list);
4244 #endif
4245 	idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
4246 	return memcg;
4247 fail:
4248 	if (memcg->id.id > 0)
4249 		idr_remove(&mem_cgroup_idr, memcg->id.id);
4250 	__mem_cgroup_free(memcg);
4251 	return NULL;
4252 }
4253 
4254 static struct cgroup_subsys_state * __ref
4255 mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
4256 {
4257 	struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
4258 	struct mem_cgroup *memcg;
4259 	long error = -ENOMEM;
4260 
4261 	memcg = mem_cgroup_alloc();
4262 	if (!memcg)
4263 		return ERR_PTR(error);
4264 
4265 	memcg->high = PAGE_COUNTER_MAX;
4266 	memcg->soft_limit = PAGE_COUNTER_MAX;
4267 	if (parent) {
4268 		memcg->swappiness = mem_cgroup_swappiness(parent);
4269 		memcg->oom_kill_disable = parent->oom_kill_disable;
4270 	}
4271 	if (parent && parent->use_hierarchy) {
4272 		memcg->use_hierarchy = true;
4273 		page_counter_init(&memcg->memory, &parent->memory);
4274 		page_counter_init(&memcg->swap, &parent->swap);
4275 		page_counter_init(&memcg->memsw, &parent->memsw);
4276 		page_counter_init(&memcg->kmem, &parent->kmem);
4277 		page_counter_init(&memcg->tcpmem, &parent->tcpmem);
4278 	} else {
4279 		page_counter_init(&memcg->memory, NULL);
4280 		page_counter_init(&memcg->swap, NULL);
4281 		page_counter_init(&memcg->memsw, NULL);
4282 		page_counter_init(&memcg->kmem, NULL);
4283 		page_counter_init(&memcg->tcpmem, NULL);
4284 		/*
4285 		 * Deeper hierachy with use_hierarchy == false doesn't make
4286 		 * much sense so let cgroup subsystem know about this
4287 		 * unfortunate state in our controller.
4288 		 */
4289 		if (parent != root_mem_cgroup)
4290 			memory_cgrp_subsys.broken_hierarchy = true;
4291 	}
4292 
4293 	/* The following stuff does not apply to the root */
4294 	if (!parent) {
4295 		root_mem_cgroup = memcg;
4296 		return &memcg->css;
4297 	}
4298 
4299 	error = memcg_online_kmem(memcg);
4300 	if (error)
4301 		goto fail;
4302 
4303 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
4304 		static_branch_inc(&memcg_sockets_enabled_key);
4305 
4306 	return &memcg->css;
4307 fail:
4308 	mem_cgroup_free(memcg);
4309 	return ERR_PTR(-ENOMEM);
4310 }
4311 
4312 static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
4313 {
4314 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4315 
4316 	/* Online state pins memcg ID, memcg ID pins CSS */
4317 	atomic_set(&memcg->id.ref, 1);
4318 	css_get(css);
4319 	return 0;
4320 }
4321 
4322 static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
4323 {
4324 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4325 	struct mem_cgroup_event *event, *tmp;
4326 
4327 	/*
4328 	 * Unregister events and notify userspace.
4329 	 * Notify userspace about cgroup removing only after rmdir of cgroup
4330 	 * directory to avoid race between userspace and kernelspace.
4331 	 */
4332 	spin_lock(&memcg->event_list_lock);
4333 	list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
4334 		list_del_init(&event->list);
4335 		schedule_work(&event->remove);
4336 	}
4337 	spin_unlock(&memcg->event_list_lock);
4338 
4339 	memcg->low = 0;
4340 
4341 	memcg_offline_kmem(memcg);
4342 	wb_memcg_offline(memcg);
4343 
4344 	mem_cgroup_id_put(memcg);
4345 }
4346 
4347 static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
4348 {
4349 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4350 
4351 	invalidate_reclaim_iterators(memcg);
4352 }
4353 
4354 static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
4355 {
4356 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4357 
4358 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
4359 		static_branch_dec(&memcg_sockets_enabled_key);
4360 
4361 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
4362 		static_branch_dec(&memcg_sockets_enabled_key);
4363 
4364 	vmpressure_cleanup(&memcg->vmpressure);
4365 	cancel_work_sync(&memcg->high_work);
4366 	mem_cgroup_remove_from_trees(memcg);
4367 	memcg_free_kmem(memcg);
4368 	mem_cgroup_free(memcg);
4369 }
4370 
4371 /**
4372  * mem_cgroup_css_reset - reset the states of a mem_cgroup
4373  * @css: the target css
4374  *
4375  * Reset the states of the mem_cgroup associated with @css.  This is
4376  * invoked when the userland requests disabling on the default hierarchy
4377  * but the memcg is pinned through dependency.  The memcg should stop
4378  * applying policies and should revert to the vanilla state as it may be
4379  * made visible again.
4380  *
4381  * The current implementation only resets the essential configurations.
4382  * This needs to be expanded to cover all the visible parts.
4383  */
4384 static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
4385 {
4386 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4387 
4388 	page_counter_limit(&memcg->memory, PAGE_COUNTER_MAX);
4389 	page_counter_limit(&memcg->swap, PAGE_COUNTER_MAX);
4390 	page_counter_limit(&memcg->memsw, PAGE_COUNTER_MAX);
4391 	page_counter_limit(&memcg->kmem, PAGE_COUNTER_MAX);
4392 	page_counter_limit(&memcg->tcpmem, PAGE_COUNTER_MAX);
4393 	memcg->low = 0;
4394 	memcg->high = PAGE_COUNTER_MAX;
4395 	memcg->soft_limit = PAGE_COUNTER_MAX;
4396 	memcg_wb_domain_size_changed(memcg);
4397 }
4398 
4399 #ifdef CONFIG_MMU
4400 /* Handlers for move charge at task migration. */
4401 static int mem_cgroup_do_precharge(unsigned long count)
4402 {
4403 	int ret;
4404 
4405 	/* Try a single bulk charge without reclaim first, kswapd may wake */
4406 	ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
4407 	if (!ret) {
4408 		mc.precharge += count;
4409 		return ret;
4410 	}
4411 
4412 	/* Try charges one by one with reclaim, but do not retry */
4413 	while (count--) {
4414 		ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
4415 		if (ret)
4416 			return ret;
4417 		mc.precharge++;
4418 		cond_resched();
4419 	}
4420 	return 0;
4421 }
4422 
4423 union mc_target {
4424 	struct page	*page;
4425 	swp_entry_t	ent;
4426 };
4427 
4428 enum mc_target_type {
4429 	MC_TARGET_NONE = 0,
4430 	MC_TARGET_PAGE,
4431 	MC_TARGET_SWAP,
4432 	MC_TARGET_DEVICE,
4433 };
4434 
4435 static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
4436 						unsigned long addr, pte_t ptent)
4437 {
4438 	struct page *page = _vm_normal_page(vma, addr, ptent, true);
4439 
4440 	if (!page || !page_mapped(page))
4441 		return NULL;
4442 	if (PageAnon(page)) {
4443 		if (!(mc.flags & MOVE_ANON))
4444 			return NULL;
4445 	} else {
4446 		if (!(mc.flags & MOVE_FILE))
4447 			return NULL;
4448 	}
4449 	if (!get_page_unless_zero(page))
4450 		return NULL;
4451 
4452 	return page;
4453 }
4454 
4455 #if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE)
4456 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4457 			pte_t ptent, swp_entry_t *entry)
4458 {
4459 	struct page *page = NULL;
4460 	swp_entry_t ent = pte_to_swp_entry(ptent);
4461 
4462 	if (!(mc.flags & MOVE_ANON) || non_swap_entry(ent))
4463 		return NULL;
4464 
4465 	/*
4466 	 * Handle MEMORY_DEVICE_PRIVATE which are ZONE_DEVICE page belonging to
4467 	 * a device and because they are not accessible by CPU they are store
4468 	 * as special swap entry in the CPU page table.
4469 	 */
4470 	if (is_device_private_entry(ent)) {
4471 		page = device_private_entry_to_page(ent);
4472 		/*
4473 		 * MEMORY_DEVICE_PRIVATE means ZONE_DEVICE page and which have
4474 		 * a refcount of 1 when free (unlike normal page)
4475 		 */
4476 		if (!page_ref_add_unless(page, 1, 1))
4477 			return NULL;
4478 		return page;
4479 	}
4480 
4481 	/*
4482 	 * Because lookup_swap_cache() updates some statistics counter,
4483 	 * we call find_get_page() with swapper_space directly.
4484 	 */
4485 	page = find_get_page(swap_address_space(ent), swp_offset(ent));
4486 	if (do_memsw_account())
4487 		entry->val = ent.val;
4488 
4489 	return page;
4490 }
4491 #else
4492 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4493 			pte_t ptent, swp_entry_t *entry)
4494 {
4495 	return NULL;
4496 }
4497 #endif
4498 
4499 static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
4500 			unsigned long addr, pte_t ptent, swp_entry_t *entry)
4501 {
4502 	struct page *page = NULL;
4503 	struct address_space *mapping;
4504 	pgoff_t pgoff;
4505 
4506 	if (!vma->vm_file) /* anonymous vma */
4507 		return NULL;
4508 	if (!(mc.flags & MOVE_FILE))
4509 		return NULL;
4510 
4511 	mapping = vma->vm_file->f_mapping;
4512 	pgoff = linear_page_index(vma, addr);
4513 
4514 	/* page is moved even if it's not RSS of this task(page-faulted). */
4515 #ifdef CONFIG_SWAP
4516 	/* shmem/tmpfs may report page out on swap: account for that too. */
4517 	if (shmem_mapping(mapping)) {
4518 		page = find_get_entry(mapping, pgoff);
4519 		if (radix_tree_exceptional_entry(page)) {
4520 			swp_entry_t swp = radix_to_swp_entry(page);
4521 			if (do_memsw_account())
4522 				*entry = swp;
4523 			page = find_get_page(swap_address_space(swp),
4524 					     swp_offset(swp));
4525 		}
4526 	} else
4527 		page = find_get_page(mapping, pgoff);
4528 #else
4529 	page = find_get_page(mapping, pgoff);
4530 #endif
4531 	return page;
4532 }
4533 
4534 /**
4535  * mem_cgroup_move_account - move account of the page
4536  * @page: the page
4537  * @compound: charge the page as compound or small page
4538  * @from: mem_cgroup which the page is moved from.
4539  * @to:	mem_cgroup which the page is moved to. @from != @to.
4540  *
4541  * The caller must make sure the page is not on LRU (isolate_page() is useful.)
4542  *
4543  * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
4544  * from old cgroup.
4545  */
4546 static int mem_cgroup_move_account(struct page *page,
4547 				   bool compound,
4548 				   struct mem_cgroup *from,
4549 				   struct mem_cgroup *to)
4550 {
4551 	unsigned long flags;
4552 	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
4553 	int ret;
4554 	bool anon;
4555 
4556 	VM_BUG_ON(from == to);
4557 	VM_BUG_ON_PAGE(PageLRU(page), page);
4558 	VM_BUG_ON(compound && !PageTransHuge(page));
4559 
4560 	/*
4561 	 * Prevent mem_cgroup_migrate() from looking at
4562 	 * page->mem_cgroup of its source page while we change it.
4563 	 */
4564 	ret = -EBUSY;
4565 	if (!trylock_page(page))
4566 		goto out;
4567 
4568 	ret = -EINVAL;
4569 	if (page->mem_cgroup != from)
4570 		goto out_unlock;
4571 
4572 	anon = PageAnon(page);
4573 
4574 	spin_lock_irqsave(&from->move_lock, flags);
4575 
4576 	if (!anon && page_mapped(page)) {
4577 		__this_cpu_sub(from->stat->count[NR_FILE_MAPPED], nr_pages);
4578 		__this_cpu_add(to->stat->count[NR_FILE_MAPPED], nr_pages);
4579 	}
4580 
4581 	/*
4582 	 * move_lock grabbed above and caller set from->moving_account, so
4583 	 * mod_memcg_page_state will serialize updates to PageDirty.
4584 	 * So mapping should be stable for dirty pages.
4585 	 */
4586 	if (!anon && PageDirty(page)) {
4587 		struct address_space *mapping = page_mapping(page);
4588 
4589 		if (mapping_cap_account_dirty(mapping)) {
4590 			__this_cpu_sub(from->stat->count[NR_FILE_DIRTY],
4591 				       nr_pages);
4592 			__this_cpu_add(to->stat->count[NR_FILE_DIRTY],
4593 				       nr_pages);
4594 		}
4595 	}
4596 
4597 	if (PageWriteback(page)) {
4598 		__this_cpu_sub(from->stat->count[NR_WRITEBACK], nr_pages);
4599 		__this_cpu_add(to->stat->count[NR_WRITEBACK], nr_pages);
4600 	}
4601 
4602 	/*
4603 	 * It is safe to change page->mem_cgroup here because the page
4604 	 * is referenced, charged, and isolated - we can't race with
4605 	 * uncharging, charging, migration, or LRU putback.
4606 	 */
4607 
4608 	/* caller should have done css_get */
4609 	page->mem_cgroup = to;
4610 	spin_unlock_irqrestore(&from->move_lock, flags);
4611 
4612 	ret = 0;
4613 
4614 	local_irq_disable();
4615 	mem_cgroup_charge_statistics(to, page, compound, nr_pages);
4616 	memcg_check_events(to, page);
4617 	mem_cgroup_charge_statistics(from, page, compound, -nr_pages);
4618 	memcg_check_events(from, page);
4619 	local_irq_enable();
4620 out_unlock:
4621 	unlock_page(page);
4622 out:
4623 	return ret;
4624 }
4625 
4626 /**
4627  * get_mctgt_type - get target type of moving charge
4628  * @vma: the vma the pte to be checked belongs
4629  * @addr: the address corresponding to the pte to be checked
4630  * @ptent: the pte to be checked
4631  * @target: the pointer the target page or swap ent will be stored(can be NULL)
4632  *
4633  * Returns
4634  *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
4635  *   1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
4636  *     move charge. if @target is not NULL, the page is stored in target->page
4637  *     with extra refcnt got(Callers should handle it).
4638  *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
4639  *     target for charge migration. if @target is not NULL, the entry is stored
4640  *     in target->ent.
4641  *   3(MC_TARGET_DEVICE): like MC_TARGET_PAGE  but page is MEMORY_DEVICE_PUBLIC
4642  *     or MEMORY_DEVICE_PRIVATE (so ZONE_DEVICE page and thus not on the lru).
4643  *     For now we such page is charge like a regular page would be as for all
4644  *     intent and purposes it is just special memory taking the place of a
4645  *     regular page.
4646  *
4647  *     See Documentations/vm/hmm.txt and include/linux/hmm.h
4648  *
4649  * Called with pte lock held.
4650  */
4651 
4652 static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
4653 		unsigned long addr, pte_t ptent, union mc_target *target)
4654 {
4655 	struct page *page = NULL;
4656 	enum mc_target_type ret = MC_TARGET_NONE;
4657 	swp_entry_t ent = { .val = 0 };
4658 
4659 	if (pte_present(ptent))
4660 		page = mc_handle_present_pte(vma, addr, ptent);
4661 	else if (is_swap_pte(ptent))
4662 		page = mc_handle_swap_pte(vma, ptent, &ent);
4663 	else if (pte_none(ptent))
4664 		page = mc_handle_file_pte(vma, addr, ptent, &ent);
4665 
4666 	if (!page && !ent.val)
4667 		return ret;
4668 	if (page) {
4669 		/*
4670 		 * Do only loose check w/o serialization.
4671 		 * mem_cgroup_move_account() checks the page is valid or
4672 		 * not under LRU exclusion.
4673 		 */
4674 		if (page->mem_cgroup == mc.from) {
4675 			ret = MC_TARGET_PAGE;
4676 			if (is_device_private_page(page) ||
4677 			    is_device_public_page(page))
4678 				ret = MC_TARGET_DEVICE;
4679 			if (target)
4680 				target->page = page;
4681 		}
4682 		if (!ret || !target)
4683 			put_page(page);
4684 	}
4685 	/*
4686 	 * There is a swap entry and a page doesn't exist or isn't charged.
4687 	 * But we cannot move a tail-page in a THP.
4688 	 */
4689 	if (ent.val && !ret && (!page || !PageTransCompound(page)) &&
4690 	    mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
4691 		ret = MC_TARGET_SWAP;
4692 		if (target)
4693 			target->ent = ent;
4694 	}
4695 	return ret;
4696 }
4697 
4698 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4699 /*
4700  * We don't consider PMD mapped swapping or file mapped pages because THP does
4701  * not support them for now.
4702  * Caller should make sure that pmd_trans_huge(pmd) is true.
4703  */
4704 static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
4705 		unsigned long addr, pmd_t pmd, union mc_target *target)
4706 {
4707 	struct page *page = NULL;
4708 	enum mc_target_type ret = MC_TARGET_NONE;
4709 
4710 	if (unlikely(is_swap_pmd(pmd))) {
4711 		VM_BUG_ON(thp_migration_supported() &&
4712 				  !is_pmd_migration_entry(pmd));
4713 		return ret;
4714 	}
4715 	page = pmd_page(pmd);
4716 	VM_BUG_ON_PAGE(!page || !PageHead(page), page);
4717 	if (!(mc.flags & MOVE_ANON))
4718 		return ret;
4719 	if (page->mem_cgroup == mc.from) {
4720 		ret = MC_TARGET_PAGE;
4721 		if (target) {
4722 			get_page(page);
4723 			target->page = page;
4724 		}
4725 	}
4726 	return ret;
4727 }
4728 #else
4729 static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
4730 		unsigned long addr, pmd_t pmd, union mc_target *target)
4731 {
4732 	return MC_TARGET_NONE;
4733 }
4734 #endif
4735 
4736 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
4737 					unsigned long addr, unsigned long end,
4738 					struct mm_walk *walk)
4739 {
4740 	struct vm_area_struct *vma = walk->vma;
4741 	pte_t *pte;
4742 	spinlock_t *ptl;
4743 
4744 	ptl = pmd_trans_huge_lock(pmd, vma);
4745 	if (ptl) {
4746 		/*
4747 		 * Note their can not be MC_TARGET_DEVICE for now as we do not
4748 		 * support transparent huge page with MEMORY_DEVICE_PUBLIC or
4749 		 * MEMORY_DEVICE_PRIVATE but this might change.
4750 		 */
4751 		if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
4752 			mc.precharge += HPAGE_PMD_NR;
4753 		spin_unlock(ptl);
4754 		return 0;
4755 	}
4756 
4757 	if (pmd_trans_unstable(pmd))
4758 		return 0;
4759 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4760 	for (; addr != end; pte++, addr += PAGE_SIZE)
4761 		if (get_mctgt_type(vma, addr, *pte, NULL))
4762 			mc.precharge++;	/* increment precharge temporarily */
4763 	pte_unmap_unlock(pte - 1, ptl);
4764 	cond_resched();
4765 
4766 	return 0;
4767 }
4768 
4769 static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
4770 {
4771 	unsigned long precharge;
4772 
4773 	struct mm_walk mem_cgroup_count_precharge_walk = {
4774 		.pmd_entry = mem_cgroup_count_precharge_pte_range,
4775 		.mm = mm,
4776 	};
4777 	down_read(&mm->mmap_sem);
4778 	walk_page_range(0, mm->highest_vm_end,
4779 			&mem_cgroup_count_precharge_walk);
4780 	up_read(&mm->mmap_sem);
4781 
4782 	precharge = mc.precharge;
4783 	mc.precharge = 0;
4784 
4785 	return precharge;
4786 }
4787 
4788 static int mem_cgroup_precharge_mc(struct mm_struct *mm)
4789 {
4790 	unsigned long precharge = mem_cgroup_count_precharge(mm);
4791 
4792 	VM_BUG_ON(mc.moving_task);
4793 	mc.moving_task = current;
4794 	return mem_cgroup_do_precharge(precharge);
4795 }
4796 
4797 /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
4798 static void __mem_cgroup_clear_mc(void)
4799 {
4800 	struct mem_cgroup *from = mc.from;
4801 	struct mem_cgroup *to = mc.to;
4802 
4803 	/* we must uncharge all the leftover precharges from mc.to */
4804 	if (mc.precharge) {
4805 		cancel_charge(mc.to, mc.precharge);
4806 		mc.precharge = 0;
4807 	}
4808 	/*
4809 	 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
4810 	 * we must uncharge here.
4811 	 */
4812 	if (mc.moved_charge) {
4813 		cancel_charge(mc.from, mc.moved_charge);
4814 		mc.moved_charge = 0;
4815 	}
4816 	/* we must fixup refcnts and charges */
4817 	if (mc.moved_swap) {
4818 		/* uncharge swap account from the old cgroup */
4819 		if (!mem_cgroup_is_root(mc.from))
4820 			page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
4821 
4822 		mem_cgroup_id_put_many(mc.from, mc.moved_swap);
4823 
4824 		/*
4825 		 * we charged both to->memory and to->memsw, so we
4826 		 * should uncharge to->memory.
4827 		 */
4828 		if (!mem_cgroup_is_root(mc.to))
4829 			page_counter_uncharge(&mc.to->memory, mc.moved_swap);
4830 
4831 		mem_cgroup_id_get_many(mc.to, mc.moved_swap);
4832 		css_put_many(&mc.to->css, mc.moved_swap);
4833 
4834 		mc.moved_swap = 0;
4835 	}
4836 	memcg_oom_recover(from);
4837 	memcg_oom_recover(to);
4838 	wake_up_all(&mc.waitq);
4839 }
4840 
4841 static void mem_cgroup_clear_mc(void)
4842 {
4843 	struct mm_struct *mm = mc.mm;
4844 
4845 	/*
4846 	 * we must clear moving_task before waking up waiters at the end of
4847 	 * task migration.
4848 	 */
4849 	mc.moving_task = NULL;
4850 	__mem_cgroup_clear_mc();
4851 	spin_lock(&mc.lock);
4852 	mc.from = NULL;
4853 	mc.to = NULL;
4854 	mc.mm = NULL;
4855 	spin_unlock(&mc.lock);
4856 
4857 	mmput(mm);
4858 }
4859 
4860 static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
4861 {
4862 	struct cgroup_subsys_state *css;
4863 	struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
4864 	struct mem_cgroup *from;
4865 	struct task_struct *leader, *p;
4866 	struct mm_struct *mm;
4867 	unsigned long move_flags;
4868 	int ret = 0;
4869 
4870 	/* charge immigration isn't supported on the default hierarchy */
4871 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
4872 		return 0;
4873 
4874 	/*
4875 	 * Multi-process migrations only happen on the default hierarchy
4876 	 * where charge immigration is not used.  Perform charge
4877 	 * immigration if @tset contains a leader and whine if there are
4878 	 * multiple.
4879 	 */
4880 	p = NULL;
4881 	cgroup_taskset_for_each_leader(leader, css, tset) {
4882 		WARN_ON_ONCE(p);
4883 		p = leader;
4884 		memcg = mem_cgroup_from_css(css);
4885 	}
4886 	if (!p)
4887 		return 0;
4888 
4889 	/*
4890 	 * We are now commited to this value whatever it is. Changes in this
4891 	 * tunable will only affect upcoming migrations, not the current one.
4892 	 * So we need to save it, and keep it going.
4893 	 */
4894 	move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
4895 	if (!move_flags)
4896 		return 0;
4897 
4898 	from = mem_cgroup_from_task(p);
4899 
4900 	VM_BUG_ON(from == memcg);
4901 
4902 	mm = get_task_mm(p);
4903 	if (!mm)
4904 		return 0;
4905 	/* We move charges only when we move a owner of the mm */
4906 	if (mm->owner == p) {
4907 		VM_BUG_ON(mc.from);
4908 		VM_BUG_ON(mc.to);
4909 		VM_BUG_ON(mc.precharge);
4910 		VM_BUG_ON(mc.moved_charge);
4911 		VM_BUG_ON(mc.moved_swap);
4912 
4913 		spin_lock(&mc.lock);
4914 		mc.mm = mm;
4915 		mc.from = from;
4916 		mc.to = memcg;
4917 		mc.flags = move_flags;
4918 		spin_unlock(&mc.lock);
4919 		/* We set mc.moving_task later */
4920 
4921 		ret = mem_cgroup_precharge_mc(mm);
4922 		if (ret)
4923 			mem_cgroup_clear_mc();
4924 	} else {
4925 		mmput(mm);
4926 	}
4927 	return ret;
4928 }
4929 
4930 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
4931 {
4932 	if (mc.to)
4933 		mem_cgroup_clear_mc();
4934 }
4935 
4936 static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
4937 				unsigned long addr, unsigned long end,
4938 				struct mm_walk *walk)
4939 {
4940 	int ret = 0;
4941 	struct vm_area_struct *vma = walk->vma;
4942 	pte_t *pte;
4943 	spinlock_t *ptl;
4944 	enum mc_target_type target_type;
4945 	union mc_target target;
4946 	struct page *page;
4947 
4948 	ptl = pmd_trans_huge_lock(pmd, vma);
4949 	if (ptl) {
4950 		if (mc.precharge < HPAGE_PMD_NR) {
4951 			spin_unlock(ptl);
4952 			return 0;
4953 		}
4954 		target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
4955 		if (target_type == MC_TARGET_PAGE) {
4956 			page = target.page;
4957 			if (!isolate_lru_page(page)) {
4958 				if (!mem_cgroup_move_account(page, true,
4959 							     mc.from, mc.to)) {
4960 					mc.precharge -= HPAGE_PMD_NR;
4961 					mc.moved_charge += HPAGE_PMD_NR;
4962 				}
4963 				putback_lru_page(page);
4964 			}
4965 			put_page(page);
4966 		} else if (target_type == MC_TARGET_DEVICE) {
4967 			page = target.page;
4968 			if (!mem_cgroup_move_account(page, true,
4969 						     mc.from, mc.to)) {
4970 				mc.precharge -= HPAGE_PMD_NR;
4971 				mc.moved_charge += HPAGE_PMD_NR;
4972 			}
4973 			put_page(page);
4974 		}
4975 		spin_unlock(ptl);
4976 		return 0;
4977 	}
4978 
4979 	if (pmd_trans_unstable(pmd))
4980 		return 0;
4981 retry:
4982 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4983 	for (; addr != end; addr += PAGE_SIZE) {
4984 		pte_t ptent = *(pte++);
4985 		bool device = false;
4986 		swp_entry_t ent;
4987 
4988 		if (!mc.precharge)
4989 			break;
4990 
4991 		switch (get_mctgt_type(vma, addr, ptent, &target)) {
4992 		case MC_TARGET_DEVICE:
4993 			device = true;
4994 			/* fall through */
4995 		case MC_TARGET_PAGE:
4996 			page = target.page;
4997 			/*
4998 			 * We can have a part of the split pmd here. Moving it
4999 			 * can be done but it would be too convoluted so simply
5000 			 * ignore such a partial THP and keep it in original
5001 			 * memcg. There should be somebody mapping the head.
5002 			 */
5003 			if (PageTransCompound(page))
5004 				goto put;
5005 			if (!device && isolate_lru_page(page))
5006 				goto put;
5007 			if (!mem_cgroup_move_account(page, false,
5008 						mc.from, mc.to)) {
5009 				mc.precharge--;
5010 				/* we uncharge from mc.from later. */
5011 				mc.moved_charge++;
5012 			}
5013 			if (!device)
5014 				putback_lru_page(page);
5015 put:			/* get_mctgt_type() gets the page */
5016 			put_page(page);
5017 			break;
5018 		case MC_TARGET_SWAP:
5019 			ent = target.ent;
5020 			if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
5021 				mc.precharge--;
5022 				/* we fixup refcnts and charges later. */
5023 				mc.moved_swap++;
5024 			}
5025 			break;
5026 		default:
5027 			break;
5028 		}
5029 	}
5030 	pte_unmap_unlock(pte - 1, ptl);
5031 	cond_resched();
5032 
5033 	if (addr != end) {
5034 		/*
5035 		 * We have consumed all precharges we got in can_attach().
5036 		 * We try charge one by one, but don't do any additional
5037 		 * charges to mc.to if we have failed in charge once in attach()
5038 		 * phase.
5039 		 */
5040 		ret = mem_cgroup_do_precharge(1);
5041 		if (!ret)
5042 			goto retry;
5043 	}
5044 
5045 	return ret;
5046 }
5047 
5048 static void mem_cgroup_move_charge(void)
5049 {
5050 	struct mm_walk mem_cgroup_move_charge_walk = {
5051 		.pmd_entry = mem_cgroup_move_charge_pte_range,
5052 		.mm = mc.mm,
5053 	};
5054 
5055 	lru_add_drain_all();
5056 	/*
5057 	 * Signal lock_page_memcg() to take the memcg's move_lock
5058 	 * while we're moving its pages to another memcg. Then wait
5059 	 * for already started RCU-only updates to finish.
5060 	 */
5061 	atomic_inc(&mc.from->moving_account);
5062 	synchronize_rcu();
5063 retry:
5064 	if (unlikely(!down_read_trylock(&mc.mm->mmap_sem))) {
5065 		/*
5066 		 * Someone who are holding the mmap_sem might be waiting in
5067 		 * waitq. So we cancel all extra charges, wake up all waiters,
5068 		 * and retry. Because we cancel precharges, we might not be able
5069 		 * to move enough charges, but moving charge is a best-effort
5070 		 * feature anyway, so it wouldn't be a big problem.
5071 		 */
5072 		__mem_cgroup_clear_mc();
5073 		cond_resched();
5074 		goto retry;
5075 	}
5076 	/*
5077 	 * When we have consumed all precharges and failed in doing
5078 	 * additional charge, the page walk just aborts.
5079 	 */
5080 	walk_page_range(0, mc.mm->highest_vm_end, &mem_cgroup_move_charge_walk);
5081 
5082 	up_read(&mc.mm->mmap_sem);
5083 	atomic_dec(&mc.from->moving_account);
5084 }
5085 
5086 static void mem_cgroup_move_task(void)
5087 {
5088 	if (mc.to) {
5089 		mem_cgroup_move_charge();
5090 		mem_cgroup_clear_mc();
5091 	}
5092 }
5093 #else	/* !CONFIG_MMU */
5094 static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
5095 {
5096 	return 0;
5097 }
5098 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
5099 {
5100 }
5101 static void mem_cgroup_move_task(void)
5102 {
5103 }
5104 #endif
5105 
5106 /*
5107  * Cgroup retains root cgroups across [un]mount cycles making it necessary
5108  * to verify whether we're attached to the default hierarchy on each mount
5109  * attempt.
5110  */
5111 static void mem_cgroup_bind(struct cgroup_subsys_state *root_css)
5112 {
5113 	/*
5114 	 * use_hierarchy is forced on the default hierarchy.  cgroup core
5115 	 * guarantees that @root doesn't have any children, so turning it
5116 	 * on for the root memcg is enough.
5117 	 */
5118 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
5119 		root_mem_cgroup->use_hierarchy = true;
5120 	else
5121 		root_mem_cgroup->use_hierarchy = false;
5122 }
5123 
5124 static u64 memory_current_read(struct cgroup_subsys_state *css,
5125 			       struct cftype *cft)
5126 {
5127 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5128 
5129 	return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
5130 }
5131 
5132 static int memory_low_show(struct seq_file *m, void *v)
5133 {
5134 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5135 	unsigned long low = READ_ONCE(memcg->low);
5136 
5137 	if (low == PAGE_COUNTER_MAX)
5138 		seq_puts(m, "max\n");
5139 	else
5140 		seq_printf(m, "%llu\n", (u64)low * PAGE_SIZE);
5141 
5142 	return 0;
5143 }
5144 
5145 static ssize_t memory_low_write(struct kernfs_open_file *of,
5146 				char *buf, size_t nbytes, loff_t off)
5147 {
5148 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5149 	unsigned long low;
5150 	int err;
5151 
5152 	buf = strstrip(buf);
5153 	err = page_counter_memparse(buf, "max", &low);
5154 	if (err)
5155 		return err;
5156 
5157 	memcg->low = low;
5158 
5159 	return nbytes;
5160 }
5161 
5162 static int memory_high_show(struct seq_file *m, void *v)
5163 {
5164 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5165 	unsigned long high = READ_ONCE(memcg->high);
5166 
5167 	if (high == PAGE_COUNTER_MAX)
5168 		seq_puts(m, "max\n");
5169 	else
5170 		seq_printf(m, "%llu\n", (u64)high * PAGE_SIZE);
5171 
5172 	return 0;
5173 }
5174 
5175 static ssize_t memory_high_write(struct kernfs_open_file *of,
5176 				 char *buf, size_t nbytes, loff_t off)
5177 {
5178 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5179 	unsigned long nr_pages;
5180 	unsigned long high;
5181 	int err;
5182 
5183 	buf = strstrip(buf);
5184 	err = page_counter_memparse(buf, "max", &high);
5185 	if (err)
5186 		return err;
5187 
5188 	memcg->high = high;
5189 
5190 	nr_pages = page_counter_read(&memcg->memory);
5191 	if (nr_pages > high)
5192 		try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
5193 					     GFP_KERNEL, true);
5194 
5195 	memcg_wb_domain_size_changed(memcg);
5196 	return nbytes;
5197 }
5198 
5199 static int memory_max_show(struct seq_file *m, void *v)
5200 {
5201 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5202 	unsigned long max = READ_ONCE(memcg->memory.limit);
5203 
5204 	if (max == PAGE_COUNTER_MAX)
5205 		seq_puts(m, "max\n");
5206 	else
5207 		seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);
5208 
5209 	return 0;
5210 }
5211 
5212 static ssize_t memory_max_write(struct kernfs_open_file *of,
5213 				char *buf, size_t nbytes, loff_t off)
5214 {
5215 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5216 	unsigned int nr_reclaims = MEM_CGROUP_RECLAIM_RETRIES;
5217 	bool drained = false;
5218 	unsigned long max;
5219 	int err;
5220 
5221 	buf = strstrip(buf);
5222 	err = page_counter_memparse(buf, "max", &max);
5223 	if (err)
5224 		return err;
5225 
5226 	xchg(&memcg->memory.limit, max);
5227 
5228 	for (;;) {
5229 		unsigned long nr_pages = page_counter_read(&memcg->memory);
5230 
5231 		if (nr_pages <= max)
5232 			break;
5233 
5234 		if (signal_pending(current)) {
5235 			err = -EINTR;
5236 			break;
5237 		}
5238 
5239 		if (!drained) {
5240 			drain_all_stock(memcg);
5241 			drained = true;
5242 			continue;
5243 		}
5244 
5245 		if (nr_reclaims) {
5246 			if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
5247 							  GFP_KERNEL, true))
5248 				nr_reclaims--;
5249 			continue;
5250 		}
5251 
5252 		mem_cgroup_event(memcg, MEMCG_OOM);
5253 		if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
5254 			break;
5255 	}
5256 
5257 	memcg_wb_domain_size_changed(memcg);
5258 	return nbytes;
5259 }
5260 
5261 static int memory_events_show(struct seq_file *m, void *v)
5262 {
5263 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5264 
5265 	seq_printf(m, "low %lu\n", memcg_sum_events(memcg, MEMCG_LOW));
5266 	seq_printf(m, "high %lu\n", memcg_sum_events(memcg, MEMCG_HIGH));
5267 	seq_printf(m, "max %lu\n", memcg_sum_events(memcg, MEMCG_MAX));
5268 	seq_printf(m, "oom %lu\n", memcg_sum_events(memcg, MEMCG_OOM));
5269 	seq_printf(m, "oom_kill %lu\n", memcg_sum_events(memcg, OOM_KILL));
5270 
5271 	return 0;
5272 }
5273 
5274 static int memory_stat_show(struct seq_file *m, void *v)
5275 {
5276 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5277 	unsigned long stat[MEMCG_NR_STAT];
5278 	unsigned long events[MEMCG_NR_EVENTS];
5279 	int i;
5280 
5281 	/*
5282 	 * Provide statistics on the state of the memory subsystem as
5283 	 * well as cumulative event counters that show past behavior.
5284 	 *
5285 	 * This list is ordered following a combination of these gradients:
5286 	 * 1) generic big picture -> specifics and details
5287 	 * 2) reflecting userspace activity -> reflecting kernel heuristics
5288 	 *
5289 	 * Current memory state:
5290 	 */
5291 
5292 	tree_stat(memcg, stat);
5293 	tree_events(memcg, events);
5294 
5295 	seq_printf(m, "anon %llu\n",
5296 		   (u64)stat[MEMCG_RSS] * PAGE_SIZE);
5297 	seq_printf(m, "file %llu\n",
5298 		   (u64)stat[MEMCG_CACHE] * PAGE_SIZE);
5299 	seq_printf(m, "kernel_stack %llu\n",
5300 		   (u64)stat[MEMCG_KERNEL_STACK_KB] * 1024);
5301 	seq_printf(m, "slab %llu\n",
5302 		   (u64)(stat[NR_SLAB_RECLAIMABLE] +
5303 			 stat[NR_SLAB_UNRECLAIMABLE]) * PAGE_SIZE);
5304 	seq_printf(m, "sock %llu\n",
5305 		   (u64)stat[MEMCG_SOCK] * PAGE_SIZE);
5306 
5307 	seq_printf(m, "shmem %llu\n",
5308 		   (u64)stat[NR_SHMEM] * PAGE_SIZE);
5309 	seq_printf(m, "file_mapped %llu\n",
5310 		   (u64)stat[NR_FILE_MAPPED] * PAGE_SIZE);
5311 	seq_printf(m, "file_dirty %llu\n",
5312 		   (u64)stat[NR_FILE_DIRTY] * PAGE_SIZE);
5313 	seq_printf(m, "file_writeback %llu\n",
5314 		   (u64)stat[NR_WRITEBACK] * PAGE_SIZE);
5315 
5316 	for (i = 0; i < NR_LRU_LISTS; i++) {
5317 		struct mem_cgroup *mi;
5318 		unsigned long val = 0;
5319 
5320 		for_each_mem_cgroup_tree(mi, memcg)
5321 			val += mem_cgroup_nr_lru_pages(mi, BIT(i));
5322 		seq_printf(m, "%s %llu\n",
5323 			   mem_cgroup_lru_names[i], (u64)val * PAGE_SIZE);
5324 	}
5325 
5326 	seq_printf(m, "slab_reclaimable %llu\n",
5327 		   (u64)stat[NR_SLAB_RECLAIMABLE] * PAGE_SIZE);
5328 	seq_printf(m, "slab_unreclaimable %llu\n",
5329 		   (u64)stat[NR_SLAB_UNRECLAIMABLE] * PAGE_SIZE);
5330 
5331 	/* Accumulated memory events */
5332 
5333 	seq_printf(m, "pgfault %lu\n", events[PGFAULT]);
5334 	seq_printf(m, "pgmajfault %lu\n", events[PGMAJFAULT]);
5335 
5336 	seq_printf(m, "pgrefill %lu\n", events[PGREFILL]);
5337 	seq_printf(m, "pgscan %lu\n", events[PGSCAN_KSWAPD] +
5338 		   events[PGSCAN_DIRECT]);
5339 	seq_printf(m, "pgsteal %lu\n", events[PGSTEAL_KSWAPD] +
5340 		   events[PGSTEAL_DIRECT]);
5341 	seq_printf(m, "pgactivate %lu\n", events[PGACTIVATE]);
5342 	seq_printf(m, "pgdeactivate %lu\n", events[PGDEACTIVATE]);
5343 	seq_printf(m, "pglazyfree %lu\n", events[PGLAZYFREE]);
5344 	seq_printf(m, "pglazyfreed %lu\n", events[PGLAZYFREED]);
5345 
5346 	seq_printf(m, "workingset_refault %lu\n",
5347 		   stat[WORKINGSET_REFAULT]);
5348 	seq_printf(m, "workingset_activate %lu\n",
5349 		   stat[WORKINGSET_ACTIVATE]);
5350 	seq_printf(m, "workingset_nodereclaim %lu\n",
5351 		   stat[WORKINGSET_NODERECLAIM]);
5352 
5353 	return 0;
5354 }
5355 
5356 static struct cftype memory_files[] = {
5357 	{
5358 		.name = "current",
5359 		.flags = CFTYPE_NOT_ON_ROOT,
5360 		.read_u64 = memory_current_read,
5361 	},
5362 	{
5363 		.name = "low",
5364 		.flags = CFTYPE_NOT_ON_ROOT,
5365 		.seq_show = memory_low_show,
5366 		.write = memory_low_write,
5367 	},
5368 	{
5369 		.name = "high",
5370 		.flags = CFTYPE_NOT_ON_ROOT,
5371 		.seq_show = memory_high_show,
5372 		.write = memory_high_write,
5373 	},
5374 	{
5375 		.name = "max",
5376 		.flags = CFTYPE_NOT_ON_ROOT,
5377 		.seq_show = memory_max_show,
5378 		.write = memory_max_write,
5379 	},
5380 	{
5381 		.name = "events",
5382 		.flags = CFTYPE_NOT_ON_ROOT,
5383 		.file_offset = offsetof(struct mem_cgroup, events_file),
5384 		.seq_show = memory_events_show,
5385 	},
5386 	{
5387 		.name = "stat",
5388 		.flags = CFTYPE_NOT_ON_ROOT,
5389 		.seq_show = memory_stat_show,
5390 	},
5391 	{ }	/* terminate */
5392 };
5393 
5394 struct cgroup_subsys memory_cgrp_subsys = {
5395 	.css_alloc = mem_cgroup_css_alloc,
5396 	.css_online = mem_cgroup_css_online,
5397 	.css_offline = mem_cgroup_css_offline,
5398 	.css_released = mem_cgroup_css_released,
5399 	.css_free = mem_cgroup_css_free,
5400 	.css_reset = mem_cgroup_css_reset,
5401 	.can_attach = mem_cgroup_can_attach,
5402 	.cancel_attach = mem_cgroup_cancel_attach,
5403 	.post_attach = mem_cgroup_move_task,
5404 	.bind = mem_cgroup_bind,
5405 	.dfl_cftypes = memory_files,
5406 	.legacy_cftypes = mem_cgroup_legacy_files,
5407 	.early_init = 0,
5408 };
5409 
5410 /**
5411  * mem_cgroup_low - check if memory consumption is below the normal range
5412  * @root: the top ancestor of the sub-tree being checked
5413  * @memcg: the memory cgroup to check
5414  *
5415  * Returns %true if memory consumption of @memcg, and that of all
5416  * ancestors up to (but not including) @root, is below the normal range.
5417  *
5418  * @root is exclusive; it is never low when looked at directly and isn't
5419  * checked when traversing the hierarchy.
5420  *
5421  * Excluding @root enables using memory.low to prioritize memory usage
5422  * between cgroups within a subtree of the hierarchy that is limited by
5423  * memory.high or memory.max.
5424  *
5425  * For example, given cgroup A with children B and C:
5426  *
5427  *    A
5428  *   / \
5429  *  B   C
5430  *
5431  * and
5432  *
5433  *  1. A/memory.current > A/memory.high
5434  *  2. A/B/memory.current < A/B/memory.low
5435  *  3. A/C/memory.current >= A/C/memory.low
5436  *
5437  * As 'A' is high, i.e. triggers reclaim from 'A', and 'B' is low, we
5438  * should reclaim from 'C' until 'A' is no longer high or until we can
5439  * no longer reclaim from 'C'.  If 'A', i.e. @root, isn't excluded by
5440  * mem_cgroup_low when reclaming from 'A', then 'B' won't be considered
5441  * low and we will reclaim indiscriminately from both 'B' and 'C'.
5442  */
5443 bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg)
5444 {
5445 	if (mem_cgroup_disabled())
5446 		return false;
5447 
5448 	if (!root)
5449 		root = root_mem_cgroup;
5450 	if (memcg == root)
5451 		return false;
5452 
5453 	for (; memcg != root; memcg = parent_mem_cgroup(memcg)) {
5454 		if (page_counter_read(&memcg->memory) >= memcg->low)
5455 			return false;
5456 	}
5457 
5458 	return true;
5459 }
5460 
5461 /**
5462  * mem_cgroup_try_charge - try charging a page
5463  * @page: page to charge
5464  * @mm: mm context of the victim
5465  * @gfp_mask: reclaim mode
5466  * @memcgp: charged memcg return
5467  * @compound: charge the page as compound or small page
5468  *
5469  * Try to charge @page to the memcg that @mm belongs to, reclaiming
5470  * pages according to @gfp_mask if necessary.
5471  *
5472  * Returns 0 on success, with *@memcgp pointing to the charged memcg.
5473  * Otherwise, an error code is returned.
5474  *
5475  * After page->mapping has been set up, the caller must finalize the
5476  * charge with mem_cgroup_commit_charge().  Or abort the transaction
5477  * with mem_cgroup_cancel_charge() in case page instantiation fails.
5478  */
5479 int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
5480 			  gfp_t gfp_mask, struct mem_cgroup **memcgp,
5481 			  bool compound)
5482 {
5483 	struct mem_cgroup *memcg = NULL;
5484 	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5485 	int ret = 0;
5486 
5487 	if (mem_cgroup_disabled())
5488 		goto out;
5489 
5490 	if (PageSwapCache(page)) {
5491 		/*
5492 		 * Every swap fault against a single page tries to charge the
5493 		 * page, bail as early as possible.  shmem_unuse() encounters
5494 		 * already charged pages, too.  The USED bit is protected by
5495 		 * the page lock, which serializes swap cache removal, which
5496 		 * in turn serializes uncharging.
5497 		 */
5498 		VM_BUG_ON_PAGE(!PageLocked(page), page);
5499 		if (compound_head(page)->mem_cgroup)
5500 			goto out;
5501 
5502 		if (do_swap_account) {
5503 			swp_entry_t ent = { .val = page_private(page), };
5504 			unsigned short id = lookup_swap_cgroup_id(ent);
5505 
5506 			rcu_read_lock();
5507 			memcg = mem_cgroup_from_id(id);
5508 			if (memcg && !css_tryget_online(&memcg->css))
5509 				memcg = NULL;
5510 			rcu_read_unlock();
5511 		}
5512 	}
5513 
5514 	if (!memcg)
5515 		memcg = get_mem_cgroup_from_mm(mm);
5516 
5517 	ret = try_charge(memcg, gfp_mask, nr_pages);
5518 
5519 	css_put(&memcg->css);
5520 out:
5521 	*memcgp = memcg;
5522 	return ret;
5523 }
5524 
5525 /**
5526  * mem_cgroup_commit_charge - commit a page charge
5527  * @page: page to charge
5528  * @memcg: memcg to charge the page to
5529  * @lrucare: page might be on LRU already
5530  * @compound: charge the page as compound or small page
5531  *
5532  * Finalize a charge transaction started by mem_cgroup_try_charge(),
5533  * after page->mapping has been set up.  This must happen atomically
5534  * as part of the page instantiation, i.e. under the page table lock
5535  * for anonymous pages, under the page lock for page and swap cache.
5536  *
5537  * In addition, the page must not be on the LRU during the commit, to
5538  * prevent racing with task migration.  If it might be, use @lrucare.
5539  *
5540  * Use mem_cgroup_cancel_charge() to cancel the transaction instead.
5541  */
5542 void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
5543 			      bool lrucare, bool compound)
5544 {
5545 	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5546 
5547 	VM_BUG_ON_PAGE(!page->mapping, page);
5548 	VM_BUG_ON_PAGE(PageLRU(page) && !lrucare, page);
5549 
5550 	if (mem_cgroup_disabled())
5551 		return;
5552 	/*
5553 	 * Swap faults will attempt to charge the same page multiple
5554 	 * times.  But reuse_swap_page() might have removed the page
5555 	 * from swapcache already, so we can't check PageSwapCache().
5556 	 */
5557 	if (!memcg)
5558 		return;
5559 
5560 	commit_charge(page, memcg, lrucare);
5561 
5562 	local_irq_disable();
5563 	mem_cgroup_charge_statistics(memcg, page, compound, nr_pages);
5564 	memcg_check_events(memcg, page);
5565 	local_irq_enable();
5566 
5567 	if (do_memsw_account() && PageSwapCache(page)) {
5568 		swp_entry_t entry = { .val = page_private(page) };
5569 		/*
5570 		 * The swap entry might not get freed for a long time,
5571 		 * let's not wait for it.  The page already received a
5572 		 * memory+swap charge, drop the swap entry duplicate.
5573 		 */
5574 		mem_cgroup_uncharge_swap(entry, nr_pages);
5575 	}
5576 }
5577 
5578 /**
5579  * mem_cgroup_cancel_charge - cancel a page charge
5580  * @page: page to charge
5581  * @memcg: memcg to charge the page to
5582  * @compound: charge the page as compound or small page
5583  *
5584  * Cancel a charge transaction started by mem_cgroup_try_charge().
5585  */
5586 void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
5587 		bool compound)
5588 {
5589 	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5590 
5591 	if (mem_cgroup_disabled())
5592 		return;
5593 	/*
5594 	 * Swap faults will attempt to charge the same page multiple
5595 	 * times.  But reuse_swap_page() might have removed the page
5596 	 * from swapcache already, so we can't check PageSwapCache().
5597 	 */
5598 	if (!memcg)
5599 		return;
5600 
5601 	cancel_charge(memcg, nr_pages);
5602 }
5603 
5604 struct uncharge_gather {
5605 	struct mem_cgroup *memcg;
5606 	unsigned long pgpgout;
5607 	unsigned long nr_anon;
5608 	unsigned long nr_file;
5609 	unsigned long nr_kmem;
5610 	unsigned long nr_huge;
5611 	unsigned long nr_shmem;
5612 	struct page *dummy_page;
5613 };
5614 
5615 static inline void uncharge_gather_clear(struct uncharge_gather *ug)
5616 {
5617 	memset(ug, 0, sizeof(*ug));
5618 }
5619 
5620 static void uncharge_batch(const struct uncharge_gather *ug)
5621 {
5622 	unsigned long nr_pages = ug->nr_anon + ug->nr_file + ug->nr_kmem;
5623 	unsigned long flags;
5624 
5625 	if (!mem_cgroup_is_root(ug->memcg)) {
5626 		page_counter_uncharge(&ug->memcg->memory, nr_pages);
5627 		if (do_memsw_account())
5628 			page_counter_uncharge(&ug->memcg->memsw, nr_pages);
5629 		if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && ug->nr_kmem)
5630 			page_counter_uncharge(&ug->memcg->kmem, ug->nr_kmem);
5631 		memcg_oom_recover(ug->memcg);
5632 	}
5633 
5634 	local_irq_save(flags);
5635 	__this_cpu_sub(ug->memcg->stat->count[MEMCG_RSS], ug->nr_anon);
5636 	__this_cpu_sub(ug->memcg->stat->count[MEMCG_CACHE], ug->nr_file);
5637 	__this_cpu_sub(ug->memcg->stat->count[MEMCG_RSS_HUGE], ug->nr_huge);
5638 	__this_cpu_sub(ug->memcg->stat->count[NR_SHMEM], ug->nr_shmem);
5639 	__this_cpu_add(ug->memcg->stat->events[PGPGOUT], ug->pgpgout);
5640 	__this_cpu_add(ug->memcg->stat->nr_page_events, nr_pages);
5641 	memcg_check_events(ug->memcg, ug->dummy_page);
5642 	local_irq_restore(flags);
5643 
5644 	if (!mem_cgroup_is_root(ug->memcg))
5645 		css_put_many(&ug->memcg->css, nr_pages);
5646 }
5647 
5648 static void uncharge_page(struct page *page, struct uncharge_gather *ug)
5649 {
5650 	VM_BUG_ON_PAGE(PageLRU(page), page);
5651 	VM_BUG_ON_PAGE(!PageHWPoison(page) && page_count(page), page);
5652 
5653 	if (!page->mem_cgroup)
5654 		return;
5655 
5656 	/*
5657 	 * Nobody should be changing or seriously looking at
5658 	 * page->mem_cgroup at this point, we have fully
5659 	 * exclusive access to the page.
5660 	 */
5661 
5662 	if (ug->memcg != page->mem_cgroup) {
5663 		if (ug->memcg) {
5664 			uncharge_batch(ug);
5665 			uncharge_gather_clear(ug);
5666 		}
5667 		ug->memcg = page->mem_cgroup;
5668 	}
5669 
5670 	if (!PageKmemcg(page)) {
5671 		unsigned int nr_pages = 1;
5672 
5673 		if (PageTransHuge(page)) {
5674 			nr_pages <<= compound_order(page);
5675 			ug->nr_huge += nr_pages;
5676 		}
5677 		if (PageAnon(page))
5678 			ug->nr_anon += nr_pages;
5679 		else {
5680 			ug->nr_file += nr_pages;
5681 			if (PageSwapBacked(page))
5682 				ug->nr_shmem += nr_pages;
5683 		}
5684 		ug->pgpgout++;
5685 	} else {
5686 		ug->nr_kmem += 1 << compound_order(page);
5687 		__ClearPageKmemcg(page);
5688 	}
5689 
5690 	ug->dummy_page = page;
5691 	page->mem_cgroup = NULL;
5692 }
5693 
5694 static void uncharge_list(struct list_head *page_list)
5695 {
5696 	struct uncharge_gather ug;
5697 	struct list_head *next;
5698 
5699 	uncharge_gather_clear(&ug);
5700 
5701 	/*
5702 	 * Note that the list can be a single page->lru; hence the
5703 	 * do-while loop instead of a simple list_for_each_entry().
5704 	 */
5705 	next = page_list->next;
5706 	do {
5707 		struct page *page;
5708 
5709 		page = list_entry(next, struct page, lru);
5710 		next = page->lru.next;
5711 
5712 		uncharge_page(page, &ug);
5713 	} while (next != page_list);
5714 
5715 	if (ug.memcg)
5716 		uncharge_batch(&ug);
5717 }
5718 
5719 /**
5720  * mem_cgroup_uncharge - uncharge a page
5721  * @page: page to uncharge
5722  *
5723  * Uncharge a page previously charged with mem_cgroup_try_charge() and
5724  * mem_cgroup_commit_charge().
5725  */
5726 void mem_cgroup_uncharge(struct page *page)
5727 {
5728 	struct uncharge_gather ug;
5729 
5730 	if (mem_cgroup_disabled())
5731 		return;
5732 
5733 	/* Don't touch page->lru of any random page, pre-check: */
5734 	if (!page->mem_cgroup)
5735 		return;
5736 
5737 	uncharge_gather_clear(&ug);
5738 	uncharge_page(page, &ug);
5739 	uncharge_batch(&ug);
5740 }
5741 
5742 /**
5743  * mem_cgroup_uncharge_list - uncharge a list of page
5744  * @page_list: list of pages to uncharge
5745  *
5746  * Uncharge a list of pages previously charged with
5747  * mem_cgroup_try_charge() and mem_cgroup_commit_charge().
5748  */
5749 void mem_cgroup_uncharge_list(struct list_head *page_list)
5750 {
5751 	if (mem_cgroup_disabled())
5752 		return;
5753 
5754 	if (!list_empty(page_list))
5755 		uncharge_list(page_list);
5756 }
5757 
5758 /**
5759  * mem_cgroup_migrate - charge a page's replacement
5760  * @oldpage: currently circulating page
5761  * @newpage: replacement page
5762  *
5763  * Charge @newpage as a replacement page for @oldpage. @oldpage will
5764  * be uncharged upon free.
5765  *
5766  * Both pages must be locked, @newpage->mapping must be set up.
5767  */
5768 void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
5769 {
5770 	struct mem_cgroup *memcg;
5771 	unsigned int nr_pages;
5772 	bool compound;
5773 	unsigned long flags;
5774 
5775 	VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
5776 	VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
5777 	VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage);
5778 	VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage),
5779 		       newpage);
5780 
5781 	if (mem_cgroup_disabled())
5782 		return;
5783 
5784 	/* Page cache replacement: new page already charged? */
5785 	if (newpage->mem_cgroup)
5786 		return;
5787 
5788 	/* Swapcache readahead pages can get replaced before being charged */
5789 	memcg = oldpage->mem_cgroup;
5790 	if (!memcg)
5791 		return;
5792 
5793 	/* Force-charge the new page. The old one will be freed soon */
5794 	compound = PageTransHuge(newpage);
5795 	nr_pages = compound ? hpage_nr_pages(newpage) : 1;
5796 
5797 	page_counter_charge(&memcg->memory, nr_pages);
5798 	if (do_memsw_account())
5799 		page_counter_charge(&memcg->memsw, nr_pages);
5800 	css_get_many(&memcg->css, nr_pages);
5801 
5802 	commit_charge(newpage, memcg, false);
5803 
5804 	local_irq_save(flags);
5805 	mem_cgroup_charge_statistics(memcg, newpage, compound, nr_pages);
5806 	memcg_check_events(memcg, newpage);
5807 	local_irq_restore(flags);
5808 }
5809 
5810 DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
5811 EXPORT_SYMBOL(memcg_sockets_enabled_key);
5812 
5813 void mem_cgroup_sk_alloc(struct sock *sk)
5814 {
5815 	struct mem_cgroup *memcg;
5816 
5817 	if (!mem_cgroup_sockets_enabled)
5818 		return;
5819 
5820 	/*
5821 	 * Socket cloning can throw us here with sk_memcg already
5822 	 * filled. It won't however, necessarily happen from
5823 	 * process context. So the test for root memcg given
5824 	 * the current task's memcg won't help us in this case.
5825 	 *
5826 	 * Respecting the original socket's memcg is a better
5827 	 * decision in this case.
5828 	 */
5829 	if (sk->sk_memcg) {
5830 		BUG_ON(mem_cgroup_is_root(sk->sk_memcg));
5831 		css_get(&sk->sk_memcg->css);
5832 		return;
5833 	}
5834 
5835 	rcu_read_lock();
5836 	memcg = mem_cgroup_from_task(current);
5837 	if (memcg == root_mem_cgroup)
5838 		goto out;
5839 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
5840 		goto out;
5841 	if (css_tryget_online(&memcg->css))
5842 		sk->sk_memcg = memcg;
5843 out:
5844 	rcu_read_unlock();
5845 }
5846 
5847 void mem_cgroup_sk_free(struct sock *sk)
5848 {
5849 	if (sk->sk_memcg)
5850 		css_put(&sk->sk_memcg->css);
5851 }
5852 
5853 /**
5854  * mem_cgroup_charge_skmem - charge socket memory
5855  * @memcg: memcg to charge
5856  * @nr_pages: number of pages to charge
5857  *
5858  * Charges @nr_pages to @memcg. Returns %true if the charge fit within
5859  * @memcg's configured limit, %false if the charge had to be forced.
5860  */
5861 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
5862 {
5863 	gfp_t gfp_mask = GFP_KERNEL;
5864 
5865 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
5866 		struct page_counter *fail;
5867 
5868 		if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
5869 			memcg->tcpmem_pressure = 0;
5870 			return true;
5871 		}
5872 		page_counter_charge(&memcg->tcpmem, nr_pages);
5873 		memcg->tcpmem_pressure = 1;
5874 		return false;
5875 	}
5876 
5877 	/* Don't block in the packet receive path */
5878 	if (in_softirq())
5879 		gfp_mask = GFP_NOWAIT;
5880 
5881 	this_cpu_add(memcg->stat->count[MEMCG_SOCK], nr_pages);
5882 
5883 	if (try_charge(memcg, gfp_mask, nr_pages) == 0)
5884 		return true;
5885 
5886 	try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages);
5887 	return false;
5888 }
5889 
5890 /**
5891  * mem_cgroup_uncharge_skmem - uncharge socket memory
5892  * @memcg - memcg to uncharge
5893  * @nr_pages - number of pages to uncharge
5894  */
5895 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
5896 {
5897 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
5898 		page_counter_uncharge(&memcg->tcpmem, nr_pages);
5899 		return;
5900 	}
5901 
5902 	this_cpu_sub(memcg->stat->count[MEMCG_SOCK], nr_pages);
5903 
5904 	refill_stock(memcg, nr_pages);
5905 }
5906 
5907 static int __init cgroup_memory(char *s)
5908 {
5909 	char *token;
5910 
5911 	while ((token = strsep(&s, ",")) != NULL) {
5912 		if (!*token)
5913 			continue;
5914 		if (!strcmp(token, "nosocket"))
5915 			cgroup_memory_nosocket = true;
5916 		if (!strcmp(token, "nokmem"))
5917 			cgroup_memory_nokmem = true;
5918 	}
5919 	return 0;
5920 }
5921 __setup("cgroup.memory=", cgroup_memory);
5922 
5923 /*
5924  * subsys_initcall() for memory controller.
5925  *
5926  * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this
5927  * context because of lock dependencies (cgroup_lock -> cpu hotplug) but
5928  * basically everything that doesn't depend on a specific mem_cgroup structure
5929  * should be initialized from here.
5930  */
5931 static int __init mem_cgroup_init(void)
5932 {
5933 	int cpu, node;
5934 
5935 #ifndef CONFIG_SLOB
5936 	/*
5937 	 * Kmem cache creation is mostly done with the slab_mutex held,
5938 	 * so use a workqueue with limited concurrency to avoid stalling
5939 	 * all worker threads in case lots of cgroups are created and
5940 	 * destroyed simultaneously.
5941 	 */
5942 	memcg_kmem_cache_wq = alloc_workqueue("memcg_kmem_cache", 0, 1);
5943 	BUG_ON(!memcg_kmem_cache_wq);
5944 #endif
5945 
5946 	cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
5947 				  memcg_hotplug_cpu_dead);
5948 
5949 	for_each_possible_cpu(cpu)
5950 		INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
5951 			  drain_local_stock);
5952 
5953 	for_each_node(node) {
5954 		struct mem_cgroup_tree_per_node *rtpn;
5955 
5956 		rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
5957 				    node_online(node) ? node : NUMA_NO_NODE);
5958 
5959 		rtpn->rb_root = RB_ROOT;
5960 		rtpn->rb_rightmost = NULL;
5961 		spin_lock_init(&rtpn->lock);
5962 		soft_limit_tree.rb_tree_per_node[node] = rtpn;
5963 	}
5964 
5965 	return 0;
5966 }
5967 subsys_initcall(mem_cgroup_init);
5968 
5969 #ifdef CONFIG_MEMCG_SWAP
5970 static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
5971 {
5972 	while (!atomic_inc_not_zero(&memcg->id.ref)) {
5973 		/*
5974 		 * The root cgroup cannot be destroyed, so it's refcount must
5975 		 * always be >= 1.
5976 		 */
5977 		if (WARN_ON_ONCE(memcg == root_mem_cgroup)) {
5978 			VM_BUG_ON(1);
5979 			break;
5980 		}
5981 		memcg = parent_mem_cgroup(memcg);
5982 		if (!memcg)
5983 			memcg = root_mem_cgroup;
5984 	}
5985 	return memcg;
5986 }
5987 
5988 /**
5989  * mem_cgroup_swapout - transfer a memsw charge to swap
5990  * @page: page whose memsw charge to transfer
5991  * @entry: swap entry to move the charge to
5992  *
5993  * Transfer the memsw charge of @page to @entry.
5994  */
5995 void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
5996 {
5997 	struct mem_cgroup *memcg, *swap_memcg;
5998 	unsigned int nr_entries;
5999 	unsigned short oldid;
6000 
6001 	VM_BUG_ON_PAGE(PageLRU(page), page);
6002 	VM_BUG_ON_PAGE(page_count(page), page);
6003 
6004 	if (!do_memsw_account())
6005 		return;
6006 
6007 	memcg = page->mem_cgroup;
6008 
6009 	/* Readahead page, never charged */
6010 	if (!memcg)
6011 		return;
6012 
6013 	/*
6014 	 * In case the memcg owning these pages has been offlined and doesn't
6015 	 * have an ID allocated to it anymore, charge the closest online
6016 	 * ancestor for the swap instead and transfer the memory+swap charge.
6017 	 */
6018 	swap_memcg = mem_cgroup_id_get_online(memcg);
6019 	nr_entries = hpage_nr_pages(page);
6020 	/* Get references for the tail pages, too */
6021 	if (nr_entries > 1)
6022 		mem_cgroup_id_get_many(swap_memcg, nr_entries - 1);
6023 	oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg),
6024 				   nr_entries);
6025 	VM_BUG_ON_PAGE(oldid, page);
6026 	mem_cgroup_swap_statistics(swap_memcg, nr_entries);
6027 
6028 	page->mem_cgroup = NULL;
6029 
6030 	if (!mem_cgroup_is_root(memcg))
6031 		page_counter_uncharge(&memcg->memory, nr_entries);
6032 
6033 	if (memcg != swap_memcg) {
6034 		if (!mem_cgroup_is_root(swap_memcg))
6035 			page_counter_charge(&swap_memcg->memsw, nr_entries);
6036 		page_counter_uncharge(&memcg->memsw, nr_entries);
6037 	}
6038 
6039 	/*
6040 	 * Interrupts should be disabled here because the caller holds the
6041 	 * mapping->tree_lock lock which is taken with interrupts-off. It is
6042 	 * important here to have the interrupts disabled because it is the
6043 	 * only synchronisation we have for udpating the per-CPU variables.
6044 	 */
6045 	VM_BUG_ON(!irqs_disabled());
6046 	mem_cgroup_charge_statistics(memcg, page, PageTransHuge(page),
6047 				     -nr_entries);
6048 	memcg_check_events(memcg, page);
6049 
6050 	if (!mem_cgroup_is_root(memcg))
6051 		css_put(&memcg->css);
6052 }
6053 
6054 /**
6055  * mem_cgroup_try_charge_swap - try charging swap space for a page
6056  * @page: page being added to swap
6057  * @entry: swap entry to charge
6058  *
6059  * Try to charge @page's memcg for the swap space at @entry.
6060  *
6061  * Returns 0 on success, -ENOMEM on failure.
6062  */
6063 int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
6064 {
6065 	unsigned int nr_pages = hpage_nr_pages(page);
6066 	struct page_counter *counter;
6067 	struct mem_cgroup *memcg;
6068 	unsigned short oldid;
6069 
6070 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) || !do_swap_account)
6071 		return 0;
6072 
6073 	memcg = page->mem_cgroup;
6074 
6075 	/* Readahead page, never charged */
6076 	if (!memcg)
6077 		return 0;
6078 
6079 	memcg = mem_cgroup_id_get_online(memcg);
6080 
6081 	if (!mem_cgroup_is_root(memcg) &&
6082 	    !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
6083 		mem_cgroup_id_put(memcg);
6084 		return -ENOMEM;
6085 	}
6086 
6087 	/* Get references for the tail pages, too */
6088 	if (nr_pages > 1)
6089 		mem_cgroup_id_get_many(memcg, nr_pages - 1);
6090 	oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages);
6091 	VM_BUG_ON_PAGE(oldid, page);
6092 	mem_cgroup_swap_statistics(memcg, nr_pages);
6093 
6094 	return 0;
6095 }
6096 
6097 /**
6098  * mem_cgroup_uncharge_swap - uncharge swap space
6099  * @entry: swap entry to uncharge
6100  * @nr_pages: the amount of swap space to uncharge
6101  */
6102 void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
6103 {
6104 	struct mem_cgroup *memcg;
6105 	unsigned short id;
6106 
6107 	if (!do_swap_account)
6108 		return;
6109 
6110 	id = swap_cgroup_record(entry, 0, nr_pages);
6111 	rcu_read_lock();
6112 	memcg = mem_cgroup_from_id(id);
6113 	if (memcg) {
6114 		if (!mem_cgroup_is_root(memcg)) {
6115 			if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
6116 				page_counter_uncharge(&memcg->swap, nr_pages);
6117 			else
6118 				page_counter_uncharge(&memcg->memsw, nr_pages);
6119 		}
6120 		mem_cgroup_swap_statistics(memcg, -nr_pages);
6121 		mem_cgroup_id_put_many(memcg, nr_pages);
6122 	}
6123 	rcu_read_unlock();
6124 }
6125 
6126 long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
6127 {
6128 	long nr_swap_pages = get_nr_swap_pages();
6129 
6130 	if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
6131 		return nr_swap_pages;
6132 	for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
6133 		nr_swap_pages = min_t(long, nr_swap_pages,
6134 				      READ_ONCE(memcg->swap.limit) -
6135 				      page_counter_read(&memcg->swap));
6136 	return nr_swap_pages;
6137 }
6138 
6139 bool mem_cgroup_swap_full(struct page *page)
6140 {
6141 	struct mem_cgroup *memcg;
6142 
6143 	VM_BUG_ON_PAGE(!PageLocked(page), page);
6144 
6145 	if (vm_swap_full())
6146 		return true;
6147 	if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
6148 		return false;
6149 
6150 	memcg = page->mem_cgroup;
6151 	if (!memcg)
6152 		return false;
6153 
6154 	for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
6155 		if (page_counter_read(&memcg->swap) * 2 >= memcg->swap.limit)
6156 			return true;
6157 
6158 	return false;
6159 }
6160 
6161 /* for remember boot option*/
6162 #ifdef CONFIG_MEMCG_SWAP_ENABLED
6163 static int really_do_swap_account __initdata = 1;
6164 #else
6165 static int really_do_swap_account __initdata;
6166 #endif
6167 
6168 static int __init enable_swap_account(char *s)
6169 {
6170 	if (!strcmp(s, "1"))
6171 		really_do_swap_account = 1;
6172 	else if (!strcmp(s, "0"))
6173 		really_do_swap_account = 0;
6174 	return 1;
6175 }
6176 __setup("swapaccount=", enable_swap_account);
6177 
6178 static u64 swap_current_read(struct cgroup_subsys_state *css,
6179 			     struct cftype *cft)
6180 {
6181 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6182 
6183 	return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
6184 }
6185 
6186 static int swap_max_show(struct seq_file *m, void *v)
6187 {
6188 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
6189 	unsigned long max = READ_ONCE(memcg->swap.limit);
6190 
6191 	if (max == PAGE_COUNTER_MAX)
6192 		seq_puts(m, "max\n");
6193 	else
6194 		seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);
6195 
6196 	return 0;
6197 }
6198 
6199 static ssize_t swap_max_write(struct kernfs_open_file *of,
6200 			      char *buf, size_t nbytes, loff_t off)
6201 {
6202 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6203 	unsigned long max;
6204 	int err;
6205 
6206 	buf = strstrip(buf);
6207 	err = page_counter_memparse(buf, "max", &max);
6208 	if (err)
6209 		return err;
6210 
6211 	mutex_lock(&memcg_limit_mutex);
6212 	err = page_counter_limit(&memcg->swap, max);
6213 	mutex_unlock(&memcg_limit_mutex);
6214 	if (err)
6215 		return err;
6216 
6217 	return nbytes;
6218 }
6219 
6220 static struct cftype swap_files[] = {
6221 	{
6222 		.name = "swap.current",
6223 		.flags = CFTYPE_NOT_ON_ROOT,
6224 		.read_u64 = swap_current_read,
6225 	},
6226 	{
6227 		.name = "swap.max",
6228 		.flags = CFTYPE_NOT_ON_ROOT,
6229 		.seq_show = swap_max_show,
6230 		.write = swap_max_write,
6231 	},
6232 	{ }	/* terminate */
6233 };
6234 
6235 static struct cftype memsw_cgroup_files[] = {
6236 	{
6237 		.name = "memsw.usage_in_bytes",
6238 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
6239 		.read_u64 = mem_cgroup_read_u64,
6240 	},
6241 	{
6242 		.name = "memsw.max_usage_in_bytes",
6243 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
6244 		.write = mem_cgroup_reset,
6245 		.read_u64 = mem_cgroup_read_u64,
6246 	},
6247 	{
6248 		.name = "memsw.limit_in_bytes",
6249 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
6250 		.write = mem_cgroup_write,
6251 		.read_u64 = mem_cgroup_read_u64,
6252 	},
6253 	{
6254 		.name = "memsw.failcnt",
6255 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
6256 		.write = mem_cgroup_reset,
6257 		.read_u64 = mem_cgroup_read_u64,
6258 	},
6259 	{ },	/* terminate */
6260 };
6261 
6262 static int __init mem_cgroup_swap_init(void)
6263 {
6264 	if (!mem_cgroup_disabled() && really_do_swap_account) {
6265 		do_swap_account = 1;
6266 		WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys,
6267 					       swap_files));
6268 		WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys,
6269 						  memsw_cgroup_files));
6270 	}
6271 	return 0;
6272 }
6273 subsys_initcall(mem_cgroup_swap_init);
6274 
6275 #endif /* CONFIG_MEMCG_SWAP */
6276