xref: /openbmc/linux/mm/memcontrol.c (revision 92a2c6b2)
1 /* memcontrol.c - Memory Controller
2  *
3  * Copyright IBM Corporation, 2007
4  * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5  *
6  * Copyright 2007 OpenVZ SWsoft Inc
7  * Author: Pavel Emelianov <xemul@openvz.org>
8  *
9  * Memory thresholds
10  * Copyright (C) 2009 Nokia Corporation
11  * Author: Kirill A. Shutemov
12  *
13  * Kernel Memory Controller
14  * Copyright (C) 2012 Parallels Inc. and Google Inc.
15  * Authors: Glauber Costa and Suleiman Souhlal
16  *
17  * This program is free software; you can redistribute it and/or modify
18  * it under the terms of the GNU General Public License as published by
19  * the Free Software Foundation; either version 2 of the License, or
20  * (at your option) any later version.
21  *
22  * This program is distributed in the hope that it will be useful,
23  * but WITHOUT ANY WARRANTY; without even the implied warranty of
24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
25  * GNU General Public License for more details.
26  */
27 
28 #include <linux/page_counter.h>
29 #include <linux/memcontrol.h>
30 #include <linux/cgroup.h>
31 #include <linux/mm.h>
32 #include <linux/hugetlb.h>
33 #include <linux/pagemap.h>
34 #include <linux/smp.h>
35 #include <linux/page-flags.h>
36 #include <linux/backing-dev.h>
37 #include <linux/bit_spinlock.h>
38 #include <linux/rcupdate.h>
39 #include <linux/limits.h>
40 #include <linux/export.h>
41 #include <linux/mutex.h>
42 #include <linux/rbtree.h>
43 #include <linux/slab.h>
44 #include <linux/swap.h>
45 #include <linux/swapops.h>
46 #include <linux/spinlock.h>
47 #include <linux/eventfd.h>
48 #include <linux/poll.h>
49 #include <linux/sort.h>
50 #include <linux/fs.h>
51 #include <linux/seq_file.h>
52 #include <linux/vmpressure.h>
53 #include <linux/mm_inline.h>
54 #include <linux/swap_cgroup.h>
55 #include <linux/cpu.h>
56 #include <linux/oom.h>
57 #include <linux/lockdep.h>
58 #include <linux/file.h>
59 #include "internal.h"
60 #include <net/sock.h>
61 #include <net/ip.h>
62 #include <net/tcp_memcontrol.h>
63 #include "slab.h"
64 
65 #include <asm/uaccess.h>
66 
67 #include <trace/events/vmscan.h>
68 
69 struct cgroup_subsys memory_cgrp_subsys __read_mostly;
70 EXPORT_SYMBOL(memory_cgrp_subsys);
71 
72 #define MEM_CGROUP_RECLAIM_RETRIES	5
73 static struct mem_cgroup *root_mem_cgroup __read_mostly;
74 
75 /* Whether the swap controller is active */
76 #ifdef CONFIG_MEMCG_SWAP
77 int do_swap_account __read_mostly;
78 #else
79 #define do_swap_account		0
80 #endif
81 
82 static const char * const mem_cgroup_stat_names[] = {
83 	"cache",
84 	"rss",
85 	"rss_huge",
86 	"mapped_file",
87 	"writeback",
88 	"swap",
89 };
90 
91 static const char * const mem_cgroup_events_names[] = {
92 	"pgpgin",
93 	"pgpgout",
94 	"pgfault",
95 	"pgmajfault",
96 };
97 
98 static const char * const mem_cgroup_lru_names[] = {
99 	"inactive_anon",
100 	"active_anon",
101 	"inactive_file",
102 	"active_file",
103 	"unevictable",
104 };
105 
106 /*
107  * Per memcg event counter is incremented at every pagein/pageout. With THP,
108  * it will be incremated by the number of pages. This counter is used for
109  * for trigger some periodic events. This is straightforward and better
110  * than using jiffies etc. to handle periodic memcg event.
111  */
112 enum mem_cgroup_events_target {
113 	MEM_CGROUP_TARGET_THRESH,
114 	MEM_CGROUP_TARGET_SOFTLIMIT,
115 	MEM_CGROUP_TARGET_NUMAINFO,
116 	MEM_CGROUP_NTARGETS,
117 };
118 #define THRESHOLDS_EVENTS_TARGET 128
119 #define SOFTLIMIT_EVENTS_TARGET 1024
120 #define NUMAINFO_EVENTS_TARGET	1024
121 
122 struct mem_cgroup_stat_cpu {
123 	long count[MEM_CGROUP_STAT_NSTATS];
124 	unsigned long events[MEMCG_NR_EVENTS];
125 	unsigned long nr_page_events;
126 	unsigned long targets[MEM_CGROUP_NTARGETS];
127 };
128 
129 struct reclaim_iter {
130 	struct mem_cgroup *position;
131 	/* scan generation, increased every round-trip */
132 	unsigned int generation;
133 };
134 
135 /*
136  * per-zone information in memory controller.
137  */
138 struct mem_cgroup_per_zone {
139 	struct lruvec		lruvec;
140 	unsigned long		lru_size[NR_LRU_LISTS];
141 
142 	struct reclaim_iter	iter[DEF_PRIORITY + 1];
143 
144 	struct rb_node		tree_node;	/* RB tree node */
145 	unsigned long		usage_in_excess;/* Set to the value by which */
146 						/* the soft limit is exceeded*/
147 	bool			on_tree;
148 	struct mem_cgroup	*memcg;		/* Back pointer, we cannot */
149 						/* use container_of	   */
150 };
151 
152 struct mem_cgroup_per_node {
153 	struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
154 };
155 
156 /*
157  * Cgroups above their limits are maintained in a RB-Tree, independent of
158  * their hierarchy representation
159  */
160 
161 struct mem_cgroup_tree_per_zone {
162 	struct rb_root rb_root;
163 	spinlock_t lock;
164 };
165 
166 struct mem_cgroup_tree_per_node {
167 	struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
168 };
169 
170 struct mem_cgroup_tree {
171 	struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
172 };
173 
174 static struct mem_cgroup_tree soft_limit_tree __read_mostly;
175 
176 struct mem_cgroup_threshold {
177 	struct eventfd_ctx *eventfd;
178 	unsigned long threshold;
179 };
180 
181 /* For threshold */
182 struct mem_cgroup_threshold_ary {
183 	/* An array index points to threshold just below or equal to usage. */
184 	int current_threshold;
185 	/* Size of entries[] */
186 	unsigned int size;
187 	/* Array of thresholds */
188 	struct mem_cgroup_threshold entries[0];
189 };
190 
191 struct mem_cgroup_thresholds {
192 	/* Primary thresholds array */
193 	struct mem_cgroup_threshold_ary *primary;
194 	/*
195 	 * Spare threshold array.
196 	 * This is needed to make mem_cgroup_unregister_event() "never fail".
197 	 * It must be able to store at least primary->size - 1 entries.
198 	 */
199 	struct mem_cgroup_threshold_ary *spare;
200 };
201 
202 /* for OOM */
203 struct mem_cgroup_eventfd_list {
204 	struct list_head list;
205 	struct eventfd_ctx *eventfd;
206 };
207 
208 /*
209  * cgroup_event represents events which userspace want to receive.
210  */
211 struct mem_cgroup_event {
212 	/*
213 	 * memcg which the event belongs to.
214 	 */
215 	struct mem_cgroup *memcg;
216 	/*
217 	 * eventfd to signal userspace about the event.
218 	 */
219 	struct eventfd_ctx *eventfd;
220 	/*
221 	 * Each of these stored in a list by the cgroup.
222 	 */
223 	struct list_head list;
224 	/*
225 	 * register_event() callback will be used to add new userspace
226 	 * waiter for changes related to this event.  Use eventfd_signal()
227 	 * on eventfd to send notification to userspace.
228 	 */
229 	int (*register_event)(struct mem_cgroup *memcg,
230 			      struct eventfd_ctx *eventfd, const char *args);
231 	/*
232 	 * unregister_event() callback will be called when userspace closes
233 	 * the eventfd or on cgroup removing.  This callback must be set,
234 	 * if you want provide notification functionality.
235 	 */
236 	void (*unregister_event)(struct mem_cgroup *memcg,
237 				 struct eventfd_ctx *eventfd);
238 	/*
239 	 * All fields below needed to unregister event when
240 	 * userspace closes eventfd.
241 	 */
242 	poll_table pt;
243 	wait_queue_head_t *wqh;
244 	wait_queue_t wait;
245 	struct work_struct remove;
246 };
247 
248 static void mem_cgroup_threshold(struct mem_cgroup *memcg);
249 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
250 
251 /*
252  * The memory controller data structure. The memory controller controls both
253  * page cache and RSS per cgroup. We would eventually like to provide
254  * statistics based on the statistics developed by Rik Van Riel for clock-pro,
255  * to help the administrator determine what knobs to tune.
256  *
257  * TODO: Add a water mark for the memory controller. Reclaim will begin when
258  * we hit the water mark. May be even add a low water mark, such that
259  * no reclaim occurs from a cgroup at it's low water mark, this is
260  * a feature that will be implemented much later in the future.
261  */
262 struct mem_cgroup {
263 	struct cgroup_subsys_state css;
264 
265 	/* Accounted resources */
266 	struct page_counter memory;
267 	struct page_counter memsw;
268 	struct page_counter kmem;
269 
270 	/* Normal memory consumption range */
271 	unsigned long low;
272 	unsigned long high;
273 
274 	unsigned long soft_limit;
275 
276 	/* vmpressure notifications */
277 	struct vmpressure vmpressure;
278 
279 	/* css_online() has been completed */
280 	int initialized;
281 
282 	/*
283 	 * Should the accounting and control be hierarchical, per subtree?
284 	 */
285 	bool use_hierarchy;
286 
287 	bool		oom_lock;
288 	atomic_t	under_oom;
289 	atomic_t	oom_wakeups;
290 
291 	int	swappiness;
292 	/* OOM-Killer disable */
293 	int		oom_kill_disable;
294 
295 	/* protect arrays of thresholds */
296 	struct mutex thresholds_lock;
297 
298 	/* thresholds for memory usage. RCU-protected */
299 	struct mem_cgroup_thresholds thresholds;
300 
301 	/* thresholds for mem+swap usage. RCU-protected */
302 	struct mem_cgroup_thresholds memsw_thresholds;
303 
304 	/* For oom notifier event fd */
305 	struct list_head oom_notify;
306 
307 	/*
308 	 * Should we move charges of a task when a task is moved into this
309 	 * mem_cgroup ? And what type of charges should we move ?
310 	 */
311 	unsigned long move_charge_at_immigrate;
312 	/*
313 	 * set > 0 if pages under this cgroup are moving to other cgroup.
314 	 */
315 	atomic_t		moving_account;
316 	/* taken only while moving_account > 0 */
317 	spinlock_t		move_lock;
318 	struct task_struct	*move_lock_task;
319 	unsigned long		move_lock_flags;
320 	/*
321 	 * percpu counter.
322 	 */
323 	struct mem_cgroup_stat_cpu __percpu *stat;
324 	/*
325 	 * used when a cpu is offlined or other synchronizations
326 	 * See mem_cgroup_read_stat().
327 	 */
328 	struct mem_cgroup_stat_cpu nocpu_base;
329 	spinlock_t pcp_counter_lock;
330 
331 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_INET)
332 	struct cg_proto tcp_mem;
333 #endif
334 #if defined(CONFIG_MEMCG_KMEM)
335         /* Index in the kmem_cache->memcg_params.memcg_caches array */
336 	int kmemcg_id;
337 	bool kmem_acct_activated;
338 	bool kmem_acct_active;
339 #endif
340 
341 	int last_scanned_node;
342 #if MAX_NUMNODES > 1
343 	nodemask_t	scan_nodes;
344 	atomic_t	numainfo_events;
345 	atomic_t	numainfo_updating;
346 #endif
347 
348 	/* List of events which userspace want to receive */
349 	struct list_head event_list;
350 	spinlock_t event_list_lock;
351 
352 	struct mem_cgroup_per_node *nodeinfo[0];
353 	/* WARNING: nodeinfo must be the last member here */
354 };
355 
356 #ifdef CONFIG_MEMCG_KMEM
357 bool memcg_kmem_is_active(struct mem_cgroup *memcg)
358 {
359 	return memcg->kmem_acct_active;
360 }
361 #endif
362 
363 /* Stuffs for move charges at task migration. */
364 /*
365  * Types of charges to be moved.
366  */
367 #define MOVE_ANON	0x1U
368 #define MOVE_FILE	0x2U
369 #define MOVE_MASK	(MOVE_ANON | MOVE_FILE)
370 
371 /* "mc" and its members are protected by cgroup_mutex */
372 static struct move_charge_struct {
373 	spinlock_t	  lock; /* for from, to */
374 	struct mem_cgroup *from;
375 	struct mem_cgroup *to;
376 	unsigned long flags;
377 	unsigned long precharge;
378 	unsigned long moved_charge;
379 	unsigned long moved_swap;
380 	struct task_struct *moving_task;	/* a task moving charges */
381 	wait_queue_head_t waitq;		/* a waitq for other context */
382 } mc = {
383 	.lock = __SPIN_LOCK_UNLOCKED(mc.lock),
384 	.waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
385 };
386 
387 /*
388  * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
389  * limit reclaim to prevent infinite loops, if they ever occur.
390  */
391 #define	MEM_CGROUP_MAX_RECLAIM_LOOPS		100
392 #define	MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS	2
393 
394 enum charge_type {
395 	MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
396 	MEM_CGROUP_CHARGE_TYPE_ANON,
397 	MEM_CGROUP_CHARGE_TYPE_SWAPOUT,	/* for accounting swapcache */
398 	MEM_CGROUP_CHARGE_TYPE_DROP,	/* a page was unused swap cache */
399 	NR_CHARGE_TYPE,
400 };
401 
402 /* for encoding cft->private value on file */
403 enum res_type {
404 	_MEM,
405 	_MEMSWAP,
406 	_OOM_TYPE,
407 	_KMEM,
408 };
409 
410 #define MEMFILE_PRIVATE(x, val)	((x) << 16 | (val))
411 #define MEMFILE_TYPE(val)	((val) >> 16 & 0xffff)
412 #define MEMFILE_ATTR(val)	((val) & 0xffff)
413 /* Used for OOM nofiier */
414 #define OOM_CONTROL		(0)
415 
416 /*
417  * The memcg_create_mutex will be held whenever a new cgroup is created.
418  * As a consequence, any change that needs to protect against new child cgroups
419  * appearing has to hold it as well.
420  */
421 static DEFINE_MUTEX(memcg_create_mutex);
422 
423 struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *s)
424 {
425 	return s ? container_of(s, struct mem_cgroup, css) : NULL;
426 }
427 
428 /* Some nice accessors for the vmpressure. */
429 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
430 {
431 	if (!memcg)
432 		memcg = root_mem_cgroup;
433 	return &memcg->vmpressure;
434 }
435 
436 struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
437 {
438 	return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
439 }
440 
441 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
442 {
443 	return (memcg == root_mem_cgroup);
444 }
445 
446 /*
447  * We restrict the id in the range of [1, 65535], so it can fit into
448  * an unsigned short.
449  */
450 #define MEM_CGROUP_ID_MAX	USHRT_MAX
451 
452 static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
453 {
454 	return memcg->css.id;
455 }
456 
457 static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
458 {
459 	struct cgroup_subsys_state *css;
460 
461 	css = css_from_id(id, &memory_cgrp_subsys);
462 	return mem_cgroup_from_css(css);
463 }
464 
465 /* Writing them here to avoid exposing memcg's inner layout */
466 #if defined(CONFIG_INET) && defined(CONFIG_MEMCG_KMEM)
467 
468 void sock_update_memcg(struct sock *sk)
469 {
470 	if (mem_cgroup_sockets_enabled) {
471 		struct mem_cgroup *memcg;
472 		struct cg_proto *cg_proto;
473 
474 		BUG_ON(!sk->sk_prot->proto_cgroup);
475 
476 		/* Socket cloning can throw us here with sk_cgrp already
477 		 * filled. It won't however, necessarily happen from
478 		 * process context. So the test for root memcg given
479 		 * the current task's memcg won't help us in this case.
480 		 *
481 		 * Respecting the original socket's memcg is a better
482 		 * decision in this case.
483 		 */
484 		if (sk->sk_cgrp) {
485 			BUG_ON(mem_cgroup_is_root(sk->sk_cgrp->memcg));
486 			css_get(&sk->sk_cgrp->memcg->css);
487 			return;
488 		}
489 
490 		rcu_read_lock();
491 		memcg = mem_cgroup_from_task(current);
492 		cg_proto = sk->sk_prot->proto_cgroup(memcg);
493 		if (!mem_cgroup_is_root(memcg) &&
494 		    memcg_proto_active(cg_proto) &&
495 		    css_tryget_online(&memcg->css)) {
496 			sk->sk_cgrp = cg_proto;
497 		}
498 		rcu_read_unlock();
499 	}
500 }
501 EXPORT_SYMBOL(sock_update_memcg);
502 
503 void sock_release_memcg(struct sock *sk)
504 {
505 	if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
506 		struct mem_cgroup *memcg;
507 		WARN_ON(!sk->sk_cgrp->memcg);
508 		memcg = sk->sk_cgrp->memcg;
509 		css_put(&sk->sk_cgrp->memcg->css);
510 	}
511 }
512 
513 struct cg_proto *tcp_proto_cgroup(struct mem_cgroup *memcg)
514 {
515 	if (!memcg || mem_cgroup_is_root(memcg))
516 		return NULL;
517 
518 	return &memcg->tcp_mem;
519 }
520 EXPORT_SYMBOL(tcp_proto_cgroup);
521 
522 #endif
523 
524 #ifdef CONFIG_MEMCG_KMEM
525 /*
526  * This will be the memcg's index in each cache's ->memcg_params.memcg_caches.
527  * The main reason for not using cgroup id for this:
528  *  this works better in sparse environments, where we have a lot of memcgs,
529  *  but only a few kmem-limited. Or also, if we have, for instance, 200
530  *  memcgs, and none but the 200th is kmem-limited, we'd have to have a
531  *  200 entry array for that.
532  *
533  * The current size of the caches array is stored in memcg_nr_cache_ids. It
534  * will double each time we have to increase it.
535  */
536 static DEFINE_IDA(memcg_cache_ida);
537 int memcg_nr_cache_ids;
538 
539 /* Protects memcg_nr_cache_ids */
540 static DECLARE_RWSEM(memcg_cache_ids_sem);
541 
542 void memcg_get_cache_ids(void)
543 {
544 	down_read(&memcg_cache_ids_sem);
545 }
546 
547 void memcg_put_cache_ids(void)
548 {
549 	up_read(&memcg_cache_ids_sem);
550 }
551 
552 /*
553  * MIN_SIZE is different than 1, because we would like to avoid going through
554  * the alloc/free process all the time. In a small machine, 4 kmem-limited
555  * cgroups is a reasonable guess. In the future, it could be a parameter or
556  * tunable, but that is strictly not necessary.
557  *
558  * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get
559  * this constant directly from cgroup, but it is understandable that this is
560  * better kept as an internal representation in cgroup.c. In any case, the
561  * cgrp_id space is not getting any smaller, and we don't have to necessarily
562  * increase ours as well if it increases.
563  */
564 #define MEMCG_CACHES_MIN_SIZE 4
565 #define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
566 
567 /*
568  * A lot of the calls to the cache allocation functions are expected to be
569  * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
570  * conditional to this static branch, we'll have to allow modules that does
571  * kmem_cache_alloc and the such to see this symbol as well
572  */
573 struct static_key memcg_kmem_enabled_key;
574 EXPORT_SYMBOL(memcg_kmem_enabled_key);
575 
576 #endif /* CONFIG_MEMCG_KMEM */
577 
578 static struct mem_cgroup_per_zone *
579 mem_cgroup_zone_zoneinfo(struct mem_cgroup *memcg, struct zone *zone)
580 {
581 	int nid = zone_to_nid(zone);
582 	int zid = zone_idx(zone);
583 
584 	return &memcg->nodeinfo[nid]->zoneinfo[zid];
585 }
586 
587 struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg)
588 {
589 	return &memcg->css;
590 }
591 
592 static struct mem_cgroup_per_zone *
593 mem_cgroup_page_zoneinfo(struct mem_cgroup *memcg, struct page *page)
594 {
595 	int nid = page_to_nid(page);
596 	int zid = page_zonenum(page);
597 
598 	return &memcg->nodeinfo[nid]->zoneinfo[zid];
599 }
600 
601 static struct mem_cgroup_tree_per_zone *
602 soft_limit_tree_node_zone(int nid, int zid)
603 {
604 	return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
605 }
606 
607 static struct mem_cgroup_tree_per_zone *
608 soft_limit_tree_from_page(struct page *page)
609 {
610 	int nid = page_to_nid(page);
611 	int zid = page_zonenum(page);
612 
613 	return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
614 }
615 
616 static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_zone *mz,
617 					 struct mem_cgroup_tree_per_zone *mctz,
618 					 unsigned long new_usage_in_excess)
619 {
620 	struct rb_node **p = &mctz->rb_root.rb_node;
621 	struct rb_node *parent = NULL;
622 	struct mem_cgroup_per_zone *mz_node;
623 
624 	if (mz->on_tree)
625 		return;
626 
627 	mz->usage_in_excess = new_usage_in_excess;
628 	if (!mz->usage_in_excess)
629 		return;
630 	while (*p) {
631 		parent = *p;
632 		mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
633 					tree_node);
634 		if (mz->usage_in_excess < mz_node->usage_in_excess)
635 			p = &(*p)->rb_left;
636 		/*
637 		 * We can't avoid mem cgroups that are over their soft
638 		 * limit by the same amount
639 		 */
640 		else if (mz->usage_in_excess >= mz_node->usage_in_excess)
641 			p = &(*p)->rb_right;
642 	}
643 	rb_link_node(&mz->tree_node, parent, p);
644 	rb_insert_color(&mz->tree_node, &mctz->rb_root);
645 	mz->on_tree = true;
646 }
647 
648 static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_zone *mz,
649 					 struct mem_cgroup_tree_per_zone *mctz)
650 {
651 	if (!mz->on_tree)
652 		return;
653 	rb_erase(&mz->tree_node, &mctz->rb_root);
654 	mz->on_tree = false;
655 }
656 
657 static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_zone *mz,
658 				       struct mem_cgroup_tree_per_zone *mctz)
659 {
660 	unsigned long flags;
661 
662 	spin_lock_irqsave(&mctz->lock, flags);
663 	__mem_cgroup_remove_exceeded(mz, mctz);
664 	spin_unlock_irqrestore(&mctz->lock, flags);
665 }
666 
667 static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
668 {
669 	unsigned long nr_pages = page_counter_read(&memcg->memory);
670 	unsigned long soft_limit = ACCESS_ONCE(memcg->soft_limit);
671 	unsigned long excess = 0;
672 
673 	if (nr_pages > soft_limit)
674 		excess = nr_pages - soft_limit;
675 
676 	return excess;
677 }
678 
679 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
680 {
681 	unsigned long excess;
682 	struct mem_cgroup_per_zone *mz;
683 	struct mem_cgroup_tree_per_zone *mctz;
684 
685 	mctz = soft_limit_tree_from_page(page);
686 	/*
687 	 * Necessary to update all ancestors when hierarchy is used.
688 	 * because their event counter is not touched.
689 	 */
690 	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
691 		mz = mem_cgroup_page_zoneinfo(memcg, page);
692 		excess = soft_limit_excess(memcg);
693 		/*
694 		 * We have to update the tree if mz is on RB-tree or
695 		 * mem is over its softlimit.
696 		 */
697 		if (excess || mz->on_tree) {
698 			unsigned long flags;
699 
700 			spin_lock_irqsave(&mctz->lock, flags);
701 			/* if on-tree, remove it */
702 			if (mz->on_tree)
703 				__mem_cgroup_remove_exceeded(mz, mctz);
704 			/*
705 			 * Insert again. mz->usage_in_excess will be updated.
706 			 * If excess is 0, no tree ops.
707 			 */
708 			__mem_cgroup_insert_exceeded(mz, mctz, excess);
709 			spin_unlock_irqrestore(&mctz->lock, flags);
710 		}
711 	}
712 }
713 
714 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
715 {
716 	struct mem_cgroup_tree_per_zone *mctz;
717 	struct mem_cgroup_per_zone *mz;
718 	int nid, zid;
719 
720 	for_each_node(nid) {
721 		for (zid = 0; zid < MAX_NR_ZONES; zid++) {
722 			mz = &memcg->nodeinfo[nid]->zoneinfo[zid];
723 			mctz = soft_limit_tree_node_zone(nid, zid);
724 			mem_cgroup_remove_exceeded(mz, mctz);
725 		}
726 	}
727 }
728 
729 static struct mem_cgroup_per_zone *
730 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
731 {
732 	struct rb_node *rightmost = NULL;
733 	struct mem_cgroup_per_zone *mz;
734 
735 retry:
736 	mz = NULL;
737 	rightmost = rb_last(&mctz->rb_root);
738 	if (!rightmost)
739 		goto done;		/* Nothing to reclaim from */
740 
741 	mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
742 	/*
743 	 * Remove the node now but someone else can add it back,
744 	 * we will to add it back at the end of reclaim to its correct
745 	 * position in the tree.
746 	 */
747 	__mem_cgroup_remove_exceeded(mz, mctz);
748 	if (!soft_limit_excess(mz->memcg) ||
749 	    !css_tryget_online(&mz->memcg->css))
750 		goto retry;
751 done:
752 	return mz;
753 }
754 
755 static struct mem_cgroup_per_zone *
756 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
757 {
758 	struct mem_cgroup_per_zone *mz;
759 
760 	spin_lock_irq(&mctz->lock);
761 	mz = __mem_cgroup_largest_soft_limit_node(mctz);
762 	spin_unlock_irq(&mctz->lock);
763 	return mz;
764 }
765 
766 /*
767  * Implementation Note: reading percpu statistics for memcg.
768  *
769  * Both of vmstat[] and percpu_counter has threshold and do periodic
770  * synchronization to implement "quick" read. There are trade-off between
771  * reading cost and precision of value. Then, we may have a chance to implement
772  * a periodic synchronizion of counter in memcg's counter.
773  *
774  * But this _read() function is used for user interface now. The user accounts
775  * memory usage by memory cgroup and he _always_ requires exact value because
776  * he accounts memory. Even if we provide quick-and-fuzzy read, we always
777  * have to visit all online cpus and make sum. So, for now, unnecessary
778  * synchronization is not implemented. (just implemented for cpu hotplug)
779  *
780  * If there are kernel internal actions which can make use of some not-exact
781  * value, and reading all cpu value can be performance bottleneck in some
782  * common workload, threashold and synchonization as vmstat[] should be
783  * implemented.
784  */
785 static long mem_cgroup_read_stat(struct mem_cgroup *memcg,
786 				 enum mem_cgroup_stat_index idx)
787 {
788 	long val = 0;
789 	int cpu;
790 
791 	get_online_cpus();
792 	for_each_online_cpu(cpu)
793 		val += per_cpu(memcg->stat->count[idx], cpu);
794 #ifdef CONFIG_HOTPLUG_CPU
795 	spin_lock(&memcg->pcp_counter_lock);
796 	val += memcg->nocpu_base.count[idx];
797 	spin_unlock(&memcg->pcp_counter_lock);
798 #endif
799 	put_online_cpus();
800 	return val;
801 }
802 
803 static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg,
804 					    enum mem_cgroup_events_index idx)
805 {
806 	unsigned long val = 0;
807 	int cpu;
808 
809 	get_online_cpus();
810 	for_each_online_cpu(cpu)
811 		val += per_cpu(memcg->stat->events[idx], cpu);
812 #ifdef CONFIG_HOTPLUG_CPU
813 	spin_lock(&memcg->pcp_counter_lock);
814 	val += memcg->nocpu_base.events[idx];
815 	spin_unlock(&memcg->pcp_counter_lock);
816 #endif
817 	put_online_cpus();
818 	return val;
819 }
820 
821 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
822 					 struct page *page,
823 					 int nr_pages)
824 {
825 	/*
826 	 * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
827 	 * counted as CACHE even if it's on ANON LRU.
828 	 */
829 	if (PageAnon(page))
830 		__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS],
831 				nr_pages);
832 	else
833 		__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE],
834 				nr_pages);
835 
836 	if (PageTransHuge(page))
837 		__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
838 				nr_pages);
839 
840 	/* pagein of a big page is an event. So, ignore page size */
841 	if (nr_pages > 0)
842 		__this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
843 	else {
844 		__this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
845 		nr_pages = -nr_pages; /* for event */
846 	}
847 
848 	__this_cpu_add(memcg->stat->nr_page_events, nr_pages);
849 }
850 
851 unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
852 {
853 	struct mem_cgroup_per_zone *mz;
854 
855 	mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
856 	return mz->lru_size[lru];
857 }
858 
859 static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
860 						  int nid,
861 						  unsigned int lru_mask)
862 {
863 	unsigned long nr = 0;
864 	int zid;
865 
866 	VM_BUG_ON((unsigned)nid >= nr_node_ids);
867 
868 	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
869 		struct mem_cgroup_per_zone *mz;
870 		enum lru_list lru;
871 
872 		for_each_lru(lru) {
873 			if (!(BIT(lru) & lru_mask))
874 				continue;
875 			mz = &memcg->nodeinfo[nid]->zoneinfo[zid];
876 			nr += mz->lru_size[lru];
877 		}
878 	}
879 	return nr;
880 }
881 
882 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
883 			unsigned int lru_mask)
884 {
885 	unsigned long nr = 0;
886 	int nid;
887 
888 	for_each_node_state(nid, N_MEMORY)
889 		nr += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
890 	return nr;
891 }
892 
893 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
894 				       enum mem_cgroup_events_target target)
895 {
896 	unsigned long val, next;
897 
898 	val = __this_cpu_read(memcg->stat->nr_page_events);
899 	next = __this_cpu_read(memcg->stat->targets[target]);
900 	/* from time_after() in jiffies.h */
901 	if ((long)next - (long)val < 0) {
902 		switch (target) {
903 		case MEM_CGROUP_TARGET_THRESH:
904 			next = val + THRESHOLDS_EVENTS_TARGET;
905 			break;
906 		case MEM_CGROUP_TARGET_SOFTLIMIT:
907 			next = val + SOFTLIMIT_EVENTS_TARGET;
908 			break;
909 		case MEM_CGROUP_TARGET_NUMAINFO:
910 			next = val + NUMAINFO_EVENTS_TARGET;
911 			break;
912 		default:
913 			break;
914 		}
915 		__this_cpu_write(memcg->stat->targets[target], next);
916 		return true;
917 	}
918 	return false;
919 }
920 
921 /*
922  * Check events in order.
923  *
924  */
925 static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
926 {
927 	/* threshold event is triggered in finer grain than soft limit */
928 	if (unlikely(mem_cgroup_event_ratelimit(memcg,
929 						MEM_CGROUP_TARGET_THRESH))) {
930 		bool do_softlimit;
931 		bool do_numainfo __maybe_unused;
932 
933 		do_softlimit = mem_cgroup_event_ratelimit(memcg,
934 						MEM_CGROUP_TARGET_SOFTLIMIT);
935 #if MAX_NUMNODES > 1
936 		do_numainfo = mem_cgroup_event_ratelimit(memcg,
937 						MEM_CGROUP_TARGET_NUMAINFO);
938 #endif
939 		mem_cgroup_threshold(memcg);
940 		if (unlikely(do_softlimit))
941 			mem_cgroup_update_tree(memcg, page);
942 #if MAX_NUMNODES > 1
943 		if (unlikely(do_numainfo))
944 			atomic_inc(&memcg->numainfo_events);
945 #endif
946 	}
947 }
948 
949 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
950 {
951 	/*
952 	 * mm_update_next_owner() may clear mm->owner to NULL
953 	 * if it races with swapoff, page migration, etc.
954 	 * So this can be called with p == NULL.
955 	 */
956 	if (unlikely(!p))
957 		return NULL;
958 
959 	return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
960 }
961 
962 static struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
963 {
964 	struct mem_cgroup *memcg = NULL;
965 
966 	rcu_read_lock();
967 	do {
968 		/*
969 		 * Page cache insertions can happen withou an
970 		 * actual mm context, e.g. during disk probing
971 		 * on boot, loopback IO, acct() writes etc.
972 		 */
973 		if (unlikely(!mm))
974 			memcg = root_mem_cgroup;
975 		else {
976 			memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
977 			if (unlikely(!memcg))
978 				memcg = root_mem_cgroup;
979 		}
980 	} while (!css_tryget_online(&memcg->css));
981 	rcu_read_unlock();
982 	return memcg;
983 }
984 
985 /**
986  * mem_cgroup_iter - iterate over memory cgroup hierarchy
987  * @root: hierarchy root
988  * @prev: previously returned memcg, NULL on first invocation
989  * @reclaim: cookie for shared reclaim walks, NULL for full walks
990  *
991  * Returns references to children of the hierarchy below @root, or
992  * @root itself, or %NULL after a full round-trip.
993  *
994  * Caller must pass the return value in @prev on subsequent
995  * invocations for reference counting, or use mem_cgroup_iter_break()
996  * to cancel a hierarchy walk before the round-trip is complete.
997  *
998  * Reclaimers can specify a zone and a priority level in @reclaim to
999  * divide up the memcgs in the hierarchy among all concurrent
1000  * reclaimers operating on the same zone and priority.
1001  */
1002 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
1003 				   struct mem_cgroup *prev,
1004 				   struct mem_cgroup_reclaim_cookie *reclaim)
1005 {
1006 	struct reclaim_iter *uninitialized_var(iter);
1007 	struct cgroup_subsys_state *css = NULL;
1008 	struct mem_cgroup *memcg = NULL;
1009 	struct mem_cgroup *pos = NULL;
1010 
1011 	if (mem_cgroup_disabled())
1012 		return NULL;
1013 
1014 	if (!root)
1015 		root = root_mem_cgroup;
1016 
1017 	if (prev && !reclaim)
1018 		pos = prev;
1019 
1020 	if (!root->use_hierarchy && root != root_mem_cgroup) {
1021 		if (prev)
1022 			goto out;
1023 		return root;
1024 	}
1025 
1026 	rcu_read_lock();
1027 
1028 	if (reclaim) {
1029 		struct mem_cgroup_per_zone *mz;
1030 
1031 		mz = mem_cgroup_zone_zoneinfo(root, reclaim->zone);
1032 		iter = &mz->iter[reclaim->priority];
1033 
1034 		if (prev && reclaim->generation != iter->generation)
1035 			goto out_unlock;
1036 
1037 		do {
1038 			pos = ACCESS_ONCE(iter->position);
1039 			/*
1040 			 * A racing update may change the position and
1041 			 * put the last reference, hence css_tryget(),
1042 			 * or retry to see the updated position.
1043 			 */
1044 		} while (pos && !css_tryget(&pos->css));
1045 	}
1046 
1047 	if (pos)
1048 		css = &pos->css;
1049 
1050 	for (;;) {
1051 		css = css_next_descendant_pre(css, &root->css);
1052 		if (!css) {
1053 			/*
1054 			 * Reclaimers share the hierarchy walk, and a
1055 			 * new one might jump in right at the end of
1056 			 * the hierarchy - make sure they see at least
1057 			 * one group and restart from the beginning.
1058 			 */
1059 			if (!prev)
1060 				continue;
1061 			break;
1062 		}
1063 
1064 		/*
1065 		 * Verify the css and acquire a reference.  The root
1066 		 * is provided by the caller, so we know it's alive
1067 		 * and kicking, and don't take an extra reference.
1068 		 */
1069 		memcg = mem_cgroup_from_css(css);
1070 
1071 		if (css == &root->css)
1072 			break;
1073 
1074 		if (css_tryget(css)) {
1075 			/*
1076 			 * Make sure the memcg is initialized:
1077 			 * mem_cgroup_css_online() orders the the
1078 			 * initialization against setting the flag.
1079 			 */
1080 			if (smp_load_acquire(&memcg->initialized))
1081 				break;
1082 
1083 			css_put(css);
1084 		}
1085 
1086 		memcg = NULL;
1087 	}
1088 
1089 	if (reclaim) {
1090 		if (cmpxchg(&iter->position, pos, memcg) == pos) {
1091 			if (memcg)
1092 				css_get(&memcg->css);
1093 			if (pos)
1094 				css_put(&pos->css);
1095 		}
1096 
1097 		/*
1098 		 * pairs with css_tryget when dereferencing iter->position
1099 		 * above.
1100 		 */
1101 		if (pos)
1102 			css_put(&pos->css);
1103 
1104 		if (!memcg)
1105 			iter->generation++;
1106 		else if (!prev)
1107 			reclaim->generation = iter->generation;
1108 	}
1109 
1110 out_unlock:
1111 	rcu_read_unlock();
1112 out:
1113 	if (prev && prev != root)
1114 		css_put(&prev->css);
1115 
1116 	return memcg;
1117 }
1118 
1119 /**
1120  * mem_cgroup_iter_break - abort a hierarchy walk prematurely
1121  * @root: hierarchy root
1122  * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
1123  */
1124 void mem_cgroup_iter_break(struct mem_cgroup *root,
1125 			   struct mem_cgroup *prev)
1126 {
1127 	if (!root)
1128 		root = root_mem_cgroup;
1129 	if (prev && prev != root)
1130 		css_put(&prev->css);
1131 }
1132 
1133 /*
1134  * Iteration constructs for visiting all cgroups (under a tree).  If
1135  * loops are exited prematurely (break), mem_cgroup_iter_break() must
1136  * be used for reference counting.
1137  */
1138 #define for_each_mem_cgroup_tree(iter, root)		\
1139 	for (iter = mem_cgroup_iter(root, NULL, NULL);	\
1140 	     iter != NULL;				\
1141 	     iter = mem_cgroup_iter(root, iter, NULL))
1142 
1143 #define for_each_mem_cgroup(iter)			\
1144 	for (iter = mem_cgroup_iter(NULL, NULL, NULL);	\
1145 	     iter != NULL;				\
1146 	     iter = mem_cgroup_iter(NULL, iter, NULL))
1147 
1148 void __mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
1149 {
1150 	struct mem_cgroup *memcg;
1151 
1152 	rcu_read_lock();
1153 	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1154 	if (unlikely(!memcg))
1155 		goto out;
1156 
1157 	switch (idx) {
1158 	case PGFAULT:
1159 		this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT]);
1160 		break;
1161 	case PGMAJFAULT:
1162 		this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
1163 		break;
1164 	default:
1165 		BUG();
1166 	}
1167 out:
1168 	rcu_read_unlock();
1169 }
1170 EXPORT_SYMBOL(__mem_cgroup_count_vm_event);
1171 
1172 /**
1173  * mem_cgroup_zone_lruvec - get the lru list vector for a zone and memcg
1174  * @zone: zone of the wanted lruvec
1175  * @memcg: memcg of the wanted lruvec
1176  *
1177  * Returns the lru list vector holding pages for the given @zone and
1178  * @mem.  This can be the global zone lruvec, if the memory controller
1179  * is disabled.
1180  */
1181 struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
1182 				      struct mem_cgroup *memcg)
1183 {
1184 	struct mem_cgroup_per_zone *mz;
1185 	struct lruvec *lruvec;
1186 
1187 	if (mem_cgroup_disabled()) {
1188 		lruvec = &zone->lruvec;
1189 		goto out;
1190 	}
1191 
1192 	mz = mem_cgroup_zone_zoneinfo(memcg, zone);
1193 	lruvec = &mz->lruvec;
1194 out:
1195 	/*
1196 	 * Since a node can be onlined after the mem_cgroup was created,
1197 	 * we have to be prepared to initialize lruvec->zone here;
1198 	 * and if offlined then reonlined, we need to reinitialize it.
1199 	 */
1200 	if (unlikely(lruvec->zone != zone))
1201 		lruvec->zone = zone;
1202 	return lruvec;
1203 }
1204 
1205 /**
1206  * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page
1207  * @page: the page
1208  * @zone: zone of the page
1209  *
1210  * This function is only safe when following the LRU page isolation
1211  * and putback protocol: the LRU lock must be held, and the page must
1212  * either be PageLRU() or the caller must have isolated/allocated it.
1213  */
1214 struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct zone *zone)
1215 {
1216 	struct mem_cgroup_per_zone *mz;
1217 	struct mem_cgroup *memcg;
1218 	struct lruvec *lruvec;
1219 
1220 	if (mem_cgroup_disabled()) {
1221 		lruvec = &zone->lruvec;
1222 		goto out;
1223 	}
1224 
1225 	memcg = page->mem_cgroup;
1226 	/*
1227 	 * Swapcache readahead pages are added to the LRU - and
1228 	 * possibly migrated - before they are charged.
1229 	 */
1230 	if (!memcg)
1231 		memcg = root_mem_cgroup;
1232 
1233 	mz = mem_cgroup_page_zoneinfo(memcg, page);
1234 	lruvec = &mz->lruvec;
1235 out:
1236 	/*
1237 	 * Since a node can be onlined after the mem_cgroup was created,
1238 	 * we have to be prepared to initialize lruvec->zone here;
1239 	 * and if offlined then reonlined, we need to reinitialize it.
1240 	 */
1241 	if (unlikely(lruvec->zone != zone))
1242 		lruvec->zone = zone;
1243 	return lruvec;
1244 }
1245 
1246 /**
1247  * mem_cgroup_update_lru_size - account for adding or removing an lru page
1248  * @lruvec: mem_cgroup per zone lru vector
1249  * @lru: index of lru list the page is sitting on
1250  * @nr_pages: positive when adding or negative when removing
1251  *
1252  * This function must be called when a page is added to or removed from an
1253  * lru list.
1254  */
1255 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
1256 				int nr_pages)
1257 {
1258 	struct mem_cgroup_per_zone *mz;
1259 	unsigned long *lru_size;
1260 
1261 	if (mem_cgroup_disabled())
1262 		return;
1263 
1264 	mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
1265 	lru_size = mz->lru_size + lru;
1266 	*lru_size += nr_pages;
1267 	VM_BUG_ON((long)(*lru_size) < 0);
1268 }
1269 
1270 bool mem_cgroup_is_descendant(struct mem_cgroup *memcg, struct mem_cgroup *root)
1271 {
1272 	if (root == memcg)
1273 		return true;
1274 	if (!root->use_hierarchy)
1275 		return false;
1276 	return cgroup_is_descendant(memcg->css.cgroup, root->css.cgroup);
1277 }
1278 
1279 bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg)
1280 {
1281 	struct mem_cgroup *task_memcg;
1282 	struct task_struct *p;
1283 	bool ret;
1284 
1285 	p = find_lock_task_mm(task);
1286 	if (p) {
1287 		task_memcg = get_mem_cgroup_from_mm(p->mm);
1288 		task_unlock(p);
1289 	} else {
1290 		/*
1291 		 * All threads may have already detached their mm's, but the oom
1292 		 * killer still needs to detect if they have already been oom
1293 		 * killed to prevent needlessly killing additional tasks.
1294 		 */
1295 		rcu_read_lock();
1296 		task_memcg = mem_cgroup_from_task(task);
1297 		css_get(&task_memcg->css);
1298 		rcu_read_unlock();
1299 	}
1300 	ret = mem_cgroup_is_descendant(task_memcg, memcg);
1301 	css_put(&task_memcg->css);
1302 	return ret;
1303 }
1304 
1305 int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
1306 {
1307 	unsigned long inactive_ratio;
1308 	unsigned long inactive;
1309 	unsigned long active;
1310 	unsigned long gb;
1311 
1312 	inactive = mem_cgroup_get_lru_size(lruvec, LRU_INACTIVE_ANON);
1313 	active = mem_cgroup_get_lru_size(lruvec, LRU_ACTIVE_ANON);
1314 
1315 	gb = (inactive + active) >> (30 - PAGE_SHIFT);
1316 	if (gb)
1317 		inactive_ratio = int_sqrt(10 * gb);
1318 	else
1319 		inactive_ratio = 1;
1320 
1321 	return inactive * inactive_ratio < active;
1322 }
1323 
1324 bool mem_cgroup_lruvec_online(struct lruvec *lruvec)
1325 {
1326 	struct mem_cgroup_per_zone *mz;
1327 	struct mem_cgroup *memcg;
1328 
1329 	if (mem_cgroup_disabled())
1330 		return true;
1331 
1332 	mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
1333 	memcg = mz->memcg;
1334 
1335 	return !!(memcg->css.flags & CSS_ONLINE);
1336 }
1337 
1338 #define mem_cgroup_from_counter(counter, member)	\
1339 	container_of(counter, struct mem_cgroup, member)
1340 
1341 /**
1342  * mem_cgroup_margin - calculate chargeable space of a memory cgroup
1343  * @memcg: the memory cgroup
1344  *
1345  * Returns the maximum amount of memory @mem can be charged with, in
1346  * pages.
1347  */
1348 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1349 {
1350 	unsigned long margin = 0;
1351 	unsigned long count;
1352 	unsigned long limit;
1353 
1354 	count = page_counter_read(&memcg->memory);
1355 	limit = ACCESS_ONCE(memcg->memory.limit);
1356 	if (count < limit)
1357 		margin = limit - count;
1358 
1359 	if (do_swap_account) {
1360 		count = page_counter_read(&memcg->memsw);
1361 		limit = ACCESS_ONCE(memcg->memsw.limit);
1362 		if (count <= limit)
1363 			margin = min(margin, limit - count);
1364 	}
1365 
1366 	return margin;
1367 }
1368 
1369 int mem_cgroup_swappiness(struct mem_cgroup *memcg)
1370 {
1371 	/* root ? */
1372 	if (mem_cgroup_disabled() || !memcg->css.parent)
1373 		return vm_swappiness;
1374 
1375 	return memcg->swappiness;
1376 }
1377 
1378 /*
1379  * A routine for checking "mem" is under move_account() or not.
1380  *
1381  * Checking a cgroup is mc.from or mc.to or under hierarchy of
1382  * moving cgroups. This is for waiting at high-memory pressure
1383  * caused by "move".
1384  */
1385 static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1386 {
1387 	struct mem_cgroup *from;
1388 	struct mem_cgroup *to;
1389 	bool ret = false;
1390 	/*
1391 	 * Unlike task_move routines, we access mc.to, mc.from not under
1392 	 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1393 	 */
1394 	spin_lock(&mc.lock);
1395 	from = mc.from;
1396 	to = mc.to;
1397 	if (!from)
1398 		goto unlock;
1399 
1400 	ret = mem_cgroup_is_descendant(from, memcg) ||
1401 		mem_cgroup_is_descendant(to, memcg);
1402 unlock:
1403 	spin_unlock(&mc.lock);
1404 	return ret;
1405 }
1406 
1407 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1408 {
1409 	if (mc.moving_task && current != mc.moving_task) {
1410 		if (mem_cgroup_under_move(memcg)) {
1411 			DEFINE_WAIT(wait);
1412 			prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1413 			/* moving charge context might have finished. */
1414 			if (mc.moving_task)
1415 				schedule();
1416 			finish_wait(&mc.waitq, &wait);
1417 			return true;
1418 		}
1419 	}
1420 	return false;
1421 }
1422 
1423 #define K(x) ((x) << (PAGE_SHIFT-10))
1424 /**
1425  * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller.
1426  * @memcg: The memory cgroup that went over limit
1427  * @p: Task that is going to be killed
1428  *
1429  * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1430  * enabled
1431  */
1432 void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
1433 {
1434 	/* oom_info_lock ensures that parallel ooms do not interleave */
1435 	static DEFINE_MUTEX(oom_info_lock);
1436 	struct mem_cgroup *iter;
1437 	unsigned int i;
1438 
1439 	if (!p)
1440 		return;
1441 
1442 	mutex_lock(&oom_info_lock);
1443 	rcu_read_lock();
1444 
1445 	pr_info("Task in ");
1446 	pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
1447 	pr_cont(" killed as a result of limit of ");
1448 	pr_cont_cgroup_path(memcg->css.cgroup);
1449 	pr_cont("\n");
1450 
1451 	rcu_read_unlock();
1452 
1453 	pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
1454 		K((u64)page_counter_read(&memcg->memory)),
1455 		K((u64)memcg->memory.limit), memcg->memory.failcnt);
1456 	pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
1457 		K((u64)page_counter_read(&memcg->memsw)),
1458 		K((u64)memcg->memsw.limit), memcg->memsw.failcnt);
1459 	pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
1460 		K((u64)page_counter_read(&memcg->kmem)),
1461 		K((u64)memcg->kmem.limit), memcg->kmem.failcnt);
1462 
1463 	for_each_mem_cgroup_tree(iter, memcg) {
1464 		pr_info("Memory cgroup stats for ");
1465 		pr_cont_cgroup_path(iter->css.cgroup);
1466 		pr_cont(":");
1467 
1468 		for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
1469 			if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
1470 				continue;
1471 			pr_cont(" %s:%ldKB", mem_cgroup_stat_names[i],
1472 				K(mem_cgroup_read_stat(iter, i)));
1473 		}
1474 
1475 		for (i = 0; i < NR_LRU_LISTS; i++)
1476 			pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
1477 				K(mem_cgroup_nr_lru_pages(iter, BIT(i))));
1478 
1479 		pr_cont("\n");
1480 	}
1481 	mutex_unlock(&oom_info_lock);
1482 }
1483 
1484 /*
1485  * This function returns the number of memcg under hierarchy tree. Returns
1486  * 1(self count) if no children.
1487  */
1488 static int mem_cgroup_count_children(struct mem_cgroup *memcg)
1489 {
1490 	int num = 0;
1491 	struct mem_cgroup *iter;
1492 
1493 	for_each_mem_cgroup_tree(iter, memcg)
1494 		num++;
1495 	return num;
1496 }
1497 
1498 /*
1499  * Return the memory (and swap, if configured) limit for a memcg.
1500  */
1501 static unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg)
1502 {
1503 	unsigned long limit;
1504 
1505 	limit = memcg->memory.limit;
1506 	if (mem_cgroup_swappiness(memcg)) {
1507 		unsigned long memsw_limit;
1508 
1509 		memsw_limit = memcg->memsw.limit;
1510 		limit = min(limit + total_swap_pages, memsw_limit);
1511 	}
1512 	return limit;
1513 }
1514 
1515 static void mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1516 				     int order)
1517 {
1518 	struct mem_cgroup *iter;
1519 	unsigned long chosen_points = 0;
1520 	unsigned long totalpages;
1521 	unsigned int points = 0;
1522 	struct task_struct *chosen = NULL;
1523 
1524 	/*
1525 	 * If current has a pending SIGKILL or is exiting, then automatically
1526 	 * select it.  The goal is to allow it to allocate so that it may
1527 	 * quickly exit and free its memory.
1528 	 */
1529 	if (fatal_signal_pending(current) || task_will_free_mem(current)) {
1530 		mark_tsk_oom_victim(current);
1531 		return;
1532 	}
1533 
1534 	check_panic_on_oom(CONSTRAINT_MEMCG, gfp_mask, order, NULL);
1535 	totalpages = mem_cgroup_get_limit(memcg) ? : 1;
1536 	for_each_mem_cgroup_tree(iter, memcg) {
1537 		struct css_task_iter it;
1538 		struct task_struct *task;
1539 
1540 		css_task_iter_start(&iter->css, &it);
1541 		while ((task = css_task_iter_next(&it))) {
1542 			switch (oom_scan_process_thread(task, totalpages, NULL,
1543 							false)) {
1544 			case OOM_SCAN_SELECT:
1545 				if (chosen)
1546 					put_task_struct(chosen);
1547 				chosen = task;
1548 				chosen_points = ULONG_MAX;
1549 				get_task_struct(chosen);
1550 				/* fall through */
1551 			case OOM_SCAN_CONTINUE:
1552 				continue;
1553 			case OOM_SCAN_ABORT:
1554 				css_task_iter_end(&it);
1555 				mem_cgroup_iter_break(memcg, iter);
1556 				if (chosen)
1557 					put_task_struct(chosen);
1558 				return;
1559 			case OOM_SCAN_OK:
1560 				break;
1561 			};
1562 			points = oom_badness(task, memcg, NULL, totalpages);
1563 			if (!points || points < chosen_points)
1564 				continue;
1565 			/* Prefer thread group leaders for display purposes */
1566 			if (points == chosen_points &&
1567 			    thread_group_leader(chosen))
1568 				continue;
1569 
1570 			if (chosen)
1571 				put_task_struct(chosen);
1572 			chosen = task;
1573 			chosen_points = points;
1574 			get_task_struct(chosen);
1575 		}
1576 		css_task_iter_end(&it);
1577 	}
1578 
1579 	if (!chosen)
1580 		return;
1581 	points = chosen_points * 1000 / totalpages;
1582 	oom_kill_process(chosen, gfp_mask, order, points, totalpages, memcg,
1583 			 NULL, "Memory cgroup out of memory");
1584 }
1585 
1586 #if MAX_NUMNODES > 1
1587 
1588 /**
1589  * test_mem_cgroup_node_reclaimable
1590  * @memcg: the target memcg
1591  * @nid: the node ID to be checked.
1592  * @noswap : specify true here if the user wants flle only information.
1593  *
1594  * This function returns whether the specified memcg contains any
1595  * reclaimable pages on a node. Returns true if there are any reclaimable
1596  * pages in the node.
1597  */
1598 static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1599 		int nid, bool noswap)
1600 {
1601 	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
1602 		return true;
1603 	if (noswap || !total_swap_pages)
1604 		return false;
1605 	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
1606 		return true;
1607 	return false;
1608 
1609 }
1610 
1611 /*
1612  * Always updating the nodemask is not very good - even if we have an empty
1613  * list or the wrong list here, we can start from some node and traverse all
1614  * nodes based on the zonelist. So update the list loosely once per 10 secs.
1615  *
1616  */
1617 static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1618 {
1619 	int nid;
1620 	/*
1621 	 * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
1622 	 * pagein/pageout changes since the last update.
1623 	 */
1624 	if (!atomic_read(&memcg->numainfo_events))
1625 		return;
1626 	if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1627 		return;
1628 
1629 	/* make a nodemask where this memcg uses memory from */
1630 	memcg->scan_nodes = node_states[N_MEMORY];
1631 
1632 	for_each_node_mask(nid, node_states[N_MEMORY]) {
1633 
1634 		if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
1635 			node_clear(nid, memcg->scan_nodes);
1636 	}
1637 
1638 	atomic_set(&memcg->numainfo_events, 0);
1639 	atomic_set(&memcg->numainfo_updating, 0);
1640 }
1641 
1642 /*
1643  * Selecting a node where we start reclaim from. Because what we need is just
1644  * reducing usage counter, start from anywhere is O,K. Considering
1645  * memory reclaim from current node, there are pros. and cons.
1646  *
1647  * Freeing memory from current node means freeing memory from a node which
1648  * we'll use or we've used. So, it may make LRU bad. And if several threads
1649  * hit limits, it will see a contention on a node. But freeing from remote
1650  * node means more costs for memory reclaim because of memory latency.
1651  *
1652  * Now, we use round-robin. Better algorithm is welcomed.
1653  */
1654 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1655 {
1656 	int node;
1657 
1658 	mem_cgroup_may_update_nodemask(memcg);
1659 	node = memcg->last_scanned_node;
1660 
1661 	node = next_node(node, memcg->scan_nodes);
1662 	if (node == MAX_NUMNODES)
1663 		node = first_node(memcg->scan_nodes);
1664 	/*
1665 	 * We call this when we hit limit, not when pages are added to LRU.
1666 	 * No LRU may hold pages because all pages are UNEVICTABLE or
1667 	 * memcg is too small and all pages are not on LRU. In that case,
1668 	 * we use curret node.
1669 	 */
1670 	if (unlikely(node == MAX_NUMNODES))
1671 		node = numa_node_id();
1672 
1673 	memcg->last_scanned_node = node;
1674 	return node;
1675 }
1676 #else
1677 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1678 {
1679 	return 0;
1680 }
1681 #endif
1682 
1683 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1684 				   struct zone *zone,
1685 				   gfp_t gfp_mask,
1686 				   unsigned long *total_scanned)
1687 {
1688 	struct mem_cgroup *victim = NULL;
1689 	int total = 0;
1690 	int loop = 0;
1691 	unsigned long excess;
1692 	unsigned long nr_scanned;
1693 	struct mem_cgroup_reclaim_cookie reclaim = {
1694 		.zone = zone,
1695 		.priority = 0,
1696 	};
1697 
1698 	excess = soft_limit_excess(root_memcg);
1699 
1700 	while (1) {
1701 		victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
1702 		if (!victim) {
1703 			loop++;
1704 			if (loop >= 2) {
1705 				/*
1706 				 * If we have not been able to reclaim
1707 				 * anything, it might because there are
1708 				 * no reclaimable pages under this hierarchy
1709 				 */
1710 				if (!total)
1711 					break;
1712 				/*
1713 				 * We want to do more targeted reclaim.
1714 				 * excess >> 2 is not to excessive so as to
1715 				 * reclaim too much, nor too less that we keep
1716 				 * coming back to reclaim from this cgroup
1717 				 */
1718 				if (total >= (excess >> 2) ||
1719 					(loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
1720 					break;
1721 			}
1722 			continue;
1723 		}
1724 		total += mem_cgroup_shrink_node_zone(victim, gfp_mask, false,
1725 						     zone, &nr_scanned);
1726 		*total_scanned += nr_scanned;
1727 		if (!soft_limit_excess(root_memcg))
1728 			break;
1729 	}
1730 	mem_cgroup_iter_break(root_memcg, victim);
1731 	return total;
1732 }
1733 
1734 #ifdef CONFIG_LOCKDEP
1735 static struct lockdep_map memcg_oom_lock_dep_map = {
1736 	.name = "memcg_oom_lock",
1737 };
1738 #endif
1739 
1740 static DEFINE_SPINLOCK(memcg_oom_lock);
1741 
1742 /*
1743  * Check OOM-Killer is already running under our hierarchy.
1744  * If someone is running, return false.
1745  */
1746 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
1747 {
1748 	struct mem_cgroup *iter, *failed = NULL;
1749 
1750 	spin_lock(&memcg_oom_lock);
1751 
1752 	for_each_mem_cgroup_tree(iter, memcg) {
1753 		if (iter->oom_lock) {
1754 			/*
1755 			 * this subtree of our hierarchy is already locked
1756 			 * so we cannot give a lock.
1757 			 */
1758 			failed = iter;
1759 			mem_cgroup_iter_break(memcg, iter);
1760 			break;
1761 		} else
1762 			iter->oom_lock = true;
1763 	}
1764 
1765 	if (failed) {
1766 		/*
1767 		 * OK, we failed to lock the whole subtree so we have
1768 		 * to clean up what we set up to the failing subtree
1769 		 */
1770 		for_each_mem_cgroup_tree(iter, memcg) {
1771 			if (iter == failed) {
1772 				mem_cgroup_iter_break(memcg, iter);
1773 				break;
1774 			}
1775 			iter->oom_lock = false;
1776 		}
1777 	} else
1778 		mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
1779 
1780 	spin_unlock(&memcg_oom_lock);
1781 
1782 	return !failed;
1783 }
1784 
1785 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1786 {
1787 	struct mem_cgroup *iter;
1788 
1789 	spin_lock(&memcg_oom_lock);
1790 	mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_);
1791 	for_each_mem_cgroup_tree(iter, memcg)
1792 		iter->oom_lock = false;
1793 	spin_unlock(&memcg_oom_lock);
1794 }
1795 
1796 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1797 {
1798 	struct mem_cgroup *iter;
1799 
1800 	for_each_mem_cgroup_tree(iter, memcg)
1801 		atomic_inc(&iter->under_oom);
1802 }
1803 
1804 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1805 {
1806 	struct mem_cgroup *iter;
1807 
1808 	/*
1809 	 * When a new child is created while the hierarchy is under oom,
1810 	 * mem_cgroup_oom_lock() may not be called. We have to use
1811 	 * atomic_add_unless() here.
1812 	 */
1813 	for_each_mem_cgroup_tree(iter, memcg)
1814 		atomic_add_unless(&iter->under_oom, -1, 0);
1815 }
1816 
1817 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1818 
1819 struct oom_wait_info {
1820 	struct mem_cgroup *memcg;
1821 	wait_queue_t	wait;
1822 };
1823 
1824 static int memcg_oom_wake_function(wait_queue_t *wait,
1825 	unsigned mode, int sync, void *arg)
1826 {
1827 	struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
1828 	struct mem_cgroup *oom_wait_memcg;
1829 	struct oom_wait_info *oom_wait_info;
1830 
1831 	oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1832 	oom_wait_memcg = oom_wait_info->memcg;
1833 
1834 	if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
1835 	    !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
1836 		return 0;
1837 	return autoremove_wake_function(wait, mode, sync, arg);
1838 }
1839 
1840 static void memcg_wakeup_oom(struct mem_cgroup *memcg)
1841 {
1842 	atomic_inc(&memcg->oom_wakeups);
1843 	/* for filtering, pass "memcg" as argument. */
1844 	__wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
1845 }
1846 
1847 static void memcg_oom_recover(struct mem_cgroup *memcg)
1848 {
1849 	if (memcg && atomic_read(&memcg->under_oom))
1850 		memcg_wakeup_oom(memcg);
1851 }
1852 
1853 static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
1854 {
1855 	if (!current->memcg_oom.may_oom)
1856 		return;
1857 	/*
1858 	 * We are in the middle of the charge context here, so we
1859 	 * don't want to block when potentially sitting on a callstack
1860 	 * that holds all kinds of filesystem and mm locks.
1861 	 *
1862 	 * Also, the caller may handle a failed allocation gracefully
1863 	 * (like optional page cache readahead) and so an OOM killer
1864 	 * invocation might not even be necessary.
1865 	 *
1866 	 * That's why we don't do anything here except remember the
1867 	 * OOM context and then deal with it at the end of the page
1868 	 * fault when the stack is unwound, the locks are released,
1869 	 * and when we know whether the fault was overall successful.
1870 	 */
1871 	css_get(&memcg->css);
1872 	current->memcg_oom.memcg = memcg;
1873 	current->memcg_oom.gfp_mask = mask;
1874 	current->memcg_oom.order = order;
1875 }
1876 
1877 /**
1878  * mem_cgroup_oom_synchronize - complete memcg OOM handling
1879  * @handle: actually kill/wait or just clean up the OOM state
1880  *
1881  * This has to be called at the end of a page fault if the memcg OOM
1882  * handler was enabled.
1883  *
1884  * Memcg supports userspace OOM handling where failed allocations must
1885  * sleep on a waitqueue until the userspace task resolves the
1886  * situation.  Sleeping directly in the charge context with all kinds
1887  * of locks held is not a good idea, instead we remember an OOM state
1888  * in the task and mem_cgroup_oom_synchronize() has to be called at
1889  * the end of the page fault to complete the OOM handling.
1890  *
1891  * Returns %true if an ongoing memcg OOM situation was detected and
1892  * completed, %false otherwise.
1893  */
1894 bool mem_cgroup_oom_synchronize(bool handle)
1895 {
1896 	struct mem_cgroup *memcg = current->memcg_oom.memcg;
1897 	struct oom_wait_info owait;
1898 	bool locked;
1899 
1900 	/* OOM is global, do not handle */
1901 	if (!memcg)
1902 		return false;
1903 
1904 	if (!handle || oom_killer_disabled)
1905 		goto cleanup;
1906 
1907 	owait.memcg = memcg;
1908 	owait.wait.flags = 0;
1909 	owait.wait.func = memcg_oom_wake_function;
1910 	owait.wait.private = current;
1911 	INIT_LIST_HEAD(&owait.wait.task_list);
1912 
1913 	prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1914 	mem_cgroup_mark_under_oom(memcg);
1915 
1916 	locked = mem_cgroup_oom_trylock(memcg);
1917 
1918 	if (locked)
1919 		mem_cgroup_oom_notify(memcg);
1920 
1921 	if (locked && !memcg->oom_kill_disable) {
1922 		mem_cgroup_unmark_under_oom(memcg);
1923 		finish_wait(&memcg_oom_waitq, &owait.wait);
1924 		mem_cgroup_out_of_memory(memcg, current->memcg_oom.gfp_mask,
1925 					 current->memcg_oom.order);
1926 	} else {
1927 		schedule();
1928 		mem_cgroup_unmark_under_oom(memcg);
1929 		finish_wait(&memcg_oom_waitq, &owait.wait);
1930 	}
1931 
1932 	if (locked) {
1933 		mem_cgroup_oom_unlock(memcg);
1934 		/*
1935 		 * There is no guarantee that an OOM-lock contender
1936 		 * sees the wakeups triggered by the OOM kill
1937 		 * uncharges.  Wake any sleepers explicitely.
1938 		 */
1939 		memcg_oom_recover(memcg);
1940 	}
1941 cleanup:
1942 	current->memcg_oom.memcg = NULL;
1943 	css_put(&memcg->css);
1944 	return true;
1945 }
1946 
1947 /**
1948  * mem_cgroup_begin_page_stat - begin a page state statistics transaction
1949  * @page: page that is going to change accounted state
1950  *
1951  * This function must mark the beginning of an accounted page state
1952  * change to prevent double accounting when the page is concurrently
1953  * being moved to another memcg:
1954  *
1955  *   memcg = mem_cgroup_begin_page_stat(page);
1956  *   if (TestClearPageState(page))
1957  *     mem_cgroup_update_page_stat(memcg, state, -1);
1958  *   mem_cgroup_end_page_stat(memcg);
1959  */
1960 struct mem_cgroup *mem_cgroup_begin_page_stat(struct page *page)
1961 {
1962 	struct mem_cgroup *memcg;
1963 	unsigned long flags;
1964 
1965 	/*
1966 	 * The RCU lock is held throughout the transaction.  The fast
1967 	 * path can get away without acquiring the memcg->move_lock
1968 	 * because page moving starts with an RCU grace period.
1969 	 *
1970 	 * The RCU lock also protects the memcg from being freed when
1971 	 * the page state that is going to change is the only thing
1972 	 * preventing the page from being uncharged.
1973 	 * E.g. end-writeback clearing PageWriteback(), which allows
1974 	 * migration to go ahead and uncharge the page before the
1975 	 * account transaction might be complete.
1976 	 */
1977 	rcu_read_lock();
1978 
1979 	if (mem_cgroup_disabled())
1980 		return NULL;
1981 again:
1982 	memcg = page->mem_cgroup;
1983 	if (unlikely(!memcg))
1984 		return NULL;
1985 
1986 	if (atomic_read(&memcg->moving_account) <= 0)
1987 		return memcg;
1988 
1989 	spin_lock_irqsave(&memcg->move_lock, flags);
1990 	if (memcg != page->mem_cgroup) {
1991 		spin_unlock_irqrestore(&memcg->move_lock, flags);
1992 		goto again;
1993 	}
1994 
1995 	/*
1996 	 * When charge migration first begins, we can have locked and
1997 	 * unlocked page stat updates happening concurrently.  Track
1998 	 * the task who has the lock for mem_cgroup_end_page_stat().
1999 	 */
2000 	memcg->move_lock_task = current;
2001 	memcg->move_lock_flags = flags;
2002 
2003 	return memcg;
2004 }
2005 
2006 /**
2007  * mem_cgroup_end_page_stat - finish a page state statistics transaction
2008  * @memcg: the memcg that was accounted against
2009  */
2010 void mem_cgroup_end_page_stat(struct mem_cgroup *memcg)
2011 {
2012 	if (memcg && memcg->move_lock_task == current) {
2013 		unsigned long flags = memcg->move_lock_flags;
2014 
2015 		memcg->move_lock_task = NULL;
2016 		memcg->move_lock_flags = 0;
2017 
2018 		spin_unlock_irqrestore(&memcg->move_lock, flags);
2019 	}
2020 
2021 	rcu_read_unlock();
2022 }
2023 
2024 /**
2025  * mem_cgroup_update_page_stat - update page state statistics
2026  * @memcg: memcg to account against
2027  * @idx: page state item to account
2028  * @val: number of pages (positive or negative)
2029  *
2030  * See mem_cgroup_begin_page_stat() for locking requirements.
2031  */
2032 void mem_cgroup_update_page_stat(struct mem_cgroup *memcg,
2033 				 enum mem_cgroup_stat_index idx, int val)
2034 {
2035 	VM_BUG_ON(!rcu_read_lock_held());
2036 
2037 	if (memcg)
2038 		this_cpu_add(memcg->stat->count[idx], val);
2039 }
2040 
2041 /*
2042  * size of first charge trial. "32" comes from vmscan.c's magic value.
2043  * TODO: maybe necessary to use big numbers in big irons.
2044  */
2045 #define CHARGE_BATCH	32U
2046 struct memcg_stock_pcp {
2047 	struct mem_cgroup *cached; /* this never be root cgroup */
2048 	unsigned int nr_pages;
2049 	struct work_struct work;
2050 	unsigned long flags;
2051 #define FLUSHING_CACHED_CHARGE	0
2052 };
2053 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
2054 static DEFINE_MUTEX(percpu_charge_mutex);
2055 
2056 /**
2057  * consume_stock: Try to consume stocked charge on this cpu.
2058  * @memcg: memcg to consume from.
2059  * @nr_pages: how many pages to charge.
2060  *
2061  * The charges will only happen if @memcg matches the current cpu's memcg
2062  * stock, and at least @nr_pages are available in that stock.  Failure to
2063  * service an allocation will refill the stock.
2064  *
2065  * returns true if successful, false otherwise.
2066  */
2067 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2068 {
2069 	struct memcg_stock_pcp *stock;
2070 	bool ret = false;
2071 
2072 	if (nr_pages > CHARGE_BATCH)
2073 		return ret;
2074 
2075 	stock = &get_cpu_var(memcg_stock);
2076 	if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
2077 		stock->nr_pages -= nr_pages;
2078 		ret = true;
2079 	}
2080 	put_cpu_var(memcg_stock);
2081 	return ret;
2082 }
2083 
2084 /*
2085  * Returns stocks cached in percpu and reset cached information.
2086  */
2087 static void drain_stock(struct memcg_stock_pcp *stock)
2088 {
2089 	struct mem_cgroup *old = stock->cached;
2090 
2091 	if (stock->nr_pages) {
2092 		page_counter_uncharge(&old->memory, stock->nr_pages);
2093 		if (do_swap_account)
2094 			page_counter_uncharge(&old->memsw, stock->nr_pages);
2095 		css_put_many(&old->css, stock->nr_pages);
2096 		stock->nr_pages = 0;
2097 	}
2098 	stock->cached = NULL;
2099 }
2100 
2101 /*
2102  * This must be called under preempt disabled or must be called by
2103  * a thread which is pinned to local cpu.
2104  */
2105 static void drain_local_stock(struct work_struct *dummy)
2106 {
2107 	struct memcg_stock_pcp *stock = this_cpu_ptr(&memcg_stock);
2108 	drain_stock(stock);
2109 	clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
2110 }
2111 
2112 /*
2113  * Cache charges(val) to local per_cpu area.
2114  * This will be consumed by consume_stock() function, later.
2115  */
2116 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2117 {
2118 	struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
2119 
2120 	if (stock->cached != memcg) { /* reset if necessary */
2121 		drain_stock(stock);
2122 		stock->cached = memcg;
2123 	}
2124 	stock->nr_pages += nr_pages;
2125 	put_cpu_var(memcg_stock);
2126 }
2127 
2128 /*
2129  * Drains all per-CPU charge caches for given root_memcg resp. subtree
2130  * of the hierarchy under it.
2131  */
2132 static void drain_all_stock(struct mem_cgroup *root_memcg)
2133 {
2134 	int cpu, curcpu;
2135 
2136 	/* If someone's already draining, avoid adding running more workers. */
2137 	if (!mutex_trylock(&percpu_charge_mutex))
2138 		return;
2139 	/* Notify other cpus that system-wide "drain" is running */
2140 	get_online_cpus();
2141 	curcpu = get_cpu();
2142 	for_each_online_cpu(cpu) {
2143 		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2144 		struct mem_cgroup *memcg;
2145 
2146 		memcg = stock->cached;
2147 		if (!memcg || !stock->nr_pages)
2148 			continue;
2149 		if (!mem_cgroup_is_descendant(memcg, root_memcg))
2150 			continue;
2151 		if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
2152 			if (cpu == curcpu)
2153 				drain_local_stock(&stock->work);
2154 			else
2155 				schedule_work_on(cpu, &stock->work);
2156 		}
2157 	}
2158 	put_cpu();
2159 	put_online_cpus();
2160 	mutex_unlock(&percpu_charge_mutex);
2161 }
2162 
2163 /*
2164  * This function drains percpu counter value from DEAD cpu and
2165  * move it to local cpu. Note that this function can be preempted.
2166  */
2167 static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *memcg, int cpu)
2168 {
2169 	int i;
2170 
2171 	spin_lock(&memcg->pcp_counter_lock);
2172 	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
2173 		long x = per_cpu(memcg->stat->count[i], cpu);
2174 
2175 		per_cpu(memcg->stat->count[i], cpu) = 0;
2176 		memcg->nocpu_base.count[i] += x;
2177 	}
2178 	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
2179 		unsigned long x = per_cpu(memcg->stat->events[i], cpu);
2180 
2181 		per_cpu(memcg->stat->events[i], cpu) = 0;
2182 		memcg->nocpu_base.events[i] += x;
2183 	}
2184 	spin_unlock(&memcg->pcp_counter_lock);
2185 }
2186 
2187 static int memcg_cpu_hotplug_callback(struct notifier_block *nb,
2188 					unsigned long action,
2189 					void *hcpu)
2190 {
2191 	int cpu = (unsigned long)hcpu;
2192 	struct memcg_stock_pcp *stock;
2193 	struct mem_cgroup *iter;
2194 
2195 	if (action == CPU_ONLINE)
2196 		return NOTIFY_OK;
2197 
2198 	if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
2199 		return NOTIFY_OK;
2200 
2201 	for_each_mem_cgroup(iter)
2202 		mem_cgroup_drain_pcp_counter(iter, cpu);
2203 
2204 	stock = &per_cpu(memcg_stock, cpu);
2205 	drain_stock(stock);
2206 	return NOTIFY_OK;
2207 }
2208 
2209 static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2210 		      unsigned int nr_pages)
2211 {
2212 	unsigned int batch = max(CHARGE_BATCH, nr_pages);
2213 	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
2214 	struct mem_cgroup *mem_over_limit;
2215 	struct page_counter *counter;
2216 	unsigned long nr_reclaimed;
2217 	bool may_swap = true;
2218 	bool drained = false;
2219 	int ret = 0;
2220 
2221 	if (mem_cgroup_is_root(memcg))
2222 		goto done;
2223 retry:
2224 	if (consume_stock(memcg, nr_pages))
2225 		goto done;
2226 
2227 	if (!do_swap_account ||
2228 	    !page_counter_try_charge(&memcg->memsw, batch, &counter)) {
2229 		if (!page_counter_try_charge(&memcg->memory, batch, &counter))
2230 			goto done_restock;
2231 		if (do_swap_account)
2232 			page_counter_uncharge(&memcg->memsw, batch);
2233 		mem_over_limit = mem_cgroup_from_counter(counter, memory);
2234 	} else {
2235 		mem_over_limit = mem_cgroup_from_counter(counter, memsw);
2236 		may_swap = false;
2237 	}
2238 
2239 	if (batch > nr_pages) {
2240 		batch = nr_pages;
2241 		goto retry;
2242 	}
2243 
2244 	/*
2245 	 * Unlike in global OOM situations, memcg is not in a physical
2246 	 * memory shortage.  Allow dying and OOM-killed tasks to
2247 	 * bypass the last charges so that they can exit quickly and
2248 	 * free their memory.
2249 	 */
2250 	if (unlikely(test_thread_flag(TIF_MEMDIE) ||
2251 		     fatal_signal_pending(current) ||
2252 		     current->flags & PF_EXITING))
2253 		goto bypass;
2254 
2255 	if (unlikely(task_in_memcg_oom(current)))
2256 		goto nomem;
2257 
2258 	if (!(gfp_mask & __GFP_WAIT))
2259 		goto nomem;
2260 
2261 	mem_cgroup_events(mem_over_limit, MEMCG_MAX, 1);
2262 
2263 	nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
2264 						    gfp_mask, may_swap);
2265 
2266 	if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
2267 		goto retry;
2268 
2269 	if (!drained) {
2270 		drain_all_stock(mem_over_limit);
2271 		drained = true;
2272 		goto retry;
2273 	}
2274 
2275 	if (gfp_mask & __GFP_NORETRY)
2276 		goto nomem;
2277 	/*
2278 	 * Even though the limit is exceeded at this point, reclaim
2279 	 * may have been able to free some pages.  Retry the charge
2280 	 * before killing the task.
2281 	 *
2282 	 * Only for regular pages, though: huge pages are rather
2283 	 * unlikely to succeed so close to the limit, and we fall back
2284 	 * to regular pages anyway in case of failure.
2285 	 */
2286 	if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
2287 		goto retry;
2288 	/*
2289 	 * At task move, charge accounts can be doubly counted. So, it's
2290 	 * better to wait until the end of task_move if something is going on.
2291 	 */
2292 	if (mem_cgroup_wait_acct_move(mem_over_limit))
2293 		goto retry;
2294 
2295 	if (nr_retries--)
2296 		goto retry;
2297 
2298 	if (gfp_mask & __GFP_NOFAIL)
2299 		goto bypass;
2300 
2301 	if (fatal_signal_pending(current))
2302 		goto bypass;
2303 
2304 	mem_cgroup_events(mem_over_limit, MEMCG_OOM, 1);
2305 
2306 	mem_cgroup_oom(mem_over_limit, gfp_mask, get_order(nr_pages));
2307 nomem:
2308 	if (!(gfp_mask & __GFP_NOFAIL))
2309 		return -ENOMEM;
2310 bypass:
2311 	return -EINTR;
2312 
2313 done_restock:
2314 	css_get_many(&memcg->css, batch);
2315 	if (batch > nr_pages)
2316 		refill_stock(memcg, batch - nr_pages);
2317 	/*
2318 	 * If the hierarchy is above the normal consumption range,
2319 	 * make the charging task trim their excess contribution.
2320 	 */
2321 	do {
2322 		if (page_counter_read(&memcg->memory) <= memcg->high)
2323 			continue;
2324 		mem_cgroup_events(memcg, MEMCG_HIGH, 1);
2325 		try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true);
2326 	} while ((memcg = parent_mem_cgroup(memcg)));
2327 done:
2328 	return ret;
2329 }
2330 
2331 static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2332 {
2333 	if (mem_cgroup_is_root(memcg))
2334 		return;
2335 
2336 	page_counter_uncharge(&memcg->memory, nr_pages);
2337 	if (do_swap_account)
2338 		page_counter_uncharge(&memcg->memsw, nr_pages);
2339 
2340 	css_put_many(&memcg->css, nr_pages);
2341 }
2342 
2343 /*
2344  * A helper function to get mem_cgroup from ID. must be called under
2345  * rcu_read_lock().  The caller is responsible for calling
2346  * css_tryget_online() if the mem_cgroup is used for charging. (dropping
2347  * refcnt from swap can be called against removed memcg.)
2348  */
2349 static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
2350 {
2351 	/* ID 0 is unused ID */
2352 	if (!id)
2353 		return NULL;
2354 	return mem_cgroup_from_id(id);
2355 }
2356 
2357 /*
2358  * try_get_mem_cgroup_from_page - look up page's memcg association
2359  * @page: the page
2360  *
2361  * Look up, get a css reference, and return the memcg that owns @page.
2362  *
2363  * The page must be locked to prevent racing with swap-in and page
2364  * cache charges.  If coming from an unlocked page table, the caller
2365  * must ensure the page is on the LRU or this can race with charging.
2366  */
2367 struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
2368 {
2369 	struct mem_cgroup *memcg;
2370 	unsigned short id;
2371 	swp_entry_t ent;
2372 
2373 	VM_BUG_ON_PAGE(!PageLocked(page), page);
2374 
2375 	memcg = page->mem_cgroup;
2376 	if (memcg) {
2377 		if (!css_tryget_online(&memcg->css))
2378 			memcg = NULL;
2379 	} else if (PageSwapCache(page)) {
2380 		ent.val = page_private(page);
2381 		id = lookup_swap_cgroup_id(ent);
2382 		rcu_read_lock();
2383 		memcg = mem_cgroup_lookup(id);
2384 		if (memcg && !css_tryget_online(&memcg->css))
2385 			memcg = NULL;
2386 		rcu_read_unlock();
2387 	}
2388 	return memcg;
2389 }
2390 
2391 static void lock_page_lru(struct page *page, int *isolated)
2392 {
2393 	struct zone *zone = page_zone(page);
2394 
2395 	spin_lock_irq(&zone->lru_lock);
2396 	if (PageLRU(page)) {
2397 		struct lruvec *lruvec;
2398 
2399 		lruvec = mem_cgroup_page_lruvec(page, zone);
2400 		ClearPageLRU(page);
2401 		del_page_from_lru_list(page, lruvec, page_lru(page));
2402 		*isolated = 1;
2403 	} else
2404 		*isolated = 0;
2405 }
2406 
2407 static void unlock_page_lru(struct page *page, int isolated)
2408 {
2409 	struct zone *zone = page_zone(page);
2410 
2411 	if (isolated) {
2412 		struct lruvec *lruvec;
2413 
2414 		lruvec = mem_cgroup_page_lruvec(page, zone);
2415 		VM_BUG_ON_PAGE(PageLRU(page), page);
2416 		SetPageLRU(page);
2417 		add_page_to_lru_list(page, lruvec, page_lru(page));
2418 	}
2419 	spin_unlock_irq(&zone->lru_lock);
2420 }
2421 
2422 static void commit_charge(struct page *page, struct mem_cgroup *memcg,
2423 			  bool lrucare)
2424 {
2425 	int isolated;
2426 
2427 	VM_BUG_ON_PAGE(page->mem_cgroup, page);
2428 
2429 	/*
2430 	 * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
2431 	 * may already be on some other mem_cgroup's LRU.  Take care of it.
2432 	 */
2433 	if (lrucare)
2434 		lock_page_lru(page, &isolated);
2435 
2436 	/*
2437 	 * Nobody should be changing or seriously looking at
2438 	 * page->mem_cgroup at this point:
2439 	 *
2440 	 * - the page is uncharged
2441 	 *
2442 	 * - the page is off-LRU
2443 	 *
2444 	 * - an anonymous fault has exclusive page access, except for
2445 	 *   a locked page table
2446 	 *
2447 	 * - a page cache insertion, a swapin fault, or a migration
2448 	 *   have the page locked
2449 	 */
2450 	page->mem_cgroup = memcg;
2451 
2452 	if (lrucare)
2453 		unlock_page_lru(page, isolated);
2454 }
2455 
2456 #ifdef CONFIG_MEMCG_KMEM
2457 int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp,
2458 		      unsigned long nr_pages)
2459 {
2460 	struct page_counter *counter;
2461 	int ret = 0;
2462 
2463 	ret = page_counter_try_charge(&memcg->kmem, nr_pages, &counter);
2464 	if (ret < 0)
2465 		return ret;
2466 
2467 	ret = try_charge(memcg, gfp, nr_pages);
2468 	if (ret == -EINTR)  {
2469 		/*
2470 		 * try_charge() chose to bypass to root due to OOM kill or
2471 		 * fatal signal.  Since our only options are to either fail
2472 		 * the allocation or charge it to this cgroup, do it as a
2473 		 * temporary condition. But we can't fail. From a kmem/slab
2474 		 * perspective, the cache has already been selected, by
2475 		 * mem_cgroup_kmem_get_cache(), so it is too late to change
2476 		 * our minds.
2477 		 *
2478 		 * This condition will only trigger if the task entered
2479 		 * memcg_charge_kmem in a sane state, but was OOM-killed
2480 		 * during try_charge() above. Tasks that were already dying
2481 		 * when the allocation triggers should have been already
2482 		 * directed to the root cgroup in memcontrol.h
2483 		 */
2484 		page_counter_charge(&memcg->memory, nr_pages);
2485 		if (do_swap_account)
2486 			page_counter_charge(&memcg->memsw, nr_pages);
2487 		css_get_many(&memcg->css, nr_pages);
2488 		ret = 0;
2489 	} else if (ret)
2490 		page_counter_uncharge(&memcg->kmem, nr_pages);
2491 
2492 	return ret;
2493 }
2494 
2495 void memcg_uncharge_kmem(struct mem_cgroup *memcg, unsigned long nr_pages)
2496 {
2497 	page_counter_uncharge(&memcg->memory, nr_pages);
2498 	if (do_swap_account)
2499 		page_counter_uncharge(&memcg->memsw, nr_pages);
2500 
2501 	page_counter_uncharge(&memcg->kmem, nr_pages);
2502 
2503 	css_put_many(&memcg->css, nr_pages);
2504 }
2505 
2506 /*
2507  * helper for acessing a memcg's index. It will be used as an index in the
2508  * child cache array in kmem_cache, and also to derive its name. This function
2509  * will return -1 when this is not a kmem-limited memcg.
2510  */
2511 int memcg_cache_id(struct mem_cgroup *memcg)
2512 {
2513 	return memcg ? memcg->kmemcg_id : -1;
2514 }
2515 
2516 static int memcg_alloc_cache_id(void)
2517 {
2518 	int id, size;
2519 	int err;
2520 
2521 	id = ida_simple_get(&memcg_cache_ida,
2522 			    0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
2523 	if (id < 0)
2524 		return id;
2525 
2526 	if (id < memcg_nr_cache_ids)
2527 		return id;
2528 
2529 	/*
2530 	 * There's no space for the new id in memcg_caches arrays,
2531 	 * so we have to grow them.
2532 	 */
2533 	down_write(&memcg_cache_ids_sem);
2534 
2535 	size = 2 * (id + 1);
2536 	if (size < MEMCG_CACHES_MIN_SIZE)
2537 		size = MEMCG_CACHES_MIN_SIZE;
2538 	else if (size > MEMCG_CACHES_MAX_SIZE)
2539 		size = MEMCG_CACHES_MAX_SIZE;
2540 
2541 	err = memcg_update_all_caches(size);
2542 	if (!err)
2543 		err = memcg_update_all_list_lrus(size);
2544 	if (!err)
2545 		memcg_nr_cache_ids = size;
2546 
2547 	up_write(&memcg_cache_ids_sem);
2548 
2549 	if (err) {
2550 		ida_simple_remove(&memcg_cache_ida, id);
2551 		return err;
2552 	}
2553 	return id;
2554 }
2555 
2556 static void memcg_free_cache_id(int id)
2557 {
2558 	ida_simple_remove(&memcg_cache_ida, id);
2559 }
2560 
2561 struct memcg_kmem_cache_create_work {
2562 	struct mem_cgroup *memcg;
2563 	struct kmem_cache *cachep;
2564 	struct work_struct work;
2565 };
2566 
2567 static void memcg_kmem_cache_create_func(struct work_struct *w)
2568 {
2569 	struct memcg_kmem_cache_create_work *cw =
2570 		container_of(w, struct memcg_kmem_cache_create_work, work);
2571 	struct mem_cgroup *memcg = cw->memcg;
2572 	struct kmem_cache *cachep = cw->cachep;
2573 
2574 	memcg_create_kmem_cache(memcg, cachep);
2575 
2576 	css_put(&memcg->css);
2577 	kfree(cw);
2578 }
2579 
2580 /*
2581  * Enqueue the creation of a per-memcg kmem_cache.
2582  */
2583 static void __memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
2584 					       struct kmem_cache *cachep)
2585 {
2586 	struct memcg_kmem_cache_create_work *cw;
2587 
2588 	cw = kmalloc(sizeof(*cw), GFP_NOWAIT);
2589 	if (!cw)
2590 		return;
2591 
2592 	css_get(&memcg->css);
2593 
2594 	cw->memcg = memcg;
2595 	cw->cachep = cachep;
2596 	INIT_WORK(&cw->work, memcg_kmem_cache_create_func);
2597 
2598 	schedule_work(&cw->work);
2599 }
2600 
2601 static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
2602 					     struct kmem_cache *cachep)
2603 {
2604 	/*
2605 	 * We need to stop accounting when we kmalloc, because if the
2606 	 * corresponding kmalloc cache is not yet created, the first allocation
2607 	 * in __memcg_schedule_kmem_cache_create will recurse.
2608 	 *
2609 	 * However, it is better to enclose the whole function. Depending on
2610 	 * the debugging options enabled, INIT_WORK(), for instance, can
2611 	 * trigger an allocation. This too, will make us recurse. Because at
2612 	 * this point we can't allow ourselves back into memcg_kmem_get_cache,
2613 	 * the safest choice is to do it like this, wrapping the whole function.
2614 	 */
2615 	current->memcg_kmem_skip_account = 1;
2616 	__memcg_schedule_kmem_cache_create(memcg, cachep);
2617 	current->memcg_kmem_skip_account = 0;
2618 }
2619 
2620 /*
2621  * Return the kmem_cache we're supposed to use for a slab allocation.
2622  * We try to use the current memcg's version of the cache.
2623  *
2624  * If the cache does not exist yet, if we are the first user of it,
2625  * we either create it immediately, if possible, or create it asynchronously
2626  * in a workqueue.
2627  * In the latter case, we will let the current allocation go through with
2628  * the original cache.
2629  *
2630  * Can't be called in interrupt context or from kernel threads.
2631  * This function needs to be called with rcu_read_lock() held.
2632  */
2633 struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep)
2634 {
2635 	struct mem_cgroup *memcg;
2636 	struct kmem_cache *memcg_cachep;
2637 	int kmemcg_id;
2638 
2639 	VM_BUG_ON(!is_root_cache(cachep));
2640 
2641 	if (current->memcg_kmem_skip_account)
2642 		return cachep;
2643 
2644 	memcg = get_mem_cgroup_from_mm(current->mm);
2645 	kmemcg_id = ACCESS_ONCE(memcg->kmemcg_id);
2646 	if (kmemcg_id < 0)
2647 		goto out;
2648 
2649 	memcg_cachep = cache_from_memcg_idx(cachep, kmemcg_id);
2650 	if (likely(memcg_cachep))
2651 		return memcg_cachep;
2652 
2653 	/*
2654 	 * If we are in a safe context (can wait, and not in interrupt
2655 	 * context), we could be be predictable and return right away.
2656 	 * This would guarantee that the allocation being performed
2657 	 * already belongs in the new cache.
2658 	 *
2659 	 * However, there are some clashes that can arrive from locking.
2660 	 * For instance, because we acquire the slab_mutex while doing
2661 	 * memcg_create_kmem_cache, this means no further allocation
2662 	 * could happen with the slab_mutex held. So it's better to
2663 	 * defer everything.
2664 	 */
2665 	memcg_schedule_kmem_cache_create(memcg, cachep);
2666 out:
2667 	css_put(&memcg->css);
2668 	return cachep;
2669 }
2670 
2671 void __memcg_kmem_put_cache(struct kmem_cache *cachep)
2672 {
2673 	if (!is_root_cache(cachep))
2674 		css_put(&cachep->memcg_params.memcg->css);
2675 }
2676 
2677 /*
2678  * We need to verify if the allocation against current->mm->owner's memcg is
2679  * possible for the given order. But the page is not allocated yet, so we'll
2680  * need a further commit step to do the final arrangements.
2681  *
2682  * It is possible for the task to switch cgroups in this mean time, so at
2683  * commit time, we can't rely on task conversion any longer.  We'll then use
2684  * the handle argument to return to the caller which cgroup we should commit
2685  * against. We could also return the memcg directly and avoid the pointer
2686  * passing, but a boolean return value gives better semantics considering
2687  * the compiled-out case as well.
2688  *
2689  * Returning true means the allocation is possible.
2690  */
2691 bool
2692 __memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **_memcg, int order)
2693 {
2694 	struct mem_cgroup *memcg;
2695 	int ret;
2696 
2697 	*_memcg = NULL;
2698 
2699 	memcg = get_mem_cgroup_from_mm(current->mm);
2700 
2701 	if (!memcg_kmem_is_active(memcg)) {
2702 		css_put(&memcg->css);
2703 		return true;
2704 	}
2705 
2706 	ret = memcg_charge_kmem(memcg, gfp, 1 << order);
2707 	if (!ret)
2708 		*_memcg = memcg;
2709 
2710 	css_put(&memcg->css);
2711 	return (ret == 0);
2712 }
2713 
2714 void __memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg,
2715 			      int order)
2716 {
2717 	VM_BUG_ON(mem_cgroup_is_root(memcg));
2718 
2719 	/* The page allocation failed. Revert */
2720 	if (!page) {
2721 		memcg_uncharge_kmem(memcg, 1 << order);
2722 		return;
2723 	}
2724 	page->mem_cgroup = memcg;
2725 }
2726 
2727 void __memcg_kmem_uncharge_pages(struct page *page, int order)
2728 {
2729 	struct mem_cgroup *memcg = page->mem_cgroup;
2730 
2731 	if (!memcg)
2732 		return;
2733 
2734 	VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
2735 
2736 	memcg_uncharge_kmem(memcg, 1 << order);
2737 	page->mem_cgroup = NULL;
2738 }
2739 
2740 struct mem_cgroup *__mem_cgroup_from_kmem(void *ptr)
2741 {
2742 	struct mem_cgroup *memcg = NULL;
2743 	struct kmem_cache *cachep;
2744 	struct page *page;
2745 
2746 	page = virt_to_head_page(ptr);
2747 	if (PageSlab(page)) {
2748 		cachep = page->slab_cache;
2749 		if (!is_root_cache(cachep))
2750 			memcg = cachep->memcg_params.memcg;
2751 	} else
2752 		/* page allocated by alloc_kmem_pages */
2753 		memcg = page->mem_cgroup;
2754 
2755 	return memcg;
2756 }
2757 #endif /* CONFIG_MEMCG_KMEM */
2758 
2759 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2760 
2761 /*
2762  * Because tail pages are not marked as "used", set it. We're under
2763  * zone->lru_lock, 'splitting on pmd' and compound_lock.
2764  * charge/uncharge will be never happen and move_account() is done under
2765  * compound_lock(), so we don't have to take care of races.
2766  */
2767 void mem_cgroup_split_huge_fixup(struct page *head)
2768 {
2769 	int i;
2770 
2771 	if (mem_cgroup_disabled())
2772 		return;
2773 
2774 	for (i = 1; i < HPAGE_PMD_NR; i++)
2775 		head[i].mem_cgroup = head->mem_cgroup;
2776 
2777 	__this_cpu_sub(head->mem_cgroup->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
2778 		       HPAGE_PMD_NR);
2779 }
2780 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
2781 
2782 /**
2783  * mem_cgroup_move_account - move account of the page
2784  * @page: the page
2785  * @nr_pages: number of regular pages (>1 for huge pages)
2786  * @from: mem_cgroup which the page is moved from.
2787  * @to:	mem_cgroup which the page is moved to. @from != @to.
2788  *
2789  * The caller must confirm following.
2790  * - page is not on LRU (isolate_page() is useful.)
2791  * - compound_lock is held when nr_pages > 1
2792  *
2793  * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
2794  * from old cgroup.
2795  */
2796 static int mem_cgroup_move_account(struct page *page,
2797 				   unsigned int nr_pages,
2798 				   struct mem_cgroup *from,
2799 				   struct mem_cgroup *to)
2800 {
2801 	unsigned long flags;
2802 	int ret;
2803 
2804 	VM_BUG_ON(from == to);
2805 	VM_BUG_ON_PAGE(PageLRU(page), page);
2806 	/*
2807 	 * The page is isolated from LRU. So, collapse function
2808 	 * will not handle this page. But page splitting can happen.
2809 	 * Do this check under compound_page_lock(). The caller should
2810 	 * hold it.
2811 	 */
2812 	ret = -EBUSY;
2813 	if (nr_pages > 1 && !PageTransHuge(page))
2814 		goto out;
2815 
2816 	/*
2817 	 * Prevent mem_cgroup_migrate() from looking at page->mem_cgroup
2818 	 * of its source page while we change it: page migration takes
2819 	 * both pages off the LRU, but page cache replacement doesn't.
2820 	 */
2821 	if (!trylock_page(page))
2822 		goto out;
2823 
2824 	ret = -EINVAL;
2825 	if (page->mem_cgroup != from)
2826 		goto out_unlock;
2827 
2828 	spin_lock_irqsave(&from->move_lock, flags);
2829 
2830 	if (!PageAnon(page) && page_mapped(page)) {
2831 		__this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED],
2832 			       nr_pages);
2833 		__this_cpu_add(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED],
2834 			       nr_pages);
2835 	}
2836 
2837 	if (PageWriteback(page)) {
2838 		__this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_WRITEBACK],
2839 			       nr_pages);
2840 		__this_cpu_add(to->stat->count[MEM_CGROUP_STAT_WRITEBACK],
2841 			       nr_pages);
2842 	}
2843 
2844 	/*
2845 	 * It is safe to change page->mem_cgroup here because the page
2846 	 * is referenced, charged, and isolated - we can't race with
2847 	 * uncharging, charging, migration, or LRU putback.
2848 	 */
2849 
2850 	/* caller should have done css_get */
2851 	page->mem_cgroup = to;
2852 	spin_unlock_irqrestore(&from->move_lock, flags);
2853 
2854 	ret = 0;
2855 
2856 	local_irq_disable();
2857 	mem_cgroup_charge_statistics(to, page, nr_pages);
2858 	memcg_check_events(to, page);
2859 	mem_cgroup_charge_statistics(from, page, -nr_pages);
2860 	memcg_check_events(from, page);
2861 	local_irq_enable();
2862 out_unlock:
2863 	unlock_page(page);
2864 out:
2865 	return ret;
2866 }
2867 
2868 #ifdef CONFIG_MEMCG_SWAP
2869 static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
2870 					 bool charge)
2871 {
2872 	int val = (charge) ? 1 : -1;
2873 	this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAP], val);
2874 }
2875 
2876 /**
2877  * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
2878  * @entry: swap entry to be moved
2879  * @from:  mem_cgroup which the entry is moved from
2880  * @to:  mem_cgroup which the entry is moved to
2881  *
2882  * It succeeds only when the swap_cgroup's record for this entry is the same
2883  * as the mem_cgroup's id of @from.
2884  *
2885  * Returns 0 on success, -EINVAL on failure.
2886  *
2887  * The caller must have charged to @to, IOW, called page_counter_charge() about
2888  * both res and memsw, and called css_get().
2889  */
2890 static int mem_cgroup_move_swap_account(swp_entry_t entry,
2891 				struct mem_cgroup *from, struct mem_cgroup *to)
2892 {
2893 	unsigned short old_id, new_id;
2894 
2895 	old_id = mem_cgroup_id(from);
2896 	new_id = mem_cgroup_id(to);
2897 
2898 	if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
2899 		mem_cgroup_swap_statistics(from, false);
2900 		mem_cgroup_swap_statistics(to, true);
2901 		return 0;
2902 	}
2903 	return -EINVAL;
2904 }
2905 #else
2906 static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
2907 				struct mem_cgroup *from, struct mem_cgroup *to)
2908 {
2909 	return -EINVAL;
2910 }
2911 #endif
2912 
2913 static DEFINE_MUTEX(memcg_limit_mutex);
2914 
2915 static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
2916 				   unsigned long limit)
2917 {
2918 	unsigned long curusage;
2919 	unsigned long oldusage;
2920 	bool enlarge = false;
2921 	int retry_count;
2922 	int ret;
2923 
2924 	/*
2925 	 * For keeping hierarchical_reclaim simple, how long we should retry
2926 	 * is depends on callers. We set our retry-count to be function
2927 	 * of # of children which we should visit in this loop.
2928 	 */
2929 	retry_count = MEM_CGROUP_RECLAIM_RETRIES *
2930 		      mem_cgroup_count_children(memcg);
2931 
2932 	oldusage = page_counter_read(&memcg->memory);
2933 
2934 	do {
2935 		if (signal_pending(current)) {
2936 			ret = -EINTR;
2937 			break;
2938 		}
2939 
2940 		mutex_lock(&memcg_limit_mutex);
2941 		if (limit > memcg->memsw.limit) {
2942 			mutex_unlock(&memcg_limit_mutex);
2943 			ret = -EINVAL;
2944 			break;
2945 		}
2946 		if (limit > memcg->memory.limit)
2947 			enlarge = true;
2948 		ret = page_counter_limit(&memcg->memory, limit);
2949 		mutex_unlock(&memcg_limit_mutex);
2950 
2951 		if (!ret)
2952 			break;
2953 
2954 		try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, true);
2955 
2956 		curusage = page_counter_read(&memcg->memory);
2957 		/* Usage is reduced ? */
2958 		if (curusage >= oldusage)
2959 			retry_count--;
2960 		else
2961 			oldusage = curusage;
2962 	} while (retry_count);
2963 
2964 	if (!ret && enlarge)
2965 		memcg_oom_recover(memcg);
2966 
2967 	return ret;
2968 }
2969 
2970 static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
2971 					 unsigned long limit)
2972 {
2973 	unsigned long curusage;
2974 	unsigned long oldusage;
2975 	bool enlarge = false;
2976 	int retry_count;
2977 	int ret;
2978 
2979 	/* see mem_cgroup_resize_res_limit */
2980 	retry_count = MEM_CGROUP_RECLAIM_RETRIES *
2981 		      mem_cgroup_count_children(memcg);
2982 
2983 	oldusage = page_counter_read(&memcg->memsw);
2984 
2985 	do {
2986 		if (signal_pending(current)) {
2987 			ret = -EINTR;
2988 			break;
2989 		}
2990 
2991 		mutex_lock(&memcg_limit_mutex);
2992 		if (limit < memcg->memory.limit) {
2993 			mutex_unlock(&memcg_limit_mutex);
2994 			ret = -EINVAL;
2995 			break;
2996 		}
2997 		if (limit > memcg->memsw.limit)
2998 			enlarge = true;
2999 		ret = page_counter_limit(&memcg->memsw, limit);
3000 		mutex_unlock(&memcg_limit_mutex);
3001 
3002 		if (!ret)
3003 			break;
3004 
3005 		try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, false);
3006 
3007 		curusage = page_counter_read(&memcg->memsw);
3008 		/* Usage is reduced ? */
3009 		if (curusage >= oldusage)
3010 			retry_count--;
3011 		else
3012 			oldusage = curusage;
3013 	} while (retry_count);
3014 
3015 	if (!ret && enlarge)
3016 		memcg_oom_recover(memcg);
3017 
3018 	return ret;
3019 }
3020 
3021 unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
3022 					    gfp_t gfp_mask,
3023 					    unsigned long *total_scanned)
3024 {
3025 	unsigned long nr_reclaimed = 0;
3026 	struct mem_cgroup_per_zone *mz, *next_mz = NULL;
3027 	unsigned long reclaimed;
3028 	int loop = 0;
3029 	struct mem_cgroup_tree_per_zone *mctz;
3030 	unsigned long excess;
3031 	unsigned long nr_scanned;
3032 
3033 	if (order > 0)
3034 		return 0;
3035 
3036 	mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
3037 	/*
3038 	 * This loop can run a while, specially if mem_cgroup's continuously
3039 	 * keep exceeding their soft limit and putting the system under
3040 	 * pressure
3041 	 */
3042 	do {
3043 		if (next_mz)
3044 			mz = next_mz;
3045 		else
3046 			mz = mem_cgroup_largest_soft_limit_node(mctz);
3047 		if (!mz)
3048 			break;
3049 
3050 		nr_scanned = 0;
3051 		reclaimed = mem_cgroup_soft_reclaim(mz->memcg, zone,
3052 						    gfp_mask, &nr_scanned);
3053 		nr_reclaimed += reclaimed;
3054 		*total_scanned += nr_scanned;
3055 		spin_lock_irq(&mctz->lock);
3056 		__mem_cgroup_remove_exceeded(mz, mctz);
3057 
3058 		/*
3059 		 * If we failed to reclaim anything from this memory cgroup
3060 		 * it is time to move on to the next cgroup
3061 		 */
3062 		next_mz = NULL;
3063 		if (!reclaimed)
3064 			next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
3065 
3066 		excess = soft_limit_excess(mz->memcg);
3067 		/*
3068 		 * One school of thought says that we should not add
3069 		 * back the node to the tree if reclaim returns 0.
3070 		 * But our reclaim could return 0, simply because due
3071 		 * to priority we are exposing a smaller subset of
3072 		 * memory to reclaim from. Consider this as a longer
3073 		 * term TODO.
3074 		 */
3075 		/* If excess == 0, no tree ops */
3076 		__mem_cgroup_insert_exceeded(mz, mctz, excess);
3077 		spin_unlock_irq(&mctz->lock);
3078 		css_put(&mz->memcg->css);
3079 		loop++;
3080 		/*
3081 		 * Could not reclaim anything and there are no more
3082 		 * mem cgroups to try or we seem to be looping without
3083 		 * reclaiming anything.
3084 		 */
3085 		if (!nr_reclaimed &&
3086 			(next_mz == NULL ||
3087 			loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
3088 			break;
3089 	} while (!nr_reclaimed);
3090 	if (next_mz)
3091 		css_put(&next_mz->memcg->css);
3092 	return nr_reclaimed;
3093 }
3094 
3095 /*
3096  * Test whether @memcg has children, dead or alive.  Note that this
3097  * function doesn't care whether @memcg has use_hierarchy enabled and
3098  * returns %true if there are child csses according to the cgroup
3099  * hierarchy.  Testing use_hierarchy is the caller's responsiblity.
3100  */
3101 static inline bool memcg_has_children(struct mem_cgroup *memcg)
3102 {
3103 	bool ret;
3104 
3105 	/*
3106 	 * The lock does not prevent addition or deletion of children, but
3107 	 * it prevents a new child from being initialized based on this
3108 	 * parent in css_online(), so it's enough to decide whether
3109 	 * hierarchically inherited attributes can still be changed or not.
3110 	 */
3111 	lockdep_assert_held(&memcg_create_mutex);
3112 
3113 	rcu_read_lock();
3114 	ret = css_next_child(NULL, &memcg->css);
3115 	rcu_read_unlock();
3116 	return ret;
3117 }
3118 
3119 /*
3120  * Reclaims as many pages from the given memcg as possible and moves
3121  * the rest to the parent.
3122  *
3123  * Caller is responsible for holding css reference for memcg.
3124  */
3125 static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
3126 {
3127 	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
3128 
3129 	/* we call try-to-free pages for make this cgroup empty */
3130 	lru_add_drain_all();
3131 	/* try to free all pages in this cgroup */
3132 	while (nr_retries && page_counter_read(&memcg->memory)) {
3133 		int progress;
3134 
3135 		if (signal_pending(current))
3136 			return -EINTR;
3137 
3138 		progress = try_to_free_mem_cgroup_pages(memcg, 1,
3139 							GFP_KERNEL, true);
3140 		if (!progress) {
3141 			nr_retries--;
3142 			/* maybe some writeback is necessary */
3143 			congestion_wait(BLK_RW_ASYNC, HZ/10);
3144 		}
3145 
3146 	}
3147 
3148 	return 0;
3149 }
3150 
3151 static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
3152 					    char *buf, size_t nbytes,
3153 					    loff_t off)
3154 {
3155 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3156 
3157 	if (mem_cgroup_is_root(memcg))
3158 		return -EINVAL;
3159 	return mem_cgroup_force_empty(memcg) ?: nbytes;
3160 }
3161 
3162 static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
3163 				     struct cftype *cft)
3164 {
3165 	return mem_cgroup_from_css(css)->use_hierarchy;
3166 }
3167 
3168 static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
3169 				      struct cftype *cft, u64 val)
3170 {
3171 	int retval = 0;
3172 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3173 	struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);
3174 
3175 	mutex_lock(&memcg_create_mutex);
3176 
3177 	if (memcg->use_hierarchy == val)
3178 		goto out;
3179 
3180 	/*
3181 	 * If parent's use_hierarchy is set, we can't make any modifications
3182 	 * in the child subtrees. If it is unset, then the change can
3183 	 * occur, provided the current cgroup has no children.
3184 	 *
3185 	 * For the root cgroup, parent_mem is NULL, we allow value to be
3186 	 * set if there are no children.
3187 	 */
3188 	if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
3189 				(val == 1 || val == 0)) {
3190 		if (!memcg_has_children(memcg))
3191 			memcg->use_hierarchy = val;
3192 		else
3193 			retval = -EBUSY;
3194 	} else
3195 		retval = -EINVAL;
3196 
3197 out:
3198 	mutex_unlock(&memcg_create_mutex);
3199 
3200 	return retval;
3201 }
3202 
3203 static unsigned long tree_stat(struct mem_cgroup *memcg,
3204 			       enum mem_cgroup_stat_index idx)
3205 {
3206 	struct mem_cgroup *iter;
3207 	long val = 0;
3208 
3209 	/* Per-cpu values can be negative, use a signed accumulator */
3210 	for_each_mem_cgroup_tree(iter, memcg)
3211 		val += mem_cgroup_read_stat(iter, idx);
3212 
3213 	if (val < 0) /* race ? */
3214 		val = 0;
3215 	return val;
3216 }
3217 
3218 static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
3219 {
3220 	u64 val;
3221 
3222 	if (mem_cgroup_is_root(memcg)) {
3223 		val = tree_stat(memcg, MEM_CGROUP_STAT_CACHE);
3224 		val += tree_stat(memcg, MEM_CGROUP_STAT_RSS);
3225 		if (swap)
3226 			val += tree_stat(memcg, MEM_CGROUP_STAT_SWAP);
3227 	} else {
3228 		if (!swap)
3229 			val = page_counter_read(&memcg->memory);
3230 		else
3231 			val = page_counter_read(&memcg->memsw);
3232 	}
3233 	return val << PAGE_SHIFT;
3234 }
3235 
3236 enum {
3237 	RES_USAGE,
3238 	RES_LIMIT,
3239 	RES_MAX_USAGE,
3240 	RES_FAILCNT,
3241 	RES_SOFT_LIMIT,
3242 };
3243 
3244 static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
3245 			       struct cftype *cft)
3246 {
3247 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3248 	struct page_counter *counter;
3249 
3250 	switch (MEMFILE_TYPE(cft->private)) {
3251 	case _MEM:
3252 		counter = &memcg->memory;
3253 		break;
3254 	case _MEMSWAP:
3255 		counter = &memcg->memsw;
3256 		break;
3257 	case _KMEM:
3258 		counter = &memcg->kmem;
3259 		break;
3260 	default:
3261 		BUG();
3262 	}
3263 
3264 	switch (MEMFILE_ATTR(cft->private)) {
3265 	case RES_USAGE:
3266 		if (counter == &memcg->memory)
3267 			return mem_cgroup_usage(memcg, false);
3268 		if (counter == &memcg->memsw)
3269 			return mem_cgroup_usage(memcg, true);
3270 		return (u64)page_counter_read(counter) * PAGE_SIZE;
3271 	case RES_LIMIT:
3272 		return (u64)counter->limit * PAGE_SIZE;
3273 	case RES_MAX_USAGE:
3274 		return (u64)counter->watermark * PAGE_SIZE;
3275 	case RES_FAILCNT:
3276 		return counter->failcnt;
3277 	case RES_SOFT_LIMIT:
3278 		return (u64)memcg->soft_limit * PAGE_SIZE;
3279 	default:
3280 		BUG();
3281 	}
3282 }
3283 
3284 #ifdef CONFIG_MEMCG_KMEM
3285 static int memcg_activate_kmem(struct mem_cgroup *memcg,
3286 			       unsigned long nr_pages)
3287 {
3288 	int err = 0;
3289 	int memcg_id;
3290 
3291 	BUG_ON(memcg->kmemcg_id >= 0);
3292 	BUG_ON(memcg->kmem_acct_activated);
3293 	BUG_ON(memcg->kmem_acct_active);
3294 
3295 	/*
3296 	 * For simplicity, we won't allow this to be disabled.  It also can't
3297 	 * be changed if the cgroup has children already, or if tasks had
3298 	 * already joined.
3299 	 *
3300 	 * If tasks join before we set the limit, a person looking at
3301 	 * kmem.usage_in_bytes will have no way to determine when it took
3302 	 * place, which makes the value quite meaningless.
3303 	 *
3304 	 * After it first became limited, changes in the value of the limit are
3305 	 * of course permitted.
3306 	 */
3307 	mutex_lock(&memcg_create_mutex);
3308 	if (cgroup_has_tasks(memcg->css.cgroup) ||
3309 	    (memcg->use_hierarchy && memcg_has_children(memcg)))
3310 		err = -EBUSY;
3311 	mutex_unlock(&memcg_create_mutex);
3312 	if (err)
3313 		goto out;
3314 
3315 	memcg_id = memcg_alloc_cache_id();
3316 	if (memcg_id < 0) {
3317 		err = memcg_id;
3318 		goto out;
3319 	}
3320 
3321 	/*
3322 	 * We couldn't have accounted to this cgroup, because it hasn't got
3323 	 * activated yet, so this should succeed.
3324 	 */
3325 	err = page_counter_limit(&memcg->kmem, nr_pages);
3326 	VM_BUG_ON(err);
3327 
3328 	static_key_slow_inc(&memcg_kmem_enabled_key);
3329 	/*
3330 	 * A memory cgroup is considered kmem-active as soon as it gets
3331 	 * kmemcg_id. Setting the id after enabling static branching will
3332 	 * guarantee no one starts accounting before all call sites are
3333 	 * patched.
3334 	 */
3335 	memcg->kmemcg_id = memcg_id;
3336 	memcg->kmem_acct_activated = true;
3337 	memcg->kmem_acct_active = true;
3338 out:
3339 	return err;
3340 }
3341 
3342 static int memcg_update_kmem_limit(struct mem_cgroup *memcg,
3343 				   unsigned long limit)
3344 {
3345 	int ret;
3346 
3347 	mutex_lock(&memcg_limit_mutex);
3348 	if (!memcg_kmem_is_active(memcg))
3349 		ret = memcg_activate_kmem(memcg, limit);
3350 	else
3351 		ret = page_counter_limit(&memcg->kmem, limit);
3352 	mutex_unlock(&memcg_limit_mutex);
3353 	return ret;
3354 }
3355 
3356 static int memcg_propagate_kmem(struct mem_cgroup *memcg)
3357 {
3358 	int ret = 0;
3359 	struct mem_cgroup *parent = parent_mem_cgroup(memcg);
3360 
3361 	if (!parent)
3362 		return 0;
3363 
3364 	mutex_lock(&memcg_limit_mutex);
3365 	/*
3366 	 * If the parent cgroup is not kmem-active now, it cannot be activated
3367 	 * after this point, because it has at least one child already.
3368 	 */
3369 	if (memcg_kmem_is_active(parent))
3370 		ret = memcg_activate_kmem(memcg, PAGE_COUNTER_MAX);
3371 	mutex_unlock(&memcg_limit_mutex);
3372 	return ret;
3373 }
3374 #else
3375 static int memcg_update_kmem_limit(struct mem_cgroup *memcg,
3376 				   unsigned long limit)
3377 {
3378 	return -EINVAL;
3379 }
3380 #endif /* CONFIG_MEMCG_KMEM */
3381 
3382 /*
3383  * The user of this function is...
3384  * RES_LIMIT.
3385  */
3386 static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
3387 				char *buf, size_t nbytes, loff_t off)
3388 {
3389 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3390 	unsigned long nr_pages;
3391 	int ret;
3392 
3393 	buf = strstrip(buf);
3394 	ret = page_counter_memparse(buf, "-1", &nr_pages);
3395 	if (ret)
3396 		return ret;
3397 
3398 	switch (MEMFILE_ATTR(of_cft(of)->private)) {
3399 	case RES_LIMIT:
3400 		if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
3401 			ret = -EINVAL;
3402 			break;
3403 		}
3404 		switch (MEMFILE_TYPE(of_cft(of)->private)) {
3405 		case _MEM:
3406 			ret = mem_cgroup_resize_limit(memcg, nr_pages);
3407 			break;
3408 		case _MEMSWAP:
3409 			ret = mem_cgroup_resize_memsw_limit(memcg, nr_pages);
3410 			break;
3411 		case _KMEM:
3412 			ret = memcg_update_kmem_limit(memcg, nr_pages);
3413 			break;
3414 		}
3415 		break;
3416 	case RES_SOFT_LIMIT:
3417 		memcg->soft_limit = nr_pages;
3418 		ret = 0;
3419 		break;
3420 	}
3421 	return ret ?: nbytes;
3422 }
3423 
3424 static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
3425 				size_t nbytes, loff_t off)
3426 {
3427 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3428 	struct page_counter *counter;
3429 
3430 	switch (MEMFILE_TYPE(of_cft(of)->private)) {
3431 	case _MEM:
3432 		counter = &memcg->memory;
3433 		break;
3434 	case _MEMSWAP:
3435 		counter = &memcg->memsw;
3436 		break;
3437 	case _KMEM:
3438 		counter = &memcg->kmem;
3439 		break;
3440 	default:
3441 		BUG();
3442 	}
3443 
3444 	switch (MEMFILE_ATTR(of_cft(of)->private)) {
3445 	case RES_MAX_USAGE:
3446 		page_counter_reset_watermark(counter);
3447 		break;
3448 	case RES_FAILCNT:
3449 		counter->failcnt = 0;
3450 		break;
3451 	default:
3452 		BUG();
3453 	}
3454 
3455 	return nbytes;
3456 }
3457 
3458 static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
3459 					struct cftype *cft)
3460 {
3461 	return mem_cgroup_from_css(css)->move_charge_at_immigrate;
3462 }
3463 
3464 #ifdef CONFIG_MMU
3465 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3466 					struct cftype *cft, u64 val)
3467 {
3468 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3469 
3470 	if (val & ~MOVE_MASK)
3471 		return -EINVAL;
3472 
3473 	/*
3474 	 * No kind of locking is needed in here, because ->can_attach() will
3475 	 * check this value once in the beginning of the process, and then carry
3476 	 * on with stale data. This means that changes to this value will only
3477 	 * affect task migrations starting after the change.
3478 	 */
3479 	memcg->move_charge_at_immigrate = val;
3480 	return 0;
3481 }
3482 #else
3483 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3484 					struct cftype *cft, u64 val)
3485 {
3486 	return -ENOSYS;
3487 }
3488 #endif
3489 
3490 #ifdef CONFIG_NUMA
3491 static int memcg_numa_stat_show(struct seq_file *m, void *v)
3492 {
3493 	struct numa_stat {
3494 		const char *name;
3495 		unsigned int lru_mask;
3496 	};
3497 
3498 	static const struct numa_stat stats[] = {
3499 		{ "total", LRU_ALL },
3500 		{ "file", LRU_ALL_FILE },
3501 		{ "anon", LRU_ALL_ANON },
3502 		{ "unevictable", BIT(LRU_UNEVICTABLE) },
3503 	};
3504 	const struct numa_stat *stat;
3505 	int nid;
3506 	unsigned long nr;
3507 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3508 
3509 	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3510 		nr = mem_cgroup_nr_lru_pages(memcg, stat->lru_mask);
3511 		seq_printf(m, "%s=%lu", stat->name, nr);
3512 		for_each_node_state(nid, N_MEMORY) {
3513 			nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
3514 							  stat->lru_mask);
3515 			seq_printf(m, " N%d=%lu", nid, nr);
3516 		}
3517 		seq_putc(m, '\n');
3518 	}
3519 
3520 	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3521 		struct mem_cgroup *iter;
3522 
3523 		nr = 0;
3524 		for_each_mem_cgroup_tree(iter, memcg)
3525 			nr += mem_cgroup_nr_lru_pages(iter, stat->lru_mask);
3526 		seq_printf(m, "hierarchical_%s=%lu", stat->name, nr);
3527 		for_each_node_state(nid, N_MEMORY) {
3528 			nr = 0;
3529 			for_each_mem_cgroup_tree(iter, memcg)
3530 				nr += mem_cgroup_node_nr_lru_pages(
3531 					iter, nid, stat->lru_mask);
3532 			seq_printf(m, " N%d=%lu", nid, nr);
3533 		}
3534 		seq_putc(m, '\n');
3535 	}
3536 
3537 	return 0;
3538 }
3539 #endif /* CONFIG_NUMA */
3540 
3541 static int memcg_stat_show(struct seq_file *m, void *v)
3542 {
3543 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3544 	unsigned long memory, memsw;
3545 	struct mem_cgroup *mi;
3546 	unsigned int i;
3547 
3548 	BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_stat_names) !=
3549 		     MEM_CGROUP_STAT_NSTATS);
3550 	BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_events_names) !=
3551 		     MEM_CGROUP_EVENTS_NSTATS);
3552 	BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
3553 
3554 	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
3555 		if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
3556 			continue;
3557 		seq_printf(m, "%s %ld\n", mem_cgroup_stat_names[i],
3558 			   mem_cgroup_read_stat(memcg, i) * PAGE_SIZE);
3559 	}
3560 
3561 	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++)
3562 		seq_printf(m, "%s %lu\n", mem_cgroup_events_names[i],
3563 			   mem_cgroup_read_events(memcg, i));
3564 
3565 	for (i = 0; i < NR_LRU_LISTS; i++)
3566 		seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
3567 			   mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);
3568 
3569 	/* Hierarchical information */
3570 	memory = memsw = PAGE_COUNTER_MAX;
3571 	for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
3572 		memory = min(memory, mi->memory.limit);
3573 		memsw = min(memsw, mi->memsw.limit);
3574 	}
3575 	seq_printf(m, "hierarchical_memory_limit %llu\n",
3576 		   (u64)memory * PAGE_SIZE);
3577 	if (do_swap_account)
3578 		seq_printf(m, "hierarchical_memsw_limit %llu\n",
3579 			   (u64)memsw * PAGE_SIZE);
3580 
3581 	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
3582 		long long val = 0;
3583 
3584 		if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
3585 			continue;
3586 		for_each_mem_cgroup_tree(mi, memcg)
3587 			val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE;
3588 		seq_printf(m, "total_%s %lld\n", mem_cgroup_stat_names[i], val);
3589 	}
3590 
3591 	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
3592 		unsigned long long val = 0;
3593 
3594 		for_each_mem_cgroup_tree(mi, memcg)
3595 			val += mem_cgroup_read_events(mi, i);
3596 		seq_printf(m, "total_%s %llu\n",
3597 			   mem_cgroup_events_names[i], val);
3598 	}
3599 
3600 	for (i = 0; i < NR_LRU_LISTS; i++) {
3601 		unsigned long long val = 0;
3602 
3603 		for_each_mem_cgroup_tree(mi, memcg)
3604 			val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE;
3605 		seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val);
3606 	}
3607 
3608 #ifdef CONFIG_DEBUG_VM
3609 	{
3610 		int nid, zid;
3611 		struct mem_cgroup_per_zone *mz;
3612 		struct zone_reclaim_stat *rstat;
3613 		unsigned long recent_rotated[2] = {0, 0};
3614 		unsigned long recent_scanned[2] = {0, 0};
3615 
3616 		for_each_online_node(nid)
3617 			for (zid = 0; zid < MAX_NR_ZONES; zid++) {
3618 				mz = &memcg->nodeinfo[nid]->zoneinfo[zid];
3619 				rstat = &mz->lruvec.reclaim_stat;
3620 
3621 				recent_rotated[0] += rstat->recent_rotated[0];
3622 				recent_rotated[1] += rstat->recent_rotated[1];
3623 				recent_scanned[0] += rstat->recent_scanned[0];
3624 				recent_scanned[1] += rstat->recent_scanned[1];
3625 			}
3626 		seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
3627 		seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
3628 		seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
3629 		seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
3630 	}
3631 #endif
3632 
3633 	return 0;
3634 }
3635 
3636 static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
3637 				      struct cftype *cft)
3638 {
3639 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3640 
3641 	return mem_cgroup_swappiness(memcg);
3642 }
3643 
3644 static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
3645 				       struct cftype *cft, u64 val)
3646 {
3647 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3648 
3649 	if (val > 100)
3650 		return -EINVAL;
3651 
3652 	if (css->parent)
3653 		memcg->swappiness = val;
3654 	else
3655 		vm_swappiness = val;
3656 
3657 	return 0;
3658 }
3659 
3660 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
3661 {
3662 	struct mem_cgroup_threshold_ary *t;
3663 	unsigned long usage;
3664 	int i;
3665 
3666 	rcu_read_lock();
3667 	if (!swap)
3668 		t = rcu_dereference(memcg->thresholds.primary);
3669 	else
3670 		t = rcu_dereference(memcg->memsw_thresholds.primary);
3671 
3672 	if (!t)
3673 		goto unlock;
3674 
3675 	usage = mem_cgroup_usage(memcg, swap);
3676 
3677 	/*
3678 	 * current_threshold points to threshold just below or equal to usage.
3679 	 * If it's not true, a threshold was crossed after last
3680 	 * call of __mem_cgroup_threshold().
3681 	 */
3682 	i = t->current_threshold;
3683 
3684 	/*
3685 	 * Iterate backward over array of thresholds starting from
3686 	 * current_threshold and check if a threshold is crossed.
3687 	 * If none of thresholds below usage is crossed, we read
3688 	 * only one element of the array here.
3689 	 */
3690 	for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
3691 		eventfd_signal(t->entries[i].eventfd, 1);
3692 
3693 	/* i = current_threshold + 1 */
3694 	i++;
3695 
3696 	/*
3697 	 * Iterate forward over array of thresholds starting from
3698 	 * current_threshold+1 and check if a threshold is crossed.
3699 	 * If none of thresholds above usage is crossed, we read
3700 	 * only one element of the array here.
3701 	 */
3702 	for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
3703 		eventfd_signal(t->entries[i].eventfd, 1);
3704 
3705 	/* Update current_threshold */
3706 	t->current_threshold = i - 1;
3707 unlock:
3708 	rcu_read_unlock();
3709 }
3710 
3711 static void mem_cgroup_threshold(struct mem_cgroup *memcg)
3712 {
3713 	while (memcg) {
3714 		__mem_cgroup_threshold(memcg, false);
3715 		if (do_swap_account)
3716 			__mem_cgroup_threshold(memcg, true);
3717 
3718 		memcg = parent_mem_cgroup(memcg);
3719 	}
3720 }
3721 
3722 static int compare_thresholds(const void *a, const void *b)
3723 {
3724 	const struct mem_cgroup_threshold *_a = a;
3725 	const struct mem_cgroup_threshold *_b = b;
3726 
3727 	if (_a->threshold > _b->threshold)
3728 		return 1;
3729 
3730 	if (_a->threshold < _b->threshold)
3731 		return -1;
3732 
3733 	return 0;
3734 }
3735 
3736 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
3737 {
3738 	struct mem_cgroup_eventfd_list *ev;
3739 
3740 	spin_lock(&memcg_oom_lock);
3741 
3742 	list_for_each_entry(ev, &memcg->oom_notify, list)
3743 		eventfd_signal(ev->eventfd, 1);
3744 
3745 	spin_unlock(&memcg_oom_lock);
3746 	return 0;
3747 }
3748 
3749 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
3750 {
3751 	struct mem_cgroup *iter;
3752 
3753 	for_each_mem_cgroup_tree(iter, memcg)
3754 		mem_cgroup_oom_notify_cb(iter);
3755 }
3756 
3757 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
3758 	struct eventfd_ctx *eventfd, const char *args, enum res_type type)
3759 {
3760 	struct mem_cgroup_thresholds *thresholds;
3761 	struct mem_cgroup_threshold_ary *new;
3762 	unsigned long threshold;
3763 	unsigned long usage;
3764 	int i, size, ret;
3765 
3766 	ret = page_counter_memparse(args, "-1", &threshold);
3767 	if (ret)
3768 		return ret;
3769 
3770 	mutex_lock(&memcg->thresholds_lock);
3771 
3772 	if (type == _MEM) {
3773 		thresholds = &memcg->thresholds;
3774 		usage = mem_cgroup_usage(memcg, false);
3775 	} else if (type == _MEMSWAP) {
3776 		thresholds = &memcg->memsw_thresholds;
3777 		usage = mem_cgroup_usage(memcg, true);
3778 	} else
3779 		BUG();
3780 
3781 	/* Check if a threshold crossed before adding a new one */
3782 	if (thresholds->primary)
3783 		__mem_cgroup_threshold(memcg, type == _MEMSWAP);
3784 
3785 	size = thresholds->primary ? thresholds->primary->size + 1 : 1;
3786 
3787 	/* Allocate memory for new array of thresholds */
3788 	new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
3789 			GFP_KERNEL);
3790 	if (!new) {
3791 		ret = -ENOMEM;
3792 		goto unlock;
3793 	}
3794 	new->size = size;
3795 
3796 	/* Copy thresholds (if any) to new array */
3797 	if (thresholds->primary) {
3798 		memcpy(new->entries, thresholds->primary->entries, (size - 1) *
3799 				sizeof(struct mem_cgroup_threshold));
3800 	}
3801 
3802 	/* Add new threshold */
3803 	new->entries[size - 1].eventfd = eventfd;
3804 	new->entries[size - 1].threshold = threshold;
3805 
3806 	/* Sort thresholds. Registering of new threshold isn't time-critical */
3807 	sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
3808 			compare_thresholds, NULL);
3809 
3810 	/* Find current threshold */
3811 	new->current_threshold = -1;
3812 	for (i = 0; i < size; i++) {
3813 		if (new->entries[i].threshold <= usage) {
3814 			/*
3815 			 * new->current_threshold will not be used until
3816 			 * rcu_assign_pointer(), so it's safe to increment
3817 			 * it here.
3818 			 */
3819 			++new->current_threshold;
3820 		} else
3821 			break;
3822 	}
3823 
3824 	/* Free old spare buffer and save old primary buffer as spare */
3825 	kfree(thresholds->spare);
3826 	thresholds->spare = thresholds->primary;
3827 
3828 	rcu_assign_pointer(thresholds->primary, new);
3829 
3830 	/* To be sure that nobody uses thresholds */
3831 	synchronize_rcu();
3832 
3833 unlock:
3834 	mutex_unlock(&memcg->thresholds_lock);
3835 
3836 	return ret;
3837 }
3838 
3839 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
3840 	struct eventfd_ctx *eventfd, const char *args)
3841 {
3842 	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
3843 }
3844 
3845 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
3846 	struct eventfd_ctx *eventfd, const char *args)
3847 {
3848 	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
3849 }
3850 
3851 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3852 	struct eventfd_ctx *eventfd, enum res_type type)
3853 {
3854 	struct mem_cgroup_thresholds *thresholds;
3855 	struct mem_cgroup_threshold_ary *new;
3856 	unsigned long usage;
3857 	int i, j, size;
3858 
3859 	mutex_lock(&memcg->thresholds_lock);
3860 
3861 	if (type == _MEM) {
3862 		thresholds = &memcg->thresholds;
3863 		usage = mem_cgroup_usage(memcg, false);
3864 	} else if (type == _MEMSWAP) {
3865 		thresholds = &memcg->memsw_thresholds;
3866 		usage = mem_cgroup_usage(memcg, true);
3867 	} else
3868 		BUG();
3869 
3870 	if (!thresholds->primary)
3871 		goto unlock;
3872 
3873 	/* Check if a threshold crossed before removing */
3874 	__mem_cgroup_threshold(memcg, type == _MEMSWAP);
3875 
3876 	/* Calculate new number of threshold */
3877 	size = 0;
3878 	for (i = 0; i < thresholds->primary->size; i++) {
3879 		if (thresholds->primary->entries[i].eventfd != eventfd)
3880 			size++;
3881 	}
3882 
3883 	new = thresholds->spare;
3884 
3885 	/* Set thresholds array to NULL if we don't have thresholds */
3886 	if (!size) {
3887 		kfree(new);
3888 		new = NULL;
3889 		goto swap_buffers;
3890 	}
3891 
3892 	new->size = size;
3893 
3894 	/* Copy thresholds and find current threshold */
3895 	new->current_threshold = -1;
3896 	for (i = 0, j = 0; i < thresholds->primary->size; i++) {
3897 		if (thresholds->primary->entries[i].eventfd == eventfd)
3898 			continue;
3899 
3900 		new->entries[j] = thresholds->primary->entries[i];
3901 		if (new->entries[j].threshold <= usage) {
3902 			/*
3903 			 * new->current_threshold will not be used
3904 			 * until rcu_assign_pointer(), so it's safe to increment
3905 			 * it here.
3906 			 */
3907 			++new->current_threshold;
3908 		}
3909 		j++;
3910 	}
3911 
3912 swap_buffers:
3913 	/* Swap primary and spare array */
3914 	thresholds->spare = thresholds->primary;
3915 	/* If all events are unregistered, free the spare array */
3916 	if (!new) {
3917 		kfree(thresholds->spare);
3918 		thresholds->spare = NULL;
3919 	}
3920 
3921 	rcu_assign_pointer(thresholds->primary, new);
3922 
3923 	/* To be sure that nobody uses thresholds */
3924 	synchronize_rcu();
3925 unlock:
3926 	mutex_unlock(&memcg->thresholds_lock);
3927 }
3928 
3929 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3930 	struct eventfd_ctx *eventfd)
3931 {
3932 	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
3933 }
3934 
3935 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3936 	struct eventfd_ctx *eventfd)
3937 {
3938 	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
3939 }
3940 
3941 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
3942 	struct eventfd_ctx *eventfd, const char *args)
3943 {
3944 	struct mem_cgroup_eventfd_list *event;
3945 
3946 	event = kmalloc(sizeof(*event),	GFP_KERNEL);
3947 	if (!event)
3948 		return -ENOMEM;
3949 
3950 	spin_lock(&memcg_oom_lock);
3951 
3952 	event->eventfd = eventfd;
3953 	list_add(&event->list, &memcg->oom_notify);
3954 
3955 	/* already in OOM ? */
3956 	if (atomic_read(&memcg->under_oom))
3957 		eventfd_signal(eventfd, 1);
3958 	spin_unlock(&memcg_oom_lock);
3959 
3960 	return 0;
3961 }
3962 
3963 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
3964 	struct eventfd_ctx *eventfd)
3965 {
3966 	struct mem_cgroup_eventfd_list *ev, *tmp;
3967 
3968 	spin_lock(&memcg_oom_lock);
3969 
3970 	list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
3971 		if (ev->eventfd == eventfd) {
3972 			list_del(&ev->list);
3973 			kfree(ev);
3974 		}
3975 	}
3976 
3977 	spin_unlock(&memcg_oom_lock);
3978 }
3979 
3980 static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
3981 {
3982 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf));
3983 
3984 	seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
3985 	seq_printf(sf, "under_oom %d\n", (bool)atomic_read(&memcg->under_oom));
3986 	return 0;
3987 }
3988 
3989 static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
3990 	struct cftype *cft, u64 val)
3991 {
3992 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3993 
3994 	/* cannot set to root cgroup and only 0 and 1 are allowed */
3995 	if (!css->parent || !((val == 0) || (val == 1)))
3996 		return -EINVAL;
3997 
3998 	memcg->oom_kill_disable = val;
3999 	if (!val)
4000 		memcg_oom_recover(memcg);
4001 
4002 	return 0;
4003 }
4004 
4005 #ifdef CONFIG_MEMCG_KMEM
4006 static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
4007 {
4008 	int ret;
4009 
4010 	ret = memcg_propagate_kmem(memcg);
4011 	if (ret)
4012 		return ret;
4013 
4014 	return mem_cgroup_sockets_init(memcg, ss);
4015 }
4016 
4017 static void memcg_deactivate_kmem(struct mem_cgroup *memcg)
4018 {
4019 	struct cgroup_subsys_state *css;
4020 	struct mem_cgroup *parent, *child;
4021 	int kmemcg_id;
4022 
4023 	if (!memcg->kmem_acct_active)
4024 		return;
4025 
4026 	/*
4027 	 * Clear the 'active' flag before clearing memcg_caches arrays entries.
4028 	 * Since we take the slab_mutex in memcg_deactivate_kmem_caches(), it
4029 	 * guarantees no cache will be created for this cgroup after we are
4030 	 * done (see memcg_create_kmem_cache()).
4031 	 */
4032 	memcg->kmem_acct_active = false;
4033 
4034 	memcg_deactivate_kmem_caches(memcg);
4035 
4036 	kmemcg_id = memcg->kmemcg_id;
4037 	BUG_ON(kmemcg_id < 0);
4038 
4039 	parent = parent_mem_cgroup(memcg);
4040 	if (!parent)
4041 		parent = root_mem_cgroup;
4042 
4043 	/*
4044 	 * Change kmemcg_id of this cgroup and all its descendants to the
4045 	 * parent's id, and then move all entries from this cgroup's list_lrus
4046 	 * to ones of the parent. After we have finished, all list_lrus
4047 	 * corresponding to this cgroup are guaranteed to remain empty. The
4048 	 * ordering is imposed by list_lru_node->lock taken by
4049 	 * memcg_drain_all_list_lrus().
4050 	 */
4051 	css_for_each_descendant_pre(css, &memcg->css) {
4052 		child = mem_cgroup_from_css(css);
4053 		BUG_ON(child->kmemcg_id != kmemcg_id);
4054 		child->kmemcg_id = parent->kmemcg_id;
4055 		if (!memcg->use_hierarchy)
4056 			break;
4057 	}
4058 	memcg_drain_all_list_lrus(kmemcg_id, parent->kmemcg_id);
4059 
4060 	memcg_free_cache_id(kmemcg_id);
4061 }
4062 
4063 static void memcg_destroy_kmem(struct mem_cgroup *memcg)
4064 {
4065 	if (memcg->kmem_acct_activated) {
4066 		memcg_destroy_kmem_caches(memcg);
4067 		static_key_slow_dec(&memcg_kmem_enabled_key);
4068 		WARN_ON(page_counter_read(&memcg->kmem));
4069 	}
4070 	mem_cgroup_sockets_destroy(memcg);
4071 }
4072 #else
4073 static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
4074 {
4075 	return 0;
4076 }
4077 
4078 static void memcg_deactivate_kmem(struct mem_cgroup *memcg)
4079 {
4080 }
4081 
4082 static void memcg_destroy_kmem(struct mem_cgroup *memcg)
4083 {
4084 }
4085 #endif
4086 
4087 /*
4088  * DO NOT USE IN NEW FILES.
4089  *
4090  * "cgroup.event_control" implementation.
4091  *
4092  * This is way over-engineered.  It tries to support fully configurable
4093  * events for each user.  Such level of flexibility is completely
4094  * unnecessary especially in the light of the planned unified hierarchy.
4095  *
4096  * Please deprecate this and replace with something simpler if at all
4097  * possible.
4098  */
4099 
4100 /*
4101  * Unregister event and free resources.
4102  *
4103  * Gets called from workqueue.
4104  */
4105 static void memcg_event_remove(struct work_struct *work)
4106 {
4107 	struct mem_cgroup_event *event =
4108 		container_of(work, struct mem_cgroup_event, remove);
4109 	struct mem_cgroup *memcg = event->memcg;
4110 
4111 	remove_wait_queue(event->wqh, &event->wait);
4112 
4113 	event->unregister_event(memcg, event->eventfd);
4114 
4115 	/* Notify userspace the event is going away. */
4116 	eventfd_signal(event->eventfd, 1);
4117 
4118 	eventfd_ctx_put(event->eventfd);
4119 	kfree(event);
4120 	css_put(&memcg->css);
4121 }
4122 
4123 /*
4124  * Gets called on POLLHUP on eventfd when user closes it.
4125  *
4126  * Called with wqh->lock held and interrupts disabled.
4127  */
4128 static int memcg_event_wake(wait_queue_t *wait, unsigned mode,
4129 			    int sync, void *key)
4130 {
4131 	struct mem_cgroup_event *event =
4132 		container_of(wait, struct mem_cgroup_event, wait);
4133 	struct mem_cgroup *memcg = event->memcg;
4134 	unsigned long flags = (unsigned long)key;
4135 
4136 	if (flags & POLLHUP) {
4137 		/*
4138 		 * If the event has been detached at cgroup removal, we
4139 		 * can simply return knowing the other side will cleanup
4140 		 * for us.
4141 		 *
4142 		 * We can't race against event freeing since the other
4143 		 * side will require wqh->lock via remove_wait_queue(),
4144 		 * which we hold.
4145 		 */
4146 		spin_lock(&memcg->event_list_lock);
4147 		if (!list_empty(&event->list)) {
4148 			list_del_init(&event->list);
4149 			/*
4150 			 * We are in atomic context, but cgroup_event_remove()
4151 			 * may sleep, so we have to call it in workqueue.
4152 			 */
4153 			schedule_work(&event->remove);
4154 		}
4155 		spin_unlock(&memcg->event_list_lock);
4156 	}
4157 
4158 	return 0;
4159 }
4160 
4161 static void memcg_event_ptable_queue_proc(struct file *file,
4162 		wait_queue_head_t *wqh, poll_table *pt)
4163 {
4164 	struct mem_cgroup_event *event =
4165 		container_of(pt, struct mem_cgroup_event, pt);
4166 
4167 	event->wqh = wqh;
4168 	add_wait_queue(wqh, &event->wait);
4169 }
4170 
4171 /*
4172  * DO NOT USE IN NEW FILES.
4173  *
4174  * Parse input and register new cgroup event handler.
4175  *
4176  * Input must be in format '<event_fd> <control_fd> <args>'.
4177  * Interpretation of args is defined by control file implementation.
4178  */
4179 static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
4180 					 char *buf, size_t nbytes, loff_t off)
4181 {
4182 	struct cgroup_subsys_state *css = of_css(of);
4183 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4184 	struct mem_cgroup_event *event;
4185 	struct cgroup_subsys_state *cfile_css;
4186 	unsigned int efd, cfd;
4187 	struct fd efile;
4188 	struct fd cfile;
4189 	const char *name;
4190 	char *endp;
4191 	int ret;
4192 
4193 	buf = strstrip(buf);
4194 
4195 	efd = simple_strtoul(buf, &endp, 10);
4196 	if (*endp != ' ')
4197 		return -EINVAL;
4198 	buf = endp + 1;
4199 
4200 	cfd = simple_strtoul(buf, &endp, 10);
4201 	if ((*endp != ' ') && (*endp != '\0'))
4202 		return -EINVAL;
4203 	buf = endp + 1;
4204 
4205 	event = kzalloc(sizeof(*event), GFP_KERNEL);
4206 	if (!event)
4207 		return -ENOMEM;
4208 
4209 	event->memcg = memcg;
4210 	INIT_LIST_HEAD(&event->list);
4211 	init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
4212 	init_waitqueue_func_entry(&event->wait, memcg_event_wake);
4213 	INIT_WORK(&event->remove, memcg_event_remove);
4214 
4215 	efile = fdget(efd);
4216 	if (!efile.file) {
4217 		ret = -EBADF;
4218 		goto out_kfree;
4219 	}
4220 
4221 	event->eventfd = eventfd_ctx_fileget(efile.file);
4222 	if (IS_ERR(event->eventfd)) {
4223 		ret = PTR_ERR(event->eventfd);
4224 		goto out_put_efile;
4225 	}
4226 
4227 	cfile = fdget(cfd);
4228 	if (!cfile.file) {
4229 		ret = -EBADF;
4230 		goto out_put_eventfd;
4231 	}
4232 
4233 	/* the process need read permission on control file */
4234 	/* AV: shouldn't we check that it's been opened for read instead? */
4235 	ret = inode_permission(file_inode(cfile.file), MAY_READ);
4236 	if (ret < 0)
4237 		goto out_put_cfile;
4238 
4239 	/*
4240 	 * Determine the event callbacks and set them in @event.  This used
4241 	 * to be done via struct cftype but cgroup core no longer knows
4242 	 * about these events.  The following is crude but the whole thing
4243 	 * is for compatibility anyway.
4244 	 *
4245 	 * DO NOT ADD NEW FILES.
4246 	 */
4247 	name = cfile.file->f_path.dentry->d_name.name;
4248 
4249 	if (!strcmp(name, "memory.usage_in_bytes")) {
4250 		event->register_event = mem_cgroup_usage_register_event;
4251 		event->unregister_event = mem_cgroup_usage_unregister_event;
4252 	} else if (!strcmp(name, "memory.oom_control")) {
4253 		event->register_event = mem_cgroup_oom_register_event;
4254 		event->unregister_event = mem_cgroup_oom_unregister_event;
4255 	} else if (!strcmp(name, "memory.pressure_level")) {
4256 		event->register_event = vmpressure_register_event;
4257 		event->unregister_event = vmpressure_unregister_event;
4258 	} else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
4259 		event->register_event = memsw_cgroup_usage_register_event;
4260 		event->unregister_event = memsw_cgroup_usage_unregister_event;
4261 	} else {
4262 		ret = -EINVAL;
4263 		goto out_put_cfile;
4264 	}
4265 
4266 	/*
4267 	 * Verify @cfile should belong to @css.  Also, remaining events are
4268 	 * automatically removed on cgroup destruction but the removal is
4269 	 * asynchronous, so take an extra ref on @css.
4270 	 */
4271 	cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,
4272 					       &memory_cgrp_subsys);
4273 	ret = -EINVAL;
4274 	if (IS_ERR(cfile_css))
4275 		goto out_put_cfile;
4276 	if (cfile_css != css) {
4277 		css_put(cfile_css);
4278 		goto out_put_cfile;
4279 	}
4280 
4281 	ret = event->register_event(memcg, event->eventfd, buf);
4282 	if (ret)
4283 		goto out_put_css;
4284 
4285 	efile.file->f_op->poll(efile.file, &event->pt);
4286 
4287 	spin_lock(&memcg->event_list_lock);
4288 	list_add(&event->list, &memcg->event_list);
4289 	spin_unlock(&memcg->event_list_lock);
4290 
4291 	fdput(cfile);
4292 	fdput(efile);
4293 
4294 	return nbytes;
4295 
4296 out_put_css:
4297 	css_put(css);
4298 out_put_cfile:
4299 	fdput(cfile);
4300 out_put_eventfd:
4301 	eventfd_ctx_put(event->eventfd);
4302 out_put_efile:
4303 	fdput(efile);
4304 out_kfree:
4305 	kfree(event);
4306 
4307 	return ret;
4308 }
4309 
4310 static struct cftype mem_cgroup_legacy_files[] = {
4311 	{
4312 		.name = "usage_in_bytes",
4313 		.private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
4314 		.read_u64 = mem_cgroup_read_u64,
4315 	},
4316 	{
4317 		.name = "max_usage_in_bytes",
4318 		.private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
4319 		.write = mem_cgroup_reset,
4320 		.read_u64 = mem_cgroup_read_u64,
4321 	},
4322 	{
4323 		.name = "limit_in_bytes",
4324 		.private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
4325 		.write = mem_cgroup_write,
4326 		.read_u64 = mem_cgroup_read_u64,
4327 	},
4328 	{
4329 		.name = "soft_limit_in_bytes",
4330 		.private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
4331 		.write = mem_cgroup_write,
4332 		.read_u64 = mem_cgroup_read_u64,
4333 	},
4334 	{
4335 		.name = "failcnt",
4336 		.private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
4337 		.write = mem_cgroup_reset,
4338 		.read_u64 = mem_cgroup_read_u64,
4339 	},
4340 	{
4341 		.name = "stat",
4342 		.seq_show = memcg_stat_show,
4343 	},
4344 	{
4345 		.name = "force_empty",
4346 		.write = mem_cgroup_force_empty_write,
4347 	},
4348 	{
4349 		.name = "use_hierarchy",
4350 		.write_u64 = mem_cgroup_hierarchy_write,
4351 		.read_u64 = mem_cgroup_hierarchy_read,
4352 	},
4353 	{
4354 		.name = "cgroup.event_control",		/* XXX: for compat */
4355 		.write = memcg_write_event_control,
4356 		.flags = CFTYPE_NO_PREFIX,
4357 		.mode = S_IWUGO,
4358 	},
4359 	{
4360 		.name = "swappiness",
4361 		.read_u64 = mem_cgroup_swappiness_read,
4362 		.write_u64 = mem_cgroup_swappiness_write,
4363 	},
4364 	{
4365 		.name = "move_charge_at_immigrate",
4366 		.read_u64 = mem_cgroup_move_charge_read,
4367 		.write_u64 = mem_cgroup_move_charge_write,
4368 	},
4369 	{
4370 		.name = "oom_control",
4371 		.seq_show = mem_cgroup_oom_control_read,
4372 		.write_u64 = mem_cgroup_oom_control_write,
4373 		.private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
4374 	},
4375 	{
4376 		.name = "pressure_level",
4377 	},
4378 #ifdef CONFIG_NUMA
4379 	{
4380 		.name = "numa_stat",
4381 		.seq_show = memcg_numa_stat_show,
4382 	},
4383 #endif
4384 #ifdef CONFIG_MEMCG_KMEM
4385 	{
4386 		.name = "kmem.limit_in_bytes",
4387 		.private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
4388 		.write = mem_cgroup_write,
4389 		.read_u64 = mem_cgroup_read_u64,
4390 	},
4391 	{
4392 		.name = "kmem.usage_in_bytes",
4393 		.private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
4394 		.read_u64 = mem_cgroup_read_u64,
4395 	},
4396 	{
4397 		.name = "kmem.failcnt",
4398 		.private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
4399 		.write = mem_cgroup_reset,
4400 		.read_u64 = mem_cgroup_read_u64,
4401 	},
4402 	{
4403 		.name = "kmem.max_usage_in_bytes",
4404 		.private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
4405 		.write = mem_cgroup_reset,
4406 		.read_u64 = mem_cgroup_read_u64,
4407 	},
4408 #ifdef CONFIG_SLABINFO
4409 	{
4410 		.name = "kmem.slabinfo",
4411 		.seq_start = slab_start,
4412 		.seq_next = slab_next,
4413 		.seq_stop = slab_stop,
4414 		.seq_show = memcg_slab_show,
4415 	},
4416 #endif
4417 #endif
4418 	{ },	/* terminate */
4419 };
4420 
4421 static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
4422 {
4423 	struct mem_cgroup_per_node *pn;
4424 	struct mem_cgroup_per_zone *mz;
4425 	int zone, tmp = node;
4426 	/*
4427 	 * This routine is called against possible nodes.
4428 	 * But it's BUG to call kmalloc() against offline node.
4429 	 *
4430 	 * TODO: this routine can waste much memory for nodes which will
4431 	 *       never be onlined. It's better to use memory hotplug callback
4432 	 *       function.
4433 	 */
4434 	if (!node_state(node, N_NORMAL_MEMORY))
4435 		tmp = -1;
4436 	pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
4437 	if (!pn)
4438 		return 1;
4439 
4440 	for (zone = 0; zone < MAX_NR_ZONES; zone++) {
4441 		mz = &pn->zoneinfo[zone];
4442 		lruvec_init(&mz->lruvec);
4443 		mz->usage_in_excess = 0;
4444 		mz->on_tree = false;
4445 		mz->memcg = memcg;
4446 	}
4447 	memcg->nodeinfo[node] = pn;
4448 	return 0;
4449 }
4450 
4451 static void free_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
4452 {
4453 	kfree(memcg->nodeinfo[node]);
4454 }
4455 
4456 static struct mem_cgroup *mem_cgroup_alloc(void)
4457 {
4458 	struct mem_cgroup *memcg;
4459 	size_t size;
4460 
4461 	size = sizeof(struct mem_cgroup);
4462 	size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);
4463 
4464 	memcg = kzalloc(size, GFP_KERNEL);
4465 	if (!memcg)
4466 		return NULL;
4467 
4468 	memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
4469 	if (!memcg->stat)
4470 		goto out_free;
4471 	spin_lock_init(&memcg->pcp_counter_lock);
4472 	return memcg;
4473 
4474 out_free:
4475 	kfree(memcg);
4476 	return NULL;
4477 }
4478 
4479 /*
4480  * At destroying mem_cgroup, references from swap_cgroup can remain.
4481  * (scanning all at force_empty is too costly...)
4482  *
4483  * Instead of clearing all references at force_empty, we remember
4484  * the number of reference from swap_cgroup and free mem_cgroup when
4485  * it goes down to 0.
4486  *
4487  * Removal of cgroup itself succeeds regardless of refs from swap.
4488  */
4489 
4490 static void __mem_cgroup_free(struct mem_cgroup *memcg)
4491 {
4492 	int node;
4493 
4494 	mem_cgroup_remove_from_trees(memcg);
4495 
4496 	for_each_node(node)
4497 		free_mem_cgroup_per_zone_info(memcg, node);
4498 
4499 	free_percpu(memcg->stat);
4500 	kfree(memcg);
4501 }
4502 
4503 /*
4504  * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
4505  */
4506 struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
4507 {
4508 	if (!memcg->memory.parent)
4509 		return NULL;
4510 	return mem_cgroup_from_counter(memcg->memory.parent, memory);
4511 }
4512 EXPORT_SYMBOL(parent_mem_cgroup);
4513 
4514 static struct cgroup_subsys_state * __ref
4515 mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
4516 {
4517 	struct mem_cgroup *memcg;
4518 	long error = -ENOMEM;
4519 	int node;
4520 
4521 	memcg = mem_cgroup_alloc();
4522 	if (!memcg)
4523 		return ERR_PTR(error);
4524 
4525 	for_each_node(node)
4526 		if (alloc_mem_cgroup_per_zone_info(memcg, node))
4527 			goto free_out;
4528 
4529 	/* root ? */
4530 	if (parent_css == NULL) {
4531 		root_mem_cgroup = memcg;
4532 		page_counter_init(&memcg->memory, NULL);
4533 		memcg->high = PAGE_COUNTER_MAX;
4534 		memcg->soft_limit = PAGE_COUNTER_MAX;
4535 		page_counter_init(&memcg->memsw, NULL);
4536 		page_counter_init(&memcg->kmem, NULL);
4537 	}
4538 
4539 	memcg->last_scanned_node = MAX_NUMNODES;
4540 	INIT_LIST_HEAD(&memcg->oom_notify);
4541 	memcg->move_charge_at_immigrate = 0;
4542 	mutex_init(&memcg->thresholds_lock);
4543 	spin_lock_init(&memcg->move_lock);
4544 	vmpressure_init(&memcg->vmpressure);
4545 	INIT_LIST_HEAD(&memcg->event_list);
4546 	spin_lock_init(&memcg->event_list_lock);
4547 #ifdef CONFIG_MEMCG_KMEM
4548 	memcg->kmemcg_id = -1;
4549 #endif
4550 
4551 	return &memcg->css;
4552 
4553 free_out:
4554 	__mem_cgroup_free(memcg);
4555 	return ERR_PTR(error);
4556 }
4557 
4558 static int
4559 mem_cgroup_css_online(struct cgroup_subsys_state *css)
4560 {
4561 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4562 	struct mem_cgroup *parent = mem_cgroup_from_css(css->parent);
4563 	int ret;
4564 
4565 	if (css->id > MEM_CGROUP_ID_MAX)
4566 		return -ENOSPC;
4567 
4568 	if (!parent)
4569 		return 0;
4570 
4571 	mutex_lock(&memcg_create_mutex);
4572 
4573 	memcg->use_hierarchy = parent->use_hierarchy;
4574 	memcg->oom_kill_disable = parent->oom_kill_disable;
4575 	memcg->swappiness = mem_cgroup_swappiness(parent);
4576 
4577 	if (parent->use_hierarchy) {
4578 		page_counter_init(&memcg->memory, &parent->memory);
4579 		memcg->high = PAGE_COUNTER_MAX;
4580 		memcg->soft_limit = PAGE_COUNTER_MAX;
4581 		page_counter_init(&memcg->memsw, &parent->memsw);
4582 		page_counter_init(&memcg->kmem, &parent->kmem);
4583 
4584 		/*
4585 		 * No need to take a reference to the parent because cgroup
4586 		 * core guarantees its existence.
4587 		 */
4588 	} else {
4589 		page_counter_init(&memcg->memory, NULL);
4590 		memcg->high = PAGE_COUNTER_MAX;
4591 		memcg->soft_limit = PAGE_COUNTER_MAX;
4592 		page_counter_init(&memcg->memsw, NULL);
4593 		page_counter_init(&memcg->kmem, NULL);
4594 		/*
4595 		 * Deeper hierachy with use_hierarchy == false doesn't make
4596 		 * much sense so let cgroup subsystem know about this
4597 		 * unfortunate state in our controller.
4598 		 */
4599 		if (parent != root_mem_cgroup)
4600 			memory_cgrp_subsys.broken_hierarchy = true;
4601 	}
4602 	mutex_unlock(&memcg_create_mutex);
4603 
4604 	ret = memcg_init_kmem(memcg, &memory_cgrp_subsys);
4605 	if (ret)
4606 		return ret;
4607 
4608 	/*
4609 	 * Make sure the memcg is initialized: mem_cgroup_iter()
4610 	 * orders reading memcg->initialized against its callers
4611 	 * reading the memcg members.
4612 	 */
4613 	smp_store_release(&memcg->initialized, 1);
4614 
4615 	return 0;
4616 }
4617 
4618 static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
4619 {
4620 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4621 	struct mem_cgroup_event *event, *tmp;
4622 
4623 	/*
4624 	 * Unregister events and notify userspace.
4625 	 * Notify userspace about cgroup removing only after rmdir of cgroup
4626 	 * directory to avoid race between userspace and kernelspace.
4627 	 */
4628 	spin_lock(&memcg->event_list_lock);
4629 	list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
4630 		list_del_init(&event->list);
4631 		schedule_work(&event->remove);
4632 	}
4633 	spin_unlock(&memcg->event_list_lock);
4634 
4635 	vmpressure_cleanup(&memcg->vmpressure);
4636 
4637 	memcg_deactivate_kmem(memcg);
4638 }
4639 
4640 static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
4641 {
4642 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4643 
4644 	memcg_destroy_kmem(memcg);
4645 	__mem_cgroup_free(memcg);
4646 }
4647 
4648 /**
4649  * mem_cgroup_css_reset - reset the states of a mem_cgroup
4650  * @css: the target css
4651  *
4652  * Reset the states of the mem_cgroup associated with @css.  This is
4653  * invoked when the userland requests disabling on the default hierarchy
4654  * but the memcg is pinned through dependency.  The memcg should stop
4655  * applying policies and should revert to the vanilla state as it may be
4656  * made visible again.
4657  *
4658  * The current implementation only resets the essential configurations.
4659  * This needs to be expanded to cover all the visible parts.
4660  */
4661 static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
4662 {
4663 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4664 
4665 	mem_cgroup_resize_limit(memcg, PAGE_COUNTER_MAX);
4666 	mem_cgroup_resize_memsw_limit(memcg, PAGE_COUNTER_MAX);
4667 	memcg_update_kmem_limit(memcg, PAGE_COUNTER_MAX);
4668 	memcg->low = 0;
4669 	memcg->high = PAGE_COUNTER_MAX;
4670 	memcg->soft_limit = PAGE_COUNTER_MAX;
4671 }
4672 
4673 #ifdef CONFIG_MMU
4674 /* Handlers for move charge at task migration. */
4675 static int mem_cgroup_do_precharge(unsigned long count)
4676 {
4677 	int ret;
4678 
4679 	/* Try a single bulk charge without reclaim first */
4680 	ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_WAIT, count);
4681 	if (!ret) {
4682 		mc.precharge += count;
4683 		return ret;
4684 	}
4685 	if (ret == -EINTR) {
4686 		cancel_charge(root_mem_cgroup, count);
4687 		return ret;
4688 	}
4689 
4690 	/* Try charges one by one with reclaim */
4691 	while (count--) {
4692 		ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_NORETRY, 1);
4693 		/*
4694 		 * In case of failure, any residual charges against
4695 		 * mc.to will be dropped by mem_cgroup_clear_mc()
4696 		 * later on.  However, cancel any charges that are
4697 		 * bypassed to root right away or they'll be lost.
4698 		 */
4699 		if (ret == -EINTR)
4700 			cancel_charge(root_mem_cgroup, 1);
4701 		if (ret)
4702 			return ret;
4703 		mc.precharge++;
4704 		cond_resched();
4705 	}
4706 	return 0;
4707 }
4708 
4709 /**
4710  * get_mctgt_type - get target type of moving charge
4711  * @vma: the vma the pte to be checked belongs
4712  * @addr: the address corresponding to the pte to be checked
4713  * @ptent: the pte to be checked
4714  * @target: the pointer the target page or swap ent will be stored(can be NULL)
4715  *
4716  * Returns
4717  *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
4718  *   1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
4719  *     move charge. if @target is not NULL, the page is stored in target->page
4720  *     with extra refcnt got(Callers should handle it).
4721  *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
4722  *     target for charge migration. if @target is not NULL, the entry is stored
4723  *     in target->ent.
4724  *
4725  * Called with pte lock held.
4726  */
4727 union mc_target {
4728 	struct page	*page;
4729 	swp_entry_t	ent;
4730 };
4731 
4732 enum mc_target_type {
4733 	MC_TARGET_NONE = 0,
4734 	MC_TARGET_PAGE,
4735 	MC_TARGET_SWAP,
4736 };
4737 
4738 static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
4739 						unsigned long addr, pte_t ptent)
4740 {
4741 	struct page *page = vm_normal_page(vma, addr, ptent);
4742 
4743 	if (!page || !page_mapped(page))
4744 		return NULL;
4745 	if (PageAnon(page)) {
4746 		if (!(mc.flags & MOVE_ANON))
4747 			return NULL;
4748 	} else {
4749 		if (!(mc.flags & MOVE_FILE))
4750 			return NULL;
4751 	}
4752 	if (!get_page_unless_zero(page))
4753 		return NULL;
4754 
4755 	return page;
4756 }
4757 
4758 #ifdef CONFIG_SWAP
4759 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4760 			unsigned long addr, pte_t ptent, swp_entry_t *entry)
4761 {
4762 	struct page *page = NULL;
4763 	swp_entry_t ent = pte_to_swp_entry(ptent);
4764 
4765 	if (!(mc.flags & MOVE_ANON) || non_swap_entry(ent))
4766 		return NULL;
4767 	/*
4768 	 * Because lookup_swap_cache() updates some statistics counter,
4769 	 * we call find_get_page() with swapper_space directly.
4770 	 */
4771 	page = find_get_page(swap_address_space(ent), ent.val);
4772 	if (do_swap_account)
4773 		entry->val = ent.val;
4774 
4775 	return page;
4776 }
4777 #else
4778 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4779 			unsigned long addr, pte_t ptent, swp_entry_t *entry)
4780 {
4781 	return NULL;
4782 }
4783 #endif
4784 
4785 static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
4786 			unsigned long addr, pte_t ptent, swp_entry_t *entry)
4787 {
4788 	struct page *page = NULL;
4789 	struct address_space *mapping;
4790 	pgoff_t pgoff;
4791 
4792 	if (!vma->vm_file) /* anonymous vma */
4793 		return NULL;
4794 	if (!(mc.flags & MOVE_FILE))
4795 		return NULL;
4796 
4797 	mapping = vma->vm_file->f_mapping;
4798 	pgoff = linear_page_index(vma, addr);
4799 
4800 	/* page is moved even if it's not RSS of this task(page-faulted). */
4801 #ifdef CONFIG_SWAP
4802 	/* shmem/tmpfs may report page out on swap: account for that too. */
4803 	if (shmem_mapping(mapping)) {
4804 		page = find_get_entry(mapping, pgoff);
4805 		if (radix_tree_exceptional_entry(page)) {
4806 			swp_entry_t swp = radix_to_swp_entry(page);
4807 			if (do_swap_account)
4808 				*entry = swp;
4809 			page = find_get_page(swap_address_space(swp), swp.val);
4810 		}
4811 	} else
4812 		page = find_get_page(mapping, pgoff);
4813 #else
4814 	page = find_get_page(mapping, pgoff);
4815 #endif
4816 	return page;
4817 }
4818 
4819 static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
4820 		unsigned long addr, pte_t ptent, union mc_target *target)
4821 {
4822 	struct page *page = NULL;
4823 	enum mc_target_type ret = MC_TARGET_NONE;
4824 	swp_entry_t ent = { .val = 0 };
4825 
4826 	if (pte_present(ptent))
4827 		page = mc_handle_present_pte(vma, addr, ptent);
4828 	else if (is_swap_pte(ptent))
4829 		page = mc_handle_swap_pte(vma, addr, ptent, &ent);
4830 	else if (pte_none(ptent))
4831 		page = mc_handle_file_pte(vma, addr, ptent, &ent);
4832 
4833 	if (!page && !ent.val)
4834 		return ret;
4835 	if (page) {
4836 		/*
4837 		 * Do only loose check w/o serialization.
4838 		 * mem_cgroup_move_account() checks the page is valid or
4839 		 * not under LRU exclusion.
4840 		 */
4841 		if (page->mem_cgroup == mc.from) {
4842 			ret = MC_TARGET_PAGE;
4843 			if (target)
4844 				target->page = page;
4845 		}
4846 		if (!ret || !target)
4847 			put_page(page);
4848 	}
4849 	/* There is a swap entry and a page doesn't exist or isn't charged */
4850 	if (ent.val && !ret &&
4851 	    mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
4852 		ret = MC_TARGET_SWAP;
4853 		if (target)
4854 			target->ent = ent;
4855 	}
4856 	return ret;
4857 }
4858 
4859 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4860 /*
4861  * We don't consider swapping or file mapped pages because THP does not
4862  * support them for now.
4863  * Caller should make sure that pmd_trans_huge(pmd) is true.
4864  */
4865 static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
4866 		unsigned long addr, pmd_t pmd, union mc_target *target)
4867 {
4868 	struct page *page = NULL;
4869 	enum mc_target_type ret = MC_TARGET_NONE;
4870 
4871 	page = pmd_page(pmd);
4872 	VM_BUG_ON_PAGE(!page || !PageHead(page), page);
4873 	if (!(mc.flags & MOVE_ANON))
4874 		return ret;
4875 	if (page->mem_cgroup == mc.from) {
4876 		ret = MC_TARGET_PAGE;
4877 		if (target) {
4878 			get_page(page);
4879 			target->page = page;
4880 		}
4881 	}
4882 	return ret;
4883 }
4884 #else
4885 static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
4886 		unsigned long addr, pmd_t pmd, union mc_target *target)
4887 {
4888 	return MC_TARGET_NONE;
4889 }
4890 #endif
4891 
4892 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
4893 					unsigned long addr, unsigned long end,
4894 					struct mm_walk *walk)
4895 {
4896 	struct vm_area_struct *vma = walk->vma;
4897 	pte_t *pte;
4898 	spinlock_t *ptl;
4899 
4900 	if (pmd_trans_huge_lock(pmd, vma, &ptl) == 1) {
4901 		if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
4902 			mc.precharge += HPAGE_PMD_NR;
4903 		spin_unlock(ptl);
4904 		return 0;
4905 	}
4906 
4907 	if (pmd_trans_unstable(pmd))
4908 		return 0;
4909 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4910 	for (; addr != end; pte++, addr += PAGE_SIZE)
4911 		if (get_mctgt_type(vma, addr, *pte, NULL))
4912 			mc.precharge++;	/* increment precharge temporarily */
4913 	pte_unmap_unlock(pte - 1, ptl);
4914 	cond_resched();
4915 
4916 	return 0;
4917 }
4918 
4919 static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
4920 {
4921 	unsigned long precharge;
4922 
4923 	struct mm_walk mem_cgroup_count_precharge_walk = {
4924 		.pmd_entry = mem_cgroup_count_precharge_pte_range,
4925 		.mm = mm,
4926 	};
4927 	down_read(&mm->mmap_sem);
4928 	walk_page_range(0, ~0UL, &mem_cgroup_count_precharge_walk);
4929 	up_read(&mm->mmap_sem);
4930 
4931 	precharge = mc.precharge;
4932 	mc.precharge = 0;
4933 
4934 	return precharge;
4935 }
4936 
4937 static int mem_cgroup_precharge_mc(struct mm_struct *mm)
4938 {
4939 	unsigned long precharge = mem_cgroup_count_precharge(mm);
4940 
4941 	VM_BUG_ON(mc.moving_task);
4942 	mc.moving_task = current;
4943 	return mem_cgroup_do_precharge(precharge);
4944 }
4945 
4946 /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
4947 static void __mem_cgroup_clear_mc(void)
4948 {
4949 	struct mem_cgroup *from = mc.from;
4950 	struct mem_cgroup *to = mc.to;
4951 
4952 	/* we must uncharge all the leftover precharges from mc.to */
4953 	if (mc.precharge) {
4954 		cancel_charge(mc.to, mc.precharge);
4955 		mc.precharge = 0;
4956 	}
4957 	/*
4958 	 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
4959 	 * we must uncharge here.
4960 	 */
4961 	if (mc.moved_charge) {
4962 		cancel_charge(mc.from, mc.moved_charge);
4963 		mc.moved_charge = 0;
4964 	}
4965 	/* we must fixup refcnts and charges */
4966 	if (mc.moved_swap) {
4967 		/* uncharge swap account from the old cgroup */
4968 		if (!mem_cgroup_is_root(mc.from))
4969 			page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
4970 
4971 		/*
4972 		 * we charged both to->memory and to->memsw, so we
4973 		 * should uncharge to->memory.
4974 		 */
4975 		if (!mem_cgroup_is_root(mc.to))
4976 			page_counter_uncharge(&mc.to->memory, mc.moved_swap);
4977 
4978 		css_put_many(&mc.from->css, mc.moved_swap);
4979 
4980 		/* we've already done css_get(mc.to) */
4981 		mc.moved_swap = 0;
4982 	}
4983 	memcg_oom_recover(from);
4984 	memcg_oom_recover(to);
4985 	wake_up_all(&mc.waitq);
4986 }
4987 
4988 static void mem_cgroup_clear_mc(void)
4989 {
4990 	/*
4991 	 * we must clear moving_task before waking up waiters at the end of
4992 	 * task migration.
4993 	 */
4994 	mc.moving_task = NULL;
4995 	__mem_cgroup_clear_mc();
4996 	spin_lock(&mc.lock);
4997 	mc.from = NULL;
4998 	mc.to = NULL;
4999 	spin_unlock(&mc.lock);
5000 }
5001 
5002 static int mem_cgroup_can_attach(struct cgroup_subsys_state *css,
5003 				 struct cgroup_taskset *tset)
5004 {
5005 	struct task_struct *p = cgroup_taskset_first(tset);
5006 	int ret = 0;
5007 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5008 	unsigned long move_flags;
5009 
5010 	/*
5011 	 * We are now commited to this value whatever it is. Changes in this
5012 	 * tunable will only affect upcoming migrations, not the current one.
5013 	 * So we need to save it, and keep it going.
5014 	 */
5015 	move_flags = ACCESS_ONCE(memcg->move_charge_at_immigrate);
5016 	if (move_flags) {
5017 		struct mm_struct *mm;
5018 		struct mem_cgroup *from = mem_cgroup_from_task(p);
5019 
5020 		VM_BUG_ON(from == memcg);
5021 
5022 		mm = get_task_mm(p);
5023 		if (!mm)
5024 			return 0;
5025 		/* We move charges only when we move a owner of the mm */
5026 		if (mm->owner == p) {
5027 			VM_BUG_ON(mc.from);
5028 			VM_BUG_ON(mc.to);
5029 			VM_BUG_ON(mc.precharge);
5030 			VM_BUG_ON(mc.moved_charge);
5031 			VM_BUG_ON(mc.moved_swap);
5032 
5033 			spin_lock(&mc.lock);
5034 			mc.from = from;
5035 			mc.to = memcg;
5036 			mc.flags = move_flags;
5037 			spin_unlock(&mc.lock);
5038 			/* We set mc.moving_task later */
5039 
5040 			ret = mem_cgroup_precharge_mc(mm);
5041 			if (ret)
5042 				mem_cgroup_clear_mc();
5043 		}
5044 		mmput(mm);
5045 	}
5046 	return ret;
5047 }
5048 
5049 static void mem_cgroup_cancel_attach(struct cgroup_subsys_state *css,
5050 				     struct cgroup_taskset *tset)
5051 {
5052 	if (mc.to)
5053 		mem_cgroup_clear_mc();
5054 }
5055 
5056 static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
5057 				unsigned long addr, unsigned long end,
5058 				struct mm_walk *walk)
5059 {
5060 	int ret = 0;
5061 	struct vm_area_struct *vma = walk->vma;
5062 	pte_t *pte;
5063 	spinlock_t *ptl;
5064 	enum mc_target_type target_type;
5065 	union mc_target target;
5066 	struct page *page;
5067 
5068 	/*
5069 	 * We don't take compound_lock() here but no race with splitting thp
5070 	 * happens because:
5071 	 *  - if pmd_trans_huge_lock() returns 1, the relevant thp is not
5072 	 *    under splitting, which means there's no concurrent thp split,
5073 	 *  - if another thread runs into split_huge_page() just after we
5074 	 *    entered this if-block, the thread must wait for page table lock
5075 	 *    to be unlocked in __split_huge_page_splitting(), where the main
5076 	 *    part of thp split is not executed yet.
5077 	 */
5078 	if (pmd_trans_huge_lock(pmd, vma, &ptl) == 1) {
5079 		if (mc.precharge < HPAGE_PMD_NR) {
5080 			spin_unlock(ptl);
5081 			return 0;
5082 		}
5083 		target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
5084 		if (target_type == MC_TARGET_PAGE) {
5085 			page = target.page;
5086 			if (!isolate_lru_page(page)) {
5087 				if (!mem_cgroup_move_account(page, HPAGE_PMD_NR,
5088 							     mc.from, mc.to)) {
5089 					mc.precharge -= HPAGE_PMD_NR;
5090 					mc.moved_charge += HPAGE_PMD_NR;
5091 				}
5092 				putback_lru_page(page);
5093 			}
5094 			put_page(page);
5095 		}
5096 		spin_unlock(ptl);
5097 		return 0;
5098 	}
5099 
5100 	if (pmd_trans_unstable(pmd))
5101 		return 0;
5102 retry:
5103 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
5104 	for (; addr != end; addr += PAGE_SIZE) {
5105 		pte_t ptent = *(pte++);
5106 		swp_entry_t ent;
5107 
5108 		if (!mc.precharge)
5109 			break;
5110 
5111 		switch (get_mctgt_type(vma, addr, ptent, &target)) {
5112 		case MC_TARGET_PAGE:
5113 			page = target.page;
5114 			if (isolate_lru_page(page))
5115 				goto put;
5116 			if (!mem_cgroup_move_account(page, 1, mc.from, mc.to)) {
5117 				mc.precharge--;
5118 				/* we uncharge from mc.from later. */
5119 				mc.moved_charge++;
5120 			}
5121 			putback_lru_page(page);
5122 put:			/* get_mctgt_type() gets the page */
5123 			put_page(page);
5124 			break;
5125 		case MC_TARGET_SWAP:
5126 			ent = target.ent;
5127 			if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
5128 				mc.precharge--;
5129 				/* we fixup refcnts and charges later. */
5130 				mc.moved_swap++;
5131 			}
5132 			break;
5133 		default:
5134 			break;
5135 		}
5136 	}
5137 	pte_unmap_unlock(pte - 1, ptl);
5138 	cond_resched();
5139 
5140 	if (addr != end) {
5141 		/*
5142 		 * We have consumed all precharges we got in can_attach().
5143 		 * We try charge one by one, but don't do any additional
5144 		 * charges to mc.to if we have failed in charge once in attach()
5145 		 * phase.
5146 		 */
5147 		ret = mem_cgroup_do_precharge(1);
5148 		if (!ret)
5149 			goto retry;
5150 	}
5151 
5152 	return ret;
5153 }
5154 
5155 static void mem_cgroup_move_charge(struct mm_struct *mm)
5156 {
5157 	struct mm_walk mem_cgroup_move_charge_walk = {
5158 		.pmd_entry = mem_cgroup_move_charge_pte_range,
5159 		.mm = mm,
5160 	};
5161 
5162 	lru_add_drain_all();
5163 	/*
5164 	 * Signal mem_cgroup_begin_page_stat() to take the memcg's
5165 	 * move_lock while we're moving its pages to another memcg.
5166 	 * Then wait for already started RCU-only updates to finish.
5167 	 */
5168 	atomic_inc(&mc.from->moving_account);
5169 	synchronize_rcu();
5170 retry:
5171 	if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
5172 		/*
5173 		 * Someone who are holding the mmap_sem might be waiting in
5174 		 * waitq. So we cancel all extra charges, wake up all waiters,
5175 		 * and retry. Because we cancel precharges, we might not be able
5176 		 * to move enough charges, but moving charge is a best-effort
5177 		 * feature anyway, so it wouldn't be a big problem.
5178 		 */
5179 		__mem_cgroup_clear_mc();
5180 		cond_resched();
5181 		goto retry;
5182 	}
5183 	/*
5184 	 * When we have consumed all precharges and failed in doing
5185 	 * additional charge, the page walk just aborts.
5186 	 */
5187 	walk_page_range(0, ~0UL, &mem_cgroup_move_charge_walk);
5188 	up_read(&mm->mmap_sem);
5189 	atomic_dec(&mc.from->moving_account);
5190 }
5191 
5192 static void mem_cgroup_move_task(struct cgroup_subsys_state *css,
5193 				 struct cgroup_taskset *tset)
5194 {
5195 	struct task_struct *p = cgroup_taskset_first(tset);
5196 	struct mm_struct *mm = get_task_mm(p);
5197 
5198 	if (mm) {
5199 		if (mc.to)
5200 			mem_cgroup_move_charge(mm);
5201 		mmput(mm);
5202 	}
5203 	if (mc.to)
5204 		mem_cgroup_clear_mc();
5205 }
5206 #else	/* !CONFIG_MMU */
5207 static int mem_cgroup_can_attach(struct cgroup_subsys_state *css,
5208 				 struct cgroup_taskset *tset)
5209 {
5210 	return 0;
5211 }
5212 static void mem_cgroup_cancel_attach(struct cgroup_subsys_state *css,
5213 				     struct cgroup_taskset *tset)
5214 {
5215 }
5216 static void mem_cgroup_move_task(struct cgroup_subsys_state *css,
5217 				 struct cgroup_taskset *tset)
5218 {
5219 }
5220 #endif
5221 
5222 /*
5223  * Cgroup retains root cgroups across [un]mount cycles making it necessary
5224  * to verify whether we're attached to the default hierarchy on each mount
5225  * attempt.
5226  */
5227 static void mem_cgroup_bind(struct cgroup_subsys_state *root_css)
5228 {
5229 	/*
5230 	 * use_hierarchy is forced on the default hierarchy.  cgroup core
5231 	 * guarantees that @root doesn't have any children, so turning it
5232 	 * on for the root memcg is enough.
5233 	 */
5234 	if (cgroup_on_dfl(root_css->cgroup))
5235 		mem_cgroup_from_css(root_css)->use_hierarchy = true;
5236 }
5237 
5238 static u64 memory_current_read(struct cgroup_subsys_state *css,
5239 			       struct cftype *cft)
5240 {
5241 	return mem_cgroup_usage(mem_cgroup_from_css(css), false);
5242 }
5243 
5244 static int memory_low_show(struct seq_file *m, void *v)
5245 {
5246 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5247 	unsigned long low = ACCESS_ONCE(memcg->low);
5248 
5249 	if (low == PAGE_COUNTER_MAX)
5250 		seq_puts(m, "infinity\n");
5251 	else
5252 		seq_printf(m, "%llu\n", (u64)low * PAGE_SIZE);
5253 
5254 	return 0;
5255 }
5256 
5257 static ssize_t memory_low_write(struct kernfs_open_file *of,
5258 				char *buf, size_t nbytes, loff_t off)
5259 {
5260 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5261 	unsigned long low;
5262 	int err;
5263 
5264 	buf = strstrip(buf);
5265 	err = page_counter_memparse(buf, "infinity", &low);
5266 	if (err)
5267 		return err;
5268 
5269 	memcg->low = low;
5270 
5271 	return nbytes;
5272 }
5273 
5274 static int memory_high_show(struct seq_file *m, void *v)
5275 {
5276 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5277 	unsigned long high = ACCESS_ONCE(memcg->high);
5278 
5279 	if (high == PAGE_COUNTER_MAX)
5280 		seq_puts(m, "infinity\n");
5281 	else
5282 		seq_printf(m, "%llu\n", (u64)high * PAGE_SIZE);
5283 
5284 	return 0;
5285 }
5286 
5287 static ssize_t memory_high_write(struct kernfs_open_file *of,
5288 				 char *buf, size_t nbytes, loff_t off)
5289 {
5290 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5291 	unsigned long high;
5292 	int err;
5293 
5294 	buf = strstrip(buf);
5295 	err = page_counter_memparse(buf, "infinity", &high);
5296 	if (err)
5297 		return err;
5298 
5299 	memcg->high = high;
5300 
5301 	return nbytes;
5302 }
5303 
5304 static int memory_max_show(struct seq_file *m, void *v)
5305 {
5306 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5307 	unsigned long max = ACCESS_ONCE(memcg->memory.limit);
5308 
5309 	if (max == PAGE_COUNTER_MAX)
5310 		seq_puts(m, "infinity\n");
5311 	else
5312 		seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);
5313 
5314 	return 0;
5315 }
5316 
5317 static ssize_t memory_max_write(struct kernfs_open_file *of,
5318 				char *buf, size_t nbytes, loff_t off)
5319 {
5320 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5321 	unsigned long max;
5322 	int err;
5323 
5324 	buf = strstrip(buf);
5325 	err = page_counter_memparse(buf, "infinity", &max);
5326 	if (err)
5327 		return err;
5328 
5329 	err = mem_cgroup_resize_limit(memcg, max);
5330 	if (err)
5331 		return err;
5332 
5333 	return nbytes;
5334 }
5335 
5336 static int memory_events_show(struct seq_file *m, void *v)
5337 {
5338 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5339 
5340 	seq_printf(m, "low %lu\n", mem_cgroup_read_events(memcg, MEMCG_LOW));
5341 	seq_printf(m, "high %lu\n", mem_cgroup_read_events(memcg, MEMCG_HIGH));
5342 	seq_printf(m, "max %lu\n", mem_cgroup_read_events(memcg, MEMCG_MAX));
5343 	seq_printf(m, "oom %lu\n", mem_cgroup_read_events(memcg, MEMCG_OOM));
5344 
5345 	return 0;
5346 }
5347 
5348 static struct cftype memory_files[] = {
5349 	{
5350 		.name = "current",
5351 		.read_u64 = memory_current_read,
5352 	},
5353 	{
5354 		.name = "low",
5355 		.flags = CFTYPE_NOT_ON_ROOT,
5356 		.seq_show = memory_low_show,
5357 		.write = memory_low_write,
5358 	},
5359 	{
5360 		.name = "high",
5361 		.flags = CFTYPE_NOT_ON_ROOT,
5362 		.seq_show = memory_high_show,
5363 		.write = memory_high_write,
5364 	},
5365 	{
5366 		.name = "max",
5367 		.flags = CFTYPE_NOT_ON_ROOT,
5368 		.seq_show = memory_max_show,
5369 		.write = memory_max_write,
5370 	},
5371 	{
5372 		.name = "events",
5373 		.flags = CFTYPE_NOT_ON_ROOT,
5374 		.seq_show = memory_events_show,
5375 	},
5376 	{ }	/* terminate */
5377 };
5378 
5379 struct cgroup_subsys memory_cgrp_subsys = {
5380 	.css_alloc = mem_cgroup_css_alloc,
5381 	.css_online = mem_cgroup_css_online,
5382 	.css_offline = mem_cgroup_css_offline,
5383 	.css_free = mem_cgroup_css_free,
5384 	.css_reset = mem_cgroup_css_reset,
5385 	.can_attach = mem_cgroup_can_attach,
5386 	.cancel_attach = mem_cgroup_cancel_attach,
5387 	.attach = mem_cgroup_move_task,
5388 	.bind = mem_cgroup_bind,
5389 	.dfl_cftypes = memory_files,
5390 	.legacy_cftypes = mem_cgroup_legacy_files,
5391 	.early_init = 0,
5392 };
5393 
5394 /**
5395  * mem_cgroup_events - count memory events against a cgroup
5396  * @memcg: the memory cgroup
5397  * @idx: the event index
5398  * @nr: the number of events to account for
5399  */
5400 void mem_cgroup_events(struct mem_cgroup *memcg,
5401 		       enum mem_cgroup_events_index idx,
5402 		       unsigned int nr)
5403 {
5404 	this_cpu_add(memcg->stat->events[idx], nr);
5405 }
5406 
5407 /**
5408  * mem_cgroup_low - check if memory consumption is below the normal range
5409  * @root: the highest ancestor to consider
5410  * @memcg: the memory cgroup to check
5411  *
5412  * Returns %true if memory consumption of @memcg, and that of all
5413  * configurable ancestors up to @root, is below the normal range.
5414  */
5415 bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg)
5416 {
5417 	if (mem_cgroup_disabled())
5418 		return false;
5419 
5420 	/*
5421 	 * The toplevel group doesn't have a configurable range, so
5422 	 * it's never low when looked at directly, and it is not
5423 	 * considered an ancestor when assessing the hierarchy.
5424 	 */
5425 
5426 	if (memcg == root_mem_cgroup)
5427 		return false;
5428 
5429 	if (page_counter_read(&memcg->memory) > memcg->low)
5430 		return false;
5431 
5432 	while (memcg != root) {
5433 		memcg = parent_mem_cgroup(memcg);
5434 
5435 		if (memcg == root_mem_cgroup)
5436 			break;
5437 
5438 		if (page_counter_read(&memcg->memory) > memcg->low)
5439 			return false;
5440 	}
5441 	return true;
5442 }
5443 
5444 /**
5445  * mem_cgroup_try_charge - try charging a page
5446  * @page: page to charge
5447  * @mm: mm context of the victim
5448  * @gfp_mask: reclaim mode
5449  * @memcgp: charged memcg return
5450  *
5451  * Try to charge @page to the memcg that @mm belongs to, reclaiming
5452  * pages according to @gfp_mask if necessary.
5453  *
5454  * Returns 0 on success, with *@memcgp pointing to the charged memcg.
5455  * Otherwise, an error code is returned.
5456  *
5457  * After page->mapping has been set up, the caller must finalize the
5458  * charge with mem_cgroup_commit_charge().  Or abort the transaction
5459  * with mem_cgroup_cancel_charge() in case page instantiation fails.
5460  */
5461 int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
5462 			  gfp_t gfp_mask, struct mem_cgroup **memcgp)
5463 {
5464 	struct mem_cgroup *memcg = NULL;
5465 	unsigned int nr_pages = 1;
5466 	int ret = 0;
5467 
5468 	if (mem_cgroup_disabled())
5469 		goto out;
5470 
5471 	if (PageSwapCache(page)) {
5472 		/*
5473 		 * Every swap fault against a single page tries to charge the
5474 		 * page, bail as early as possible.  shmem_unuse() encounters
5475 		 * already charged pages, too.  The USED bit is protected by
5476 		 * the page lock, which serializes swap cache removal, which
5477 		 * in turn serializes uncharging.
5478 		 */
5479 		if (page->mem_cgroup)
5480 			goto out;
5481 	}
5482 
5483 	if (PageTransHuge(page)) {
5484 		nr_pages <<= compound_order(page);
5485 		VM_BUG_ON_PAGE(!PageTransHuge(page), page);
5486 	}
5487 
5488 	if (do_swap_account && PageSwapCache(page))
5489 		memcg = try_get_mem_cgroup_from_page(page);
5490 	if (!memcg)
5491 		memcg = get_mem_cgroup_from_mm(mm);
5492 
5493 	ret = try_charge(memcg, gfp_mask, nr_pages);
5494 
5495 	css_put(&memcg->css);
5496 
5497 	if (ret == -EINTR) {
5498 		memcg = root_mem_cgroup;
5499 		ret = 0;
5500 	}
5501 out:
5502 	*memcgp = memcg;
5503 	return ret;
5504 }
5505 
5506 /**
5507  * mem_cgroup_commit_charge - commit a page charge
5508  * @page: page to charge
5509  * @memcg: memcg to charge the page to
5510  * @lrucare: page might be on LRU already
5511  *
5512  * Finalize a charge transaction started by mem_cgroup_try_charge(),
5513  * after page->mapping has been set up.  This must happen atomically
5514  * as part of the page instantiation, i.e. under the page table lock
5515  * for anonymous pages, under the page lock for page and swap cache.
5516  *
5517  * In addition, the page must not be on the LRU during the commit, to
5518  * prevent racing with task migration.  If it might be, use @lrucare.
5519  *
5520  * Use mem_cgroup_cancel_charge() to cancel the transaction instead.
5521  */
5522 void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
5523 			      bool lrucare)
5524 {
5525 	unsigned int nr_pages = 1;
5526 
5527 	VM_BUG_ON_PAGE(!page->mapping, page);
5528 	VM_BUG_ON_PAGE(PageLRU(page) && !lrucare, page);
5529 
5530 	if (mem_cgroup_disabled())
5531 		return;
5532 	/*
5533 	 * Swap faults will attempt to charge the same page multiple
5534 	 * times.  But reuse_swap_page() might have removed the page
5535 	 * from swapcache already, so we can't check PageSwapCache().
5536 	 */
5537 	if (!memcg)
5538 		return;
5539 
5540 	commit_charge(page, memcg, lrucare);
5541 
5542 	if (PageTransHuge(page)) {
5543 		nr_pages <<= compound_order(page);
5544 		VM_BUG_ON_PAGE(!PageTransHuge(page), page);
5545 	}
5546 
5547 	local_irq_disable();
5548 	mem_cgroup_charge_statistics(memcg, page, nr_pages);
5549 	memcg_check_events(memcg, page);
5550 	local_irq_enable();
5551 
5552 	if (do_swap_account && PageSwapCache(page)) {
5553 		swp_entry_t entry = { .val = page_private(page) };
5554 		/*
5555 		 * The swap entry might not get freed for a long time,
5556 		 * let's not wait for it.  The page already received a
5557 		 * memory+swap charge, drop the swap entry duplicate.
5558 		 */
5559 		mem_cgroup_uncharge_swap(entry);
5560 	}
5561 }
5562 
5563 /**
5564  * mem_cgroup_cancel_charge - cancel a page charge
5565  * @page: page to charge
5566  * @memcg: memcg to charge the page to
5567  *
5568  * Cancel a charge transaction started by mem_cgroup_try_charge().
5569  */
5570 void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg)
5571 {
5572 	unsigned int nr_pages = 1;
5573 
5574 	if (mem_cgroup_disabled())
5575 		return;
5576 	/*
5577 	 * Swap faults will attempt to charge the same page multiple
5578 	 * times.  But reuse_swap_page() might have removed the page
5579 	 * from swapcache already, so we can't check PageSwapCache().
5580 	 */
5581 	if (!memcg)
5582 		return;
5583 
5584 	if (PageTransHuge(page)) {
5585 		nr_pages <<= compound_order(page);
5586 		VM_BUG_ON_PAGE(!PageTransHuge(page), page);
5587 	}
5588 
5589 	cancel_charge(memcg, nr_pages);
5590 }
5591 
5592 static void uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout,
5593 			   unsigned long nr_anon, unsigned long nr_file,
5594 			   unsigned long nr_huge, struct page *dummy_page)
5595 {
5596 	unsigned long nr_pages = nr_anon + nr_file;
5597 	unsigned long flags;
5598 
5599 	if (!mem_cgroup_is_root(memcg)) {
5600 		page_counter_uncharge(&memcg->memory, nr_pages);
5601 		if (do_swap_account)
5602 			page_counter_uncharge(&memcg->memsw, nr_pages);
5603 		memcg_oom_recover(memcg);
5604 	}
5605 
5606 	local_irq_save(flags);
5607 	__this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS], nr_anon);
5608 	__this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_CACHE], nr_file);
5609 	__this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], nr_huge);
5610 	__this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT], pgpgout);
5611 	__this_cpu_add(memcg->stat->nr_page_events, nr_pages);
5612 	memcg_check_events(memcg, dummy_page);
5613 	local_irq_restore(flags);
5614 
5615 	if (!mem_cgroup_is_root(memcg))
5616 		css_put_many(&memcg->css, nr_pages);
5617 }
5618 
5619 static void uncharge_list(struct list_head *page_list)
5620 {
5621 	struct mem_cgroup *memcg = NULL;
5622 	unsigned long nr_anon = 0;
5623 	unsigned long nr_file = 0;
5624 	unsigned long nr_huge = 0;
5625 	unsigned long pgpgout = 0;
5626 	struct list_head *next;
5627 	struct page *page;
5628 
5629 	next = page_list->next;
5630 	do {
5631 		unsigned int nr_pages = 1;
5632 
5633 		page = list_entry(next, struct page, lru);
5634 		next = page->lru.next;
5635 
5636 		VM_BUG_ON_PAGE(PageLRU(page), page);
5637 		VM_BUG_ON_PAGE(page_count(page), page);
5638 
5639 		if (!page->mem_cgroup)
5640 			continue;
5641 
5642 		/*
5643 		 * Nobody should be changing or seriously looking at
5644 		 * page->mem_cgroup at this point, we have fully
5645 		 * exclusive access to the page.
5646 		 */
5647 
5648 		if (memcg != page->mem_cgroup) {
5649 			if (memcg) {
5650 				uncharge_batch(memcg, pgpgout, nr_anon, nr_file,
5651 					       nr_huge, page);
5652 				pgpgout = nr_anon = nr_file = nr_huge = 0;
5653 			}
5654 			memcg = page->mem_cgroup;
5655 		}
5656 
5657 		if (PageTransHuge(page)) {
5658 			nr_pages <<= compound_order(page);
5659 			VM_BUG_ON_PAGE(!PageTransHuge(page), page);
5660 			nr_huge += nr_pages;
5661 		}
5662 
5663 		if (PageAnon(page))
5664 			nr_anon += nr_pages;
5665 		else
5666 			nr_file += nr_pages;
5667 
5668 		page->mem_cgroup = NULL;
5669 
5670 		pgpgout++;
5671 	} while (next != page_list);
5672 
5673 	if (memcg)
5674 		uncharge_batch(memcg, pgpgout, nr_anon, nr_file,
5675 			       nr_huge, page);
5676 }
5677 
5678 /**
5679  * mem_cgroup_uncharge - uncharge a page
5680  * @page: page to uncharge
5681  *
5682  * Uncharge a page previously charged with mem_cgroup_try_charge() and
5683  * mem_cgroup_commit_charge().
5684  */
5685 void mem_cgroup_uncharge(struct page *page)
5686 {
5687 	if (mem_cgroup_disabled())
5688 		return;
5689 
5690 	/* Don't touch page->lru of any random page, pre-check: */
5691 	if (!page->mem_cgroup)
5692 		return;
5693 
5694 	INIT_LIST_HEAD(&page->lru);
5695 	uncharge_list(&page->lru);
5696 }
5697 
5698 /**
5699  * mem_cgroup_uncharge_list - uncharge a list of page
5700  * @page_list: list of pages to uncharge
5701  *
5702  * Uncharge a list of pages previously charged with
5703  * mem_cgroup_try_charge() and mem_cgroup_commit_charge().
5704  */
5705 void mem_cgroup_uncharge_list(struct list_head *page_list)
5706 {
5707 	if (mem_cgroup_disabled())
5708 		return;
5709 
5710 	if (!list_empty(page_list))
5711 		uncharge_list(page_list);
5712 }
5713 
5714 /**
5715  * mem_cgroup_migrate - migrate a charge to another page
5716  * @oldpage: currently charged page
5717  * @newpage: page to transfer the charge to
5718  * @lrucare: either or both pages might be on the LRU already
5719  *
5720  * Migrate the charge from @oldpage to @newpage.
5721  *
5722  * Both pages must be locked, @newpage->mapping must be set up.
5723  */
5724 void mem_cgroup_migrate(struct page *oldpage, struct page *newpage,
5725 			bool lrucare)
5726 {
5727 	struct mem_cgroup *memcg;
5728 	int isolated;
5729 
5730 	VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
5731 	VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
5732 	VM_BUG_ON_PAGE(!lrucare && PageLRU(oldpage), oldpage);
5733 	VM_BUG_ON_PAGE(!lrucare && PageLRU(newpage), newpage);
5734 	VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage);
5735 	VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage),
5736 		       newpage);
5737 
5738 	if (mem_cgroup_disabled())
5739 		return;
5740 
5741 	/* Page cache replacement: new page already charged? */
5742 	if (newpage->mem_cgroup)
5743 		return;
5744 
5745 	/*
5746 	 * Swapcache readahead pages can get migrated before being
5747 	 * charged, and migration from compaction can happen to an
5748 	 * uncharged page when the PFN walker finds a page that
5749 	 * reclaim just put back on the LRU but has not released yet.
5750 	 */
5751 	memcg = oldpage->mem_cgroup;
5752 	if (!memcg)
5753 		return;
5754 
5755 	if (lrucare)
5756 		lock_page_lru(oldpage, &isolated);
5757 
5758 	oldpage->mem_cgroup = NULL;
5759 
5760 	if (lrucare)
5761 		unlock_page_lru(oldpage, isolated);
5762 
5763 	commit_charge(newpage, memcg, lrucare);
5764 }
5765 
5766 /*
5767  * subsys_initcall() for memory controller.
5768  *
5769  * Some parts like hotcpu_notifier() have to be initialized from this context
5770  * because of lock dependencies (cgroup_lock -> cpu hotplug) but basically
5771  * everything that doesn't depend on a specific mem_cgroup structure should
5772  * be initialized from here.
5773  */
5774 static int __init mem_cgroup_init(void)
5775 {
5776 	int cpu, node;
5777 
5778 	hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
5779 
5780 	for_each_possible_cpu(cpu)
5781 		INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
5782 			  drain_local_stock);
5783 
5784 	for_each_node(node) {
5785 		struct mem_cgroup_tree_per_node *rtpn;
5786 		int zone;
5787 
5788 		rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
5789 				    node_online(node) ? node : NUMA_NO_NODE);
5790 
5791 		for (zone = 0; zone < MAX_NR_ZONES; zone++) {
5792 			struct mem_cgroup_tree_per_zone *rtpz;
5793 
5794 			rtpz = &rtpn->rb_tree_per_zone[zone];
5795 			rtpz->rb_root = RB_ROOT;
5796 			spin_lock_init(&rtpz->lock);
5797 		}
5798 		soft_limit_tree.rb_tree_per_node[node] = rtpn;
5799 	}
5800 
5801 	return 0;
5802 }
5803 subsys_initcall(mem_cgroup_init);
5804 
5805 #ifdef CONFIG_MEMCG_SWAP
5806 /**
5807  * mem_cgroup_swapout - transfer a memsw charge to swap
5808  * @page: page whose memsw charge to transfer
5809  * @entry: swap entry to move the charge to
5810  *
5811  * Transfer the memsw charge of @page to @entry.
5812  */
5813 void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
5814 {
5815 	struct mem_cgroup *memcg;
5816 	unsigned short oldid;
5817 
5818 	VM_BUG_ON_PAGE(PageLRU(page), page);
5819 	VM_BUG_ON_PAGE(page_count(page), page);
5820 
5821 	if (!do_swap_account)
5822 		return;
5823 
5824 	memcg = page->mem_cgroup;
5825 
5826 	/* Readahead page, never charged */
5827 	if (!memcg)
5828 		return;
5829 
5830 	oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg));
5831 	VM_BUG_ON_PAGE(oldid, page);
5832 	mem_cgroup_swap_statistics(memcg, true);
5833 
5834 	page->mem_cgroup = NULL;
5835 
5836 	if (!mem_cgroup_is_root(memcg))
5837 		page_counter_uncharge(&memcg->memory, 1);
5838 
5839 	/* XXX: caller holds IRQ-safe mapping->tree_lock */
5840 	VM_BUG_ON(!irqs_disabled());
5841 
5842 	mem_cgroup_charge_statistics(memcg, page, -1);
5843 	memcg_check_events(memcg, page);
5844 }
5845 
5846 /**
5847  * mem_cgroup_uncharge_swap - uncharge a swap entry
5848  * @entry: swap entry to uncharge
5849  *
5850  * Drop the memsw charge associated with @entry.
5851  */
5852 void mem_cgroup_uncharge_swap(swp_entry_t entry)
5853 {
5854 	struct mem_cgroup *memcg;
5855 	unsigned short id;
5856 
5857 	if (!do_swap_account)
5858 		return;
5859 
5860 	id = swap_cgroup_record(entry, 0);
5861 	rcu_read_lock();
5862 	memcg = mem_cgroup_lookup(id);
5863 	if (memcg) {
5864 		if (!mem_cgroup_is_root(memcg))
5865 			page_counter_uncharge(&memcg->memsw, 1);
5866 		mem_cgroup_swap_statistics(memcg, false);
5867 		css_put(&memcg->css);
5868 	}
5869 	rcu_read_unlock();
5870 }
5871 
5872 /* for remember boot option*/
5873 #ifdef CONFIG_MEMCG_SWAP_ENABLED
5874 static int really_do_swap_account __initdata = 1;
5875 #else
5876 static int really_do_swap_account __initdata;
5877 #endif
5878 
5879 static int __init enable_swap_account(char *s)
5880 {
5881 	if (!strcmp(s, "1"))
5882 		really_do_swap_account = 1;
5883 	else if (!strcmp(s, "0"))
5884 		really_do_swap_account = 0;
5885 	return 1;
5886 }
5887 __setup("swapaccount=", enable_swap_account);
5888 
5889 static struct cftype memsw_cgroup_files[] = {
5890 	{
5891 		.name = "memsw.usage_in_bytes",
5892 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
5893 		.read_u64 = mem_cgroup_read_u64,
5894 	},
5895 	{
5896 		.name = "memsw.max_usage_in_bytes",
5897 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
5898 		.write = mem_cgroup_reset,
5899 		.read_u64 = mem_cgroup_read_u64,
5900 	},
5901 	{
5902 		.name = "memsw.limit_in_bytes",
5903 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
5904 		.write = mem_cgroup_write,
5905 		.read_u64 = mem_cgroup_read_u64,
5906 	},
5907 	{
5908 		.name = "memsw.failcnt",
5909 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
5910 		.write = mem_cgroup_reset,
5911 		.read_u64 = mem_cgroup_read_u64,
5912 	},
5913 	{ },	/* terminate */
5914 };
5915 
5916 static int __init mem_cgroup_swap_init(void)
5917 {
5918 	if (!mem_cgroup_disabled() && really_do_swap_account) {
5919 		do_swap_account = 1;
5920 		WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys,
5921 						  memsw_cgroup_files));
5922 	}
5923 	return 0;
5924 }
5925 subsys_initcall(mem_cgroup_swap_init);
5926 
5927 #endif /* CONFIG_MEMCG_SWAP */
5928