1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2a38e4082SDave Chinner /*
3a38e4082SDave Chinner * Copyright (c) 2013 Red Hat, Inc. and Parallels Inc. All rights reserved.
4a38e4082SDave Chinner * Authors: David Chinner and Glauber Costa
5a38e4082SDave Chinner *
6a38e4082SDave Chinner * Generic LRU infrastructure
7a38e4082SDave Chinner */
8a38e4082SDave Chinner #include <linux/kernel.h>
9a38e4082SDave Chinner #include <linux/module.h>
103b1d58a4SDave Chinner #include <linux/mm.h>
11a38e4082SDave Chinner #include <linux/list_lru.h>
125ca302c8SGlauber Costa #include <linux/slab.h>
13c0a5b560SVladimir Davydov #include <linux/mutex.h>
1460d3fd32SVladimir Davydov #include <linux/memcontrol.h>
154d96ba35SRoman Gushchin #include "slab.h"
1688f2ef73SMuchun Song #include "internal.h"
17c0a5b560SVladimir Davydov
1884c07d11SKirill Tkhai #ifdef CONFIG_MEMCG_KMEM
193eef1127SMuchun Song static LIST_HEAD(memcg_list_lrus);
20c0a5b560SVladimir Davydov static DEFINE_MUTEX(list_lrus_mutex);
21c0a5b560SVladimir Davydov
list_lru_memcg_aware(struct list_lru * lru)223eef1127SMuchun Song static inline bool list_lru_memcg_aware(struct list_lru *lru)
233eef1127SMuchun Song {
243eef1127SMuchun Song return lru->memcg_aware;
253eef1127SMuchun Song }
263eef1127SMuchun Song
list_lru_register(struct list_lru * lru)27c0a5b560SVladimir Davydov static void list_lru_register(struct list_lru *lru)
28c0a5b560SVladimir Davydov {
293eef1127SMuchun Song if (!list_lru_memcg_aware(lru))
303eef1127SMuchun Song return;
313eef1127SMuchun Song
32c0a5b560SVladimir Davydov mutex_lock(&list_lrus_mutex);
333eef1127SMuchun Song list_add(&lru->list, &memcg_list_lrus);
34c0a5b560SVladimir Davydov mutex_unlock(&list_lrus_mutex);
35c0a5b560SVladimir Davydov }
36c0a5b560SVladimir Davydov
list_lru_unregister(struct list_lru * lru)37c0a5b560SVladimir Davydov static void list_lru_unregister(struct list_lru *lru)
38c0a5b560SVladimir Davydov {
393eef1127SMuchun Song if (!list_lru_memcg_aware(lru))
403eef1127SMuchun Song return;
413eef1127SMuchun Song
42c0a5b560SVladimir Davydov mutex_lock(&list_lrus_mutex);
43c0a5b560SVladimir Davydov list_del(&lru->list);
44c0a5b560SVladimir Davydov mutex_unlock(&list_lrus_mutex);
45c0a5b560SVladimir Davydov }
46c0a5b560SVladimir Davydov
lru_shrinker_id(struct list_lru * lru)47fae91d6dSKirill Tkhai static int lru_shrinker_id(struct list_lru *lru)
48fae91d6dSKirill Tkhai {
49fae91d6dSKirill Tkhai return lru->shrinker_id;
50fae91d6dSKirill Tkhai }
51fae91d6dSKirill Tkhai
5260d3fd32SVladimir Davydov static inline struct list_lru_one *
list_lru_from_memcg_idx(struct list_lru * lru,int nid,int idx)536a6b7b77SMuchun Song list_lru_from_memcg_idx(struct list_lru *lru, int nid, int idx)
5460d3fd32SVladimir Davydov {
55bbca91ccSMuchun Song if (list_lru_memcg_aware(lru) && idx >= 0) {
56d7011070SMuchun Song struct list_lru_memcg *mlru = xa_load(&lru->xa, idx);
576a6b7b77SMuchun Song
585abc1e37SMuchun Song return mlru ? &mlru->node[nid] : NULL;
595abc1e37SMuchun Song }
60bbca91ccSMuchun Song return &lru->node[nid].lru;
6160d3fd32SVladimir Davydov }
6260d3fd32SVladimir Davydov
6360d3fd32SVladimir Davydov static inline struct list_lru_one *
list_lru_from_kmem(struct list_lru * lru,int nid,void * ptr,struct mem_cgroup ** memcg_ptr)646a6b7b77SMuchun Song list_lru_from_kmem(struct list_lru *lru, int nid, void *ptr,
6544bd4a47SKirill Tkhai struct mem_cgroup **memcg_ptr)
6660d3fd32SVladimir Davydov {
676a6b7b77SMuchun Song struct list_lru_node *nlru = &lru->node[nid];
6844bd4a47SKirill Tkhai struct list_lru_one *l = &nlru->lru;
6944bd4a47SKirill Tkhai struct mem_cgroup *memcg = NULL;
7060d3fd32SVladimir Davydov
71bbca91ccSMuchun Song if (!list_lru_memcg_aware(lru))
7244bd4a47SKirill Tkhai goto out;
7360d3fd32SVladimir Davydov
74*fc4db90fSRoman Gushchin memcg = mem_cgroup_from_slab_obj(ptr);
7560d3fd32SVladimir Davydov if (!memcg)
7644bd4a47SKirill Tkhai goto out;
7760d3fd32SVladimir Davydov
787c52f65dSMuchun Song l = list_lru_from_memcg_idx(lru, nid, memcg_kmem_id(memcg));
7944bd4a47SKirill Tkhai out:
8044bd4a47SKirill Tkhai if (memcg_ptr)
8144bd4a47SKirill Tkhai *memcg_ptr = memcg;
8244bd4a47SKirill Tkhai return l;
8360d3fd32SVladimir Davydov }
8460d3fd32SVladimir Davydov #else
list_lru_register(struct list_lru * lru)85e0295238SKirill Tkhai static void list_lru_register(struct list_lru *lru)
86e0295238SKirill Tkhai {
87e0295238SKirill Tkhai }
88e0295238SKirill Tkhai
list_lru_unregister(struct list_lru * lru)89e0295238SKirill Tkhai static void list_lru_unregister(struct list_lru *lru)
90e0295238SKirill Tkhai {
91e0295238SKirill Tkhai }
92e0295238SKirill Tkhai
lru_shrinker_id(struct list_lru * lru)93fae91d6dSKirill Tkhai static int lru_shrinker_id(struct list_lru *lru)
94fae91d6dSKirill Tkhai {
95fae91d6dSKirill Tkhai return -1;
96fae91d6dSKirill Tkhai }
97fae91d6dSKirill Tkhai
list_lru_memcg_aware(struct list_lru * lru)9860d3fd32SVladimir Davydov static inline bool list_lru_memcg_aware(struct list_lru *lru)
9960d3fd32SVladimir Davydov {
10060d3fd32SVladimir Davydov return false;
10160d3fd32SVladimir Davydov }
10260d3fd32SVladimir Davydov
10360d3fd32SVladimir Davydov static inline struct list_lru_one *
list_lru_from_memcg_idx(struct list_lru * lru,int nid,int idx)1046a6b7b77SMuchun Song list_lru_from_memcg_idx(struct list_lru *lru, int nid, int idx)
10560d3fd32SVladimir Davydov {
1066a6b7b77SMuchun Song return &lru->node[nid].lru;
10760d3fd32SVladimir Davydov }
10860d3fd32SVladimir Davydov
10960d3fd32SVladimir Davydov static inline struct list_lru_one *
list_lru_from_kmem(struct list_lru * lru,int nid,void * ptr,struct mem_cgroup ** memcg_ptr)1106a6b7b77SMuchun Song list_lru_from_kmem(struct list_lru *lru, int nid, void *ptr,
11144bd4a47SKirill Tkhai struct mem_cgroup **memcg_ptr)
11260d3fd32SVladimir Davydov {
11344bd4a47SKirill Tkhai if (memcg_ptr)
11444bd4a47SKirill Tkhai *memcg_ptr = NULL;
1156a6b7b77SMuchun Song return &lru->node[nid].lru;
11660d3fd32SVladimir Davydov }
11784c07d11SKirill Tkhai #endif /* CONFIG_MEMCG_KMEM */
11860d3fd32SVladimir Davydov
list_lru_add(struct list_lru * lru,struct list_head * item)119a38e4082SDave Chinner bool list_lru_add(struct list_lru *lru, struct list_head *item)
120a38e4082SDave Chinner {
1213b1d58a4SDave Chinner int nid = page_to_nid(virt_to_page(item));
1223b1d58a4SDave Chinner struct list_lru_node *nlru = &lru->node[nid];
123fae91d6dSKirill Tkhai struct mem_cgroup *memcg;
12460d3fd32SVladimir Davydov struct list_lru_one *l;
1253b1d58a4SDave Chinner
1263b1d58a4SDave Chinner spin_lock(&nlru->lock);
127a38e4082SDave Chinner if (list_empty(item)) {
1286a6b7b77SMuchun Song l = list_lru_from_kmem(lru, nid, item, &memcg);
12960d3fd32SVladimir Davydov list_add_tail(item, &l->list);
130fae91d6dSKirill Tkhai /* Set shrinker bit if the first element was added */
131fae91d6dSKirill Tkhai if (!l->nr_items++)
1322bfd3637SYang Shi set_shrinker_bit(memcg, nid,
133fae91d6dSKirill Tkhai lru_shrinker_id(lru));
1342c80cd57SSahitya Tummala nlru->nr_items++;
1353b1d58a4SDave Chinner spin_unlock(&nlru->lock);
136a38e4082SDave Chinner return true;
137a38e4082SDave Chinner }
1383b1d58a4SDave Chinner spin_unlock(&nlru->lock);
139a38e4082SDave Chinner return false;
140a38e4082SDave Chinner }
141a38e4082SDave Chinner EXPORT_SYMBOL_GPL(list_lru_add);
142a38e4082SDave Chinner
list_lru_del(struct list_lru * lru,struct list_head * item)143a38e4082SDave Chinner bool list_lru_del(struct list_lru *lru, struct list_head *item)
144a38e4082SDave Chinner {
1453b1d58a4SDave Chinner int nid = page_to_nid(virt_to_page(item));
1463b1d58a4SDave Chinner struct list_lru_node *nlru = &lru->node[nid];
14760d3fd32SVladimir Davydov struct list_lru_one *l;
1483b1d58a4SDave Chinner
1493b1d58a4SDave Chinner spin_lock(&nlru->lock);
150a38e4082SDave Chinner if (!list_empty(item)) {
1516a6b7b77SMuchun Song l = list_lru_from_kmem(lru, nid, item, NULL);
152a38e4082SDave Chinner list_del_init(item);
15360d3fd32SVladimir Davydov l->nr_items--;
1542c80cd57SSahitya Tummala nlru->nr_items--;
1553b1d58a4SDave Chinner spin_unlock(&nlru->lock);
156a38e4082SDave Chinner return true;
157a38e4082SDave Chinner }
1583b1d58a4SDave Chinner spin_unlock(&nlru->lock);
159a38e4082SDave Chinner return false;
160a38e4082SDave Chinner }
161a38e4082SDave Chinner EXPORT_SYMBOL_GPL(list_lru_del);
162a38e4082SDave Chinner
list_lru_isolate(struct list_lru_one * list,struct list_head * item)1633f97b163SVladimir Davydov void list_lru_isolate(struct list_lru_one *list, struct list_head *item)
1643f97b163SVladimir Davydov {
1653f97b163SVladimir Davydov list_del_init(item);
1663f97b163SVladimir Davydov list->nr_items--;
1673f97b163SVladimir Davydov }
1683f97b163SVladimir Davydov EXPORT_SYMBOL_GPL(list_lru_isolate);
1693f97b163SVladimir Davydov
list_lru_isolate_move(struct list_lru_one * list,struct list_head * item,struct list_head * head)1703f97b163SVladimir Davydov void list_lru_isolate_move(struct list_lru_one *list, struct list_head *item,
1713f97b163SVladimir Davydov struct list_head *head)
1723f97b163SVladimir Davydov {
1733f97b163SVladimir Davydov list_move(item, head);
1743f97b163SVladimir Davydov list->nr_items--;
1753f97b163SVladimir Davydov }
1763f97b163SVladimir Davydov EXPORT_SYMBOL_GPL(list_lru_isolate_move);
1773f97b163SVladimir Davydov
list_lru_count_one(struct list_lru * lru,int nid,struct mem_cgroup * memcg)178930eaac5SAndrew Morton unsigned long list_lru_count_one(struct list_lru *lru,
179930eaac5SAndrew Morton int nid, struct mem_cgroup *memcg)
180a38e4082SDave Chinner {
18160d3fd32SVladimir Davydov struct list_lru_one *l;
18241d17431SMuchun Song long count;
1833b1d58a4SDave Chinner
1840c7c1bedSKirill Tkhai rcu_read_lock();
1857c52f65dSMuchun Song l = list_lru_from_memcg_idx(lru, nid, memcg_kmem_id(memcg));
1865abc1e37SMuchun Song count = l ? READ_ONCE(l->nr_items) : 0;
1870c7c1bedSKirill Tkhai rcu_read_unlock();
1883b1d58a4SDave Chinner
18941d17431SMuchun Song if (unlikely(count < 0))
19041d17431SMuchun Song count = 0;
19141d17431SMuchun Song
1923b1d58a4SDave Chinner return count;
1933b1d58a4SDave Chinner }
19460d3fd32SVladimir Davydov EXPORT_SYMBOL_GPL(list_lru_count_one);
19560d3fd32SVladimir Davydov
list_lru_count_node(struct list_lru * lru,int nid)19660d3fd32SVladimir Davydov unsigned long list_lru_count_node(struct list_lru *lru, int nid)
19760d3fd32SVladimir Davydov {
1982c80cd57SSahitya Tummala struct list_lru_node *nlru;
19960d3fd32SVladimir Davydov
2002c80cd57SSahitya Tummala nlru = &lru->node[nid];
2012c80cd57SSahitya Tummala return nlru->nr_items;
20260d3fd32SVladimir Davydov }
2036a4f496fSGlauber Costa EXPORT_SYMBOL_GPL(list_lru_count_node);
2043b1d58a4SDave Chinner
20560d3fd32SVladimir Davydov static unsigned long
__list_lru_walk_one(struct list_lru * lru,int nid,int memcg_idx,list_lru_walk_cb isolate,void * cb_arg,unsigned long * nr_to_walk)2066a6b7b77SMuchun Song __list_lru_walk_one(struct list_lru *lru, int nid, int memcg_idx,
20760d3fd32SVladimir Davydov list_lru_walk_cb isolate, void *cb_arg,
20860d3fd32SVladimir Davydov unsigned long *nr_to_walk)
2093b1d58a4SDave Chinner {
2106a6b7b77SMuchun Song struct list_lru_node *nlru = &lru->node[nid];
21160d3fd32SVladimir Davydov struct list_lru_one *l;
212a38e4082SDave Chinner struct list_head *item, *n;
2133b1d58a4SDave Chinner unsigned long isolated = 0;
214a38e4082SDave Chinner
215a38e4082SDave Chinner restart:
2165abc1e37SMuchun Song l = list_lru_from_memcg_idx(lru, nid, memcg_idx);
2175abc1e37SMuchun Song if (!l)
2185abc1e37SMuchun Song goto out;
2195abc1e37SMuchun Song
22060d3fd32SVladimir Davydov list_for_each_safe(item, n, &l->list) {
221a38e4082SDave Chinner enum lru_status ret;
2225cedf721SDave Chinner
2235cedf721SDave Chinner /*
2245cedf721SDave Chinner * decrement nr_to_walk first so that we don't livelock if we
2253dc5f032SEthon Paul * get stuck on large numbers of LRU_RETRY items
2265cedf721SDave Chinner */
227c56b097aSRussell King if (!*nr_to_walk)
2285cedf721SDave Chinner break;
229c56b097aSRussell King --*nr_to_walk;
2305cedf721SDave Chinner
2313f97b163SVladimir Davydov ret = isolate(item, l, &nlru->lock, cb_arg);
232a38e4082SDave Chinner switch (ret) {
233449dd698SJohannes Weiner case LRU_REMOVED_RETRY:
234449dd698SJohannes Weiner assert_spin_locked(&nlru->lock);
235e4a9bc58SJoe Perches fallthrough;
236a38e4082SDave Chinner case LRU_REMOVED:
2373b1d58a4SDave Chinner isolated++;
2382c80cd57SSahitya Tummala nlru->nr_items--;
239449dd698SJohannes Weiner /*
240449dd698SJohannes Weiner * If the lru lock has been dropped, our list
241449dd698SJohannes Weiner * traversal is now invalid and so we have to
242449dd698SJohannes Weiner * restart from scratch.
243449dd698SJohannes Weiner */
244449dd698SJohannes Weiner if (ret == LRU_REMOVED_RETRY)
245449dd698SJohannes Weiner goto restart;
246a38e4082SDave Chinner break;
247a38e4082SDave Chinner case LRU_ROTATE:
24860d3fd32SVladimir Davydov list_move_tail(item, &l->list);
249a38e4082SDave Chinner break;
250a38e4082SDave Chinner case LRU_SKIP:
251a38e4082SDave Chinner break;
252a38e4082SDave Chinner case LRU_RETRY:
2535cedf721SDave Chinner /*
2545cedf721SDave Chinner * The lru lock has been dropped, our list traversal is
2555cedf721SDave Chinner * now invalid and so we have to restart from scratch.
2565cedf721SDave Chinner */
257449dd698SJohannes Weiner assert_spin_locked(&nlru->lock);
258a38e4082SDave Chinner goto restart;
259a38e4082SDave Chinner default:
260a38e4082SDave Chinner BUG();
261a38e4082SDave Chinner }
262a38e4082SDave Chinner }
2635abc1e37SMuchun Song out:
2643b1d58a4SDave Chinner return isolated;
2653b1d58a4SDave Chinner }
26660d3fd32SVladimir Davydov
26760d3fd32SVladimir Davydov unsigned long
list_lru_walk_one(struct list_lru * lru,int nid,struct mem_cgroup * memcg,list_lru_walk_cb isolate,void * cb_arg,unsigned long * nr_to_walk)26860d3fd32SVladimir Davydov list_lru_walk_one(struct list_lru *lru, int nid, struct mem_cgroup *memcg,
26960d3fd32SVladimir Davydov list_lru_walk_cb isolate, void *cb_arg,
27060d3fd32SVladimir Davydov unsigned long *nr_to_walk)
27160d3fd32SVladimir Davydov {
2726cfe57a9SSebastian Andrzej Siewior struct list_lru_node *nlru = &lru->node[nid];
2736cfe57a9SSebastian Andrzej Siewior unsigned long ret;
2746cfe57a9SSebastian Andrzej Siewior
2756cfe57a9SSebastian Andrzej Siewior spin_lock(&nlru->lock);
2767c52f65dSMuchun Song ret = __list_lru_walk_one(lru, nid, memcg_kmem_id(memcg), isolate,
2776a6b7b77SMuchun Song cb_arg, nr_to_walk);
2786cfe57a9SSebastian Andrzej Siewior spin_unlock(&nlru->lock);
2796cfe57a9SSebastian Andrzej Siewior return ret;
28060d3fd32SVladimir Davydov }
28160d3fd32SVladimir Davydov EXPORT_SYMBOL_GPL(list_lru_walk_one);
28260d3fd32SVladimir Davydov
2836b51e881SSebastian Andrzej Siewior unsigned long
list_lru_walk_one_irq(struct list_lru * lru,int nid,struct mem_cgroup * memcg,list_lru_walk_cb isolate,void * cb_arg,unsigned long * nr_to_walk)2846b51e881SSebastian Andrzej Siewior list_lru_walk_one_irq(struct list_lru *lru, int nid, struct mem_cgroup *memcg,
2856b51e881SSebastian Andrzej Siewior list_lru_walk_cb isolate, void *cb_arg,
2866b51e881SSebastian Andrzej Siewior unsigned long *nr_to_walk)
2876b51e881SSebastian Andrzej Siewior {
2886b51e881SSebastian Andrzej Siewior struct list_lru_node *nlru = &lru->node[nid];
2896b51e881SSebastian Andrzej Siewior unsigned long ret;
2906b51e881SSebastian Andrzej Siewior
2916b51e881SSebastian Andrzej Siewior spin_lock_irq(&nlru->lock);
2927c52f65dSMuchun Song ret = __list_lru_walk_one(lru, nid, memcg_kmem_id(memcg), isolate,
2936a6b7b77SMuchun Song cb_arg, nr_to_walk);
2946b51e881SSebastian Andrzej Siewior spin_unlock_irq(&nlru->lock);
2956b51e881SSebastian Andrzej Siewior return ret;
2966b51e881SSebastian Andrzej Siewior }
2976b51e881SSebastian Andrzej Siewior
list_lru_walk_node(struct list_lru * lru,int nid,list_lru_walk_cb isolate,void * cb_arg,unsigned long * nr_to_walk)29860d3fd32SVladimir Davydov unsigned long list_lru_walk_node(struct list_lru *lru, int nid,
29960d3fd32SVladimir Davydov list_lru_walk_cb isolate, void *cb_arg,
30060d3fd32SVladimir Davydov unsigned long *nr_to_walk)
30160d3fd32SVladimir Davydov {
30260d3fd32SVladimir Davydov long isolated = 0;
30360d3fd32SVladimir Davydov
30487a5ffc1SSebastian Andrzej Siewior isolated += list_lru_walk_one(lru, nid, NULL, isolate, cb_arg,
30560d3fd32SVladimir Davydov nr_to_walk);
306bbca91ccSMuchun Song
307bbca91ccSMuchun Song #ifdef CONFIG_MEMCG_KMEM
30860d3fd32SVladimir Davydov if (*nr_to_walk > 0 && list_lru_memcg_aware(lru)) {
309d7011070SMuchun Song struct list_lru_memcg *mlru;
310bbca91ccSMuchun Song unsigned long index;
311bbca91ccSMuchun Song
312bbca91ccSMuchun Song xa_for_each(&lru->xa, index, mlru) {
3136cfe57a9SSebastian Andrzej Siewior struct list_lru_node *nlru = &lru->node[nid];
3146cfe57a9SSebastian Andrzej Siewior
3156cfe57a9SSebastian Andrzej Siewior spin_lock(&nlru->lock);
316bbca91ccSMuchun Song isolated += __list_lru_walk_one(lru, nid, index,
3176e018968SSebastian Andrzej Siewior isolate, cb_arg,
3186e018968SSebastian Andrzej Siewior nr_to_walk);
3196cfe57a9SSebastian Andrzej Siewior spin_unlock(&nlru->lock);
3206cfe57a9SSebastian Andrzej Siewior
32160d3fd32SVladimir Davydov if (*nr_to_walk <= 0)
32260d3fd32SVladimir Davydov break;
32360d3fd32SVladimir Davydov }
32460d3fd32SVladimir Davydov }
325bbca91ccSMuchun Song #endif
326bbca91ccSMuchun Song
32760d3fd32SVladimir Davydov return isolated;
32860d3fd32SVladimir Davydov }
3293b1d58a4SDave Chinner EXPORT_SYMBOL_GPL(list_lru_walk_node);
3303b1d58a4SDave Chinner
init_one_lru(struct list_lru_one * l)33160d3fd32SVladimir Davydov static void init_one_lru(struct list_lru_one *l)
33260d3fd32SVladimir Davydov {
33360d3fd32SVladimir Davydov INIT_LIST_HEAD(&l->list);
33460d3fd32SVladimir Davydov l->nr_items = 0;
33560d3fd32SVladimir Davydov }
33660d3fd32SVladimir Davydov
33784c07d11SKirill Tkhai #ifdef CONFIG_MEMCG_KMEM
memcg_init_list_lru_one(gfp_t gfp)338d7011070SMuchun Song static struct list_lru_memcg *memcg_init_list_lru_one(gfp_t gfp)
33988f2ef73SMuchun Song {
34088f2ef73SMuchun Song int nid;
341d7011070SMuchun Song struct list_lru_memcg *mlru;
34288f2ef73SMuchun Song
34388f2ef73SMuchun Song mlru = kmalloc(struct_size(mlru, node, nr_node_ids), gfp);
34488f2ef73SMuchun Song if (!mlru)
34588f2ef73SMuchun Song return NULL;
34688f2ef73SMuchun Song
34788f2ef73SMuchun Song for_each_node(nid)
34888f2ef73SMuchun Song init_one_lru(&mlru->node[nid]);
34988f2ef73SMuchun Song
35088f2ef73SMuchun Song return mlru;
35188f2ef73SMuchun Song }
35288f2ef73SMuchun Song
memcg_list_lru_free(struct list_lru * lru,int src_idx)3535abc1e37SMuchun Song static void memcg_list_lru_free(struct list_lru *lru, int src_idx)
35460d3fd32SVladimir Davydov {
355d7011070SMuchun Song struct list_lru_memcg *mlru = xa_erase_irq(&lru->xa, src_idx);
3565abc1e37SMuchun Song
3575abc1e37SMuchun Song /*
3585abc1e37SMuchun Song * The __list_lru_walk_one() can walk the list of this node.
3595abc1e37SMuchun Song * We need kvfree_rcu() here. And the walking of the list
3605abc1e37SMuchun Song * is under lru->node[nid]->lock, which can serve as a RCU
3615abc1e37SMuchun Song * read-side critical section.
3625abc1e37SMuchun Song */
3635abc1e37SMuchun Song if (mlru)
3645abc1e37SMuchun Song kvfree_rcu(mlru, rcu);
36560d3fd32SVladimir Davydov }
36660d3fd32SVladimir Davydov
memcg_init_list_lru(struct list_lru * lru,bool memcg_aware)367bbca91ccSMuchun Song static inline void memcg_init_list_lru(struct list_lru *lru, bool memcg_aware)
36860d3fd32SVladimir Davydov {
369bbca91ccSMuchun Song if (memcg_aware)
370bbca91ccSMuchun Song xa_init_flags(&lru->xa, XA_FLAGS_LOCK_IRQ);
3716a6b7b77SMuchun Song lru->memcg_aware = memcg_aware;
37260d3fd32SVladimir Davydov }
37360d3fd32SVladimir Davydov
memcg_destroy_list_lru(struct list_lru * lru)3746a6b7b77SMuchun Song static void memcg_destroy_list_lru(struct list_lru *lru)
37560d3fd32SVladimir Davydov {
376bbca91ccSMuchun Song XA_STATE(xas, &lru->xa, 0);
377d7011070SMuchun Song struct list_lru_memcg *mlru;
3786a6b7b77SMuchun Song
3796a6b7b77SMuchun Song if (!list_lru_memcg_aware(lru))
3806a6b7b77SMuchun Song return;
3816a6b7b77SMuchun Song
382bbca91ccSMuchun Song xas_lock_irq(&xas);
383bbca91ccSMuchun Song xas_for_each(&xas, mlru, ULONG_MAX) {
384bbca91ccSMuchun Song kfree(mlru);
385bbca91ccSMuchun Song xas_store(&xas, NULL);
3860c7c1bedSKirill Tkhai }
387bbca91ccSMuchun Song xas_unlock_irq(&xas);
38860d3fd32SVladimir Davydov }
3892788cf0cSVladimir Davydov
memcg_reparent_list_lru_node(struct list_lru * lru,int nid,int src_idx,struct mem_cgroup * dst_memcg)3901f391eb2SMuchun Song static void memcg_reparent_list_lru_node(struct list_lru *lru, int nid,
3919bec5c35SKirill Tkhai int src_idx, struct mem_cgroup *dst_memcg)
3922788cf0cSVladimir Davydov {
3933b82c4dcSKirill Tkhai struct list_lru_node *nlru = &lru->node[nid];
3949bec5c35SKirill Tkhai int dst_idx = dst_memcg->kmemcg_id;
3952788cf0cSVladimir Davydov struct list_lru_one *src, *dst;
3962788cf0cSVladimir Davydov
3972788cf0cSVladimir Davydov /*
3982788cf0cSVladimir Davydov * Since list_lru_{add,del} may be called under an IRQ-safe lock,
3992788cf0cSVladimir Davydov * we have to use IRQ-safe primitives here to avoid deadlock.
4002788cf0cSVladimir Davydov */
4012788cf0cSVladimir Davydov spin_lock_irq(&nlru->lock);
4022788cf0cSVladimir Davydov
4036a6b7b77SMuchun Song src = list_lru_from_memcg_idx(lru, nid, src_idx);
4045abc1e37SMuchun Song if (!src)
4055abc1e37SMuchun Song goto out;
4066a6b7b77SMuchun Song dst = list_lru_from_memcg_idx(lru, nid, dst_idx);
4072788cf0cSVladimir Davydov
4082788cf0cSVladimir Davydov list_splice_init(&src->list, &dst->list);
4098199be00SYang Shi
4108199be00SYang Shi if (src->nr_items) {
4112788cf0cSVladimir Davydov dst->nr_items += src->nr_items;
4122bfd3637SYang Shi set_shrinker_bit(dst_memcg, nid, lru_shrinker_id(lru));
4132788cf0cSVladimir Davydov src->nr_items = 0;
4148199be00SYang Shi }
4155abc1e37SMuchun Song out:
4162788cf0cSVladimir Davydov spin_unlock_irq(&nlru->lock);
4172788cf0cSVladimir Davydov }
4182788cf0cSVladimir Davydov
memcg_reparent_list_lru(struct list_lru * lru,int src_idx,struct mem_cgroup * dst_memcg)4191f391eb2SMuchun Song static void memcg_reparent_list_lru(struct list_lru *lru,
4209bec5c35SKirill Tkhai int src_idx, struct mem_cgroup *dst_memcg)
4212788cf0cSVladimir Davydov {
4222788cf0cSVladimir Davydov int i;
4232788cf0cSVladimir Davydov
424145949a1SRaghavendra K T for_each_node(i)
4251f391eb2SMuchun Song memcg_reparent_list_lru_node(lru, i, src_idx, dst_memcg);
4265abc1e37SMuchun Song
4275abc1e37SMuchun Song memcg_list_lru_free(lru, src_idx);
4282788cf0cSVladimir Davydov }
4292788cf0cSVladimir Davydov
memcg_reparent_list_lrus(struct mem_cgroup * memcg,struct mem_cgroup * parent)4301f391eb2SMuchun Song void memcg_reparent_list_lrus(struct mem_cgroup *memcg, struct mem_cgroup *parent)
4312788cf0cSVladimir Davydov {
4325abc1e37SMuchun Song struct cgroup_subsys_state *css;
4332788cf0cSVladimir Davydov struct list_lru *lru;
4341f391eb2SMuchun Song int src_idx = memcg->kmemcg_id;
4355abc1e37SMuchun Song
4365abc1e37SMuchun Song /*
4375abc1e37SMuchun Song * Change kmemcg_id of this cgroup and all its descendants to the
4385abc1e37SMuchun Song * parent's id, and then move all entries from this cgroup's list_lrus
4395abc1e37SMuchun Song * to ones of the parent.
4405abc1e37SMuchun Song *
4415abc1e37SMuchun Song * After we have finished, all list_lrus corresponding to this cgroup
4425abc1e37SMuchun Song * are guaranteed to remain empty. So we can safely free this cgroup's
4435abc1e37SMuchun Song * list lrus in memcg_list_lru_free().
4445abc1e37SMuchun Song *
4455abc1e37SMuchun Song * Changing ->kmemcg_id to the parent can prevent memcg_list_lru_alloc()
4465abc1e37SMuchun Song * from allocating list lrus for this cgroup after memcg_list_lru_free()
4475abc1e37SMuchun Song * call.
4485abc1e37SMuchun Song */
4495abc1e37SMuchun Song rcu_read_lock();
4501f391eb2SMuchun Song css_for_each_descendant_pre(css, &memcg->css) {
4511f391eb2SMuchun Song struct mem_cgroup *child;
4525abc1e37SMuchun Song
4531f391eb2SMuchun Song child = mem_cgroup_from_css(css);
454bbca91ccSMuchun Song WRITE_ONCE(child->kmemcg_id, parent->kmemcg_id);
4555abc1e37SMuchun Song }
4565abc1e37SMuchun Song rcu_read_unlock();
4572788cf0cSVladimir Davydov
4582788cf0cSVladimir Davydov mutex_lock(&list_lrus_mutex);
4593eef1127SMuchun Song list_for_each_entry(lru, &memcg_list_lrus, list)
4601f391eb2SMuchun Song memcg_reparent_list_lru(lru, src_idx, parent);
4612788cf0cSVladimir Davydov mutex_unlock(&list_lrus_mutex);
4622788cf0cSVladimir Davydov }
46388f2ef73SMuchun Song
memcg_list_lru_allocated(struct mem_cgroup * memcg,struct list_lru * lru)464bbca91ccSMuchun Song static inline bool memcg_list_lru_allocated(struct mem_cgroup *memcg,
46588f2ef73SMuchun Song struct list_lru *lru)
46688f2ef73SMuchun Song {
467bbca91ccSMuchun Song int idx = memcg->kmemcg_id;
46888f2ef73SMuchun Song
469bbca91ccSMuchun Song return idx < 0 || xa_load(&lru->xa, idx);
47088f2ef73SMuchun Song }
47188f2ef73SMuchun Song
memcg_list_lru_alloc(struct mem_cgroup * memcg,struct list_lru * lru,gfp_t gfp)47288f2ef73SMuchun Song int memcg_list_lru_alloc(struct mem_cgroup *memcg, struct list_lru *lru,
47388f2ef73SMuchun Song gfp_t gfp)
47488f2ef73SMuchun Song {
47588f2ef73SMuchun Song int i;
47688f2ef73SMuchun Song unsigned long flags;
47788f2ef73SMuchun Song struct list_lru_memcg_table {
478d7011070SMuchun Song struct list_lru_memcg *mlru;
47988f2ef73SMuchun Song struct mem_cgroup *memcg;
48088f2ef73SMuchun Song } *table;
481bbca91ccSMuchun Song XA_STATE(xas, &lru->xa, 0);
48288f2ef73SMuchun Song
48388f2ef73SMuchun Song if (!list_lru_memcg_aware(lru) || memcg_list_lru_allocated(memcg, lru))
48488f2ef73SMuchun Song return 0;
48588f2ef73SMuchun Song
48688f2ef73SMuchun Song gfp &= GFP_RECLAIM_MASK;
48788f2ef73SMuchun Song table = kmalloc_array(memcg->css.cgroup->level, sizeof(*table), gfp);
48888f2ef73SMuchun Song if (!table)
48988f2ef73SMuchun Song return -ENOMEM;
49088f2ef73SMuchun Song
49188f2ef73SMuchun Song /*
49288f2ef73SMuchun Song * Because the list_lru can be reparented to the parent cgroup's
49388f2ef73SMuchun Song * list_lru, we should make sure that this cgroup and all its
494d7011070SMuchun Song * ancestors have allocated list_lru_memcg.
49588f2ef73SMuchun Song */
49688f2ef73SMuchun Song for (i = 0; memcg; memcg = parent_mem_cgroup(memcg), i++) {
49788f2ef73SMuchun Song if (memcg_list_lru_allocated(memcg, lru))
49888f2ef73SMuchun Song break;
49988f2ef73SMuchun Song
50088f2ef73SMuchun Song table[i].memcg = memcg;
50188f2ef73SMuchun Song table[i].mlru = memcg_init_list_lru_one(gfp);
50288f2ef73SMuchun Song if (!table[i].mlru) {
50388f2ef73SMuchun Song while (i--)
50488f2ef73SMuchun Song kfree(table[i].mlru);
50588f2ef73SMuchun Song kfree(table);
50688f2ef73SMuchun Song return -ENOMEM;
50788f2ef73SMuchun Song }
50888f2ef73SMuchun Song }
50988f2ef73SMuchun Song
510bbca91ccSMuchun Song xas_lock_irqsave(&xas, flags);
51188f2ef73SMuchun Song while (i--) {
512bbca91ccSMuchun Song int index = READ_ONCE(table[i].memcg->kmemcg_id);
513d7011070SMuchun Song struct list_lru_memcg *mlru = table[i].mlru;
51488f2ef73SMuchun Song
515bbca91ccSMuchun Song xas_set(&xas, index);
516bbca91ccSMuchun Song retry:
517bbca91ccSMuchun Song if (unlikely(index < 0 || xas_error(&xas) || xas_load(&xas))) {
5185abc1e37SMuchun Song kfree(mlru);
519bbca91ccSMuchun Song } else {
520bbca91ccSMuchun Song xas_store(&xas, mlru);
521bbca91ccSMuchun Song if (xas_error(&xas) == -ENOMEM) {
522bbca91ccSMuchun Song xas_unlock_irqrestore(&xas, flags);
523bbca91ccSMuchun Song if (xas_nomem(&xas, gfp))
524bbca91ccSMuchun Song xas_set_err(&xas, 0);
525bbca91ccSMuchun Song xas_lock_irqsave(&xas, flags);
526bbca91ccSMuchun Song /*
527bbca91ccSMuchun Song * The xas lock has been released, this memcg
528bbca91ccSMuchun Song * can be reparented before us. So reload
529bbca91ccSMuchun Song * memcg id. More details see the comments
530bbca91ccSMuchun Song * in memcg_reparent_list_lrus().
531bbca91ccSMuchun Song */
532bbca91ccSMuchun Song index = READ_ONCE(table[i].memcg->kmemcg_id);
533bbca91ccSMuchun Song if (index < 0)
534bbca91ccSMuchun Song xas_set_err(&xas, 0);
535bbca91ccSMuchun Song else if (!xas_error(&xas) && index != xas.xa_index)
536bbca91ccSMuchun Song xas_set(&xas, index);
537bbca91ccSMuchun Song goto retry;
53888f2ef73SMuchun Song }
539bbca91ccSMuchun Song }
540bbca91ccSMuchun Song }
541bbca91ccSMuchun Song /* xas_nomem() is used to free memory instead of memory allocation. */
542bbca91ccSMuchun Song if (xas.xa_alloc)
543bbca91ccSMuchun Song xas_nomem(&xas, gfp);
544bbca91ccSMuchun Song xas_unlock_irqrestore(&xas, flags);
54588f2ef73SMuchun Song kfree(table);
54688f2ef73SMuchun Song
547bbca91ccSMuchun Song return xas_error(&xas);
54888f2ef73SMuchun Song }
54960d3fd32SVladimir Davydov #else
memcg_init_list_lru(struct list_lru * lru,bool memcg_aware)550bbca91ccSMuchun Song static inline void memcg_init_list_lru(struct list_lru *lru, bool memcg_aware)
55160d3fd32SVladimir Davydov {
55260d3fd32SVladimir Davydov }
55360d3fd32SVladimir Davydov
memcg_destroy_list_lru(struct list_lru * lru)55460d3fd32SVladimir Davydov static void memcg_destroy_list_lru(struct list_lru *lru)
55560d3fd32SVladimir Davydov {
55660d3fd32SVladimir Davydov }
55784c07d11SKirill Tkhai #endif /* CONFIG_MEMCG_KMEM */
55860d3fd32SVladimir Davydov
__list_lru_init(struct list_lru * lru,bool memcg_aware,struct lock_class_key * key,struct shrinker * shrinker)55960d3fd32SVladimir Davydov int __list_lru_init(struct list_lru *lru, bool memcg_aware,
560c92e8e10SKirill Tkhai struct lock_class_key *key, struct shrinker *shrinker)
561a38e4082SDave Chinner {
5623b1d58a4SDave Chinner int i;
56360d3fd32SVladimir Davydov
564c92e8e10SKirill Tkhai #ifdef CONFIG_MEMCG_KMEM
565c92e8e10SKirill Tkhai if (shrinker)
566c92e8e10SKirill Tkhai lru->shrinker_id = shrinker->id;
567c92e8e10SKirill Tkhai else
568c92e8e10SKirill Tkhai lru->shrinker_id = -1;
569c92e8e10SKirill Tkhai #endif
5705ca302c8SGlauber Costa
571b9726c26SAlexey Dobriyan lru->node = kcalloc(nr_node_ids, sizeof(*lru->node), GFP_KERNEL);
5725ca302c8SGlauber Costa if (!lru->node)
573bbca91ccSMuchun Song return -ENOMEM;
574a38e4082SDave Chinner
575145949a1SRaghavendra K T for_each_node(i) {
5763b1d58a4SDave Chinner spin_lock_init(&lru->node[i].lock);
577449dd698SJohannes Weiner if (key)
578449dd698SJohannes Weiner lockdep_set_class(&lru->node[i].lock, key);
57960d3fd32SVladimir Davydov init_one_lru(&lru->node[i].lru);
5803b1d58a4SDave Chinner }
58160d3fd32SVladimir Davydov
582bbca91ccSMuchun Song memcg_init_list_lru(lru, memcg_aware);
583c0a5b560SVladimir Davydov list_lru_register(lru);
584bbca91ccSMuchun Song
585bbca91ccSMuchun Song return 0;
586a38e4082SDave Chinner }
58760d3fd32SVladimir Davydov EXPORT_SYMBOL_GPL(__list_lru_init);
5885ca302c8SGlauber Costa
list_lru_destroy(struct list_lru * lru)5895ca302c8SGlauber Costa void list_lru_destroy(struct list_lru *lru)
5905ca302c8SGlauber Costa {
591c0a5b560SVladimir Davydov /* Already destroyed or not yet initialized? */
592c0a5b560SVladimir Davydov if (!lru->node)
593c0a5b560SVladimir Davydov return;
59460d3fd32SVladimir Davydov
595c0a5b560SVladimir Davydov list_lru_unregister(lru);
59660d3fd32SVladimir Davydov
59760d3fd32SVladimir Davydov memcg_destroy_list_lru(lru);
5985ca302c8SGlauber Costa kfree(lru->node);
599c0a5b560SVladimir Davydov lru->node = NULL;
60060d3fd32SVladimir Davydov
601c92e8e10SKirill Tkhai #ifdef CONFIG_MEMCG_KMEM
602c92e8e10SKirill Tkhai lru->shrinker_id = -1;
603c92e8e10SKirill Tkhai #endif
6045ca302c8SGlauber Costa }
6055ca302c8SGlauber Costa EXPORT_SYMBOL_GPL(list_lru_destroy);
606