1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
267afa38eSTim Chen /*
367afa38eSTim Chen * Manage cache of swap slots to be used for and returned from
467afa38eSTim Chen * swap.
567afa38eSTim Chen *
667afa38eSTim Chen * Copyright(c) 2016 Intel Corporation.
767afa38eSTim Chen *
867afa38eSTim Chen * Author: Tim Chen <tim.c.chen@linux.intel.com>
967afa38eSTim Chen *
1067afa38eSTim Chen * We allocate the swap slots from the global pool and put
1167afa38eSTim Chen * it into local per cpu caches. This has the advantage
1267afa38eSTim Chen * of no needing to acquire the swap_info lock every time
1367afa38eSTim Chen * we need a new slot.
1467afa38eSTim Chen *
1567afa38eSTim Chen * There is also opportunity to simply return the slot
1667afa38eSTim Chen * to local caches without needing to acquire swap_info
1767afa38eSTim Chen * lock. We do not reuse the returned slots directly but
1867afa38eSTim Chen * move them back to the global pool in a batch. This
19f0953a1bSIngo Molnar * allows the slots to coalesce and reduce fragmentation.
2067afa38eSTim Chen *
2167afa38eSTim Chen * The swap entry allocated is marked with SWAP_HAS_CACHE
2267afa38eSTim Chen * flag in map_count that prevents it from being allocated
2367afa38eSTim Chen * again from the global pool.
2467afa38eSTim Chen *
2567afa38eSTim Chen * The swap slots cache is protected by a mutex instead of
2667afa38eSTim Chen * a spin lock as when we search for slots with scan_swap_map,
2767afa38eSTim Chen * we can possibly sleep.
2867afa38eSTim Chen */
2967afa38eSTim Chen
3067afa38eSTim Chen #include <linux/swap_slots.h>
3167afa38eSTim Chen #include <linux/cpu.h>
3267afa38eSTim Chen #include <linux/cpumask.h>
338581fd40SJakub Kicinski #include <linux/slab.h>
3467afa38eSTim Chen #include <linux/vmalloc.h>
3567afa38eSTim Chen #include <linux/mutex.h>
3654f180d3SHuang Ying #include <linux/mm.h>
3767afa38eSTim Chen
3867afa38eSTim Chen static DEFINE_PER_CPU(struct swap_slots_cache, swp_slots);
3967afa38eSTim Chen static bool swap_slot_cache_active;
40ba81f838SHuang Ying bool swap_slot_cache_enabled;
4167afa38eSTim Chen static bool swap_slot_cache_initialized;
4231f21da1SColin Ian King static DEFINE_MUTEX(swap_slots_cache_mutex);
4367afa38eSTim Chen /* Serialize swap slots cache enable/disable operations */
4431f21da1SColin Ian King static DEFINE_MUTEX(swap_slots_cache_enable_mutex);
4567afa38eSTim Chen
4667afa38eSTim Chen static void __drain_swap_slots_cache(unsigned int type);
4767afa38eSTim Chen
48e0f3ebbaSZhen Lei #define use_swap_slot_cache (swap_slot_cache_active && swap_slot_cache_enabled)
4967afa38eSTim Chen #define SLOTS_CACHE 0x1
5067afa38eSTim Chen #define SLOTS_CACHE_RET 0x2
5167afa38eSTim Chen
deactivate_swap_slots_cache(void)5267afa38eSTim Chen static void deactivate_swap_slots_cache(void)
5367afa38eSTim Chen {
5467afa38eSTim Chen mutex_lock(&swap_slots_cache_mutex);
5567afa38eSTim Chen swap_slot_cache_active = false;
5667afa38eSTim Chen __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
5767afa38eSTim Chen mutex_unlock(&swap_slots_cache_mutex);
5867afa38eSTim Chen }
5967afa38eSTim Chen
reactivate_swap_slots_cache(void)6067afa38eSTim Chen static void reactivate_swap_slots_cache(void)
6167afa38eSTim Chen {
6267afa38eSTim Chen mutex_lock(&swap_slots_cache_mutex);
6367afa38eSTim Chen swap_slot_cache_active = true;
6467afa38eSTim Chen mutex_unlock(&swap_slots_cache_mutex);
6567afa38eSTim Chen }
6667afa38eSTim Chen
6767afa38eSTim Chen /* Must not be called with cpu hot plug lock */
disable_swap_slots_cache_lock(void)6867afa38eSTim Chen void disable_swap_slots_cache_lock(void)
6967afa38eSTim Chen {
7067afa38eSTim Chen mutex_lock(&swap_slots_cache_enable_mutex);
7167afa38eSTim Chen swap_slot_cache_enabled = false;
7267afa38eSTim Chen if (swap_slot_cache_initialized) {
7367afa38eSTim Chen /* serialize with cpu hotplug operations */
747625eccdSSebastian Andrzej Siewior cpus_read_lock();
7567afa38eSTim Chen __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
767625eccdSSebastian Andrzej Siewior cpus_read_unlock();
7767afa38eSTim Chen }
7867afa38eSTim Chen }
7967afa38eSTim Chen
__reenable_swap_slots_cache(void)8067afa38eSTim Chen static void __reenable_swap_slots_cache(void)
8167afa38eSTim Chen {
8267afa38eSTim Chen swap_slot_cache_enabled = has_usable_swap();
8367afa38eSTim Chen }
8467afa38eSTim Chen
reenable_swap_slots_cache_unlock(void)8567afa38eSTim Chen void reenable_swap_slots_cache_unlock(void)
8667afa38eSTim Chen {
8767afa38eSTim Chen __reenable_swap_slots_cache();
8867afa38eSTim Chen mutex_unlock(&swap_slots_cache_enable_mutex);
8967afa38eSTim Chen }
9067afa38eSTim Chen
check_cache_active(void)9167afa38eSTim Chen static bool check_cache_active(void)
9267afa38eSTim Chen {
9367afa38eSTim Chen long pages;
9467afa38eSTim Chen
95e0f3ebbaSZhen Lei if (!swap_slot_cache_enabled)
9667afa38eSTim Chen return false;
9767afa38eSTim Chen
9867afa38eSTim Chen pages = get_nr_swap_pages();
9967afa38eSTim Chen if (!swap_slot_cache_active) {
10067afa38eSTim Chen if (pages > num_online_cpus() *
10167afa38eSTim Chen THRESHOLD_ACTIVATE_SWAP_SLOTS_CACHE)
10267afa38eSTim Chen reactivate_swap_slots_cache();
10367afa38eSTim Chen goto out;
10467afa38eSTim Chen }
10567afa38eSTim Chen
10667afa38eSTim Chen /* if global pool of slot caches too low, deactivate cache */
10767afa38eSTim Chen if (pages < num_online_cpus() * THRESHOLD_DEACTIVATE_SWAP_SLOTS_CACHE)
10867afa38eSTim Chen deactivate_swap_slots_cache();
10967afa38eSTim Chen out:
11067afa38eSTim Chen return swap_slot_cache_active;
11167afa38eSTim Chen }
11267afa38eSTim Chen
alloc_swap_slot_cache(unsigned int cpu)11367afa38eSTim Chen static int alloc_swap_slot_cache(unsigned int cpu)
11467afa38eSTim Chen {
11567afa38eSTim Chen struct swap_slots_cache *cache;
11667afa38eSTim Chen swp_entry_t *slots, *slots_ret;
11767afa38eSTim Chen
11867afa38eSTim Chen /*
11967afa38eSTim Chen * Do allocation outside swap_slots_cache_mutex
120e2e3fdc7SMatthew Wilcox (Oracle) * as kvzalloc could trigger reclaim and folio_alloc_swap,
12167afa38eSTim Chen * which can lock swap_slots_cache_mutex.
12267afa38eSTim Chen */
123778e1cddSKees Cook slots = kvcalloc(SWAP_SLOTS_CACHE_SIZE, sizeof(swp_entry_t),
12454f180d3SHuang Ying GFP_KERNEL);
12567afa38eSTim Chen if (!slots)
12667afa38eSTim Chen return -ENOMEM;
12767afa38eSTim Chen
128778e1cddSKees Cook slots_ret = kvcalloc(SWAP_SLOTS_CACHE_SIZE, sizeof(swp_entry_t),
12954f180d3SHuang Ying GFP_KERNEL);
13067afa38eSTim Chen if (!slots_ret) {
13154f180d3SHuang Ying kvfree(slots);
13267afa38eSTim Chen return -ENOMEM;
13367afa38eSTim Chen }
13467afa38eSTim Chen
13567afa38eSTim Chen mutex_lock(&swap_slots_cache_mutex);
13667afa38eSTim Chen cache = &per_cpu(swp_slots, cpu);
137f90eae2aSZhen Lei if (cache->slots || cache->slots_ret) {
13867afa38eSTim Chen /* cache already allocated */
139f90eae2aSZhen Lei mutex_unlock(&swap_slots_cache_mutex);
140f90eae2aSZhen Lei
141f90eae2aSZhen Lei kvfree(slots);
142f90eae2aSZhen Lei kvfree(slots_ret);
143f90eae2aSZhen Lei
144f90eae2aSZhen Lei return 0;
145f90eae2aSZhen Lei }
146f90eae2aSZhen Lei
14767afa38eSTim Chen if (!cache->lock_initialized) {
14867afa38eSTim Chen mutex_init(&cache->alloc_lock);
14967afa38eSTim Chen spin_lock_init(&cache->free_lock);
15067afa38eSTim Chen cache->lock_initialized = true;
15167afa38eSTim Chen }
15267afa38eSTim Chen cache->nr = 0;
15367afa38eSTim Chen cache->cur = 0;
15467afa38eSTim Chen cache->n_ret = 0;
155a2e16731STim Chen /*
156a2e16731STim Chen * We initialized alloc_lock and free_lock earlier. We use
157a2e16731STim Chen * !cache->slots or !cache->slots_ret to know if it is safe to acquire
158a2e16731STim Chen * the corresponding lock and use the cache. Memory barrier below
159a2e16731STim Chen * ensures the assumption.
160a2e16731STim Chen */
161a2e16731STim Chen mb();
16267afa38eSTim Chen cache->slots = slots;
16367afa38eSTim Chen cache->slots_ret = slots_ret;
16467afa38eSTim Chen mutex_unlock(&swap_slots_cache_mutex);
16567afa38eSTim Chen return 0;
16667afa38eSTim Chen }
16767afa38eSTim Chen
drain_slots_cache_cpu(unsigned int cpu,unsigned int type,bool free_slots)16867afa38eSTim Chen static void drain_slots_cache_cpu(unsigned int cpu, unsigned int type,
16967afa38eSTim Chen bool free_slots)
17067afa38eSTim Chen {
17167afa38eSTim Chen struct swap_slots_cache *cache;
17267afa38eSTim Chen swp_entry_t *slots = NULL;
17367afa38eSTim Chen
17467afa38eSTim Chen cache = &per_cpu(swp_slots, cpu);
17567afa38eSTim Chen if ((type & SLOTS_CACHE) && cache->slots) {
17667afa38eSTim Chen mutex_lock(&cache->alloc_lock);
17767afa38eSTim Chen swapcache_free_entries(cache->slots + cache->cur, cache->nr);
17867afa38eSTim Chen cache->cur = 0;
17967afa38eSTim Chen cache->nr = 0;
18067afa38eSTim Chen if (free_slots && cache->slots) {
18154f180d3SHuang Ying kvfree(cache->slots);
18267afa38eSTim Chen cache->slots = NULL;
18367afa38eSTim Chen }
18467afa38eSTim Chen mutex_unlock(&cache->alloc_lock);
18567afa38eSTim Chen }
18667afa38eSTim Chen if ((type & SLOTS_CACHE_RET) && cache->slots_ret) {
18767afa38eSTim Chen spin_lock_irq(&cache->free_lock);
18867afa38eSTim Chen swapcache_free_entries(cache->slots_ret, cache->n_ret);
18967afa38eSTim Chen cache->n_ret = 0;
19067afa38eSTim Chen if (free_slots && cache->slots_ret) {
19167afa38eSTim Chen slots = cache->slots_ret;
19267afa38eSTim Chen cache->slots_ret = NULL;
19367afa38eSTim Chen }
19467afa38eSTim Chen spin_unlock_irq(&cache->free_lock);
19554f180d3SHuang Ying kvfree(slots);
19667afa38eSTim Chen }
19767afa38eSTim Chen }
19867afa38eSTim Chen
__drain_swap_slots_cache(unsigned int type)19967afa38eSTim Chen static void __drain_swap_slots_cache(unsigned int type)
20067afa38eSTim Chen {
20167afa38eSTim Chen unsigned int cpu;
20267afa38eSTim Chen
20367afa38eSTim Chen /*
20467afa38eSTim Chen * This function is called during
20567afa38eSTim Chen * 1) swapoff, when we have to make sure no
20667afa38eSTim Chen * left over slots are in cache when we remove
20767afa38eSTim Chen * a swap device;
20867afa38eSTim Chen * 2) disabling of swap slot cache, when we run low
20967afa38eSTim Chen * on swap slots when allocating memory and need
21067afa38eSTim Chen * to return swap slots to global pool.
21167afa38eSTim Chen *
21267afa38eSTim Chen * We cannot acquire cpu hot plug lock here as
21367afa38eSTim Chen * this function can be invoked in the cpu
21467afa38eSTim Chen * hot plug path:
21567afa38eSTim Chen * cpu_up -> lock cpu_hotplug -> cpu hotplug state callback
216e2e3fdc7SMatthew Wilcox (Oracle) * -> memory allocation -> direct reclaim -> folio_alloc_swap
21767afa38eSTim Chen * -> drain_swap_slots_cache
21867afa38eSTim Chen *
21967afa38eSTim Chen * Hence the loop over current online cpu below could miss cpu that
22067afa38eSTim Chen * is being brought online but not yet marked as online.
22167afa38eSTim Chen * That is okay as we do not schedule and run anything on a
22267afa38eSTim Chen * cpu before it has been marked online. Hence, we will not
22367afa38eSTim Chen * fill any swap slots in slots cache of such cpu.
22467afa38eSTim Chen * There are no slots on such cpu that need to be drained.
22567afa38eSTim Chen */
22667afa38eSTim Chen for_each_online_cpu(cpu)
22767afa38eSTim Chen drain_slots_cache_cpu(cpu, type, false);
22867afa38eSTim Chen }
22967afa38eSTim Chen
free_slot_cache(unsigned int cpu)23067afa38eSTim Chen static int free_slot_cache(unsigned int cpu)
23167afa38eSTim Chen {
23267afa38eSTim Chen mutex_lock(&swap_slots_cache_mutex);
23367afa38eSTim Chen drain_slots_cache_cpu(cpu, SLOTS_CACHE | SLOTS_CACHE_RET, true);
23467afa38eSTim Chen mutex_unlock(&swap_slots_cache_mutex);
23567afa38eSTim Chen return 0;
23667afa38eSTim Chen }
23767afa38eSTim Chen
enable_swap_slots_cache(void)238f3bc52cbSMiaohe Lin void enable_swap_slots_cache(void)
23967afa38eSTim Chen {
24067afa38eSTim Chen mutex_lock(&swap_slots_cache_enable_mutex);
241d69a9575SZhen Lei if (!swap_slot_cache_initialized) {
242d69a9575SZhen Lei int ret;
24367afa38eSTim Chen
24467afa38eSTim Chen ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "swap_slots_cache",
24567afa38eSTim Chen alloc_swap_slot_cache, free_slot_cache);
2469b7a8143STim Chen if (WARN_ONCE(ret < 0, "Cache allocation failed (%s), operating "
2479b7a8143STim Chen "without swap slots cache.\n", __func__))
24867afa38eSTim Chen goto out_unlock;
2499b7a8143STim Chen
25067afa38eSTim Chen swap_slot_cache_initialized = true;
251d69a9575SZhen Lei }
252d69a9575SZhen Lei
25367afa38eSTim Chen __reenable_swap_slots_cache();
25467afa38eSTim Chen out_unlock:
25567afa38eSTim Chen mutex_unlock(&swap_slots_cache_enable_mutex);
25667afa38eSTim Chen }
25767afa38eSTim Chen
25867afa38eSTim Chen /* called with swap slot cache's alloc lock held */
refill_swap_slots_cache(struct swap_slots_cache * cache)25967afa38eSTim Chen static int refill_swap_slots_cache(struct swap_slots_cache *cache)
26067afa38eSTim Chen {
261f19c2568SMiaohe Lin if (!use_swap_slot_cache)
26267afa38eSTim Chen return 0;
26367afa38eSTim Chen
26467afa38eSTim Chen cache->cur = 0;
26567afa38eSTim Chen if (swap_slot_cache_active)
2665d5e8f19SHuang Ying cache->nr = get_swap_pages(SWAP_SLOTS_CACHE_SIZE,
2675d5e8f19SHuang Ying cache->slots, 1);
26867afa38eSTim Chen
26967afa38eSTim Chen return cache->nr;
27067afa38eSTim Chen }
27167afa38eSTim Chen
free_swap_slot(swp_entry_t entry)272bc4a68adSMiaohe Lin void free_swap_slot(swp_entry_t entry)
27367afa38eSTim Chen {
27467afa38eSTim Chen struct swap_slots_cache *cache;
27567afa38eSTim Chen
276f07e0f84SSebastian Andrzej Siewior cache = raw_cpu_ptr(&swp_slots);
277a2e16731STim Chen if (likely(use_swap_slot_cache && cache->slots_ret)) {
27867afa38eSTim Chen spin_lock_irq(&cache->free_lock);
27967afa38eSTim Chen /* Swap slots cache may be deactivated before acquiring lock */
280f07e0f84SSebastian Andrzej Siewior if (!use_swap_slot_cache || !cache->slots_ret) {
28167afa38eSTim Chen spin_unlock_irq(&cache->free_lock);
28267afa38eSTim Chen goto direct_free;
28367afa38eSTim Chen }
28467afa38eSTim Chen if (cache->n_ret >= SWAP_SLOTS_CACHE_SIZE) {
28567afa38eSTim Chen /*
28667afa38eSTim Chen * Return slots to global pool.
28767afa38eSTim Chen * The current swap_map value is SWAP_HAS_CACHE.
28867afa38eSTim Chen * Set it to 0 to indicate it is available for
28967afa38eSTim Chen * allocation in global pool
29067afa38eSTim Chen */
29167afa38eSTim Chen swapcache_free_entries(cache->slots_ret, cache->n_ret);
29267afa38eSTim Chen cache->n_ret = 0;
29367afa38eSTim Chen }
29467afa38eSTim Chen cache->slots_ret[cache->n_ret++] = entry;
29567afa38eSTim Chen spin_unlock_irq(&cache->free_lock);
29667afa38eSTim Chen } else {
29767afa38eSTim Chen direct_free:
29867afa38eSTim Chen swapcache_free_entries(&entry, 1);
29967afa38eSTim Chen }
30067afa38eSTim Chen }
30167afa38eSTim Chen
folio_alloc_swap(struct folio * folio)302e2e3fdc7SMatthew Wilcox (Oracle) swp_entry_t folio_alloc_swap(struct folio *folio)
30367afa38eSTim Chen {
3042406b76fSWei Yang swp_entry_t entry;
30567afa38eSTim Chen struct swap_slots_cache *cache;
30667afa38eSTim Chen
30738d8b4e6SHuang Ying entry.val = 0;
30838d8b4e6SHuang Ying
309e2e3fdc7SMatthew Wilcox (Oracle) if (folio_test_large(folio)) {
310d0637c50SBarry Song if (IS_ENABLED(CONFIG_THP_SWAP) && arch_thp_swp_supported())
311e2e3fdc7SMatthew Wilcox (Oracle) get_swap_pages(1, &entry, folio_nr_pages(folio));
312bb98f2c5STejun Heo goto out;
31338d8b4e6SHuang Ying }
31438d8b4e6SHuang Ying
31567afa38eSTim Chen /*
31667afa38eSTim Chen * Preemption is allowed here, because we may sleep
31767afa38eSTim Chen * in refill_swap_slots_cache(). But it is safe, because
31867afa38eSTim Chen * accesses to the per-CPU data structure are protected by the
31967afa38eSTim Chen * mutex cache->alloc_lock.
32067afa38eSTim Chen *
32167afa38eSTim Chen * The alloc path here does not touch cache->slots_ret
32267afa38eSTim Chen * so cache->free_lock is not taken.
32367afa38eSTim Chen */
32467afa38eSTim Chen cache = raw_cpu_ptr(&swp_slots);
32567afa38eSTim Chen
326a2e16731STim Chen if (likely(check_cache_active() && cache->slots)) {
32767afa38eSTim Chen mutex_lock(&cache->alloc_lock);
32867afa38eSTim Chen if (cache->slots) {
32967afa38eSTim Chen repeat:
33067afa38eSTim Chen if (cache->nr) {
3312406b76fSWei Yang entry = cache->slots[cache->cur];
3322406b76fSWei Yang cache->slots[cache->cur++].val = 0;
33367afa38eSTim Chen cache->nr--;
3342406b76fSWei Yang } else if (refill_swap_slots_cache(cache)) {
33567afa38eSTim Chen goto repeat;
33667afa38eSTim Chen }
33767afa38eSTim Chen }
33867afa38eSTim Chen mutex_unlock(&cache->alloc_lock);
33967afa38eSTim Chen if (entry.val)
340bb98f2c5STejun Heo goto out;
34167afa38eSTim Chen }
34267afa38eSTim Chen
3435d5e8f19SHuang Ying get_swap_pages(1, &entry, 1);
344bb98f2c5STejun Heo out:
345e2e3fdc7SMatthew Wilcox (Oracle) if (mem_cgroup_try_charge_swap(folio, entry)) {
346*4081f744SMatthew Wilcox (Oracle) put_swap_folio(folio, entry);
347bb98f2c5STejun Heo entry.val = 0;
348bb98f2c5STejun Heo }
34967afa38eSTim Chen return entry;
35067afa38eSTim Chen }
351