86 int add_to_swap_cache(struct folio *folio, swp_entry_t entry,  in add_to_swap_cache()  argument
91 	XA_STATE_ORDER(xas, &address_space->i_pages, idx, folio_order(folio));  in add_to_swap_cache()
92 	unsigned long i, nr = folio_nr_pages(folio);  in add_to_swap_cache()
97 	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);  in add_to_swap_cache()
98 	VM_BUG_ON_FOLIO(folio_test_swapcache(folio), folio);  in add_to_swap_cache()
99 	VM_BUG_ON_FOLIO(!folio_test_swapbacked(folio), folio);  in add_to_swap_cache()
101 	folio_ref_add(folio, nr);  in add_to_swap_cache()
102 	folio_set_swapcache(folio);  in add_to_swap_cache()
103 	folio->swap = entry;  in add_to_swap_cache()
111 			VM_BUG_ON_FOLIO(xas.xa_index != idx + i, folio);  in add_to_swap_cache()
117 			xas_store(&xas, folio);  in add_to_swap_cache()
121 		__node_stat_mod_folio(folio, NR_FILE_PAGES, nr);  in add_to_swap_cache()
122 		__lruvec_stat_mod_folio(folio, NR_SWAPCACHE, nr);  in add_to_swap_cache()
130 	folio_clear_swapcache(folio);  in add_to_swap_cache()
131 	folio_ref_sub(folio, nr);  in add_to_swap_cache()
139 void __delete_from_swap_cache(struct folio *folio,  in __delete_from_swap_cache()  argument
144 	long nr = folio_nr_pages(folio);  in __delete_from_swap_cache()
150 	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);  in __delete_from_swap_cache()
151 	VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio);  in __delete_from_swap_cache()
152 	VM_BUG_ON_FOLIO(folio_test_writeback(folio), folio);  in __delete_from_swap_cache()
156 		VM_BUG_ON_PAGE(entry != folio, entry);  in __delete_from_swap_cache()
159 	folio->swap.val = 0;  in __delete_from_swap_cache()
160 	folio_clear_swapcache(folio);  in __delete_from_swap_cache()
162 	__node_stat_mod_folio(folio, NR_FILE_PAGES, -nr);  in __delete_from_swap_cache()
163 	__lruvec_stat_mod_folio(folio, NR_SWAPCACHE, -nr);  in __delete_from_swap_cache()
176 bool add_to_swap(struct folio *folio)  in add_to_swap()  argument
181 	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);  in add_to_swap()
182 	VM_BUG_ON_FOLIO(!folio_test_uptodate(folio), folio);  in add_to_swap()
184 	entry = folio_alloc_swap(folio);  in add_to_swap()
199 	err = add_to_swap_cache(folio, entry,  in add_to_swap()
218 	folio_mark_dirty(folio);  in add_to_swap()
223 	put_swap_folio(folio, entry);  in add_to_swap()
233 void delete_from_swap_cache(struct folio *folio)  in delete_from_swap_cache()  argument
235 	swp_entry_t entry = folio->swap;  in delete_from_swap_cache()
239 	__delete_from_swap_cache(folio, entry, NULL);  in delete_from_swap_cache()
242 	put_swap_folio(folio, entry);  in delete_from_swap_cache()
243 	folio_ref_sub(folio, folio_nr_pages(folio));  in delete_from_swap_cache()
286 	struct folio *folio = page_folio(page);  in free_swap_cache()  local
288 	if (folio_test_swapcache(folio) && !folio_mapped(folio) &&  in free_swap_cache()
289 	    folio_trylock(folio)) {  in free_swap_cache()
290 		folio_free_swap(folio);  in free_swap_cache()
291 		folio_unlock(folio);  in free_swap_cache()
331 struct folio *swap_cache_get_folio(swp_entry_t entry,  in swap_cache_get_folio()
334 	struct folio *folio;  in swap_cache_get_folio()  local
336 	folio = filemap_get_folio(swap_address_space(entry), swp_offset(entry));  in swap_cache_get_folio()
337 	if (!IS_ERR(folio)) {  in swap_cache_get_folio()
345 		if (unlikely(folio_test_large(folio)))  in swap_cache_get_folio()
346 			return folio;  in swap_cache_get_folio()
348 		readahead = folio_test_clear_readahead(folio);  in swap_cache_get_folio()
368 		folio = NULL;  in swap_cache_get_folio()
371 	return folio;  in swap_cache_get_folio()
384 struct folio *filemap_get_incore_folio(struct address_space *mapping,  in filemap_get_incore_folio()
389 	struct folio *folio = filemap_get_entry(mapping, index);  in filemap_get_incore_folio()  local
391 	if (!folio)  in filemap_get_incore_folio()
393 	if (!xa_is_value(folio))  in filemap_get_incore_folio()
394 		return folio;  in filemap_get_incore_folio()
398 	swp = radix_to_swp_entry(folio);  in filemap_get_incore_folio()
407 	folio = filemap_get_folio(swap_address_space(swp), index);  in filemap_get_incore_folio()
409 	return folio;  in filemap_get_incore_folio()
417 	struct folio *folio;  in __read_swap_cache_async()  local
433 		folio = filemap_get_folio(swap_address_space(entry),  in __read_swap_cache_async()
435 		if (!IS_ERR(folio)) {  in __read_swap_cache_async()
436 			page = folio_file_page(folio, swp_offset(entry));  in __read_swap_cache_async()
456 		folio = vma_alloc_folio(gfp_mask, 0, vma, addr, false);  in __read_swap_cache_async()
457 		if (!folio)  in __read_swap_cache_async()
467 		folio_put(folio);  in __read_swap_cache_async()
485 	__folio_set_locked(folio);  in __read_swap_cache_async()
486 	__folio_set_swapbacked(folio);  in __read_swap_cache_async()
488 	if (mem_cgroup_swapin_charge_folio(folio, NULL, gfp_mask, entry))  in __read_swap_cache_async()
492 	if (add_to_swap_cache(folio, entry, gfp_mask & GFP_RECLAIM_MASK, &shadow))  in __read_swap_cache_async()
498 		workingset_refault(folio, shadow);  in __read_swap_cache_async()
501 	folio_add_lru(folio);  in __read_swap_cache_async()
503 	page = &folio->page;  in __read_swap_cache_async()
509 	put_swap_folio(folio, entry);  in __read_swap_cache_async()
510 	folio_unlock(folio);  in __read_swap_cache_async()
511 	folio_put(folio);  in __read_swap_cache_async()