81 static void __page_cache_release(struct folio *folio)  in __page_cache_release()  argument
83 	if (folio_test_lru(folio)) {  in __page_cache_release()
87 		lruvec = folio_lruvec_lock_irqsave(folio, &flags);  in __page_cache_release()
88 		lruvec_del_folio(lruvec, folio);  in __page_cache_release()
89 		__folio_clear_lru_flags(folio);  in __page_cache_release()
93 	if (unlikely(folio_test_mlocked(folio))) {  in __page_cache_release()
94 		long nr_pages = folio_nr_pages(folio);  in __page_cache_release()
96 		__folio_clear_mlocked(folio);  in __page_cache_release()
97 		zone_stat_mod_folio(folio, NR_MLOCK, -nr_pages);  in __page_cache_release()
102 static void __folio_put_small(struct folio *folio)  in __folio_put_small()  argument
104 	__page_cache_release(folio);  in __folio_put_small()
105 	mem_cgroup_uncharge(folio);  in __folio_put_small()
106 	free_unref_page(&folio->page, 0);  in __folio_put_small()
109 static void __folio_put_large(struct folio *folio)  in __folio_put_large()  argument
117 	if (!folio_test_hugetlb(folio))  in __folio_put_large()
118 		__page_cache_release(folio);  in __folio_put_large()
119 	destroy_large_folio(folio);  in __folio_put_large()
122 void __folio_put(struct folio *folio)  in __folio_put()  argument
124 	if (unlikely(folio_is_zone_device(folio)))  in __folio_put()
125 		free_zone_device_page(&folio->page);  in __folio_put()
126 	else if (unlikely(folio_test_large(folio)))  in __folio_put()
127 		__folio_put_large(folio);  in __folio_put()
129 		__folio_put_small(folio);  in __folio_put()
141 	struct folio *folio, *next;  in put_pages_list()  local
143 	list_for_each_entry_safe(folio, next, pages, lru) {  in put_pages_list()
144 		if (!folio_put_testzero(folio)) {  in put_pages_list()
145 			list_del(&folio->lru);  in put_pages_list()
148 		if (folio_test_large(folio)) {  in put_pages_list()
149 			list_del(&folio->lru);  in put_pages_list()
150 			__folio_put_large(folio);  in put_pages_list()
161 typedef void (*move_fn_t)(struct lruvec *lruvec, struct folio *folio);
163 static void lru_add_fn(struct lruvec *lruvec, struct folio *folio)  in lru_add_fn()  argument
165 	int was_unevictable = folio_test_clear_unevictable(folio);  in lru_add_fn()
166 	long nr_pages = folio_nr_pages(folio);  in lru_add_fn()
168 	VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);  in lru_add_fn()
181 	if (folio_evictable(folio)) {  in lru_add_fn()
185 		folio_clear_active(folio);  in lru_add_fn()
186 		folio_set_unevictable(folio);  in lru_add_fn()
194 		folio->mlock_count = 0;  in lru_add_fn()
199 	lruvec_add_folio(lruvec, folio);  in lru_add_fn()
200 	trace_mm_lru_insertion(folio);  in lru_add_fn()
210 		struct folio *folio = fbatch->folios[i];  in folio_batch_move_lru()  local
213 		if (move_fn != lru_add_fn && !folio_test_clear_lru(folio))  in folio_batch_move_lru()
216 		lruvec = folio_lruvec_relock_irqsave(folio, lruvec, &flags);  in folio_batch_move_lru()
217 		move_fn(lruvec, folio);  in folio_batch_move_lru()
219 		folio_set_lru(folio);  in folio_batch_move_lru()
229 		struct folio *folio, move_fn_t move_fn)  in folio_batch_add_and_move()  argument
231 	if (folio_batch_add(fbatch, folio) && !folio_test_large(folio) &&  in folio_batch_add_and_move()
237 static void lru_move_tail_fn(struct lruvec *lruvec, struct folio *folio)  in lru_move_tail_fn()  argument
239 	if (!folio_test_unevictable(folio)) {  in lru_move_tail_fn()
240 		lruvec_del_folio(lruvec, folio);  in lru_move_tail_fn()
241 		folio_clear_active(folio);  in lru_move_tail_fn()
242 		lruvec_add_folio_tail(lruvec, folio);  in lru_move_tail_fn()
243 		__count_vm_events(PGROTATED, folio_nr_pages(folio));  in lru_move_tail_fn()
254 void folio_rotate_reclaimable(struct folio *folio)  in folio_rotate_reclaimable()  argument
256 	if (!folio_test_locked(folio) && !folio_test_dirty(folio) &&  in folio_rotate_reclaimable()
257 	    !folio_test_unevictable(folio) && folio_test_lru(folio)) {  in folio_rotate_reclaimable()
261 		folio_get(folio);  in folio_rotate_reclaimable()
264 		folio_batch_add_and_move(fbatch, folio, lru_move_tail_fn);  in folio_rotate_reclaimable()
321 void lru_note_cost_refault(struct folio *folio)  in lru_note_cost_refault()  argument
323 	lru_note_cost(folio_lruvec(folio), folio_is_file_lru(folio),  in lru_note_cost_refault()
324 		      folio_nr_pages(folio), 0);  in lru_note_cost_refault()
327 static void folio_activate_fn(struct lruvec *lruvec, struct folio *folio)  in folio_activate_fn()  argument
329 	if (!folio_test_active(folio) && !folio_test_unevictable(folio)) {  in folio_activate_fn()
330 		long nr_pages = folio_nr_pages(folio);  in folio_activate_fn()
332 		lruvec_del_folio(lruvec, folio);  in folio_activate_fn()
333 		folio_set_active(folio);  in folio_activate_fn()
334 		lruvec_add_folio(lruvec, folio);  in folio_activate_fn()
335 		trace_mm_lru_activate(folio);  in folio_activate_fn()
352 void folio_activate(struct folio *folio)  in folio_activate()  argument
354 	if (folio_test_lru(folio) && !folio_test_active(folio) &&  in folio_activate()
355 	    !folio_test_unevictable(folio)) {  in folio_activate()
358 		folio_get(folio);  in folio_activate()
361 		folio_batch_add_and_move(fbatch, folio, folio_activate_fn);  in folio_activate()
371 void folio_activate(struct folio *folio)  in folio_activate()  argument
375 	if (folio_test_clear_lru(folio)) {  in folio_activate()
376 		lruvec = folio_lruvec_lock_irq(folio);  in folio_activate()
377 		folio_activate_fn(lruvec, folio);  in folio_activate()
379 		folio_set_lru(folio);  in folio_activate()
384 static void __lru_cache_activate_folio(struct folio *folio)  in __lru_cache_activate_folio()  argument
403 		struct folio *batch_folio = fbatch->folios[i];  in __lru_cache_activate_folio()
405 		if (batch_folio == folio) {  in __lru_cache_activate_folio()
406 			folio_set_active(folio);  in __lru_cache_activate_folio()
415 static void folio_inc_refs(struct folio *folio)  in folio_inc_refs()  argument
417 	unsigned long new_flags, old_flags = READ_ONCE(folio->flags);  in folio_inc_refs()
419 	if (folio_test_unevictable(folio))  in folio_inc_refs()
422 	if (!folio_test_referenced(folio)) {  in folio_inc_refs()
423 		folio_set_referenced(folio);  in folio_inc_refs()
427 	if (!folio_test_workingset(folio)) {  in folio_inc_refs()
428 		folio_set_workingset(folio);  in folio_inc_refs()
440 	} while (!try_cmpxchg(&folio->flags, &old_flags, new_flags));  in folio_inc_refs()
443 static void folio_inc_refs(struct folio *folio)  in folio_inc_refs()  argument
458 void folio_mark_accessed(struct folio *folio)  in folio_mark_accessed()  argument
461 		folio_inc_refs(folio);  in folio_mark_accessed()
465 	if (!folio_test_referenced(folio)) {  in folio_mark_accessed()
466 		folio_set_referenced(folio);  in folio_mark_accessed()
467 	} else if (folio_test_unevictable(folio)) {  in folio_mark_accessed()
473 	} else if (!folio_test_active(folio)) {  in folio_mark_accessed()
480 		if (folio_test_lru(folio))  in folio_mark_accessed()
481 			folio_activate(folio);  in folio_mark_accessed()
483 			__lru_cache_activate_folio(folio);  in folio_mark_accessed()
484 		folio_clear_referenced(folio);  in folio_mark_accessed()
485 		workingset_activation(folio);  in folio_mark_accessed()
487 	if (folio_test_idle(folio))  in folio_mark_accessed()
488 		folio_clear_idle(folio);  in folio_mark_accessed()
501 void folio_add_lru(struct folio *folio)  in folio_add_lru()  argument
505 	VM_BUG_ON_FOLIO(folio_test_active(folio) &&  in folio_add_lru()
506 			folio_test_unevictable(folio), folio);  in folio_add_lru()
507 	VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);  in folio_add_lru()
510 	if (lru_gen_enabled() && !folio_test_unevictable(folio) &&  in folio_add_lru()
512 		folio_set_active(folio);  in folio_add_lru()
514 	folio_get(folio);  in folio_add_lru()
517 	folio_batch_add_and_move(fbatch, folio, lru_add_fn);  in folio_add_lru()
530 void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma)  in folio_add_lru_vma()  argument
532 	VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);  in folio_add_lru_vma()
535 		mlock_new_folio(folio);  in folio_add_lru_vma()
537 		folio_add_lru(folio);  in folio_add_lru_vma()
561 static void lru_deactivate_file_fn(struct lruvec *lruvec, struct folio *folio)  in lru_deactivate_file_fn()  argument
563 	bool active = folio_test_active(folio);  in lru_deactivate_file_fn()
564 	long nr_pages = folio_nr_pages(folio);  in lru_deactivate_file_fn()
566 	if (folio_test_unevictable(folio))  in lru_deactivate_file_fn()
570 	if (folio_mapped(folio))  in lru_deactivate_file_fn()
573 	lruvec_del_folio(lruvec, folio);  in lru_deactivate_file_fn()
574 	folio_clear_active(folio);  in lru_deactivate_file_fn()
575 	folio_clear_referenced(folio);  in lru_deactivate_file_fn()
577 	if (folio_test_writeback(folio) || folio_test_dirty(folio)) {  in lru_deactivate_file_fn()
584 		lruvec_add_folio(lruvec, folio);  in lru_deactivate_file_fn()
585 		folio_set_reclaim(folio);  in lru_deactivate_file_fn()
591 		lruvec_add_folio_tail(lruvec, folio);  in lru_deactivate_file_fn()
602 static void lru_deactivate_fn(struct lruvec *lruvec, struct folio *folio)  in lru_deactivate_fn()  argument
604 	if (!folio_test_unevictable(folio) && (folio_test_active(folio) || lru_gen_enabled())) {  in lru_deactivate_fn()
605 		long nr_pages = folio_nr_pages(folio);  in lru_deactivate_fn()
607 		lruvec_del_folio(lruvec, folio);  in lru_deactivate_fn()
608 		folio_clear_active(folio);  in lru_deactivate_fn()
609 		folio_clear_referenced(folio);  in lru_deactivate_fn()
610 		lruvec_add_folio(lruvec, folio);  in lru_deactivate_fn()
618 static void lru_lazyfree_fn(struct lruvec *lruvec, struct folio *folio)  in lru_lazyfree_fn()  argument
620 	if (folio_test_anon(folio) && folio_test_swapbacked(folio) &&  in lru_lazyfree_fn()
621 	    !folio_test_swapcache(folio) && !folio_test_unevictable(folio)) {  in lru_lazyfree_fn()
622 		long nr_pages = folio_nr_pages(folio);  in lru_lazyfree_fn()
624 		lruvec_del_folio(lruvec, folio);  in lru_lazyfree_fn()
625 		folio_clear_active(folio);  in lru_lazyfree_fn()
626 		folio_clear_referenced(folio);  in lru_lazyfree_fn()
632 		folio_clear_swapbacked(folio);  in lru_lazyfree_fn()
633 		lruvec_add_folio(lruvec, folio);  in lru_lazyfree_fn()
690 void deactivate_file_folio(struct folio *folio)  in deactivate_file_folio()  argument
695 	if (folio_test_unevictable(folio))  in deactivate_file_folio()
698 	folio_get(folio);  in deactivate_file_folio()
701 	folio_batch_add_and_move(fbatch, folio, lru_deactivate_file_fn);  in deactivate_file_folio()
713 void folio_deactivate(struct folio *folio)  in folio_deactivate()  argument
715 	if (folio_test_lru(folio) && !folio_test_unevictable(folio) &&  in folio_deactivate()
716 	    (folio_test_active(folio) || lru_gen_enabled())) {  in folio_deactivate()
719 		folio_get(folio);  in folio_deactivate()
722 		folio_batch_add_and_move(fbatch, folio, lru_deactivate_fn);  in folio_deactivate()
734 void folio_mark_lazyfree(struct folio *folio)  in folio_mark_lazyfree()  argument
736 	if (folio_test_lru(folio) && folio_test_anon(folio) &&  in folio_mark_lazyfree()
737 	    folio_test_swapbacked(folio) && !folio_test_swapcache(folio) &&  in folio_mark_lazyfree()
738 	    !folio_test_unevictable(folio)) {  in folio_mark_lazyfree()
741 		folio_get(folio);  in folio_mark_lazyfree()
744 		folio_batch_add_and_move(fbatch, folio, lru_lazyfree_fn);  in folio_mark_lazyfree()
970 		struct folio *folio;  in release_pages()  local
973 		folio = page_folio(encoded_page_ptr(encoded[i]));  in release_pages()
985 		if (is_huge_zero_page(&folio->page))  in release_pages()
988 		if (folio_is_zone_device(folio)) {  in release_pages()
993 			if (put_devmap_managed_page(&folio->page))  in release_pages()
995 			if (folio_put_testzero(folio))  in release_pages()
996 				free_zone_device_page(&folio->page);  in release_pages()
1000 		if (!folio_put_testzero(folio))  in release_pages()
1003 		if (folio_test_large(folio)) {  in release_pages()
1008 			__folio_put_large(folio);  in release_pages()
1012 		if (folio_test_lru(folio)) {  in release_pages()
1015 			lruvec = folio_lruvec_relock_irqsave(folio, lruvec,  in release_pages()
1020 			lruvec_del_folio(lruvec, folio);  in release_pages()
1021 			__folio_clear_lru_flags(folio);  in release_pages()
1030 		if (unlikely(folio_test_mlocked(folio))) {  in release_pages()
1031 			__folio_clear_mlocked(folio);  in release_pages()
1032 			zone_stat_sub_folio(folio, NR_MLOCK);  in release_pages()
1036 		list_add(&folio->lru, &pages_to_free);  in release_pages()
1081 		struct folio *folio = fbatch->folios[i];  in folio_batch_remove_exceptionals()  local
1082 		if (!xa_is_value(folio))  in folio_batch_remove_exceptionals()
1083 			fbatch->folios[j++] = folio;  in folio_batch_remove_exceptionals()