1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */
2b2e18538SRik van Riel #ifndef LINUX_MM_INLINE_H
3b2e18538SRik van Riel #define LINUX_MM_INLINE_H
4b2e18538SRik van Riel
536090defSArnd Bergmann #include <linux/atomic.h>
62c888cfbSRik van Riel #include <linux/huge_mm.h>
76e543d57SLisa Du #include <linux/swap.h>
817fca131SArnd Bergmann #include <linux/string.h>
9999dad82SPeter Xu #include <linux/userfaultfd_k.h>
10999dad82SPeter Xu #include <linux/swapops.h>
112c888cfbSRik van Riel
12b2e18538SRik van Riel /**
13889a3747SMatthew Wilcox (Oracle) * folio_is_file_lru - Should the folio be on a file LRU or anon LRU?
14889a3747SMatthew Wilcox (Oracle) * @folio: The folio to test.
15b2e18538SRik van Riel *
16b2e18538SRik van Riel * We would like to get this info without a page flag, but the state
17889a3747SMatthew Wilcox (Oracle) * needs to survive until the folio is last deleted from the LRU, which
18b2e18538SRik van Riel * could be as far down as __page_cache_release.
19889a3747SMatthew Wilcox (Oracle) *
20889a3747SMatthew Wilcox (Oracle) * Return: An integer (not a boolean!) used to sort a folio onto the
21889a3747SMatthew Wilcox (Oracle) * right LRU list and to account folios correctly.
22889a3747SMatthew Wilcox (Oracle) * 1 if @folio is a regular filesystem backed page cache folio
23889a3747SMatthew Wilcox (Oracle) * or a lazily freed anonymous folio (e.g. via MADV_FREE).
24889a3747SMatthew Wilcox (Oracle) * 0 if @folio is a normal anonymous folio, a tmpfs folio or otherwise
25889a3747SMatthew Wilcox (Oracle) * ram or swap backed folio.
26b2e18538SRik van Riel */
folio_is_file_lru(struct folio * folio)27889a3747SMatthew Wilcox (Oracle) static inline int folio_is_file_lru(struct folio *folio)
28889a3747SMatthew Wilcox (Oracle) {
29889a3747SMatthew Wilcox (Oracle) return !folio_test_swapbacked(folio);
30889a3747SMatthew Wilcox (Oracle) }
31889a3747SMatthew Wilcox (Oracle)
page_is_file_lru(struct page * page)329de4f22aSHuang Ying static inline int page_is_file_lru(struct page *page)
33b2e18538SRik van Riel {
34889a3747SMatthew Wilcox (Oracle) return folio_is_file_lru(page_folio(page));
35b2e18538SRik van Riel }
36b2e18538SRik van Riel
__update_lru_size(struct lruvec * lruvec,enum lru_list lru,enum zone_type zid,long nr_pages)37aa1b6790SYu Zhao static __always_inline void __update_lru_size(struct lruvec *lruvec,
38599d0c95SMel Gorman enum lru_list lru, enum zone_type zid,
39889a3747SMatthew Wilcox (Oracle) long nr_pages)
409d5e6a9fSHugh Dickins {
41599d0c95SMel Gorman struct pglist_data *pgdat = lruvec_pgdat(lruvec);
42599d0c95SMel Gorman
43ec1c86b2SYu Zhao lockdep_assert_held(&lruvec->lru_lock);
44ec1c86b2SYu Zhao WARN_ON_ONCE(nr_pages != (int)nr_pages);
45ec1c86b2SYu Zhao
46e0ee0e71SJohannes Weiner __mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages);
4771c799f4SMinchan Kim __mod_zone_page_state(&pgdat->node_zones[zid],
4871c799f4SMinchan Kim NR_ZONE_LRU_BASE + lru, nr_pages);
49aa1b6790SYu Zhao }
50aa1b6790SYu Zhao
update_lru_size(struct lruvec * lruvec,enum lru_list lru,enum zone_type zid,long nr_pages)51aa1b6790SYu Zhao static __always_inline void update_lru_size(struct lruvec *lruvec,
52aa1b6790SYu Zhao enum lru_list lru, enum zone_type zid,
53aa1b6790SYu Zhao long nr_pages)
54aa1b6790SYu Zhao {
55aa1b6790SYu Zhao __update_lru_size(lruvec, lru, zid, nr_pages);
567ee36a14SMel Gorman #ifdef CONFIG_MEMCG
57b4536f0cSMichal Hocko mem_cgroup_update_lru_size(lruvec, lru, zid, nr_pages);
589d5e6a9fSHugh Dickins #endif
599d5e6a9fSHugh Dickins }
609d5e6a9fSHugh Dickins
61401a8e1cSJohannes Weiner /**
62889a3747SMatthew Wilcox (Oracle) * __folio_clear_lru_flags - Clear page lru flags before releasing a page.
63889a3747SMatthew Wilcox (Oracle) * @folio: The folio that was on lru and now has a zero reference.
641c1c53d4SHugh Dickins */
__folio_clear_lru_flags(struct folio * folio)65889a3747SMatthew Wilcox (Oracle) static __always_inline void __folio_clear_lru_flags(struct folio *folio)
661da177e4SLinus Torvalds {
67889a3747SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(!folio_test_lru(folio), folio);
68bc711271SYu Zhao
69889a3747SMatthew Wilcox (Oracle) __folio_clear_lru(folio);
70b69408e8SChristoph Lameter
7187560179SYu Zhao /* this shouldn't happen, so leave the flags to bad_page() */
72889a3747SMatthew Wilcox (Oracle) if (folio_test_active(folio) && folio_test_unevictable(folio))
7387560179SYu Zhao return;
7487560179SYu Zhao
75889a3747SMatthew Wilcox (Oracle) __folio_clear_active(folio);
76889a3747SMatthew Wilcox (Oracle) __folio_clear_unevictable(folio);
77889a3747SMatthew Wilcox (Oracle) }
78889a3747SMatthew Wilcox (Oracle)
79b69408e8SChristoph Lameter /**
80889a3747SMatthew Wilcox (Oracle) * folio_lru_list - Which LRU list should a folio be on?
81889a3747SMatthew Wilcox (Oracle) * @folio: The folio to test.
82b69408e8SChristoph Lameter *
83889a3747SMatthew Wilcox (Oracle) * Return: The LRU list a folio should be on, as an index
84b69408e8SChristoph Lameter * into the array of LRU lists.
85b69408e8SChristoph Lameter */
folio_lru_list(struct folio * folio)86889a3747SMatthew Wilcox (Oracle) static __always_inline enum lru_list folio_lru_list(struct folio *folio)
87b69408e8SChristoph Lameter {
88401a8e1cSJohannes Weiner enum lru_list lru;
89b69408e8SChristoph Lameter
90889a3747SMatthew Wilcox (Oracle) VM_BUG_ON_FOLIO(folio_test_active(folio) && folio_test_unevictable(folio), folio);
91bc711271SYu Zhao
92889a3747SMatthew Wilcox (Oracle) if (folio_test_unevictable(folio))
93c1770e34SYu Zhao return LRU_UNEVICTABLE;
94c1770e34SYu Zhao
95889a3747SMatthew Wilcox (Oracle) lru = folio_is_file_lru(folio) ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON;
96889a3747SMatthew Wilcox (Oracle) if (folio_test_active(folio))
97b69408e8SChristoph Lameter lru += LRU_ACTIVE;
98c1770e34SYu Zhao
99b69408e8SChristoph Lameter return lru;
100b69408e8SChristoph Lameter }
101f90d8191SYu Zhao
102ec1c86b2SYu Zhao #ifdef CONFIG_LRU_GEN
103ec1c86b2SYu Zhao
104354ed597SYu Zhao #ifdef CONFIG_LRU_GEN_ENABLED
lru_gen_enabled(void)105ec1c86b2SYu Zhao static inline bool lru_gen_enabled(void)
106ec1c86b2SYu Zhao {
107354ed597SYu Zhao DECLARE_STATIC_KEY_TRUE(lru_gen_caps[NR_LRU_GEN_CAPS]);
108354ed597SYu Zhao
109354ed597SYu Zhao return static_branch_likely(&lru_gen_caps[LRU_GEN_CORE]);
110ec1c86b2SYu Zhao }
111354ed597SYu Zhao #else
lru_gen_enabled(void)112354ed597SYu Zhao static inline bool lru_gen_enabled(void)
113354ed597SYu Zhao {
114354ed597SYu Zhao DECLARE_STATIC_KEY_FALSE(lru_gen_caps[NR_LRU_GEN_CAPS]);
115354ed597SYu Zhao
116354ed597SYu Zhao return static_branch_unlikely(&lru_gen_caps[LRU_GEN_CORE]);
117354ed597SYu Zhao }
118354ed597SYu Zhao #endif
119ec1c86b2SYu Zhao
lru_gen_in_fault(void)120ec1c86b2SYu Zhao static inline bool lru_gen_in_fault(void)
121ec1c86b2SYu Zhao {
122ec1c86b2SYu Zhao return current->in_lru_fault;
123ec1c86b2SYu Zhao }
124ec1c86b2SYu Zhao
lru_gen_from_seq(unsigned long seq)125ec1c86b2SYu Zhao static inline int lru_gen_from_seq(unsigned long seq)
126ec1c86b2SYu Zhao {
127ec1c86b2SYu Zhao return seq % MAX_NR_GENS;
128ec1c86b2SYu Zhao }
129ec1c86b2SYu Zhao
lru_hist_from_seq(unsigned long seq)130ac35a490SYu Zhao static inline int lru_hist_from_seq(unsigned long seq)
131ac35a490SYu Zhao {
132ac35a490SYu Zhao return seq % NR_HIST_GENS;
133ac35a490SYu Zhao }
134ac35a490SYu Zhao
lru_tier_from_refs(int refs)135ac35a490SYu Zhao static inline int lru_tier_from_refs(int refs)
136ac35a490SYu Zhao {
137ac35a490SYu Zhao VM_WARN_ON_ONCE(refs > BIT(LRU_REFS_WIDTH));
138ac35a490SYu Zhao
139ac35a490SYu Zhao /* see the comment in folio_lru_refs() */
140ac35a490SYu Zhao return order_base_2(refs + 1);
141ac35a490SYu Zhao }
142ac35a490SYu Zhao
folio_lru_refs(struct folio * folio)143ac35a490SYu Zhao static inline int folio_lru_refs(struct folio *folio)
144ac35a490SYu Zhao {
145ac35a490SYu Zhao unsigned long flags = READ_ONCE(folio->flags);
146ac35a490SYu Zhao bool workingset = flags & BIT(PG_workingset);
147ac35a490SYu Zhao
148ac35a490SYu Zhao /*
149ac35a490SYu Zhao * Return the number of accesses beyond PG_referenced, i.e., N-1 if the
150ac35a490SYu Zhao * total number of accesses is N>1, since N=0,1 both map to the first
151ac35a490SYu Zhao * tier. lru_tier_from_refs() will account for this off-by-one. Also see
152ac35a490SYu Zhao * the comment on MAX_NR_TIERS.
153ac35a490SYu Zhao */
154ac35a490SYu Zhao return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + workingset;
155ac35a490SYu Zhao }
156ac35a490SYu Zhao
folio_lru_gen(struct folio * folio)157ec1c86b2SYu Zhao static inline int folio_lru_gen(struct folio *folio)
158ec1c86b2SYu Zhao {
159ec1c86b2SYu Zhao unsigned long flags = READ_ONCE(folio->flags);
160ec1c86b2SYu Zhao
161ec1c86b2SYu Zhao return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
162ec1c86b2SYu Zhao }
163ec1c86b2SYu Zhao
lru_gen_is_active(struct lruvec * lruvec,int gen)164ec1c86b2SYu Zhao static inline bool lru_gen_is_active(struct lruvec *lruvec, int gen)
165ec1c86b2SYu Zhao {
166ec1c86b2SYu Zhao unsigned long max_seq = lruvec->lrugen.max_seq;
167ec1c86b2SYu Zhao
168ec1c86b2SYu Zhao VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);
169ec1c86b2SYu Zhao
170ec1c86b2SYu Zhao /* see the comment on MIN_NR_GENS */
171ec1c86b2SYu Zhao return gen == lru_gen_from_seq(max_seq) || gen == lru_gen_from_seq(max_seq - 1);
172ec1c86b2SYu Zhao }
173ec1c86b2SYu Zhao
lru_gen_update_size(struct lruvec * lruvec,struct folio * folio,int old_gen,int new_gen)174ec1c86b2SYu Zhao static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio,
175ec1c86b2SYu Zhao int old_gen, int new_gen)
176ec1c86b2SYu Zhao {
177ec1c86b2SYu Zhao int type = folio_is_file_lru(folio);
178ec1c86b2SYu Zhao int zone = folio_zonenum(folio);
179ec1c86b2SYu Zhao int delta = folio_nr_pages(folio);
180ec1c86b2SYu Zhao enum lru_list lru = type * LRU_INACTIVE_FILE;
181391655feSYu Zhao struct lru_gen_folio *lrugen = &lruvec->lrugen;
182ec1c86b2SYu Zhao
183ec1c86b2SYu Zhao VM_WARN_ON_ONCE(old_gen != -1 && old_gen >= MAX_NR_GENS);
184ec1c86b2SYu Zhao VM_WARN_ON_ONCE(new_gen != -1 && new_gen >= MAX_NR_GENS);
185ec1c86b2SYu Zhao VM_WARN_ON_ONCE(old_gen == -1 && new_gen == -1);
186ec1c86b2SYu Zhao
187ec1c86b2SYu Zhao if (old_gen >= 0)
188ec1c86b2SYu Zhao WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone],
189ec1c86b2SYu Zhao lrugen->nr_pages[old_gen][type][zone] - delta);
190ec1c86b2SYu Zhao if (new_gen >= 0)
191ec1c86b2SYu Zhao WRITE_ONCE(lrugen->nr_pages[new_gen][type][zone],
192ec1c86b2SYu Zhao lrugen->nr_pages[new_gen][type][zone] + delta);
193ec1c86b2SYu Zhao
194ec1c86b2SYu Zhao /* addition */
195ec1c86b2SYu Zhao if (old_gen < 0) {
196ec1c86b2SYu Zhao if (lru_gen_is_active(lruvec, new_gen))
197ec1c86b2SYu Zhao lru += LRU_ACTIVE;
198ec1c86b2SYu Zhao __update_lru_size(lruvec, lru, zone, delta);
199ec1c86b2SYu Zhao return;
200ec1c86b2SYu Zhao }
201ec1c86b2SYu Zhao
202ec1c86b2SYu Zhao /* deletion */
203ec1c86b2SYu Zhao if (new_gen < 0) {
204ec1c86b2SYu Zhao if (lru_gen_is_active(lruvec, old_gen))
205ec1c86b2SYu Zhao lru += LRU_ACTIVE;
206ec1c86b2SYu Zhao __update_lru_size(lruvec, lru, zone, -delta);
207ec1c86b2SYu Zhao return;
208ec1c86b2SYu Zhao }
209ac35a490SYu Zhao
210ac35a490SYu Zhao /* promotion */
211ac35a490SYu Zhao if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) {
212ac35a490SYu Zhao __update_lru_size(lruvec, lru, zone, -delta);
213ac35a490SYu Zhao __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, delta);
214ac35a490SYu Zhao }
215ac35a490SYu Zhao
216ac35a490SYu Zhao /* demotion requires isolation, e.g., lru_deactivate_fn() */
217ac35a490SYu Zhao VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen));
218ec1c86b2SYu Zhao }
219ec1c86b2SYu Zhao
lru_gen_add_folio(struct lruvec * lruvec,struct folio * folio,bool reclaiming)220ec1c86b2SYu Zhao static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
221ec1c86b2SYu Zhao {
222ec1c86b2SYu Zhao unsigned long seq;
223ec1c86b2SYu Zhao unsigned long flags;
224ec1c86b2SYu Zhao int gen = folio_lru_gen(folio);
225ec1c86b2SYu Zhao int type = folio_is_file_lru(folio);
226ec1c86b2SYu Zhao int zone = folio_zonenum(folio);
227391655feSYu Zhao struct lru_gen_folio *lrugen = &lruvec->lrugen;
228ec1c86b2SYu Zhao
229ec1c86b2SYu Zhao VM_WARN_ON_ONCE_FOLIO(gen != -1, folio);
230ec1c86b2SYu Zhao
231354ed597SYu Zhao if (folio_test_unevictable(folio) || !lrugen->enabled)
232ec1c86b2SYu Zhao return false;
233ec1c86b2SYu Zhao /*
234*b2ce691bSYu Zhao * There are four common cases for this page:
235*b2ce691bSYu Zhao * 1. If it's hot, i.e., freshly faulted in, add it to the youngest
236*b2ce691bSYu Zhao * generation, and it's protected over the rest below.
237*b2ce691bSYu Zhao * 2. If it can't be evicted immediately, i.e., a dirty page pending
238*b2ce691bSYu Zhao * writeback, add it to the second youngest generation.
239*b2ce691bSYu Zhao * 3. If it should be evicted first, e.g., cold and clean from
240*b2ce691bSYu Zhao * folio_rotate_reclaimable(), add it to the oldest generation.
241*b2ce691bSYu Zhao * 4. Everything else falls between 2 & 3 above and is added to the
242*b2ce691bSYu Zhao * second oldest generation if it's considered inactive, or the
243*b2ce691bSYu Zhao * oldest generation otherwise. See lru_gen_is_active().
244ec1c86b2SYu Zhao */
245ec1c86b2SYu Zhao if (folio_test_active(folio))
246ec1c86b2SYu Zhao seq = lrugen->max_seq;
247ec1c86b2SYu Zhao else if ((type == LRU_GEN_ANON && !folio_test_swapcache(folio)) ||
248ec1c86b2SYu Zhao (folio_test_reclaim(folio) &&
249ec1c86b2SYu Zhao (folio_test_dirty(folio) || folio_test_writeback(folio))))
250*b2ce691bSYu Zhao seq = lrugen->max_seq - 1;
251*b2ce691bSYu Zhao else if (reclaiming || lrugen->min_seq[type] + MIN_NR_GENS >= lrugen->max_seq)
252ec1c86b2SYu Zhao seq = lrugen->min_seq[type];
253*b2ce691bSYu Zhao else
254*b2ce691bSYu Zhao seq = lrugen->min_seq[type] + 1;
255ec1c86b2SYu Zhao
256ec1c86b2SYu Zhao gen = lru_gen_from_seq(seq);
257ec1c86b2SYu Zhao flags = (gen + 1UL) << LRU_GEN_PGOFF;
258ec1c86b2SYu Zhao /* see the comment on MIN_NR_GENS about PG_active */
259ec1c86b2SYu Zhao set_mask_bits(&folio->flags, LRU_GEN_MASK | BIT(PG_active), flags);
260ec1c86b2SYu Zhao
261ec1c86b2SYu Zhao lru_gen_update_size(lruvec, folio, -1, gen);
262ec1c86b2SYu Zhao /* for folio_rotate_reclaimable() */
263ec1c86b2SYu Zhao if (reclaiming)
2646df1b221SYu Zhao list_add_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
265ec1c86b2SYu Zhao else
2666df1b221SYu Zhao list_add(&folio->lru, &lrugen->folios[gen][type][zone]);
267ec1c86b2SYu Zhao
268ec1c86b2SYu Zhao return true;
269ec1c86b2SYu Zhao }
270ec1c86b2SYu Zhao
lru_gen_del_folio(struct lruvec * lruvec,struct folio * folio,bool reclaiming)271ec1c86b2SYu Zhao static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
272ec1c86b2SYu Zhao {
273ec1c86b2SYu Zhao unsigned long flags;
274ec1c86b2SYu Zhao int gen = folio_lru_gen(folio);
275ec1c86b2SYu Zhao
276ec1c86b2SYu Zhao if (gen < 0)
277ec1c86b2SYu Zhao return false;
278ec1c86b2SYu Zhao
279ec1c86b2SYu Zhao VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
280ec1c86b2SYu Zhao VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
281ec1c86b2SYu Zhao
282ec1c86b2SYu Zhao /* for folio_migrate_flags() */
283ec1c86b2SYu Zhao flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0;
284ec1c86b2SYu Zhao flags = set_mask_bits(&folio->flags, LRU_GEN_MASK, flags);
285ec1c86b2SYu Zhao gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
286ec1c86b2SYu Zhao
287ec1c86b2SYu Zhao lru_gen_update_size(lruvec, folio, gen, -1);
288ec1c86b2SYu Zhao list_del(&folio->lru);
289ec1c86b2SYu Zhao
290ec1c86b2SYu Zhao return true;
291ec1c86b2SYu Zhao }
292ec1c86b2SYu Zhao
293ec1c86b2SYu Zhao #else /* !CONFIG_LRU_GEN */
294ec1c86b2SYu Zhao
lru_gen_enabled(void)295ec1c86b2SYu Zhao static inline bool lru_gen_enabled(void)
296ec1c86b2SYu Zhao {
297ec1c86b2SYu Zhao return false;
298ec1c86b2SYu Zhao }
299ec1c86b2SYu Zhao
lru_gen_in_fault(void)300ec1c86b2SYu Zhao static inline bool lru_gen_in_fault(void)
301ec1c86b2SYu Zhao {
302ec1c86b2SYu Zhao return false;
303ec1c86b2SYu Zhao }
304ec1c86b2SYu Zhao
lru_gen_add_folio(struct lruvec * lruvec,struct folio * folio,bool reclaiming)305ec1c86b2SYu Zhao static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
306ec1c86b2SYu Zhao {
307ec1c86b2SYu Zhao return false;
308ec1c86b2SYu Zhao }
309ec1c86b2SYu Zhao
lru_gen_del_folio(struct lruvec * lruvec,struct folio * folio,bool reclaiming)310ec1c86b2SYu Zhao static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
311ec1c86b2SYu Zhao {
312ec1c86b2SYu Zhao return false;
313ec1c86b2SYu Zhao }
314ec1c86b2SYu Zhao
315ec1c86b2SYu Zhao #endif /* CONFIG_LRU_GEN */
316ec1c86b2SYu Zhao
317889a3747SMatthew Wilcox (Oracle) static __always_inline
lruvec_add_folio(struct lruvec * lruvec,struct folio * folio)318889a3747SMatthew Wilcox (Oracle) void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio)
319889a3747SMatthew Wilcox (Oracle) {
320889a3747SMatthew Wilcox (Oracle) enum lru_list lru = folio_lru_list(folio);
321889a3747SMatthew Wilcox (Oracle)
322ec1c86b2SYu Zhao if (lru_gen_add_folio(lruvec, folio, false))
323ec1c86b2SYu Zhao return;
324ec1c86b2SYu Zhao
325889a3747SMatthew Wilcox (Oracle) update_lru_size(lruvec, lru, folio_zonenum(folio),
326889a3747SMatthew Wilcox (Oracle) folio_nr_pages(folio));
32707ca7606SHugh Dickins if (lru != LRU_UNEVICTABLE)
328889a3747SMatthew Wilcox (Oracle) list_add(&folio->lru, &lruvec->lists[lru]);
329889a3747SMatthew Wilcox (Oracle) }
330889a3747SMatthew Wilcox (Oracle)
331889a3747SMatthew Wilcox (Oracle) static __always_inline
lruvec_add_folio_tail(struct lruvec * lruvec,struct folio * folio)332889a3747SMatthew Wilcox (Oracle) void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio)
333889a3747SMatthew Wilcox (Oracle) {
334889a3747SMatthew Wilcox (Oracle) enum lru_list lru = folio_lru_list(folio);
335889a3747SMatthew Wilcox (Oracle)
336ec1c86b2SYu Zhao if (lru_gen_add_folio(lruvec, folio, true))
337ec1c86b2SYu Zhao return;
338ec1c86b2SYu Zhao
339889a3747SMatthew Wilcox (Oracle) update_lru_size(lruvec, lru, folio_zonenum(folio),
340889a3747SMatthew Wilcox (Oracle) folio_nr_pages(folio));
34107ca7606SHugh Dickins /* This is not expected to be used on LRU_UNEVICTABLE */
342889a3747SMatthew Wilcox (Oracle) list_add_tail(&folio->lru, &lruvec->lists[lru]);
343f90d8191SYu Zhao }
344f90d8191SYu Zhao
345889a3747SMatthew Wilcox (Oracle) static __always_inline
lruvec_del_folio(struct lruvec * lruvec,struct folio * folio)346889a3747SMatthew Wilcox (Oracle) void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio)
347889a3747SMatthew Wilcox (Oracle) {
34807ca7606SHugh Dickins enum lru_list lru = folio_lru_list(folio);
34907ca7606SHugh Dickins
350ec1c86b2SYu Zhao if (lru_gen_del_folio(lruvec, folio, false))
351ec1c86b2SYu Zhao return;
352ec1c86b2SYu Zhao
35307ca7606SHugh Dickins if (lru != LRU_UNEVICTABLE)
354889a3747SMatthew Wilcox (Oracle) list_del(&folio->lru);
35507ca7606SHugh Dickins update_lru_size(lruvec, lru, folio_zonenum(folio),
356889a3747SMatthew Wilcox (Oracle) -folio_nr_pages(folio));
357f90d8191SYu Zhao }
358f90d8191SYu Zhao
35917fca131SArnd Bergmann #ifdef CONFIG_ANON_VMA_NAME
36017fca131SArnd Bergmann /*
3615c26f6acSSuren Baghdasaryan * mmap_lock should be read-locked when calling anon_vma_name(). Caller should
3625c26f6acSSuren Baghdasaryan * either keep holding the lock while using the returned pointer or it should
3635c26f6acSSuren Baghdasaryan * raise anon_vma_name refcount before releasing the lock.
36417fca131SArnd Bergmann */
3655c26f6acSSuren Baghdasaryan extern struct anon_vma_name *anon_vma_name(struct vm_area_struct *vma);
3665c26f6acSSuren Baghdasaryan extern struct anon_vma_name *anon_vma_name_alloc(const char *name);
3675c26f6acSSuren Baghdasaryan extern void anon_vma_name_free(struct kref *kref);
36817fca131SArnd Bergmann
36917fca131SArnd Bergmann /* mmap_lock should be read-locked */
anon_vma_name_get(struct anon_vma_name * anon_name)3705c26f6acSSuren Baghdasaryan static inline void anon_vma_name_get(struct anon_vma_name *anon_name)
37117fca131SArnd Bergmann {
3725c26f6acSSuren Baghdasaryan if (anon_name)
3735c26f6acSSuren Baghdasaryan kref_get(&anon_name->kref);
3745c26f6acSSuren Baghdasaryan }
37517fca131SArnd Bergmann
anon_vma_name_put(struct anon_vma_name * anon_name)3765c26f6acSSuren Baghdasaryan static inline void anon_vma_name_put(struct anon_vma_name *anon_name)
3775c26f6acSSuren Baghdasaryan {
3785c26f6acSSuren Baghdasaryan if (anon_name)
3795c26f6acSSuren Baghdasaryan kref_put(&anon_name->kref, anon_vma_name_free);
3805c26f6acSSuren Baghdasaryan }
3815c26f6acSSuren Baghdasaryan
38296403e11SSuren Baghdasaryan static inline
anon_vma_name_reuse(struct anon_vma_name * anon_name)38396403e11SSuren Baghdasaryan struct anon_vma_name *anon_vma_name_reuse(struct anon_vma_name *anon_name)
38496403e11SSuren Baghdasaryan {
38596403e11SSuren Baghdasaryan /* Prevent anon_name refcount saturation early on */
38696403e11SSuren Baghdasaryan if (kref_read(&anon_name->kref) < REFCOUNT_MAX) {
38796403e11SSuren Baghdasaryan anon_vma_name_get(anon_name);
38896403e11SSuren Baghdasaryan return anon_name;
38996403e11SSuren Baghdasaryan
39096403e11SSuren Baghdasaryan }
39196403e11SSuren Baghdasaryan return anon_vma_name_alloc(anon_name->name);
39296403e11SSuren Baghdasaryan }
39396403e11SSuren Baghdasaryan
dup_anon_vma_name(struct vm_area_struct * orig_vma,struct vm_area_struct * new_vma)3945c26f6acSSuren Baghdasaryan static inline void dup_anon_vma_name(struct vm_area_struct *orig_vma,
3955c26f6acSSuren Baghdasaryan struct vm_area_struct *new_vma)
3965c26f6acSSuren Baghdasaryan {
3975c26f6acSSuren Baghdasaryan struct anon_vma_name *anon_name = anon_vma_name(orig_vma);
3985c26f6acSSuren Baghdasaryan
39996403e11SSuren Baghdasaryan if (anon_name)
40096403e11SSuren Baghdasaryan new_vma->anon_name = anon_vma_name_reuse(anon_name);
4015c26f6acSSuren Baghdasaryan }
4025c26f6acSSuren Baghdasaryan
free_anon_vma_name(struct vm_area_struct * vma)4035c26f6acSSuren Baghdasaryan static inline void free_anon_vma_name(struct vm_area_struct *vma)
4045c26f6acSSuren Baghdasaryan {
4055c26f6acSSuren Baghdasaryan /*
4065c26f6acSSuren Baghdasaryan * Not using anon_vma_name because it generates a warning if mmap_lock
4075c26f6acSSuren Baghdasaryan * is not held, which might be the case here.
4085c26f6acSSuren Baghdasaryan */
4095c26f6acSSuren Baghdasaryan anon_vma_name_put(vma->anon_name);
4105c26f6acSSuren Baghdasaryan }
4115c26f6acSSuren Baghdasaryan
anon_vma_name_eq(struct anon_vma_name * anon_name1,struct anon_vma_name * anon_name2)4125c26f6acSSuren Baghdasaryan static inline bool anon_vma_name_eq(struct anon_vma_name *anon_name1,
4135c26f6acSSuren Baghdasaryan struct anon_vma_name *anon_name2)
4145c26f6acSSuren Baghdasaryan {
4155c26f6acSSuren Baghdasaryan if (anon_name1 == anon_name2)
41617fca131SArnd Bergmann return true;
41717fca131SArnd Bergmann
4185c26f6acSSuren Baghdasaryan return anon_name1 && anon_name2 &&
4195c26f6acSSuren Baghdasaryan !strcmp(anon_name1->name, anon_name2->name);
42017fca131SArnd Bergmann }
4215c26f6acSSuren Baghdasaryan
42217fca131SArnd Bergmann #else /* CONFIG_ANON_VMA_NAME */
anon_vma_name(struct vm_area_struct * vma)4235c26f6acSSuren Baghdasaryan static inline struct anon_vma_name *anon_vma_name(struct vm_area_struct *vma)
42417fca131SArnd Bergmann {
42517fca131SArnd Bergmann return NULL;
42617fca131SArnd Bergmann }
4275c26f6acSSuren Baghdasaryan
anon_vma_name_alloc(const char * name)4285c26f6acSSuren Baghdasaryan static inline struct anon_vma_name *anon_vma_name_alloc(const char *name)
4295c26f6acSSuren Baghdasaryan {
4305c26f6acSSuren Baghdasaryan return NULL;
4315c26f6acSSuren Baghdasaryan }
4325c26f6acSSuren Baghdasaryan
anon_vma_name_get(struct anon_vma_name * anon_name)4335c26f6acSSuren Baghdasaryan static inline void anon_vma_name_get(struct anon_vma_name *anon_name) {}
anon_vma_name_put(struct anon_vma_name * anon_name)4345c26f6acSSuren Baghdasaryan static inline void anon_vma_name_put(struct anon_vma_name *anon_name) {}
dup_anon_vma_name(struct vm_area_struct * orig_vma,struct vm_area_struct * new_vma)4355c26f6acSSuren Baghdasaryan static inline void dup_anon_vma_name(struct vm_area_struct *orig_vma,
43617fca131SArnd Bergmann struct vm_area_struct *new_vma) {}
free_anon_vma_name(struct vm_area_struct * vma)4375c26f6acSSuren Baghdasaryan static inline void free_anon_vma_name(struct vm_area_struct *vma) {}
4385c26f6acSSuren Baghdasaryan
anon_vma_name_eq(struct anon_vma_name * anon_name1,struct anon_vma_name * anon_name2)4395c26f6acSSuren Baghdasaryan static inline bool anon_vma_name_eq(struct anon_vma_name *anon_name1,
4405c26f6acSSuren Baghdasaryan struct anon_vma_name *anon_name2)
44117fca131SArnd Bergmann {
44217fca131SArnd Bergmann return true;
44317fca131SArnd Bergmann }
4445c26f6acSSuren Baghdasaryan
44517fca131SArnd Bergmann #endif /* CONFIG_ANON_VMA_NAME */
44617fca131SArnd Bergmann
init_tlb_flush_pending(struct mm_struct * mm)44736090defSArnd Bergmann static inline void init_tlb_flush_pending(struct mm_struct *mm)
44836090defSArnd Bergmann {
44936090defSArnd Bergmann atomic_set(&mm->tlb_flush_pending, 0);
45036090defSArnd Bergmann }
45136090defSArnd Bergmann
inc_tlb_flush_pending(struct mm_struct * mm)45236090defSArnd Bergmann static inline void inc_tlb_flush_pending(struct mm_struct *mm)
45336090defSArnd Bergmann {
45436090defSArnd Bergmann atomic_inc(&mm->tlb_flush_pending);
45536090defSArnd Bergmann /*
45636090defSArnd Bergmann * The only time this value is relevant is when there are indeed pages
45736090defSArnd Bergmann * to flush. And we'll only flush pages after changing them, which
45836090defSArnd Bergmann * requires the PTL.
45936090defSArnd Bergmann *
46036090defSArnd Bergmann * So the ordering here is:
46136090defSArnd Bergmann *
46236090defSArnd Bergmann * atomic_inc(&mm->tlb_flush_pending);
46336090defSArnd Bergmann * spin_lock(&ptl);
46436090defSArnd Bergmann * ...
46536090defSArnd Bergmann * set_pte_at();
46636090defSArnd Bergmann * spin_unlock(&ptl);
46736090defSArnd Bergmann *
46836090defSArnd Bergmann * spin_lock(&ptl)
46936090defSArnd Bergmann * mm_tlb_flush_pending();
47036090defSArnd Bergmann * ....
47136090defSArnd Bergmann * spin_unlock(&ptl);
47236090defSArnd Bergmann *
47336090defSArnd Bergmann * flush_tlb_range();
47436090defSArnd Bergmann * atomic_dec(&mm->tlb_flush_pending);
47536090defSArnd Bergmann *
47636090defSArnd Bergmann * Where the increment if constrained by the PTL unlock, it thus
47736090defSArnd Bergmann * ensures that the increment is visible if the PTE modification is
47836090defSArnd Bergmann * visible. After all, if there is no PTE modification, nobody cares
47936090defSArnd Bergmann * about TLB flushes either.
48036090defSArnd Bergmann *
48136090defSArnd Bergmann * This very much relies on users (mm_tlb_flush_pending() and
48236090defSArnd Bergmann * mm_tlb_flush_nested()) only caring about _specific_ PTEs (and
48336090defSArnd Bergmann * therefore specific PTLs), because with SPLIT_PTE_PTLOCKS and RCpc
48436090defSArnd Bergmann * locks (PPC) the unlock of one doesn't order against the lock of
48536090defSArnd Bergmann * another PTL.
48636090defSArnd Bergmann *
48736090defSArnd Bergmann * The decrement is ordered by the flush_tlb_range(), such that
48836090defSArnd Bergmann * mm_tlb_flush_pending() will not return false unless all flushes have
48936090defSArnd Bergmann * completed.
49036090defSArnd Bergmann */
49136090defSArnd Bergmann }
49236090defSArnd Bergmann
dec_tlb_flush_pending(struct mm_struct * mm)49336090defSArnd Bergmann static inline void dec_tlb_flush_pending(struct mm_struct *mm)
49436090defSArnd Bergmann {
49536090defSArnd Bergmann /*
49636090defSArnd Bergmann * See inc_tlb_flush_pending().
49736090defSArnd Bergmann *
49836090defSArnd Bergmann * This cannot be smp_mb__before_atomic() because smp_mb() simply does
49936090defSArnd Bergmann * not order against TLB invalidate completion, which is what we need.
50036090defSArnd Bergmann *
50136090defSArnd Bergmann * Therefore we must rely on tlb_flush_*() to guarantee order.
50236090defSArnd Bergmann */
50336090defSArnd Bergmann atomic_dec(&mm->tlb_flush_pending);
50436090defSArnd Bergmann }
50536090defSArnd Bergmann
mm_tlb_flush_pending(struct mm_struct * mm)50636090defSArnd Bergmann static inline bool mm_tlb_flush_pending(struct mm_struct *mm)
50736090defSArnd Bergmann {
50836090defSArnd Bergmann /*
50936090defSArnd Bergmann * Must be called after having acquired the PTL; orders against that
51036090defSArnd Bergmann * PTLs release and therefore ensures that if we observe the modified
51136090defSArnd Bergmann * PTE we must also observe the increment from inc_tlb_flush_pending().
51236090defSArnd Bergmann *
51336090defSArnd Bergmann * That is, it only guarantees to return true if there is a flush
51436090defSArnd Bergmann * pending for _this_ PTL.
51536090defSArnd Bergmann */
51636090defSArnd Bergmann return atomic_read(&mm->tlb_flush_pending);
51736090defSArnd Bergmann }
51836090defSArnd Bergmann
mm_tlb_flush_nested(struct mm_struct * mm)51936090defSArnd Bergmann static inline bool mm_tlb_flush_nested(struct mm_struct *mm)
52036090defSArnd Bergmann {
52136090defSArnd Bergmann /*
52236090defSArnd Bergmann * Similar to mm_tlb_flush_pending(), we must have acquired the PTL
52336090defSArnd Bergmann * for which there is a TLB flush pending in order to guarantee
52436090defSArnd Bergmann * we've seen both that PTE modification and the increment.
52536090defSArnd Bergmann *
52636090defSArnd Bergmann * (no requirement on actually still holding the PTL, that is irrelevant)
52736090defSArnd Bergmann */
52836090defSArnd Bergmann return atomic_read(&mm->tlb_flush_pending) > 1;
52936090defSArnd Bergmann }
53036090defSArnd Bergmann
531f92cedfaSAndrew Morton #ifdef CONFIG_MMU
532999dad82SPeter Xu /*
533af19487fSAxel Rasmussen * Computes the pte marker to copy from the given source entry into dst_vma.
534af19487fSAxel Rasmussen * If no marker should be copied, returns 0.
535af19487fSAxel Rasmussen * The caller should insert a new pte created with make_pte_marker().
536af19487fSAxel Rasmussen */
copy_pte_marker(swp_entry_t entry,struct vm_area_struct * dst_vma)537af19487fSAxel Rasmussen static inline pte_marker copy_pte_marker(
538af19487fSAxel Rasmussen swp_entry_t entry, struct vm_area_struct *dst_vma)
539af19487fSAxel Rasmussen {
540af19487fSAxel Rasmussen pte_marker srcm = pte_marker_get(entry);
541af19487fSAxel Rasmussen /* Always copy error entries. */
542af19487fSAxel Rasmussen pte_marker dstm = srcm & PTE_MARKER_POISONED;
543af19487fSAxel Rasmussen
544af19487fSAxel Rasmussen /* Only copy PTE markers if UFFD register matches. */
545af19487fSAxel Rasmussen if ((srcm & PTE_MARKER_UFFD_WP) && userfaultfd_wp(dst_vma))
546af19487fSAxel Rasmussen dstm |= PTE_MARKER_UFFD_WP;
547af19487fSAxel Rasmussen
548af19487fSAxel Rasmussen return dstm;
549af19487fSAxel Rasmussen }
550f92cedfaSAndrew Morton #endif
551af19487fSAxel Rasmussen
552af19487fSAxel Rasmussen /*
553999dad82SPeter Xu * If this pte is wr-protected by uffd-wp in any form, arm the special pte to
554999dad82SPeter Xu * replace a none pte. NOTE! This should only be called when *pte is already
555999dad82SPeter Xu * cleared so we will never accidentally replace something valuable. Meanwhile
556999dad82SPeter Xu * none pte also means we are not demoting the pte so tlb flushed is not needed.
557999dad82SPeter Xu * E.g., when pte cleared the caller should have taken care of the tlb flush.
558999dad82SPeter Xu *
559999dad82SPeter Xu * Must be called with pgtable lock held so that no thread will see the none
560999dad82SPeter Xu * pte, and if they see it, they'll fault and serialize at the pgtable lock.
561999dad82SPeter Xu *
562999dad82SPeter Xu * This function is a no-op if PTE_MARKER_UFFD_WP is not enabled.
563999dad82SPeter Xu */
564999dad82SPeter Xu static inline void
pte_install_uffd_wp_if_needed(struct vm_area_struct * vma,unsigned long addr,pte_t * pte,pte_t pteval)565999dad82SPeter Xu pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr,
566999dad82SPeter Xu pte_t *pte, pte_t pteval)
567999dad82SPeter Xu {
568999dad82SPeter Xu #ifdef CONFIG_PTE_MARKER_UFFD_WP
569999dad82SPeter Xu bool arm_uffd_pte = false;
570999dad82SPeter Xu
571999dad82SPeter Xu /* The current status of the pte should be "cleared" before calling */
572c33c7948SRyan Roberts WARN_ON_ONCE(!pte_none(ptep_get(pte)));
573999dad82SPeter Xu
5742bad466cSPeter Xu /*
5752bad466cSPeter Xu * NOTE: userfaultfd_wp_unpopulated() doesn't need this whole
5762bad466cSPeter Xu * thing, because when zapping either it means it's dropping the
5772bad466cSPeter Xu * page, or in TTU where the present pte will be quickly replaced
5782bad466cSPeter Xu * with a swap pte. There's no way of leaking the bit.
5792bad466cSPeter Xu */
580999dad82SPeter Xu if (vma_is_anonymous(vma) || !userfaultfd_wp(vma))
581999dad82SPeter Xu return;
582999dad82SPeter Xu
583999dad82SPeter Xu /* A uffd-wp wr-protected normal pte */
584999dad82SPeter Xu if (unlikely(pte_present(pteval) && pte_uffd_wp(pteval)))
585999dad82SPeter Xu arm_uffd_pte = true;
586999dad82SPeter Xu
587999dad82SPeter Xu /*
588999dad82SPeter Xu * A uffd-wp wr-protected swap pte. Note: this should even cover an
589999dad82SPeter Xu * existing pte marker with uffd-wp bit set.
590999dad82SPeter Xu */
591999dad82SPeter Xu if (unlikely(pte_swp_uffd_wp_any(pteval)))
592999dad82SPeter Xu arm_uffd_pte = true;
593999dad82SPeter Xu
594999dad82SPeter Xu if (unlikely(arm_uffd_pte))
595999dad82SPeter Xu set_pte_at(vma->vm_mm, addr, pte,
596999dad82SPeter Xu make_pte_marker(PTE_MARKER_UFFD_WP));
597999dad82SPeter Xu #endif
598999dad82SPeter Xu }
59936090defSArnd Bergmann
vma_has_recency(struct vm_area_struct * vma)6008788f678SYu Zhao static inline bool vma_has_recency(struct vm_area_struct *vma)
6018788f678SYu Zhao {
6028788f678SYu Zhao if (vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ))
6038788f678SYu Zhao return false;
6048788f678SYu Zhao
60517e81022SYu Zhao if (vma->vm_file && (vma->vm_file->f_mode & FMODE_NOREUSE))
60617e81022SYu Zhao return false;
60717e81022SYu Zhao
6088788f678SYu Zhao return true;
6098788f678SYu Zhao }
6108788f678SYu Zhao
611b2e18538SRik van Riel #endif
612