xref: /openbmc/linux/include/linux/page-flags.h (revision 4d75f5c664195b970e1cd2fd25b65b5eff257a0a)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * Macros for manipulating and testing page->flags
4  */
5 
6 #ifndef PAGE_FLAGS_H
7 #define PAGE_FLAGS_H
8 
9 #include <linux/types.h>
10 #include <linux/bug.h>
11 #include <linux/mmdebug.h>
12 #ifndef __GENERATING_BOUNDS_H
13 #include <linux/mm_types.h>
14 #include <generated/bounds.h>
15 #endif /* !__GENERATING_BOUNDS_H */
16 
17 /*
18  * Various page->flags bits:
19  *
20  * PG_reserved is set for special pages. The "struct page" of such a page
21  * should in general not be touched (e.g. set dirty) except by its owner.
22  * Pages marked as PG_reserved include:
23  * - Pages part of the kernel image (including vDSO) and similar (e.g. BIOS,
24  *   initrd, HW tables)
25  * - Pages reserved or allocated early during boot (before the page allocator
26  *   was initialized). This includes (depending on the architecture) the
27  *   initial vmemmap, initial page tables, crashkernel, elfcorehdr, and much
28  *   much more. Once (if ever) freed, PG_reserved is cleared and they will
29  *   be given to the page allocator.
30  * - Pages falling into physical memory gaps - not IORESOURCE_SYSRAM. Trying
31  *   to read/write these pages might end badly. Don't touch!
32  * - The zero page(s)
33  * - Pages not added to the page allocator when onlining a section because
34  *   they were excluded via the online_page_callback() or because they are
35  *   PG_hwpoison.
36  * - Pages allocated in the context of kexec/kdump (loaded kernel image,
37  *   control pages, vmcoreinfo)
38  * - MMIO/DMA pages. Some architectures don't allow to ioremap pages that are
39  *   not marked PG_reserved (as they might be in use by somebody else who does
40  *   not respect the caching strategy).
41  * - Pages part of an offline section (struct pages of offline sections should
42  *   not be trusted as they will be initialized when first onlined).
43  * - MCA pages on ia64
44  * - Pages holding CPU notes for POWER Firmware Assisted Dump
45  * - Device memory (e.g. PMEM, DAX, HMM)
46  * Some PG_reserved pages will be excluded from the hibernation image.
47  * PG_reserved does in general not hinder anybody from dumping or swapping
48  * and is no longer required for remap_pfn_range(). ioremap might require it.
49  * Consequently, PG_reserved for a page mapped into user space can indicate
50  * the zero page, the vDSO, MMIO pages or device memory.
51  *
52  * The PG_private bitflag is set on pagecache pages if they contain filesystem
53  * specific data (which is normally at page->private). It can be used by
54  * private allocations for its own usage.
55  *
56  * During initiation of disk I/O, PG_locked is set. This bit is set before I/O
57  * and cleared when writeback _starts_ or when read _completes_. PG_writeback
58  * is set before writeback starts and cleared when it finishes.
59  *
60  * PG_locked also pins a page in pagecache, and blocks truncation of the file
61  * while it is held.
62  *
63  * page_waitqueue(page) is a wait queue of all tasks waiting for the page
64  * to become unlocked.
65  *
66  * PG_swapbacked is set when a page uses swap as a backing storage.  This are
67  * usually PageAnon or shmem pages but please note that even anonymous pages
68  * might lose their PG_swapbacked flag when they simply can be dropped (e.g. as
69  * a result of MADV_FREE).
70  *
71  * PG_referenced, PG_reclaim are used for page reclaim for anonymous and
72  * file-backed pagecache (see mm/vmscan.c).
73  *
74  * PG_error is set to indicate that an I/O error occurred on this page.
75  *
76  * PG_arch_1 is an architecture specific page state bit.  The generic code
77  * guarantees that this bit is cleared for a page when it first is entered into
78  * the page cache.
79  *
80  * PG_hwpoison indicates that a page got corrupted in hardware and contains
81  * data with incorrect ECC bits that triggered a machine check. Accessing is
82  * not safe since it may cause another machine check. Don't touch!
83  */
84 
85 /*
86  * Don't use the pageflags directly.  Use the PageFoo macros.
87  *
88  * The page flags field is split into two parts, the main flags area
89  * which extends from the low bits upwards, and the fields area which
90  * extends from the high bits downwards.
91  *
92  *  | FIELD | ... | FLAGS |
93  *  N-1           ^       0
94  *               (NR_PAGEFLAGS)
95  *
96  * The fields area is reserved for fields mapping zone, node (for NUMA) and
97  * SPARSEMEM section (for variants of SPARSEMEM that require section ids like
98  * SPARSEMEM_EXTREME with !SPARSEMEM_VMEMMAP).
99  */
100 enum pageflags {
101 	PG_locked,		/* Page is locked. Don't touch. */
102 	PG_writeback,		/* Page is under writeback */
103 	PG_referenced,
104 	PG_uptodate,
105 	PG_dirty,
106 	PG_lru,
107 	PG_head,		/* Must be in bit 6 */
108 	PG_waiters,		/* Page has waiters, check its waitqueue. Must be bit #7 and in the same byte as "PG_locked" */
109 	PG_active,
110 	PG_workingset,
111 	PG_error,
112 	PG_slab,
113 	PG_owner_priv_1,	/* Owner use. If pagecache, fs may use*/
114 	PG_arch_1,
115 	PG_reserved,
116 	PG_private,		/* If pagecache, has fs-private data */
117 	PG_private_2,		/* If pagecache, has fs aux data */
118 	PG_mappedtodisk,	/* Has blocks allocated on-disk */
119 	PG_reclaim,		/* To be reclaimed asap */
120 	PG_swapbacked,		/* Page is backed by RAM/swap */
121 	PG_unevictable,		/* Page is "unevictable"  */
122 #ifdef CONFIG_MMU
123 	PG_mlocked,		/* Page is vma mlocked */
124 #endif
125 #ifdef CONFIG_ARCH_USES_PG_UNCACHED
126 	PG_uncached,		/* Page has been mapped as uncached */
127 #endif
128 #ifdef CONFIG_MEMORY_FAILURE
129 	PG_hwpoison,		/* hardware poisoned page. Don't touch */
130 #endif
131 #if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT)
132 	PG_young,
133 	PG_idle,
134 #endif
135 #ifdef CONFIG_ARCH_USES_PG_ARCH_X
136 	PG_arch_2,
137 	PG_arch_3,
138 #endif
139 	__NR_PAGEFLAGS,
140 
141 	PG_readahead = PG_reclaim,
142 
143 	/*
144 	 * Depending on the way an anonymous folio can be mapped into a page
145 	 * table (e.g., single PMD/PUD/CONT of the head page vs. PTE-mapped
146 	 * THP), PG_anon_exclusive may be set only for the head page or for
147 	 * tail pages of an anonymous folio. For now, we only expect it to be
148 	 * set on tail pages for PTE-mapped THP.
149 	 */
150 	PG_anon_exclusive = PG_mappedtodisk,
151 
152 	/* Filesystems */
153 	PG_checked = PG_owner_priv_1,
154 
155 	/* SwapBacked */
156 	PG_swapcache = PG_owner_priv_1,	/* Swap page: swp_entry_t in private */
157 
158 	/* Two page bits are conscripted by FS-Cache to maintain local caching
159 	 * state.  These bits are set on pages belonging to the netfs's inodes
160 	 * when those inodes are being locally cached.
161 	 */
162 	PG_fscache = PG_private_2,	/* page backed by cache */
163 
164 	/* XEN */
165 	/* Pinned in Xen as a read-only pagetable page. */
166 	PG_pinned = PG_owner_priv_1,
167 	/* Pinned as part of domain save (see xen_mm_pin_all()). */
168 	PG_savepinned = PG_dirty,
169 	/* Has a grant mapping of another (foreign) domain's page. */
170 	PG_foreign = PG_owner_priv_1,
171 	/* Remapped by swiotlb-xen. */
172 	PG_xen_remapped = PG_owner_priv_1,
173 
174 	/* non-lru isolated movable page */
175 	PG_isolated = PG_reclaim,
176 
177 	/* Only valid for buddy pages. Used to track pages that are reported */
178 	PG_reported = PG_uptodate,
179 
180 #ifdef CONFIG_MEMORY_HOTPLUG
181 	/* For self-hosted memmap pages */
182 	PG_vmemmap_self_hosted = PG_owner_priv_1,
183 #endif
184 
185 	/*
186 	 * Flags only valid for compound pages.  Stored in first tail page's
187 	 * flags word.  Cannot use the first 8 flags or any flag marked as
188 	 * PF_ANY.
189 	 */
190 
191 	/* At least one page in this folio has the hwpoison flag set */
192 	PG_has_hwpoisoned = PG_error,
193 	PG_large_rmappable = PG_workingset, /* anon or file-backed */
194 };
195 
196 #define PAGEFLAGS_MASK		((1UL << NR_PAGEFLAGS) - 1)
197 
198 #ifndef __GENERATING_BOUNDS_H
199 
200 #ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
201 DECLARE_STATIC_KEY_FALSE(hugetlb_optimize_vmemmap_key);
202 
203 /*
204  * Return the real head page struct iff the @page is a fake head page, otherwise
205  * return the @page itself. See Documentation/mm/vmemmap_dedup.rst.
206  */
page_fixed_fake_head(const struct page * page)207 static __always_inline const struct page *page_fixed_fake_head(const struct page *page)
208 {
209 	if (!static_branch_unlikely(&hugetlb_optimize_vmemmap_key))
210 		return page;
211 
212 	/*
213 	 * Only addresses aligned with PAGE_SIZE of struct page may be fake head
214 	 * struct page. The alignment check aims to avoid access the fields (
215 	 * e.g. compound_head) of the @page[1]. It can avoid touch a (possibly)
216 	 * cold cacheline in some cases.
217 	 */
218 	if (IS_ALIGNED((unsigned long)page, PAGE_SIZE) &&
219 	    test_bit(PG_head, &page->flags)) {
220 		/*
221 		 * We can safely access the field of the @page[1] with PG_head
222 		 * because the @page is a compound page composed with at least
223 		 * two contiguous pages.
224 		 */
225 		unsigned long head = READ_ONCE(page[1].compound_head);
226 
227 		if (likely(head & 1))
228 			return (const struct page *)(head - 1);
229 	}
230 	return page;
231 }
232 #else
page_fixed_fake_head(const struct page * page)233 static inline const struct page *page_fixed_fake_head(const struct page *page)
234 {
235 	return page;
236 }
237 #endif
238 
page_is_fake_head(struct page * page)239 static __always_inline int page_is_fake_head(struct page *page)
240 {
241 	return page_fixed_fake_head(page) != page;
242 }
243 
_compound_head(const struct page * page)244 static inline unsigned long _compound_head(const struct page *page)
245 {
246 	unsigned long head = READ_ONCE(page->compound_head);
247 
248 	if (unlikely(head & 1))
249 		return head - 1;
250 	return (unsigned long)page_fixed_fake_head(page);
251 }
252 
253 #define compound_head(page)	((typeof(page))_compound_head(page))
254 
255 /**
256  * page_folio - Converts from page to folio.
257  * @p: The page.
258  *
259  * Every page is part of a folio.  This function cannot be called on a
260  * NULL pointer.
261  *
262  * Context: No reference, nor lock is required on @page.  If the caller
263  * does not hold a reference, this call may race with a folio split, so
264  * it should re-check the folio still contains this page after gaining
265  * a reference on the folio.
266  * Return: The folio which contains this page.
267  */
268 #define page_folio(p)		(_Generic((p),				\
269 	const struct page *:	(const struct folio *)_compound_head(p), \
270 	struct page *:		(struct folio *)_compound_head(p)))
271 
272 /**
273  * folio_page - Return a page from a folio.
274  * @folio: The folio.
275  * @n: The page number to return.
276  *
277  * @n is relative to the start of the folio.  This function does not
278  * check that the page number lies within @folio; the caller is presumed
279  * to have a reference to the page.
280  */
281 #define folio_page(folio, n)	nth_page(&(folio)->page, n)
282 
PageTail(struct page * page)283 static __always_inline int PageTail(struct page *page)
284 {
285 	return READ_ONCE(page->compound_head) & 1 || page_is_fake_head(page);
286 }
287 
PageCompound(struct page * page)288 static __always_inline int PageCompound(struct page *page)
289 {
290 	return test_bit(PG_head, &page->flags) ||
291 	       READ_ONCE(page->compound_head) & 1;
292 }
293 
294 #define	PAGE_POISON_PATTERN	-1l
PagePoisoned(const struct page * page)295 static inline int PagePoisoned(const struct page *page)
296 {
297 	return READ_ONCE(page->flags) == PAGE_POISON_PATTERN;
298 }
299 
300 #ifdef CONFIG_DEBUG_VM
301 void page_init_poison(struct page *page, size_t size);
302 #else
page_init_poison(struct page * page,size_t size)303 static inline void page_init_poison(struct page *page, size_t size)
304 {
305 }
306 #endif
307 
folio_flags(struct folio * folio,unsigned n)308 static unsigned long *folio_flags(struct folio *folio, unsigned n)
309 {
310 	struct page *page = &folio->page;
311 
312 	VM_BUG_ON_PGFLAGS(PageTail(page), page);
313 	VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags), page);
314 	return &page[n].flags;
315 }
316 
317 /*
318  * Page flags policies wrt compound pages
319  *
320  * PF_POISONED_CHECK
321  *     check if this struct page poisoned/uninitialized
322  *
323  * PF_ANY:
324  *     the page flag is relevant for small, head and tail pages.
325  *
326  * PF_HEAD:
327  *     for compound page all operations related to the page flag applied to
328  *     head page.
329  *
330  * PF_ONLY_HEAD:
331  *     for compound page, callers only ever operate on the head page.
332  *
333  * PF_NO_TAIL:
334  *     modifications of the page flag must be done on small or head pages,
335  *     checks can be done on tail pages too.
336  *
337  * PF_NO_COMPOUND:
338  *     the page flag is not relevant for compound pages.
339  *
340  * PF_SECOND:
341  *     the page flag is stored in the first tail page.
342  */
343 #define PF_POISONED_CHECK(page) ({					\
344 		VM_BUG_ON_PGFLAGS(PagePoisoned(page), page);		\
345 		page; })
346 #define PF_ANY(page, enforce)	PF_POISONED_CHECK(page)
347 #define PF_HEAD(page, enforce)	PF_POISONED_CHECK(compound_head(page))
348 #define PF_ONLY_HEAD(page, enforce) ({					\
349 		VM_BUG_ON_PGFLAGS(PageTail(page), page);		\
350 		PF_POISONED_CHECK(page); })
351 #define PF_NO_TAIL(page, enforce) ({					\
352 		VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page);	\
353 		PF_POISONED_CHECK(compound_head(page)); })
354 #define PF_NO_COMPOUND(page, enforce) ({				\
355 		VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page);	\
356 		PF_POISONED_CHECK(page); })
357 #define PF_SECOND(page, enforce) ({					\
358 		VM_BUG_ON_PGFLAGS(!PageHead(page), page);		\
359 		PF_POISONED_CHECK(&page[1]); })
360 
361 /* Which page is the flag stored in */
362 #define FOLIO_PF_ANY		0
363 #define FOLIO_PF_HEAD		0
364 #define FOLIO_PF_ONLY_HEAD	0
365 #define FOLIO_PF_NO_TAIL	0
366 #define FOLIO_PF_NO_COMPOUND	0
367 #define FOLIO_PF_SECOND		1
368 
369 /*
370  * Macros to create function definitions for page flags
371  */
372 #define TESTPAGEFLAG(uname, lname, policy)				\
373 static __always_inline bool folio_test_##lname(struct folio *folio)	\
374 { return test_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); }	\
375 static __always_inline int Page##uname(struct page *page)		\
376 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
377 
378 #define SETPAGEFLAG(uname, lname, policy)				\
379 static __always_inline							\
380 void folio_set_##lname(struct folio *folio)				\
381 { set_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); }		\
382 static __always_inline void SetPage##uname(struct page *page)		\
383 { set_bit(PG_##lname, &policy(page, 1)->flags); }
384 
385 #define CLEARPAGEFLAG(uname, lname, policy)				\
386 static __always_inline							\
387 void folio_clear_##lname(struct folio *folio)				\
388 { clear_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); }		\
389 static __always_inline void ClearPage##uname(struct page *page)		\
390 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
391 
392 #define __SETPAGEFLAG(uname, lname, policy)				\
393 static __always_inline							\
394 void __folio_set_##lname(struct folio *folio)				\
395 { __set_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); }		\
396 static __always_inline void __SetPage##uname(struct page *page)		\
397 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
398 
399 #define __CLEARPAGEFLAG(uname, lname, policy)				\
400 static __always_inline							\
401 void __folio_clear_##lname(struct folio *folio)				\
402 { __clear_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); }	\
403 static __always_inline void __ClearPage##uname(struct page *page)	\
404 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
405 
406 #define TESTSETFLAG(uname, lname, policy)				\
407 static __always_inline							\
408 bool folio_test_set_##lname(struct folio *folio)			\
409 { return test_and_set_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
410 static __always_inline int TestSetPage##uname(struct page *page)	\
411 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
412 
413 #define TESTCLEARFLAG(uname, lname, policy)				\
414 static __always_inline							\
415 bool folio_test_clear_##lname(struct folio *folio)			\
416 { return test_and_clear_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
417 static __always_inline int TestClearPage##uname(struct page *page)	\
418 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
419 
420 #define PAGEFLAG(uname, lname, policy)					\
421 	TESTPAGEFLAG(uname, lname, policy)				\
422 	SETPAGEFLAG(uname, lname, policy)				\
423 	CLEARPAGEFLAG(uname, lname, policy)
424 
425 #define __PAGEFLAG(uname, lname, policy)				\
426 	TESTPAGEFLAG(uname, lname, policy)				\
427 	__SETPAGEFLAG(uname, lname, policy)				\
428 	__CLEARPAGEFLAG(uname, lname, policy)
429 
430 #define TESTSCFLAG(uname, lname, policy)				\
431 	TESTSETFLAG(uname, lname, policy)				\
432 	TESTCLEARFLAG(uname, lname, policy)
433 
434 #define FOLIO_TEST_FLAG_FALSE(name)					\
435 static inline bool folio_test_##name(const struct folio *folio)		\
436 { return false; }
437 #define FOLIO_SET_FLAG_NOOP(name)					\
438 static inline void folio_set_##name(struct folio *folio) { }
439 #define FOLIO_CLEAR_FLAG_NOOP(name)					\
440 static inline void folio_clear_##name(struct folio *folio) { }
441 #define __FOLIO_SET_FLAG_NOOP(name)					\
442 static inline void __folio_set_##name(struct folio *folio) { }
443 #define __FOLIO_CLEAR_FLAG_NOOP(name)					\
444 static inline void __folio_clear_##name(struct folio *folio) { }
445 #define FOLIO_TEST_SET_FLAG_FALSE(name)					\
446 static inline bool folio_test_set_##name(struct folio *folio)		\
447 { return false; }
448 #define FOLIO_TEST_CLEAR_FLAG_FALSE(name)				\
449 static inline bool folio_test_clear_##name(struct folio *folio)		\
450 { return false; }
451 
452 #define FOLIO_FLAG_FALSE(name)						\
453 FOLIO_TEST_FLAG_FALSE(name)						\
454 FOLIO_SET_FLAG_NOOP(name)						\
455 FOLIO_CLEAR_FLAG_NOOP(name)
456 
457 #define TESTPAGEFLAG_FALSE(uname, lname)				\
458 FOLIO_TEST_FLAG_FALSE(lname)						\
459 static inline int Page##uname(const struct page *page) { return 0; }
460 
461 #define SETPAGEFLAG_NOOP(uname, lname)					\
462 FOLIO_SET_FLAG_NOOP(lname)						\
463 static inline void SetPage##uname(struct page *page) {  }
464 
465 #define CLEARPAGEFLAG_NOOP(uname, lname)				\
466 FOLIO_CLEAR_FLAG_NOOP(lname)						\
467 static inline void ClearPage##uname(struct page *page) {  }
468 
469 #define __CLEARPAGEFLAG_NOOP(uname, lname)				\
470 __FOLIO_CLEAR_FLAG_NOOP(lname)						\
471 static inline void __ClearPage##uname(struct page *page) {  }
472 
473 #define TESTSETFLAG_FALSE(uname, lname)					\
474 FOLIO_TEST_SET_FLAG_FALSE(lname)					\
475 static inline int TestSetPage##uname(struct page *page) { return 0; }
476 
477 #define TESTCLEARFLAG_FALSE(uname, lname)				\
478 FOLIO_TEST_CLEAR_FLAG_FALSE(lname)					\
479 static inline int TestClearPage##uname(struct page *page) { return 0; }
480 
481 #define PAGEFLAG_FALSE(uname, lname) TESTPAGEFLAG_FALSE(uname, lname)	\
482 	SETPAGEFLAG_NOOP(uname, lname) CLEARPAGEFLAG_NOOP(uname, lname)
483 
484 #define TESTSCFLAG_FALSE(uname, lname)					\
485 	TESTSETFLAG_FALSE(uname, lname) TESTCLEARFLAG_FALSE(uname, lname)
486 
487 __PAGEFLAG(Locked, locked, PF_NO_TAIL)
488 PAGEFLAG(Waiters, waiters, PF_ONLY_HEAD)
489 PAGEFLAG(Error, error, PF_NO_TAIL) TESTCLEARFLAG(Error, error, PF_NO_TAIL)
490 PAGEFLAG(Referenced, referenced, PF_HEAD)
491 	TESTCLEARFLAG(Referenced, referenced, PF_HEAD)
492 	__SETPAGEFLAG(Referenced, referenced, PF_HEAD)
493 PAGEFLAG(Dirty, dirty, PF_HEAD) TESTSCFLAG(Dirty, dirty, PF_HEAD)
494 	__CLEARPAGEFLAG(Dirty, dirty, PF_HEAD)
495 PAGEFLAG(LRU, lru, PF_HEAD) __CLEARPAGEFLAG(LRU, lru, PF_HEAD)
496 	TESTCLEARFLAG(LRU, lru, PF_HEAD)
497 PAGEFLAG(Active, active, PF_HEAD) __CLEARPAGEFLAG(Active, active, PF_HEAD)
498 	TESTCLEARFLAG(Active, active, PF_HEAD)
499 PAGEFLAG(Workingset, workingset, PF_HEAD)
500 	TESTCLEARFLAG(Workingset, workingset, PF_HEAD)
501 __PAGEFLAG(Slab, slab, PF_NO_TAIL)
502 PAGEFLAG(Checked, checked, PF_NO_COMPOUND)	   /* Used by some filesystems */
503 
504 /* Xen */
505 PAGEFLAG(Pinned, pinned, PF_NO_COMPOUND)
506 	TESTSCFLAG(Pinned, pinned, PF_NO_COMPOUND)
507 PAGEFLAG(SavePinned, savepinned, PF_NO_COMPOUND);
508 PAGEFLAG(Foreign, foreign, PF_NO_COMPOUND);
PAGEFLAG(XenRemapped,xen_remapped,PF_NO_COMPOUND)509 PAGEFLAG(XenRemapped, xen_remapped, PF_NO_COMPOUND)
510 	TESTCLEARFLAG(XenRemapped, xen_remapped, PF_NO_COMPOUND)
511 
512 PAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
513 	__CLEARPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
514 	__SETPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
515 PAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
516 	__CLEARPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
517 	__SETPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
518 
519 /*
520  * Private page markings that may be used by the filesystem that owns the page
521  * for its own purposes.
522  * - PG_private and PG_private_2 cause release_folio() and co to be invoked
523  */
524 PAGEFLAG(Private, private, PF_ANY)
525 PAGEFLAG(Private2, private_2, PF_ANY) TESTSCFLAG(Private2, private_2, PF_ANY)
526 PAGEFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
527 	TESTCLEARFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
528 
529 /*
530  * Only test-and-set exist for PG_writeback.  The unconditional operators are
531  * risky: they bypass page accounting.
532  */
533 TESTPAGEFLAG(Writeback, writeback, PF_NO_TAIL)
534 	TESTSCFLAG(Writeback, writeback, PF_NO_TAIL)
535 PAGEFLAG(MappedToDisk, mappedtodisk, PF_NO_TAIL)
536 
537 /* PG_readahead is only used for reads; PG_reclaim is only for writes */
538 PAGEFLAG(Reclaim, reclaim, PF_NO_TAIL)
539 	TESTCLEARFLAG(Reclaim, reclaim, PF_NO_TAIL)
540 PAGEFLAG(Readahead, readahead, PF_NO_COMPOUND)
541 	TESTCLEARFLAG(Readahead, readahead, PF_NO_COMPOUND)
542 
543 #ifdef CONFIG_HIGHMEM
544 /*
545  * Must use a macro here due to header dependency issues. page_zone() is not
546  * available at this point.
547  */
548 #define PageHighMem(__p) is_highmem_idx(page_zonenum(__p))
549 #define folio_test_highmem(__f)	is_highmem_idx(folio_zonenum(__f))
550 #else
551 PAGEFLAG_FALSE(HighMem, highmem)
552 #endif
553 
554 /* Does kmap_local_folio() only allow access to one page of the folio? */
555 #ifdef CONFIG_DEBUG_KMAP_LOCAL_FORCE_MAP
556 #define folio_test_partial_kmap(f)	true
557 #else
558 #define folio_test_partial_kmap(f)	folio_test_highmem(f)
559 #endif
560 
561 #ifdef CONFIG_SWAP
562 static __always_inline bool folio_test_swapcache(struct folio *folio)
563 {
564 	return folio_test_swapbacked(folio) &&
565 			test_bit(PG_swapcache, folio_flags(folio, 0));
566 }
567 
PageSwapCache(struct page * page)568 static __always_inline bool PageSwapCache(struct page *page)
569 {
570 	return folio_test_swapcache(page_folio(page));
571 }
572 
573 SETPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
574 CLEARPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
575 #else
576 PAGEFLAG_FALSE(SwapCache, swapcache)
577 #endif
578 
579 PAGEFLAG(Unevictable, unevictable, PF_HEAD)
580 	__CLEARPAGEFLAG(Unevictable, unevictable, PF_HEAD)
581 	TESTCLEARFLAG(Unevictable, unevictable, PF_HEAD)
582 
583 #ifdef CONFIG_MMU
584 PAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
585 	__CLEARPAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
586 	TESTSCFLAG(Mlocked, mlocked, PF_NO_TAIL)
587 #else
588 PAGEFLAG_FALSE(Mlocked, mlocked) __CLEARPAGEFLAG_NOOP(Mlocked, mlocked)
589 	TESTSCFLAG_FALSE(Mlocked, mlocked)
590 #endif
591 
592 #ifdef CONFIG_ARCH_USES_PG_UNCACHED
593 PAGEFLAG(Uncached, uncached, PF_NO_COMPOUND)
594 #else
595 PAGEFLAG_FALSE(Uncached, uncached)
596 #endif
597 
598 #ifdef CONFIG_MEMORY_FAILURE
599 PAGEFLAG(HWPoison, hwpoison, PF_ANY)
600 TESTSCFLAG(HWPoison, hwpoison, PF_ANY)
601 #define __PG_HWPOISON (1UL << PG_hwpoison)
602 #define MAGIC_HWPOISON	0x48575053U	/* HWPS */
603 extern void SetPageHWPoisonTakenOff(struct page *page);
604 extern void ClearPageHWPoisonTakenOff(struct page *page);
605 extern bool take_page_off_buddy(struct page *page);
606 extern bool put_page_back_buddy(struct page *page);
607 #else
608 PAGEFLAG_FALSE(HWPoison, hwpoison)
609 #define __PG_HWPOISON 0
610 #endif
611 
612 #if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT)
TESTPAGEFLAG(Young,young,PF_ANY)613 TESTPAGEFLAG(Young, young, PF_ANY)
614 SETPAGEFLAG(Young, young, PF_ANY)
615 TESTCLEARFLAG(Young, young, PF_ANY)
616 PAGEFLAG(Idle, idle, PF_ANY)
617 #endif
618 
619 /*
620  * PageReported() is used to track reported free pages within the Buddy
621  * allocator. We can use the non-atomic version of the test and set
622  * operations as both should be shielded with the zone lock to prevent
623  * any possible races on the setting or clearing of the bit.
624  */
625 __PAGEFLAG(Reported, reported, PF_NO_COMPOUND)
626 
627 #ifdef CONFIG_MEMORY_HOTPLUG
628 PAGEFLAG(VmemmapSelfHosted, vmemmap_self_hosted, PF_ANY)
629 #else
630 PAGEFLAG_FALSE(VmemmapSelfHosted, vmemmap_self_hosted)
631 #endif
632 
633 /*
634  * On an anonymous page mapped into a user virtual memory area,
635  * page->mapping points to its anon_vma, not to a struct address_space;
636  * with the PAGE_MAPPING_ANON bit set to distinguish it.  See rmap.h.
637  *
638  * On an anonymous page in a VM_MERGEABLE area, if CONFIG_KSM is enabled,
639  * the PAGE_MAPPING_MOVABLE bit may be set along with the PAGE_MAPPING_ANON
640  * bit; and then page->mapping points, not to an anon_vma, but to a private
641  * structure which KSM associates with that merged page.  See ksm.h.
642  *
643  * PAGE_MAPPING_KSM without PAGE_MAPPING_ANON is used for non-lru movable
644  * page and then page->mapping points to a struct movable_operations.
645  *
646  * Please note that, confusingly, "page_mapping" refers to the inode
647  * address_space which maps the page from disk; whereas "page_mapped"
648  * refers to user virtual address space into which the page is mapped.
649  *
650  * For slab pages, since slab reuses the bits in struct page to store its
651  * internal states, the page->mapping does not exist as such, nor do these
652  * flags below.  So in order to avoid testing non-existent bits, please
653  * make sure that PageSlab(page) actually evaluates to false before calling
654  * the following functions (e.g., PageAnon).  See mm/slab.h.
655  */
656 #define PAGE_MAPPING_ANON	0x1
657 #define PAGE_MAPPING_MOVABLE	0x2
658 #define PAGE_MAPPING_KSM	(PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
659 #define PAGE_MAPPING_FLAGS	(PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
660 
661 /*
662  * Different with flags above, this flag is used only for fsdax mode.  It
663  * indicates that this page->mapping is now under reflink case.
664  */
665 #define PAGE_MAPPING_DAX_SHARED	((void *)0x1)
666 
667 static __always_inline bool folio_mapping_flags(struct folio *folio)
668 {
669 	return ((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS) != 0;
670 }
671 
PageMappingFlags(struct page * page)672 static __always_inline int PageMappingFlags(struct page *page)
673 {
674 	return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0;
675 }
676 
folio_test_anon(struct folio * folio)677 static __always_inline bool folio_test_anon(struct folio *folio)
678 {
679 	return ((unsigned long)folio->mapping & PAGE_MAPPING_ANON) != 0;
680 }
681 
PageAnon(struct page * page)682 static __always_inline bool PageAnon(struct page *page)
683 {
684 	return folio_test_anon(page_folio(page));
685 }
686 
__folio_test_movable(const struct folio * folio)687 static __always_inline bool __folio_test_movable(const struct folio *folio)
688 {
689 	return ((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS) ==
690 			PAGE_MAPPING_MOVABLE;
691 }
692 
__PageMovable(struct page * page)693 static __always_inline int __PageMovable(struct page *page)
694 {
695 	return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
696 				PAGE_MAPPING_MOVABLE;
697 }
698 
699 #ifdef CONFIG_KSM
700 /*
701  * A KSM page is one of those write-protected "shared pages" or "merged pages"
702  * which KSM maps into multiple mms, wherever identical anonymous page content
703  * is found in VM_MERGEABLE vmas.  It's a PageAnon page, pointing not to any
704  * anon_vma, but to that page's node of the stable tree.
705  */
folio_test_ksm(struct folio * folio)706 static __always_inline bool folio_test_ksm(struct folio *folio)
707 {
708 	return ((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS) ==
709 				PAGE_MAPPING_KSM;
710 }
711 
PageKsm(struct page * page)712 static __always_inline bool PageKsm(struct page *page)
713 {
714 	return folio_test_ksm(page_folio(page));
715 }
716 #else
717 TESTPAGEFLAG_FALSE(Ksm, ksm)
718 #endif
719 
720 u64 stable_page_flags(struct page *page);
721 
722 /**
723  * folio_test_uptodate - Is this folio up to date?
724  * @folio: The folio.
725  *
726  * The uptodate flag is set on a folio when every byte in the folio is
727  * at least as new as the corresponding bytes on storage.  Anonymous
728  * and CoW folios are always uptodate.  If the folio is not uptodate,
729  * some of the bytes in it may be; see the is_partially_uptodate()
730  * address_space operation.
731  */
folio_test_uptodate(struct folio * folio)732 static inline bool folio_test_uptodate(struct folio *folio)
733 {
734 	bool ret = test_bit(PG_uptodate, folio_flags(folio, 0));
735 	/*
736 	 * Must ensure that the data we read out of the folio is loaded
737 	 * _after_ we've loaded folio->flags to check the uptodate bit.
738 	 * We can skip the barrier if the folio is not uptodate, because
739 	 * we wouldn't be reading anything from it.
740 	 *
741 	 * See folio_mark_uptodate() for the other side of the story.
742 	 */
743 	if (ret)
744 		smp_rmb();
745 
746 	return ret;
747 }
748 
PageUptodate(struct page * page)749 static inline int PageUptodate(struct page *page)
750 {
751 	return folio_test_uptodate(page_folio(page));
752 }
753 
__folio_mark_uptodate(struct folio * folio)754 static __always_inline void __folio_mark_uptodate(struct folio *folio)
755 {
756 	smp_wmb();
757 	__set_bit(PG_uptodate, folio_flags(folio, 0));
758 }
759 
folio_mark_uptodate(struct folio * folio)760 static __always_inline void folio_mark_uptodate(struct folio *folio)
761 {
762 	/*
763 	 * Memory barrier must be issued before setting the PG_uptodate bit,
764 	 * so that all previous stores issued in order to bring the folio
765 	 * uptodate are actually visible before folio_test_uptodate becomes true.
766 	 */
767 	smp_wmb();
768 	set_bit(PG_uptodate, folio_flags(folio, 0));
769 }
770 
__SetPageUptodate(struct page * page)771 static __always_inline void __SetPageUptodate(struct page *page)
772 {
773 	__folio_mark_uptodate((struct folio *)page);
774 }
775 
SetPageUptodate(struct page * page)776 static __always_inline void SetPageUptodate(struct page *page)
777 {
778 	folio_mark_uptodate((struct folio *)page);
779 }
780 
781 CLEARPAGEFLAG(Uptodate, uptodate, PF_NO_TAIL)
782 
783 bool __folio_start_writeback(struct folio *folio, bool keep_write);
784 bool set_page_writeback(struct page *page);
785 
786 #define folio_start_writeback(folio)			\
787 	__folio_start_writeback(folio, false)
788 #define folio_start_writeback_keepwrite(folio)	\
789 	__folio_start_writeback(folio, true)
790 
test_set_page_writeback(struct page * page)791 static inline bool test_set_page_writeback(struct page *page)
792 {
793 	return set_page_writeback(page);
794 }
795 
folio_test_head(struct folio * folio)796 static __always_inline bool folio_test_head(struct folio *folio)
797 {
798 	return test_bit(PG_head, folio_flags(folio, FOLIO_PF_ANY));
799 }
800 
PageHead(struct page * page)801 static __always_inline int PageHead(struct page *page)
802 {
803 	PF_POISONED_CHECK(page);
804 	return test_bit(PG_head, &page->flags) && !page_is_fake_head(page);
805 }
806 
__SETPAGEFLAG(Head,head,PF_ANY)807 __SETPAGEFLAG(Head, head, PF_ANY)
808 __CLEARPAGEFLAG(Head, head, PF_ANY)
809 CLEARPAGEFLAG(Head, head, PF_ANY)
810 
811 /**
812  * folio_test_large() - Does this folio contain more than one page?
813  * @folio: The folio to test.
814  *
815  * Return: True if the folio is larger than one page.
816  */
817 static inline bool folio_test_large(struct folio *folio)
818 {
819 	return folio_test_head(folio);
820 }
821 
set_compound_head(struct page * page,struct page * head)822 static __always_inline void set_compound_head(struct page *page, struct page *head)
823 {
824 	WRITE_ONCE(page->compound_head, (unsigned long)head + 1);
825 }
826 
clear_compound_head(struct page * page)827 static __always_inline void clear_compound_head(struct page *page)
828 {
829 	WRITE_ONCE(page->compound_head, 0);
830 }
831 
832 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
ClearPageCompound(struct page * page)833 static inline void ClearPageCompound(struct page *page)
834 {
835 	BUG_ON(!PageHead(page));
836 	ClearPageHead(page);
837 }
PAGEFLAG(LargeRmappable,large_rmappable,PF_SECOND)838 PAGEFLAG(LargeRmappable, large_rmappable, PF_SECOND)
839 #else
840 TESTPAGEFLAG_FALSE(LargeRmappable, large_rmappable)
841 #endif
842 
843 #define PG_head_mask ((1UL << PG_head))
844 
845 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
846 /*
847  * PageHuge() only returns true for hugetlbfs pages, but not for
848  * normal or transparent huge pages.
849  *
850  * PageTransHuge() returns true for both transparent huge and
851  * hugetlbfs pages, but not normal pages. PageTransHuge() can only be
852  * called only in the core VM paths where hugetlbfs pages can't exist.
853  */
854 static inline int PageTransHuge(struct page *page)
855 {
856 	VM_BUG_ON_PAGE(PageTail(page), page);
857 	return PageHead(page);
858 }
859 
860 /*
861  * PageTransCompound returns true for both transparent huge pages
862  * and hugetlbfs pages, so it should only be called when it's known
863  * that hugetlbfs pages aren't involved.
864  */
PageTransCompound(struct page * page)865 static inline int PageTransCompound(struct page *page)
866 {
867 	return PageCompound(page);
868 }
869 
870 /*
871  * PageTransTail returns true for both transparent huge pages
872  * and hugetlbfs pages, so it should only be called when it's known
873  * that hugetlbfs pages aren't involved.
874  */
PageTransTail(struct page * page)875 static inline int PageTransTail(struct page *page)
876 {
877 	return PageTail(page);
878 }
879 #else
880 TESTPAGEFLAG_FALSE(TransHuge, transhuge)
881 TESTPAGEFLAG_FALSE(TransCompound, transcompound)
882 TESTPAGEFLAG_FALSE(TransCompoundMap, transcompoundmap)
883 TESTPAGEFLAG_FALSE(TransTail, transtail)
884 #endif
885 
886 #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
887 /*
888  * PageHasHWPoisoned indicates that at least one subpage is hwpoisoned in the
889  * compound page.
890  *
891  * This flag is set by hwpoison handler.  Cleared by THP split or free page.
892  */
PAGEFLAG(HasHWPoisoned,has_hwpoisoned,PF_SECOND)893 PAGEFLAG(HasHWPoisoned, has_hwpoisoned, PF_SECOND)
894 	TESTSCFLAG(HasHWPoisoned, has_hwpoisoned, PF_SECOND)
895 #else
896 PAGEFLAG_FALSE(HasHWPoisoned, has_hwpoisoned)
897 	TESTSCFLAG_FALSE(HasHWPoisoned, has_hwpoisoned)
898 #endif
899 
900 /*
901  * For pages that are never mapped to userspace (and aren't PageSlab),
902  * page_type may be used.  Because it is initialised to -1, we invert the
903  * sense of the bit, so __SetPageFoo *clears* the bit used for PageFoo, and
904  * __ClearPageFoo *sets* the bit used for PageFoo.  We reserve a few high and
905  * low bits so that an underflow or overflow of _mapcount won't be
906  * mistaken for a page type value.
907  */
908 
909 #define PAGE_TYPE_BASE	0xf0000000
910 /* Reserve		0x0000007f to catch underflows of _mapcount */
911 #define PAGE_MAPCOUNT_RESERVE	-128
912 #define PG_buddy	0x00000080
913 #define PG_offline	0x00000100
914 #define PG_table	0x00000200
915 #define PG_guard	0x00000400
916 #define PG_hugetlb	0x00000800
917 
918 #define PageType(page, flag)						\
919 	((page->page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE)
920 #define folio_test_type(folio, flag)					\
921 	((folio->page.page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE)
922 
923 static inline int page_type_has_type(unsigned int page_type)
924 {
925 	return (int)page_type < PAGE_MAPCOUNT_RESERVE;
926 }
927 
page_has_type(struct page * page)928 static inline int page_has_type(struct page *page)
929 {
930 	return page_type_has_type(page->page_type);
931 }
932 
933 #define FOLIO_TYPE_OPS(lname, fname)					\
934 static __always_inline bool folio_test_##fname(const struct folio *folio)\
935 {									\
936 	return folio_test_type(folio, PG_##lname);			\
937 }									\
938 static __always_inline void __folio_set_##fname(struct folio *folio)	\
939 {									\
940 	VM_BUG_ON_FOLIO(!folio_test_type(folio, 0), folio);		\
941 	folio->page.page_type &= ~PG_##lname;				\
942 }									\
943 static __always_inline void __folio_clear_##fname(struct folio *folio)	\
944 {									\
945 	VM_BUG_ON_FOLIO(!folio_test_##fname(folio), folio);		\
946 	folio->page.page_type |= PG_##lname;				\
947 }
948 
949 #define PAGE_TYPE_OPS(uname, lname, fname)				\
950 FOLIO_TYPE_OPS(lname, fname)						\
951 static __always_inline int Page##uname(const struct page *page)		\
952 {									\
953 	return PageType(page, PG_##lname);				\
954 }									\
955 static __always_inline void __SetPage##uname(struct page *page)		\
956 {									\
957 	VM_BUG_ON_PAGE(!PageType(page, 0), page);			\
958 	page->page_type &= ~PG_##lname;					\
959 }									\
960 static __always_inline void __ClearPage##uname(struct page *page)	\
961 {									\
962 	VM_BUG_ON_PAGE(!Page##uname(page), page);			\
963 	page->page_type |= PG_##lname;					\
964 }
965 
966 /*
967  * PageBuddy() indicates that the page is free and in the buddy system
968  * (see mm/page_alloc.c).
969  */
970 PAGE_TYPE_OPS(Buddy, buddy, buddy)
971 
972 /*
973  * PageOffline() indicates that the page is logically offline although the
974  * containing section is online. (e.g. inflated in a balloon driver or
975  * not onlined when onlining the section).
976  * The content of these pages is effectively stale. Such pages should not
977  * be touched (read/write/dump/save) except by their owner.
978  *
979  * If a driver wants to allow to offline unmovable PageOffline() pages without
980  * putting them back to the buddy, it can do so via the memory notifier by
981  * decrementing the reference count in MEM_GOING_OFFLINE and incrementing the
982  * reference count in MEM_CANCEL_OFFLINE. When offlining, the PageOffline()
983  * pages (now with a reference count of zero) are treated like free pages,
984  * allowing the containing memory block to get offlined. A driver that
985  * relies on this feature is aware that re-onlining the memory block will
986  * require to re-set the pages PageOffline() and not giving them to the
987  * buddy via online_page_callback_t.
988  *
989  * There are drivers that mark a page PageOffline() and expect there won't be
990  * any further access to page content. PFN walkers that read content of random
991  * pages should check PageOffline() and synchronize with such drivers using
992  * page_offline_freeze()/page_offline_thaw().
993  */
994 PAGE_TYPE_OPS(Offline, offline, offline)
995 
996 extern void page_offline_freeze(void);
997 extern void page_offline_thaw(void);
998 extern void page_offline_begin(void);
999 extern void page_offline_end(void);
1000 
1001 /*
1002  * Marks pages in use as page tables.
1003  */
PAGE_TYPE_OPS(Table,table,pgtable)1004 PAGE_TYPE_OPS(Table, table, pgtable)
1005 
1006 /*
1007  * Marks guardpages used with debug_pagealloc.
1008  */
1009 PAGE_TYPE_OPS(Guard, guard, guard)
1010 
1011 #ifdef CONFIG_HUGETLB_PAGE
1012 FOLIO_TYPE_OPS(hugetlb, hugetlb)
1013 #else
1014 FOLIO_TEST_FLAG_FALSE(hugetlb)
1015 #endif
1016 
1017 /**
1018  * PageHuge - Determine if the page belongs to hugetlbfs
1019  * @page: The page to test.
1020  *
1021  * Context: Any context.
1022  * Return: True for hugetlbfs pages, false for anon pages or pages
1023  * belonging to other filesystems.
1024  */
1025 static inline bool PageHuge(const struct page *page)
1026 {
1027 	return folio_test_hugetlb(page_folio(page));
1028 }
1029 
1030 /*
1031  * Check if a page is currently marked HWPoisoned. Note that this check is
1032  * best effort only and inherently racy: there is no way to synchronize with
1033  * failing hardware.
1034  */
is_page_hwpoison(struct page * page)1035 static inline bool is_page_hwpoison(struct page *page)
1036 {
1037 	if (PageHWPoison(page))
1038 		return true;
1039 	return PageHuge(page) && PageHWPoison(compound_head(page));
1040 }
1041 
1042 extern bool is_free_buddy_page(struct page *page);
1043 
1044 PAGEFLAG(Isolated, isolated, PF_ANY);
1045 
PageAnonExclusive(struct page * page)1046 static __always_inline int PageAnonExclusive(struct page *page)
1047 {
1048 	VM_BUG_ON_PGFLAGS(!PageAnon(page), page);
1049 	VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);
1050 	return test_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);
1051 }
1052 
SetPageAnonExclusive(struct page * page)1053 static __always_inline void SetPageAnonExclusive(struct page *page)
1054 {
1055 	VM_BUG_ON_PGFLAGS(!PageAnon(page) || PageKsm(page), page);
1056 	VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);
1057 	set_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);
1058 }
1059 
ClearPageAnonExclusive(struct page * page)1060 static __always_inline void ClearPageAnonExclusive(struct page *page)
1061 {
1062 	VM_BUG_ON_PGFLAGS(!PageAnon(page) || PageKsm(page), page);
1063 	VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);
1064 	clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);
1065 }
1066 
__ClearPageAnonExclusive(struct page * page)1067 static __always_inline void __ClearPageAnonExclusive(struct page *page)
1068 {
1069 	VM_BUG_ON_PGFLAGS(!PageAnon(page), page);
1070 	VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);
1071 	__clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);
1072 }
1073 
1074 #ifdef CONFIG_MMU
1075 #define __PG_MLOCKED		(1UL << PG_mlocked)
1076 #else
1077 #define __PG_MLOCKED		0
1078 #endif
1079 
1080 /*
1081  * Flags checked when a page is freed.  Pages being freed should not have
1082  * these flags set.  If they are, there is a problem.
1083  */
1084 #define PAGE_FLAGS_CHECK_AT_FREE				\
1085 	(1UL << PG_lru		| 1UL << PG_locked	|	\
1086 	 1UL << PG_private	| 1UL << PG_private_2	|	\
1087 	 1UL << PG_writeback	| 1UL << PG_reserved	|	\
1088 	 1UL << PG_slab		| 1UL << PG_active 	|	\
1089 	 1UL << PG_unevictable	| __PG_MLOCKED | LRU_GEN_MASK)
1090 
1091 /*
1092  * Flags checked when a page is prepped for return by the page allocator.
1093  * Pages being prepped should not have these flags set.  If they are set,
1094  * there has been a kernel bug or struct page corruption.
1095  *
1096  * __PG_HWPOISON is exceptional because it needs to be kept beyond page's
1097  * alloc-free cycle to prevent from reusing the page.
1098  */
1099 #define PAGE_FLAGS_CHECK_AT_PREP	\
1100 	((PAGEFLAGS_MASK & ~__PG_HWPOISON) | LRU_GEN_MASK | LRU_REFS_MASK)
1101 
1102 /*
1103  * Flags stored in the second page of a compound page.  They may overlap
1104  * the CHECK_AT_FREE flags above, so need to be cleared.
1105  */
1106 #define PAGE_FLAGS_SECOND						\
1107 	(0xffUL /* order */		| 1UL << PG_has_hwpoisoned |	\
1108 	 1UL << PG_large_rmappable)
1109 
1110 #define PAGE_FLAGS_PRIVATE				\
1111 	(1UL << PG_private | 1UL << PG_private_2)
1112 /**
1113  * page_has_private - Determine if page has private stuff
1114  * @page: The page to be checked
1115  *
1116  * Determine if a page has private stuff, indicating that release routines
1117  * should be invoked upon it.
1118  */
page_has_private(struct page * page)1119 static inline int page_has_private(struct page *page)
1120 {
1121 	return !!(page->flags & PAGE_FLAGS_PRIVATE);
1122 }
1123 
folio_has_private(struct folio * folio)1124 static inline bool folio_has_private(struct folio *folio)
1125 {
1126 	return page_has_private(&folio->page);
1127 }
1128 
1129 #undef PF_ANY
1130 #undef PF_HEAD
1131 #undef PF_ONLY_HEAD
1132 #undef PF_NO_TAIL
1133 #undef PF_NO_COMPOUND
1134 #undef PF_SECOND
1135 #endif /* !__GENERATING_BOUNDS_H */
1136 
1137 #endif	/* PAGE_FLAGS_H */
1138