1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_MM_H 3 #define _LINUX_MM_H 4 5 #include <linux/errno.h> 6 #include <linux/mmdebug.h> 7 #include <linux/gfp.h> 8 #include <linux/bug.h> 9 #include <linux/list.h> 10 #include <linux/mmzone.h> 11 #include <linux/rbtree.h> 12 #include <linux/atomic.h> 13 #include <linux/debug_locks.h> 14 #include <linux/mm_types.h> 15 #include <linux/mmap_lock.h> 16 #include <linux/range.h> 17 #include <linux/pfn.h> 18 #include <linux/percpu-refcount.h> 19 #include <linux/bit_spinlock.h> 20 #include <linux/shrinker.h> 21 #include <linux/resource.h> 22 #include <linux/page_ext.h> 23 #include <linux/err.h> 24 #include <linux/page-flags.h> 25 #include <linux/page_ref.h> 26 #include <linux/overflow.h> 27 #include <linux/sizes.h> 28 #include <linux/sched.h> 29 #include <linux/pgtable.h> 30 #include <linux/kasan.h> 31 #include <linux/memremap.h> 32 #include <linux/slab.h> 33 34 struct mempolicy; 35 struct anon_vma; 36 struct anon_vma_chain; 37 struct user_struct; 38 struct pt_regs; 39 40 extern int sysctl_page_lock_unfairness; 41 42 void mm_core_init(void); 43 void init_mm_internals(void); 44 45 #ifndef CONFIG_NUMA /* Don't use mapnrs, do it properly */ 46 extern unsigned long max_mapnr; 47 48 static inline void set_max_mapnr(unsigned long limit) 49 { 50 max_mapnr = limit; 51 } 52 #else 53 static inline void set_max_mapnr(unsigned long limit) { } 54 #endif 55 56 extern atomic_long_t _totalram_pages; 57 static inline unsigned long totalram_pages(void) 58 { 59 return (unsigned long)atomic_long_read(&_totalram_pages); 60 } 61 62 static inline void totalram_pages_inc(void) 63 { 64 atomic_long_inc(&_totalram_pages); 65 } 66 67 static inline void totalram_pages_dec(void) 68 { 69 atomic_long_dec(&_totalram_pages); 70 } 71 72 static inline void totalram_pages_add(long count) 73 { 74 atomic_long_add(count, &_totalram_pages); 75 } 76 77 extern void * high_memory; 78 extern int page_cluster; 79 extern const int page_cluster_max; 80 81 #ifdef CONFIG_SYSCTL 82 extern int sysctl_legacy_va_layout; 83 #else 84 #define sysctl_legacy_va_layout 0 85 #endif 86 87 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS 88 extern const int mmap_rnd_bits_min; 89 extern const int mmap_rnd_bits_max; 90 extern int mmap_rnd_bits __read_mostly; 91 #endif 92 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS 93 extern const int mmap_rnd_compat_bits_min; 94 extern const int mmap_rnd_compat_bits_max; 95 extern int mmap_rnd_compat_bits __read_mostly; 96 #endif 97 98 #include <asm/page.h> 99 #include <asm/processor.h> 100 101 #ifndef __pa_symbol 102 #define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0)) 103 #endif 104 105 #ifndef page_to_virt 106 #define page_to_virt(x) __va(PFN_PHYS(page_to_pfn(x))) 107 #endif 108 109 #ifndef lm_alias 110 #define lm_alias(x) __va(__pa_symbol(x)) 111 #endif 112 113 /* 114 * To prevent common memory management code establishing 115 * a zero page mapping on a read fault. 116 * This macro should be defined within <asm/pgtable.h>. 117 * s390 does this to prevent multiplexing of hardware bits 118 * related to the physical page in case of virtualization. 119 */ 120 #ifndef mm_forbids_zeropage 121 #define mm_forbids_zeropage(X) (0) 122 #endif 123 124 /* 125 * On some architectures it is expensive to call memset() for small sizes. 126 * If an architecture decides to implement their own version of 127 * mm_zero_struct_page they should wrap the defines below in a #ifndef and 128 * define their own version of this macro in <asm/pgtable.h> 129 */ 130 #if BITS_PER_LONG == 64 131 /* This function must be updated when the size of struct page grows above 96 132 * or reduces below 56. The idea that compiler optimizes out switch() 133 * statement, and only leaves move/store instructions. Also the compiler can 134 * combine write statements if they are both assignments and can be reordered, 135 * this can result in several of the writes here being dropped. 136 */ 137 #define mm_zero_struct_page(pp) __mm_zero_struct_page(pp) 138 static inline void __mm_zero_struct_page(struct page *page) 139 { 140 unsigned long *_pp = (void *)page; 141 142 /* Check that struct page is either 56, 64, 72, 80, 88 or 96 bytes */ 143 BUILD_BUG_ON(sizeof(struct page) & 7); 144 BUILD_BUG_ON(sizeof(struct page) < 56); 145 BUILD_BUG_ON(sizeof(struct page) > 96); 146 147 switch (sizeof(struct page)) { 148 case 96: 149 _pp[11] = 0; 150 fallthrough; 151 case 88: 152 _pp[10] = 0; 153 fallthrough; 154 case 80: 155 _pp[9] = 0; 156 fallthrough; 157 case 72: 158 _pp[8] = 0; 159 fallthrough; 160 case 64: 161 _pp[7] = 0; 162 fallthrough; 163 case 56: 164 _pp[6] = 0; 165 _pp[5] = 0; 166 _pp[4] = 0; 167 _pp[3] = 0; 168 _pp[2] = 0; 169 _pp[1] = 0; 170 _pp[0] = 0; 171 } 172 } 173 #else 174 #define mm_zero_struct_page(pp) ((void)memset((pp), 0, sizeof(struct page))) 175 #endif 176 177 /* 178 * Default maximum number of active map areas, this limits the number of vmas 179 * per mm struct. Users can overwrite this number by sysctl but there is a 180 * problem. 181 * 182 * When a program's coredump is generated as ELF format, a section is created 183 * per a vma. In ELF, the number of sections is represented in unsigned short. 184 * This means the number of sections should be smaller than 65535 at coredump. 185 * Because the kernel adds some informative sections to a image of program at 186 * generating coredump, we need some margin. The number of extra sections is 187 * 1-3 now and depends on arch. We use "5" as safe margin, here. 188 * 189 * ELF extended numbering allows more than 65535 sections, so 16-bit bound is 190 * not a hard limit any more. Although some userspace tools can be surprised by 191 * that. 192 */ 193 #define MAPCOUNT_ELF_CORE_MARGIN (5) 194 #define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN) 195 196 extern int sysctl_max_map_count; 197 198 extern unsigned long sysctl_user_reserve_kbytes; 199 extern unsigned long sysctl_admin_reserve_kbytes; 200 201 extern int sysctl_overcommit_memory; 202 extern int sysctl_overcommit_ratio; 203 extern unsigned long sysctl_overcommit_kbytes; 204 205 int overcommit_ratio_handler(struct ctl_table *, int, void *, size_t *, 206 loff_t *); 207 int overcommit_kbytes_handler(struct ctl_table *, int, void *, size_t *, 208 loff_t *); 209 int overcommit_policy_handler(struct ctl_table *, int, void *, size_t *, 210 loff_t *); 211 212 #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) 213 #define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n)) 214 #define folio_page_idx(folio, p) (page_to_pfn(p) - folio_pfn(folio)) 215 #else 216 #define nth_page(page,n) ((page) + (n)) 217 #define folio_page_idx(folio, p) ((p) - &(folio)->page) 218 #endif 219 220 /* to align the pointer to the (next) page boundary */ 221 #define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE) 222 223 /* to align the pointer to the (prev) page boundary */ 224 #define PAGE_ALIGN_DOWN(addr) ALIGN_DOWN(addr, PAGE_SIZE) 225 226 /* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */ 227 #define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), PAGE_SIZE) 228 229 #define lru_to_page(head) (list_entry((head)->prev, struct page, lru)) 230 static inline struct folio *lru_to_folio(struct list_head *head) 231 { 232 return list_entry((head)->prev, struct folio, lru); 233 } 234 235 void setup_initial_init_mm(void *start_code, void *end_code, 236 void *end_data, void *brk); 237 238 /* 239 * Linux kernel virtual memory manager primitives. 240 * The idea being to have a "virtual" mm in the same way 241 * we have a virtual fs - giving a cleaner interface to the 242 * mm details, and allowing different kinds of memory mappings 243 * (from shared memory to executable loading to arbitrary 244 * mmap() functions). 245 */ 246 247 struct vm_area_struct *vm_area_alloc(struct mm_struct *); 248 struct vm_area_struct *vm_area_dup(struct vm_area_struct *); 249 void vm_area_free(struct vm_area_struct *); 250 /* Use only if VMA has no other users */ 251 void __vm_area_free(struct vm_area_struct *vma); 252 253 #ifndef CONFIG_MMU 254 extern struct rb_root nommu_region_tree; 255 extern struct rw_semaphore nommu_region_sem; 256 257 extern unsigned int kobjsize(const void *objp); 258 #endif 259 260 /* 261 * vm_flags in vm_area_struct, see mm_types.h. 262 * When changing, update also include/trace/events/mmflags.h 263 */ 264 #define VM_NONE 0x00000000 265 266 #define VM_READ 0x00000001 /* currently active flags */ 267 #define VM_WRITE 0x00000002 268 #define VM_EXEC 0x00000004 269 #define VM_SHARED 0x00000008 270 271 /* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */ 272 #define VM_MAYREAD 0x00000010 /* limits for mprotect() etc */ 273 #define VM_MAYWRITE 0x00000020 274 #define VM_MAYEXEC 0x00000040 275 #define VM_MAYSHARE 0x00000080 276 277 #define VM_GROWSDOWN 0x00000100 /* general info on the segment */ 278 #ifdef CONFIG_MMU 279 #define VM_UFFD_MISSING 0x00000200 /* missing pages tracking */ 280 #else /* CONFIG_MMU */ 281 #define VM_MAYOVERLAY 0x00000200 /* nommu: R/O MAP_PRIVATE mapping that might overlay a file mapping */ 282 #define VM_UFFD_MISSING 0 283 #endif /* CONFIG_MMU */ 284 #define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */ 285 #define VM_UFFD_WP 0x00001000 /* wrprotect pages tracking */ 286 287 #define VM_LOCKED 0x00002000 288 #define VM_IO 0x00004000 /* Memory mapped I/O or similar */ 289 290 /* Used by sys_madvise() */ 291 #define VM_SEQ_READ 0x00008000 /* App will access data sequentially */ 292 #define VM_RAND_READ 0x00010000 /* App will not benefit from clustered reads */ 293 294 #define VM_DONTCOPY 0x00020000 /* Do not copy this vma on fork */ 295 #define VM_DONTEXPAND 0x00040000 /* Cannot expand with mremap() */ 296 #define VM_LOCKONFAULT 0x00080000 /* Lock the pages covered when they are faulted in */ 297 #define VM_ACCOUNT 0x00100000 /* Is a VM accounted object */ 298 #define VM_NORESERVE 0x00200000 /* should the VM suppress accounting */ 299 #define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */ 300 #define VM_SYNC 0x00800000 /* Synchronous page faults */ 301 #define VM_ARCH_1 0x01000000 /* Architecture-specific flag */ 302 #define VM_WIPEONFORK 0x02000000 /* Wipe VMA contents in child. */ 303 #define VM_DONTDUMP 0x04000000 /* Do not include in the core dump */ 304 305 #ifdef CONFIG_MEM_SOFT_DIRTY 306 # define VM_SOFTDIRTY 0x08000000 /* Not soft dirty clean area */ 307 #else 308 # define VM_SOFTDIRTY 0 309 #endif 310 311 #define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */ 312 #define VM_HUGEPAGE 0x20000000 /* MADV_HUGEPAGE marked this vma */ 313 #define VM_NOHUGEPAGE 0x40000000 /* MADV_NOHUGEPAGE marked this vma */ 314 #define VM_MERGEABLE 0x80000000 /* KSM may merge identical pages */ 315 316 #ifdef CONFIG_ARCH_USES_HIGH_VMA_FLAGS 317 #define VM_HIGH_ARCH_BIT_0 32 /* bit only usable on 64-bit architectures */ 318 #define VM_HIGH_ARCH_BIT_1 33 /* bit only usable on 64-bit architectures */ 319 #define VM_HIGH_ARCH_BIT_2 34 /* bit only usable on 64-bit architectures */ 320 #define VM_HIGH_ARCH_BIT_3 35 /* bit only usable on 64-bit architectures */ 321 #define VM_HIGH_ARCH_BIT_4 36 /* bit only usable on 64-bit architectures */ 322 #define VM_HIGH_ARCH_BIT_5 37 /* bit only usable on 64-bit architectures */ 323 #define VM_HIGH_ARCH_0 BIT(VM_HIGH_ARCH_BIT_0) 324 #define VM_HIGH_ARCH_1 BIT(VM_HIGH_ARCH_BIT_1) 325 #define VM_HIGH_ARCH_2 BIT(VM_HIGH_ARCH_BIT_2) 326 #define VM_HIGH_ARCH_3 BIT(VM_HIGH_ARCH_BIT_3) 327 #define VM_HIGH_ARCH_4 BIT(VM_HIGH_ARCH_BIT_4) 328 #define VM_HIGH_ARCH_5 BIT(VM_HIGH_ARCH_BIT_5) 329 #endif /* CONFIG_ARCH_USES_HIGH_VMA_FLAGS */ 330 331 #ifdef CONFIG_ARCH_HAS_PKEYS 332 # define VM_PKEY_SHIFT VM_HIGH_ARCH_BIT_0 333 # define VM_PKEY_BIT0 VM_HIGH_ARCH_0 /* A protection key is a 4-bit value */ 334 # define VM_PKEY_BIT1 VM_HIGH_ARCH_1 /* on x86 and 5-bit value on ppc64 */ 335 # define VM_PKEY_BIT2 VM_HIGH_ARCH_2 336 # define VM_PKEY_BIT3 VM_HIGH_ARCH_3 337 #ifdef CONFIG_PPC 338 # define VM_PKEY_BIT4 VM_HIGH_ARCH_4 339 #else 340 # define VM_PKEY_BIT4 0 341 #endif 342 #endif /* CONFIG_ARCH_HAS_PKEYS */ 343 344 #ifdef CONFIG_X86_USER_SHADOW_STACK 345 /* 346 * VM_SHADOW_STACK should not be set with VM_SHARED because of lack of 347 * support core mm. 348 * 349 * These VMAs will get a single end guard page. This helps userspace protect 350 * itself from attacks. A single page is enough for current shadow stack archs 351 * (x86). See the comments near alloc_shstk() in arch/x86/kernel/shstk.c 352 * for more details on the guard size. 353 */ 354 # define VM_SHADOW_STACK VM_HIGH_ARCH_5 355 #else 356 # define VM_SHADOW_STACK VM_NONE 357 #endif 358 359 #if defined(CONFIG_X86) 360 # define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */ 361 #elif defined(CONFIG_PPC) 362 # define VM_SAO VM_ARCH_1 /* Strong Access Ordering (powerpc) */ 363 #elif defined(CONFIG_PARISC) 364 # define VM_GROWSUP VM_ARCH_1 365 #elif defined(CONFIG_IA64) 366 # define VM_GROWSUP VM_ARCH_1 367 #elif defined(CONFIG_SPARC64) 368 # define VM_SPARC_ADI VM_ARCH_1 /* Uses ADI tag for access control */ 369 # define VM_ARCH_CLEAR VM_SPARC_ADI 370 #elif defined(CONFIG_ARM64) 371 # define VM_ARM64_BTI VM_ARCH_1 /* BTI guarded page, a.k.a. GP bit */ 372 # define VM_ARCH_CLEAR VM_ARM64_BTI 373 #elif !defined(CONFIG_MMU) 374 # define VM_MAPPED_COPY VM_ARCH_1 /* T if mapped copy of data (nommu mmap) */ 375 #endif 376 377 #if defined(CONFIG_ARM64_MTE) 378 # define VM_MTE VM_HIGH_ARCH_0 /* Use Tagged memory for access control */ 379 # define VM_MTE_ALLOWED VM_HIGH_ARCH_1 /* Tagged memory permitted */ 380 #else 381 # define VM_MTE VM_NONE 382 # define VM_MTE_ALLOWED VM_NONE 383 #endif 384 385 #ifndef VM_GROWSUP 386 # define VM_GROWSUP VM_NONE 387 #endif 388 389 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR 390 # define VM_UFFD_MINOR_BIT 38 391 # define VM_UFFD_MINOR BIT(VM_UFFD_MINOR_BIT) /* UFFD minor faults */ 392 #else /* !CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */ 393 # define VM_UFFD_MINOR VM_NONE 394 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */ 395 396 /* Bits set in the VMA until the stack is in its final location */ 397 #define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY) 398 399 #define TASK_EXEC ((current->personality & READ_IMPLIES_EXEC) ? VM_EXEC : 0) 400 401 /* Common data flag combinations */ 402 #define VM_DATA_FLAGS_TSK_EXEC (VM_READ | VM_WRITE | TASK_EXEC | \ 403 VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) 404 #define VM_DATA_FLAGS_NON_EXEC (VM_READ | VM_WRITE | VM_MAYREAD | \ 405 VM_MAYWRITE | VM_MAYEXEC) 406 #define VM_DATA_FLAGS_EXEC (VM_READ | VM_WRITE | VM_EXEC | \ 407 VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) 408 409 #ifndef VM_DATA_DEFAULT_FLAGS /* arch can override this */ 410 #define VM_DATA_DEFAULT_FLAGS VM_DATA_FLAGS_EXEC 411 #endif 412 413 #ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */ 414 #define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS 415 #endif 416 417 #define VM_STARTGAP_FLAGS (VM_GROWSDOWN | VM_SHADOW_STACK) 418 419 #ifdef CONFIG_STACK_GROWSUP 420 #define VM_STACK VM_GROWSUP 421 #define VM_STACK_EARLY VM_GROWSDOWN 422 #else 423 #define VM_STACK VM_GROWSDOWN 424 #define VM_STACK_EARLY 0 425 #endif 426 427 #define VM_STACK_FLAGS (VM_STACK | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT) 428 429 /* VMA basic access permission flags */ 430 #define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC) 431 432 433 /* 434 * Special vmas that are non-mergable, non-mlock()able. 435 */ 436 #define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_MIXEDMAP) 437 438 /* This mask prevents VMA from being scanned with khugepaged */ 439 #define VM_NO_KHUGEPAGED (VM_SPECIAL | VM_HUGETLB) 440 441 /* This mask defines which mm->def_flags a process can inherit its parent */ 442 #define VM_INIT_DEF_MASK VM_NOHUGEPAGE 443 444 /* This mask represents all the VMA flag bits used by mlock */ 445 #define VM_LOCKED_MASK (VM_LOCKED | VM_LOCKONFAULT) 446 447 /* Arch-specific flags to clear when updating VM flags on protection change */ 448 #ifndef VM_ARCH_CLEAR 449 # define VM_ARCH_CLEAR VM_NONE 450 #endif 451 #define VM_FLAGS_CLEAR (ARCH_VM_PKEY_FLAGS | VM_ARCH_CLEAR) 452 453 /* 454 * mapping from the currently active vm_flags protection bits (the 455 * low four bits) to a page protection mask.. 456 */ 457 458 /* 459 * The default fault flags that should be used by most of the 460 * arch-specific page fault handlers. 461 */ 462 #define FAULT_FLAG_DEFAULT (FAULT_FLAG_ALLOW_RETRY | \ 463 FAULT_FLAG_KILLABLE | \ 464 FAULT_FLAG_INTERRUPTIBLE) 465 466 /** 467 * fault_flag_allow_retry_first - check ALLOW_RETRY the first time 468 * @flags: Fault flags. 469 * 470 * This is mostly used for places where we want to try to avoid taking 471 * the mmap_lock for too long a time when waiting for another condition 472 * to change, in which case we can try to be polite to release the 473 * mmap_lock in the first round to avoid potential starvation of other 474 * processes that would also want the mmap_lock. 475 * 476 * Return: true if the page fault allows retry and this is the first 477 * attempt of the fault handling; false otherwise. 478 */ 479 static inline bool fault_flag_allow_retry_first(enum fault_flag flags) 480 { 481 return (flags & FAULT_FLAG_ALLOW_RETRY) && 482 (!(flags & FAULT_FLAG_TRIED)); 483 } 484 485 #define FAULT_FLAG_TRACE \ 486 { FAULT_FLAG_WRITE, "WRITE" }, \ 487 { FAULT_FLAG_MKWRITE, "MKWRITE" }, \ 488 { FAULT_FLAG_ALLOW_RETRY, "ALLOW_RETRY" }, \ 489 { FAULT_FLAG_RETRY_NOWAIT, "RETRY_NOWAIT" }, \ 490 { FAULT_FLAG_KILLABLE, "KILLABLE" }, \ 491 { FAULT_FLAG_TRIED, "TRIED" }, \ 492 { FAULT_FLAG_USER, "USER" }, \ 493 { FAULT_FLAG_REMOTE, "REMOTE" }, \ 494 { FAULT_FLAG_INSTRUCTION, "INSTRUCTION" }, \ 495 { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }, \ 496 { FAULT_FLAG_VMA_LOCK, "VMA_LOCK" } 497 498 /* 499 * vm_fault is filled by the pagefault handler and passed to the vma's 500 * ->fault function. The vma's ->fault is responsible for returning a bitmask 501 * of VM_FAULT_xxx flags that give details about how the fault was handled. 502 * 503 * MM layer fills up gfp_mask for page allocations but fault handler might 504 * alter it if its implementation requires a different allocation context. 505 * 506 * pgoff should be used in favour of virtual_address, if possible. 507 */ 508 struct vm_fault { 509 const struct { 510 struct vm_area_struct *vma; /* Target VMA */ 511 gfp_t gfp_mask; /* gfp mask to be used for allocations */ 512 pgoff_t pgoff; /* Logical page offset based on vma */ 513 unsigned long address; /* Faulting virtual address - masked */ 514 unsigned long real_address; /* Faulting virtual address - unmasked */ 515 }; 516 enum fault_flag flags; /* FAULT_FLAG_xxx flags 517 * XXX: should really be 'const' */ 518 pmd_t *pmd; /* Pointer to pmd entry matching 519 * the 'address' */ 520 pud_t *pud; /* Pointer to pud entry matching 521 * the 'address' 522 */ 523 union { 524 pte_t orig_pte; /* Value of PTE at the time of fault */ 525 pmd_t orig_pmd; /* Value of PMD at the time of fault, 526 * used by PMD fault only. 527 */ 528 }; 529 530 struct page *cow_page; /* Page handler may use for COW fault */ 531 struct page *page; /* ->fault handlers should return a 532 * page here, unless VM_FAULT_NOPAGE 533 * is set (which is also implied by 534 * VM_FAULT_ERROR). 535 */ 536 /* These three entries are valid only while holding ptl lock */ 537 pte_t *pte; /* Pointer to pte entry matching 538 * the 'address'. NULL if the page 539 * table hasn't been allocated. 540 */ 541 spinlock_t *ptl; /* Page table lock. 542 * Protects pte page table if 'pte' 543 * is not NULL, otherwise pmd. 544 */ 545 pgtable_t prealloc_pte; /* Pre-allocated pte page table. 546 * vm_ops->map_pages() sets up a page 547 * table from atomic context. 548 * do_fault_around() pre-allocates 549 * page table to avoid allocation from 550 * atomic context. 551 */ 552 }; 553 554 /* 555 * These are the virtual MM functions - opening of an area, closing and 556 * unmapping it (needed to keep files on disk up-to-date etc), pointer 557 * to the functions called when a no-page or a wp-page exception occurs. 558 */ 559 struct vm_operations_struct { 560 void (*open)(struct vm_area_struct * area); 561 /** 562 * @close: Called when the VMA is being removed from the MM. 563 * Context: User context. May sleep. Caller holds mmap_lock. 564 */ 565 void (*close)(struct vm_area_struct * area); 566 /* Called any time before splitting to check if it's allowed */ 567 int (*may_split)(struct vm_area_struct *area, unsigned long addr); 568 int (*mremap)(struct vm_area_struct *area); 569 /* 570 * Called by mprotect() to make driver-specific permission 571 * checks before mprotect() is finalised. The VMA must not 572 * be modified. Returns 0 if mprotect() can proceed. 573 */ 574 int (*mprotect)(struct vm_area_struct *vma, unsigned long start, 575 unsigned long end, unsigned long newflags); 576 vm_fault_t (*fault)(struct vm_fault *vmf); 577 vm_fault_t (*huge_fault)(struct vm_fault *vmf, unsigned int order); 578 vm_fault_t (*map_pages)(struct vm_fault *vmf, 579 pgoff_t start_pgoff, pgoff_t end_pgoff); 580 unsigned long (*pagesize)(struct vm_area_struct * area); 581 582 /* notification that a previously read-only page is about to become 583 * writable, if an error is returned it will cause a SIGBUS */ 584 vm_fault_t (*page_mkwrite)(struct vm_fault *vmf); 585 586 /* same as page_mkwrite when using VM_PFNMAP|VM_MIXEDMAP */ 587 vm_fault_t (*pfn_mkwrite)(struct vm_fault *vmf); 588 589 /* called by access_process_vm when get_user_pages() fails, typically 590 * for use by special VMAs. See also generic_access_phys() for a generic 591 * implementation useful for any iomem mapping. 592 */ 593 int (*access)(struct vm_area_struct *vma, unsigned long addr, 594 void *buf, int len, int write); 595 596 /* Called by the /proc/PID/maps code to ask the vma whether it 597 * has a special name. Returning non-NULL will also cause this 598 * vma to be dumped unconditionally. */ 599 const char *(*name)(struct vm_area_struct *vma); 600 601 #ifdef CONFIG_NUMA 602 /* 603 * set_policy() op must add a reference to any non-NULL @new mempolicy 604 * to hold the policy upon return. Caller should pass NULL @new to 605 * remove a policy and fall back to surrounding context--i.e. do not 606 * install a MPOL_DEFAULT policy, nor the task or system default 607 * mempolicy. 608 */ 609 int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new); 610 611 /* 612 * get_policy() op must add reference [mpol_get()] to any policy at 613 * (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure 614 * in mm/mempolicy.c will do this automatically. 615 * get_policy() must NOT add a ref if the policy at (vma,addr) is not 616 * marked as MPOL_SHARED. vma policies are protected by the mmap_lock. 617 * If no [shared/vma] mempolicy exists at the addr, get_policy() op 618 * must return NULL--i.e., do not "fallback" to task or system default 619 * policy. 620 */ 621 struct mempolicy *(*get_policy)(struct vm_area_struct *vma, 622 unsigned long addr); 623 #endif 624 /* 625 * Called by vm_normal_page() for special PTEs to find the 626 * page for @addr. This is useful if the default behavior 627 * (using pte_page()) would not find the correct page. 628 */ 629 struct page *(*find_special_page)(struct vm_area_struct *vma, 630 unsigned long addr); 631 }; 632 633 #ifdef CONFIG_NUMA_BALANCING 634 static inline void vma_numab_state_init(struct vm_area_struct *vma) 635 { 636 vma->numab_state = NULL; 637 } 638 static inline void vma_numab_state_free(struct vm_area_struct *vma) 639 { 640 kfree(vma->numab_state); 641 } 642 #else 643 static inline void vma_numab_state_init(struct vm_area_struct *vma) {} 644 static inline void vma_numab_state_free(struct vm_area_struct *vma) {} 645 #endif /* CONFIG_NUMA_BALANCING */ 646 647 #ifdef CONFIG_PER_VMA_LOCK 648 /* 649 * Try to read-lock a vma. The function is allowed to occasionally yield false 650 * locked result to avoid performance overhead, in which case we fall back to 651 * using mmap_lock. The function should never yield false unlocked result. 652 */ 653 static inline bool vma_start_read(struct vm_area_struct *vma) 654 { 655 /* 656 * Check before locking. A race might cause false locked result. 657 * We can use READ_ONCE() for the mm_lock_seq here, and don't need 658 * ACQUIRE semantics, because this is just a lockless check whose result 659 * we don't rely on for anything - the mm_lock_seq read against which we 660 * need ordering is below. 661 */ 662 if (READ_ONCE(vma->vm_lock_seq) == READ_ONCE(vma->vm_mm->mm_lock_seq)) 663 return false; 664 665 if (unlikely(down_read_trylock(&vma->vm_lock->lock) == 0)) 666 return false; 667 668 /* 669 * Overflow might produce false locked result. 670 * False unlocked result is impossible because we modify and check 671 * vma->vm_lock_seq under vma->vm_lock protection and mm->mm_lock_seq 672 * modification invalidates all existing locks. 673 * 674 * We must use ACQUIRE semantics for the mm_lock_seq so that if we are 675 * racing with vma_end_write_all(), we only start reading from the VMA 676 * after it has been unlocked. 677 * This pairs with RELEASE semantics in vma_end_write_all(). 678 */ 679 if (unlikely(vma->vm_lock_seq == smp_load_acquire(&vma->vm_mm->mm_lock_seq))) { 680 up_read(&vma->vm_lock->lock); 681 return false; 682 } 683 return true; 684 } 685 686 static inline void vma_end_read(struct vm_area_struct *vma) 687 { 688 rcu_read_lock(); /* keeps vma alive till the end of up_read */ 689 up_read(&vma->vm_lock->lock); 690 rcu_read_unlock(); 691 } 692 693 /* WARNING! Can only be used if mmap_lock is expected to be write-locked */ 694 static bool __is_vma_write_locked(struct vm_area_struct *vma, int *mm_lock_seq) 695 { 696 mmap_assert_write_locked(vma->vm_mm); 697 698 /* 699 * current task is holding mmap_write_lock, both vma->vm_lock_seq and 700 * mm->mm_lock_seq can't be concurrently modified. 701 */ 702 *mm_lock_seq = vma->vm_mm->mm_lock_seq; 703 return (vma->vm_lock_seq == *mm_lock_seq); 704 } 705 706 /* 707 * Begin writing to a VMA. 708 * Exclude concurrent readers under the per-VMA lock until the currently 709 * write-locked mmap_lock is dropped or downgraded. 710 */ 711 static inline void vma_start_write(struct vm_area_struct *vma) 712 { 713 int mm_lock_seq; 714 715 if (__is_vma_write_locked(vma, &mm_lock_seq)) 716 return; 717 718 down_write(&vma->vm_lock->lock); 719 /* 720 * We should use WRITE_ONCE() here because we can have concurrent reads 721 * from the early lockless pessimistic check in vma_start_read(). 722 * We don't really care about the correctness of that early check, but 723 * we should use WRITE_ONCE() for cleanliness and to keep KCSAN happy. 724 */ 725 WRITE_ONCE(vma->vm_lock_seq, mm_lock_seq); 726 up_write(&vma->vm_lock->lock); 727 } 728 729 static inline void vma_assert_write_locked(struct vm_area_struct *vma) 730 { 731 int mm_lock_seq; 732 733 VM_BUG_ON_VMA(!__is_vma_write_locked(vma, &mm_lock_seq), vma); 734 } 735 736 static inline void vma_assert_locked(struct vm_area_struct *vma) 737 { 738 if (!rwsem_is_locked(&vma->vm_lock->lock)) 739 vma_assert_write_locked(vma); 740 } 741 742 static inline void vma_mark_detached(struct vm_area_struct *vma, bool detached) 743 { 744 /* When detaching vma should be write-locked */ 745 if (detached) 746 vma_assert_write_locked(vma); 747 vma->detached = detached; 748 } 749 750 static inline void release_fault_lock(struct vm_fault *vmf) 751 { 752 if (vmf->flags & FAULT_FLAG_VMA_LOCK) 753 vma_end_read(vmf->vma); 754 else 755 mmap_read_unlock(vmf->vma->vm_mm); 756 } 757 758 static inline void assert_fault_locked(struct vm_fault *vmf) 759 { 760 if (vmf->flags & FAULT_FLAG_VMA_LOCK) 761 vma_assert_locked(vmf->vma); 762 else 763 mmap_assert_locked(vmf->vma->vm_mm); 764 } 765 766 struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, 767 unsigned long address); 768 769 #else /* CONFIG_PER_VMA_LOCK */ 770 771 static inline bool vma_start_read(struct vm_area_struct *vma) 772 { return false; } 773 static inline void vma_end_read(struct vm_area_struct *vma) {} 774 static inline void vma_start_write(struct vm_area_struct *vma) {} 775 static inline void vma_assert_write_locked(struct vm_area_struct *vma) 776 { mmap_assert_write_locked(vma->vm_mm); } 777 static inline void vma_mark_detached(struct vm_area_struct *vma, 778 bool detached) {} 779 780 static inline struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, 781 unsigned long address) 782 { 783 return NULL; 784 } 785 786 static inline void release_fault_lock(struct vm_fault *vmf) 787 { 788 mmap_read_unlock(vmf->vma->vm_mm); 789 } 790 791 static inline void assert_fault_locked(struct vm_fault *vmf) 792 { 793 mmap_assert_locked(vmf->vma->vm_mm); 794 } 795 796 #endif /* CONFIG_PER_VMA_LOCK */ 797 798 extern const struct vm_operations_struct vma_dummy_vm_ops; 799 800 /* 801 * WARNING: vma_init does not initialize vma->vm_lock. 802 * Use vm_area_alloc()/vm_area_free() if vma needs locking. 803 */ 804 static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm) 805 { 806 memset(vma, 0, sizeof(*vma)); 807 vma->vm_mm = mm; 808 vma->vm_ops = &vma_dummy_vm_ops; 809 INIT_LIST_HEAD(&vma->anon_vma_chain); 810 vma_mark_detached(vma, false); 811 vma_numab_state_init(vma); 812 } 813 814 /* Use when VMA is not part of the VMA tree and needs no locking */ 815 static inline void vm_flags_init(struct vm_area_struct *vma, 816 vm_flags_t flags) 817 { 818 ACCESS_PRIVATE(vma, __vm_flags) = flags; 819 } 820 821 /* 822 * Use when VMA is part of the VMA tree and modifications need coordination 823 * Note: vm_flags_reset and vm_flags_reset_once do not lock the vma and 824 * it should be locked explicitly beforehand. 825 */ 826 static inline void vm_flags_reset(struct vm_area_struct *vma, 827 vm_flags_t flags) 828 { 829 vma_assert_write_locked(vma); 830 vm_flags_init(vma, flags); 831 } 832 833 static inline void vm_flags_reset_once(struct vm_area_struct *vma, 834 vm_flags_t flags) 835 { 836 vma_assert_write_locked(vma); 837 WRITE_ONCE(ACCESS_PRIVATE(vma, __vm_flags), flags); 838 } 839 840 static inline void vm_flags_set(struct vm_area_struct *vma, 841 vm_flags_t flags) 842 { 843 vma_start_write(vma); 844 ACCESS_PRIVATE(vma, __vm_flags) |= flags; 845 } 846 847 static inline void vm_flags_clear(struct vm_area_struct *vma, 848 vm_flags_t flags) 849 { 850 vma_start_write(vma); 851 ACCESS_PRIVATE(vma, __vm_flags) &= ~flags; 852 } 853 854 /* 855 * Use only if VMA is not part of the VMA tree or has no other users and 856 * therefore needs no locking. 857 */ 858 static inline void __vm_flags_mod(struct vm_area_struct *vma, 859 vm_flags_t set, vm_flags_t clear) 860 { 861 vm_flags_init(vma, (vma->vm_flags | set) & ~clear); 862 } 863 864 /* 865 * Use only when the order of set/clear operations is unimportant, otherwise 866 * use vm_flags_{set|clear} explicitly. 867 */ 868 static inline void vm_flags_mod(struct vm_area_struct *vma, 869 vm_flags_t set, vm_flags_t clear) 870 { 871 vma_start_write(vma); 872 __vm_flags_mod(vma, set, clear); 873 } 874 875 static inline void vma_set_anonymous(struct vm_area_struct *vma) 876 { 877 vma->vm_ops = NULL; 878 } 879 880 static inline bool vma_is_anonymous(struct vm_area_struct *vma) 881 { 882 return !vma->vm_ops; 883 } 884 885 /* 886 * Indicate if the VMA is a heap for the given task; for 887 * /proc/PID/maps that is the heap of the main task. 888 */ 889 static inline bool vma_is_initial_heap(const struct vm_area_struct *vma) 890 { 891 return vma->vm_start <= vma->vm_mm->brk && 892 vma->vm_end >= vma->vm_mm->start_brk; 893 } 894 895 /* 896 * Indicate if the VMA is a stack for the given task; for 897 * /proc/PID/maps that is the stack of the main task. 898 */ 899 static inline bool vma_is_initial_stack(const struct vm_area_struct *vma) 900 { 901 /* 902 * We make no effort to guess what a given thread considers to be 903 * its "stack". It's not even well-defined for programs written 904 * languages like Go. 905 */ 906 return vma->vm_start <= vma->vm_mm->start_stack && 907 vma->vm_end >= vma->vm_mm->start_stack; 908 } 909 910 static inline bool vma_is_temporary_stack(struct vm_area_struct *vma) 911 { 912 int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP); 913 914 if (!maybe_stack) 915 return false; 916 917 if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) == 918 VM_STACK_INCOMPLETE_SETUP) 919 return true; 920 921 return false; 922 } 923 924 static inline bool vma_is_foreign(struct vm_area_struct *vma) 925 { 926 if (!current->mm) 927 return true; 928 929 if (current->mm != vma->vm_mm) 930 return true; 931 932 return false; 933 } 934 935 static inline bool vma_is_accessible(struct vm_area_struct *vma) 936 { 937 return vma->vm_flags & VM_ACCESS_FLAGS; 938 } 939 940 static inline 941 struct vm_area_struct *vma_find(struct vma_iterator *vmi, unsigned long max) 942 { 943 return mas_find(&vmi->mas, max - 1); 944 } 945 946 static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi) 947 { 948 /* 949 * Uses mas_find() to get the first VMA when the iterator starts. 950 * Calling mas_next() could skip the first entry. 951 */ 952 return mas_find(&vmi->mas, ULONG_MAX); 953 } 954 955 static inline 956 struct vm_area_struct *vma_iter_next_range(struct vma_iterator *vmi) 957 { 958 return mas_next_range(&vmi->mas, ULONG_MAX); 959 } 960 961 962 static inline struct vm_area_struct *vma_prev(struct vma_iterator *vmi) 963 { 964 return mas_prev(&vmi->mas, 0); 965 } 966 967 static inline 968 struct vm_area_struct *vma_iter_prev_range(struct vma_iterator *vmi) 969 { 970 return mas_prev_range(&vmi->mas, 0); 971 } 972 973 static inline unsigned long vma_iter_addr(struct vma_iterator *vmi) 974 { 975 return vmi->mas.index; 976 } 977 978 static inline unsigned long vma_iter_end(struct vma_iterator *vmi) 979 { 980 return vmi->mas.last + 1; 981 } 982 static inline int vma_iter_bulk_alloc(struct vma_iterator *vmi, 983 unsigned long count) 984 { 985 return mas_expected_entries(&vmi->mas, count); 986 } 987 988 /* Free any unused preallocations */ 989 static inline void vma_iter_free(struct vma_iterator *vmi) 990 { 991 mas_destroy(&vmi->mas); 992 } 993 994 static inline int vma_iter_bulk_store(struct vma_iterator *vmi, 995 struct vm_area_struct *vma) 996 { 997 vmi->mas.index = vma->vm_start; 998 vmi->mas.last = vma->vm_end - 1; 999 mas_store(&vmi->mas, vma); 1000 if (unlikely(mas_is_err(&vmi->mas))) 1001 return -ENOMEM; 1002 1003 return 0; 1004 } 1005 1006 static inline void vma_iter_invalidate(struct vma_iterator *vmi) 1007 { 1008 mas_pause(&vmi->mas); 1009 } 1010 1011 static inline void vma_iter_set(struct vma_iterator *vmi, unsigned long addr) 1012 { 1013 mas_set(&vmi->mas, addr); 1014 } 1015 1016 #define for_each_vma(__vmi, __vma) \ 1017 while (((__vma) = vma_next(&(__vmi))) != NULL) 1018 1019 /* The MM code likes to work with exclusive end addresses */ 1020 #define for_each_vma_range(__vmi, __vma, __end) \ 1021 while (((__vma) = vma_find(&(__vmi), (__end))) != NULL) 1022 1023 #ifdef CONFIG_SHMEM 1024 /* 1025 * The vma_is_shmem is not inline because it is used only by slow 1026 * paths in userfault. 1027 */ 1028 bool vma_is_shmem(struct vm_area_struct *vma); 1029 bool vma_is_anon_shmem(struct vm_area_struct *vma); 1030 #else 1031 static inline bool vma_is_shmem(struct vm_area_struct *vma) { return false; } 1032 static inline bool vma_is_anon_shmem(struct vm_area_struct *vma) { return false; } 1033 #endif 1034 1035 int vma_is_stack_for_current(struct vm_area_struct *vma); 1036 1037 /* flush_tlb_range() takes a vma, not a mm, and can care about flags */ 1038 #define TLB_FLUSH_VMA(mm,flags) { .vm_mm = (mm), .vm_flags = (flags) } 1039 1040 struct mmu_gather; 1041 struct inode; 1042 1043 /* 1044 * compound_order() can be called without holding a reference, which means 1045 * that niceties like page_folio() don't work. These callers should be 1046 * prepared to handle wild return values. For example, PG_head may be 1047 * set before the order is initialised, or this may be a tail page. 1048 * See compaction.c for some good examples. 1049 */ 1050 static inline unsigned int compound_order(struct page *page) 1051 { 1052 struct folio *folio = (struct folio *)page; 1053 1054 if (!test_bit(PG_head, &folio->flags)) 1055 return 0; 1056 return folio->_flags_1 & 0xff; 1057 } 1058 1059 /** 1060 * folio_order - The allocation order of a folio. 1061 * @folio: The folio. 1062 * 1063 * A folio is composed of 2^order pages. See get_order() for the definition 1064 * of order. 1065 * 1066 * Return: The order of the folio. 1067 */ 1068 static inline unsigned int folio_order(struct folio *folio) 1069 { 1070 if (!folio_test_large(folio)) 1071 return 0; 1072 return folio->_flags_1 & 0xff; 1073 } 1074 1075 #include <linux/huge_mm.h> 1076 1077 /* 1078 * Methods to modify the page usage count. 1079 * 1080 * What counts for a page usage: 1081 * - cache mapping (page->mapping) 1082 * - private data (page->private) 1083 * - page mapped in a task's page tables, each mapping 1084 * is counted separately 1085 * 1086 * Also, many kernel routines increase the page count before a critical 1087 * routine so they can be sure the page doesn't go away from under them. 1088 */ 1089 1090 /* 1091 * Drop a ref, return true if the refcount fell to zero (the page has no users) 1092 */ 1093 static inline int put_page_testzero(struct page *page) 1094 { 1095 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); 1096 return page_ref_dec_and_test(page); 1097 } 1098 1099 static inline int folio_put_testzero(struct folio *folio) 1100 { 1101 return put_page_testzero(&folio->page); 1102 } 1103 1104 /* 1105 * Try to grab a ref unless the page has a refcount of zero, return false if 1106 * that is the case. 1107 * This can be called when MMU is off so it must not access 1108 * any of the virtual mappings. 1109 */ 1110 static inline bool get_page_unless_zero(struct page *page) 1111 { 1112 return page_ref_add_unless(page, 1, 0); 1113 } 1114 1115 static inline struct folio *folio_get_nontail_page(struct page *page) 1116 { 1117 if (unlikely(!get_page_unless_zero(page))) 1118 return NULL; 1119 return (struct folio *)page; 1120 } 1121 1122 extern int page_is_ram(unsigned long pfn); 1123 1124 enum { 1125 REGION_INTERSECTS, 1126 REGION_DISJOINT, 1127 REGION_MIXED, 1128 }; 1129 1130 int region_intersects(resource_size_t offset, size_t size, unsigned long flags, 1131 unsigned long desc); 1132 1133 /* Support for virtually mapped pages */ 1134 struct page *vmalloc_to_page(const void *addr); 1135 unsigned long vmalloc_to_pfn(const void *addr); 1136 1137 /* 1138 * Determine if an address is within the vmalloc range 1139 * 1140 * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there 1141 * is no special casing required. 1142 */ 1143 #ifdef CONFIG_MMU 1144 extern bool is_vmalloc_addr(const void *x); 1145 extern int is_vmalloc_or_module_addr(const void *x); 1146 #else 1147 static inline bool is_vmalloc_addr(const void *x) 1148 { 1149 return false; 1150 } 1151 static inline int is_vmalloc_or_module_addr(const void *x) 1152 { 1153 return 0; 1154 } 1155 #endif 1156 1157 /* 1158 * How many times the entire folio is mapped as a single unit (eg by a 1159 * PMD or PUD entry). This is probably not what you want, except for 1160 * debugging purposes - it does not include PTE-mapped sub-pages; look 1161 * at folio_mapcount() or page_mapcount() or total_mapcount() instead. 1162 */ 1163 static inline int folio_entire_mapcount(struct folio *folio) 1164 { 1165 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 1166 return atomic_read(&folio->_entire_mapcount) + 1; 1167 } 1168 1169 /* 1170 * The atomic page->_mapcount, starts from -1: so that transitions 1171 * both from it and to it can be tracked, using atomic_inc_and_test 1172 * and atomic_add_negative(-1). 1173 */ 1174 static inline void page_mapcount_reset(struct page *page) 1175 { 1176 atomic_set(&(page)->_mapcount, -1); 1177 } 1178 1179 /** 1180 * page_mapcount() - Number of times this precise page is mapped. 1181 * @page: The page. 1182 * 1183 * The number of times this page is mapped. If this page is part of 1184 * a large folio, it includes the number of times this page is mapped 1185 * as part of that folio. 1186 * 1187 * The result is undefined for pages which cannot be mapped into userspace. 1188 * For example SLAB or special types of pages. See function page_has_type(). 1189 * They use this field in struct page differently. 1190 */ 1191 static inline int page_mapcount(struct page *page) 1192 { 1193 int mapcount = atomic_read(&page->_mapcount) + 1; 1194 1195 if (unlikely(PageCompound(page))) 1196 mapcount += folio_entire_mapcount(page_folio(page)); 1197 1198 return mapcount; 1199 } 1200 1201 int folio_total_mapcount(struct folio *folio); 1202 1203 /** 1204 * folio_mapcount() - Calculate the number of mappings of this folio. 1205 * @folio: The folio. 1206 * 1207 * A large folio tracks both how many times the entire folio is mapped, 1208 * and how many times each individual page in the folio is mapped. 1209 * This function calculates the total number of times the folio is 1210 * mapped. 1211 * 1212 * Return: The number of times this folio is mapped. 1213 */ 1214 static inline int folio_mapcount(struct folio *folio) 1215 { 1216 if (likely(!folio_test_large(folio))) 1217 return atomic_read(&folio->_mapcount) + 1; 1218 return folio_total_mapcount(folio); 1219 } 1220 1221 static inline int total_mapcount(struct page *page) 1222 { 1223 if (likely(!PageCompound(page))) 1224 return atomic_read(&page->_mapcount) + 1; 1225 return folio_total_mapcount(page_folio(page)); 1226 } 1227 1228 static inline bool folio_large_is_mapped(struct folio *folio) 1229 { 1230 /* 1231 * Reading _entire_mapcount below could be omitted if hugetlb 1232 * participated in incrementing nr_pages_mapped when compound mapped. 1233 */ 1234 return atomic_read(&folio->_nr_pages_mapped) > 0 || 1235 atomic_read(&folio->_entire_mapcount) >= 0; 1236 } 1237 1238 /** 1239 * folio_mapped - Is this folio mapped into userspace? 1240 * @folio: The folio. 1241 * 1242 * Return: True if any page in this folio is referenced by user page tables. 1243 */ 1244 static inline bool folio_mapped(struct folio *folio) 1245 { 1246 if (likely(!folio_test_large(folio))) 1247 return atomic_read(&folio->_mapcount) >= 0; 1248 return folio_large_is_mapped(folio); 1249 } 1250 1251 /* 1252 * Return true if this page is mapped into pagetables. 1253 * For compound page it returns true if any sub-page of compound page is mapped, 1254 * even if this particular sub-page is not itself mapped by any PTE or PMD. 1255 */ 1256 static inline bool page_mapped(struct page *page) 1257 { 1258 if (likely(!PageCompound(page))) 1259 return atomic_read(&page->_mapcount) >= 0; 1260 return folio_large_is_mapped(page_folio(page)); 1261 } 1262 1263 static inline struct page *virt_to_head_page(const void *x) 1264 { 1265 struct page *page = virt_to_page(x); 1266 1267 return compound_head(page); 1268 } 1269 1270 static inline struct folio *virt_to_folio(const void *x) 1271 { 1272 struct page *page = virt_to_page(x); 1273 1274 return page_folio(page); 1275 } 1276 1277 void __folio_put(struct folio *folio); 1278 1279 void put_pages_list(struct list_head *pages); 1280 1281 void split_page(struct page *page, unsigned int order); 1282 void folio_copy(struct folio *dst, struct folio *src); 1283 1284 unsigned long nr_free_buffer_pages(void); 1285 1286 void destroy_large_folio(struct folio *folio); 1287 1288 /* Returns the number of bytes in this potentially compound page. */ 1289 static inline unsigned long page_size(struct page *page) 1290 { 1291 return PAGE_SIZE << compound_order(page); 1292 } 1293 1294 /* Returns the number of bits needed for the number of bytes in a page */ 1295 static inline unsigned int page_shift(struct page *page) 1296 { 1297 return PAGE_SHIFT + compound_order(page); 1298 } 1299 1300 /** 1301 * thp_order - Order of a transparent huge page. 1302 * @page: Head page of a transparent huge page. 1303 */ 1304 static inline unsigned int thp_order(struct page *page) 1305 { 1306 VM_BUG_ON_PGFLAGS(PageTail(page), page); 1307 return compound_order(page); 1308 } 1309 1310 /** 1311 * thp_size - Size of a transparent huge page. 1312 * @page: Head page of a transparent huge page. 1313 * 1314 * Return: Number of bytes in this page. 1315 */ 1316 static inline unsigned long thp_size(struct page *page) 1317 { 1318 return PAGE_SIZE << thp_order(page); 1319 } 1320 1321 #ifdef CONFIG_MMU 1322 /* 1323 * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when 1324 * servicing faults for write access. In the normal case, do always want 1325 * pte_mkwrite. But get_user_pages can cause write faults for mappings 1326 * that do not have writing enabled, when used by access_process_vm. 1327 */ 1328 static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) 1329 { 1330 if (likely(vma->vm_flags & VM_WRITE)) 1331 pte = pte_mkwrite(pte, vma); 1332 return pte; 1333 } 1334 1335 vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page); 1336 void set_pte_range(struct vm_fault *vmf, struct folio *folio, 1337 struct page *page, unsigned int nr, unsigned long addr); 1338 1339 vm_fault_t finish_fault(struct vm_fault *vmf); 1340 vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf); 1341 #endif 1342 1343 /* 1344 * Multiple processes may "see" the same page. E.g. for untouched 1345 * mappings of /dev/null, all processes see the same page full of 1346 * zeroes, and text pages of executables and shared libraries have 1347 * only one copy in memory, at most, normally. 1348 * 1349 * For the non-reserved pages, page_count(page) denotes a reference count. 1350 * page_count() == 0 means the page is free. page->lru is then used for 1351 * freelist management in the buddy allocator. 1352 * page_count() > 0 means the page has been allocated. 1353 * 1354 * Pages are allocated by the slab allocator in order to provide memory 1355 * to kmalloc and kmem_cache_alloc. In this case, the management of the 1356 * page, and the fields in 'struct page' are the responsibility of mm/slab.c 1357 * unless a particular usage is carefully commented. (the responsibility of 1358 * freeing the kmalloc memory is the caller's, of course). 1359 * 1360 * A page may be used by anyone else who does a __get_free_page(). 1361 * In this case, page_count still tracks the references, and should only 1362 * be used through the normal accessor functions. The top bits of page->flags 1363 * and page->virtual store page management information, but all other fields 1364 * are unused and could be used privately, carefully. The management of this 1365 * page is the responsibility of the one who allocated it, and those who have 1366 * subsequently been given references to it. 1367 * 1368 * The other pages (we may call them "pagecache pages") are completely 1369 * managed by the Linux memory manager: I/O, buffers, swapping etc. 1370 * The following discussion applies only to them. 1371 * 1372 * A pagecache page contains an opaque `private' member, which belongs to the 1373 * page's address_space. Usually, this is the address of a circular list of 1374 * the page's disk buffers. PG_private must be set to tell the VM to call 1375 * into the filesystem to release these pages. 1376 * 1377 * A page may belong to an inode's memory mapping. In this case, page->mapping 1378 * is the pointer to the inode, and page->index is the file offset of the page, 1379 * in units of PAGE_SIZE. 1380 * 1381 * If pagecache pages are not associated with an inode, they are said to be 1382 * anonymous pages. These may become associated with the swapcache, and in that 1383 * case PG_swapcache is set, and page->private is an offset into the swapcache. 1384 * 1385 * In either case (swapcache or inode backed), the pagecache itself holds one 1386 * reference to the page. Setting PG_private should also increment the 1387 * refcount. The each user mapping also has a reference to the page. 1388 * 1389 * The pagecache pages are stored in a per-mapping radix tree, which is 1390 * rooted at mapping->i_pages, and indexed by offset. 1391 * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space 1392 * lists, we instead now tag pages as dirty/writeback in the radix tree. 1393 * 1394 * All pagecache pages may be subject to I/O: 1395 * - inode pages may need to be read from disk, 1396 * - inode pages which have been modified and are MAP_SHARED may need 1397 * to be written back to the inode on disk, 1398 * - anonymous pages (including MAP_PRIVATE file mappings) which have been 1399 * modified may need to be swapped out to swap space and (later) to be read 1400 * back into memory. 1401 */ 1402 1403 #if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_FS_DAX) 1404 DECLARE_STATIC_KEY_FALSE(devmap_managed_key); 1405 1406 bool __put_devmap_managed_page_refs(struct page *page, int refs); 1407 static inline bool put_devmap_managed_page_refs(struct page *page, int refs) 1408 { 1409 if (!static_branch_unlikely(&devmap_managed_key)) 1410 return false; 1411 if (!is_zone_device_page(page)) 1412 return false; 1413 return __put_devmap_managed_page_refs(page, refs); 1414 } 1415 #else /* CONFIG_ZONE_DEVICE && CONFIG_FS_DAX */ 1416 static inline bool put_devmap_managed_page_refs(struct page *page, int refs) 1417 { 1418 return false; 1419 } 1420 #endif /* CONFIG_ZONE_DEVICE && CONFIG_FS_DAX */ 1421 1422 static inline bool put_devmap_managed_page(struct page *page) 1423 { 1424 return put_devmap_managed_page_refs(page, 1); 1425 } 1426 1427 /* 127: arbitrary random number, small enough to assemble well */ 1428 #define folio_ref_zero_or_close_to_overflow(folio) \ 1429 ((unsigned int) folio_ref_count(folio) + 127u <= 127u) 1430 1431 /** 1432 * folio_get - Increment the reference count on a folio. 1433 * @folio: The folio. 1434 * 1435 * Context: May be called in any context, as long as you know that 1436 * you have a refcount on the folio. If you do not already have one, 1437 * folio_try_get() may be the right interface for you to use. 1438 */ 1439 static inline void folio_get(struct folio *folio) 1440 { 1441 VM_BUG_ON_FOLIO(folio_ref_zero_or_close_to_overflow(folio), folio); 1442 folio_ref_inc(folio); 1443 } 1444 1445 static inline void get_page(struct page *page) 1446 { 1447 folio_get(page_folio(page)); 1448 } 1449 1450 static inline __must_check bool try_get_page(struct page *page) 1451 { 1452 page = compound_head(page); 1453 if (WARN_ON_ONCE(page_ref_count(page) <= 0)) 1454 return false; 1455 page_ref_inc(page); 1456 return true; 1457 } 1458 1459 /** 1460 * folio_put - Decrement the reference count on a folio. 1461 * @folio: The folio. 1462 * 1463 * If the folio's reference count reaches zero, the memory will be 1464 * released back to the page allocator and may be used by another 1465 * allocation immediately. Do not access the memory or the struct folio 1466 * after calling folio_put() unless you can be sure that it wasn't the 1467 * last reference. 1468 * 1469 * Context: May be called in process or interrupt context, but not in NMI 1470 * context. May be called while holding a spinlock. 1471 */ 1472 static inline void folio_put(struct folio *folio) 1473 { 1474 if (folio_put_testzero(folio)) 1475 __folio_put(folio); 1476 } 1477 1478 /** 1479 * folio_put_refs - Reduce the reference count on a folio. 1480 * @folio: The folio. 1481 * @refs: The amount to subtract from the folio's reference count. 1482 * 1483 * If the folio's reference count reaches zero, the memory will be 1484 * released back to the page allocator and may be used by another 1485 * allocation immediately. Do not access the memory or the struct folio 1486 * after calling folio_put_refs() unless you can be sure that these weren't 1487 * the last references. 1488 * 1489 * Context: May be called in process or interrupt context, but not in NMI 1490 * context. May be called while holding a spinlock. 1491 */ 1492 static inline void folio_put_refs(struct folio *folio, int refs) 1493 { 1494 if (folio_ref_sub_and_test(folio, refs)) 1495 __folio_put(folio); 1496 } 1497 1498 /* 1499 * union release_pages_arg - an array of pages or folios 1500 * 1501 * release_pages() releases a simple array of multiple pages, and 1502 * accepts various different forms of said page array: either 1503 * a regular old boring array of pages, an array of folios, or 1504 * an array of encoded page pointers. 1505 * 1506 * The transparent union syntax for this kind of "any of these 1507 * argument types" is all kinds of ugly, so look away. 1508 */ 1509 typedef union { 1510 struct page **pages; 1511 struct folio **folios; 1512 struct encoded_page **encoded_pages; 1513 } release_pages_arg __attribute__ ((__transparent_union__)); 1514 1515 void release_pages(release_pages_arg, int nr); 1516 1517 /** 1518 * folios_put - Decrement the reference count on an array of folios. 1519 * @folios: The folios. 1520 * @nr: How many folios there are. 1521 * 1522 * Like folio_put(), but for an array of folios. This is more efficient 1523 * than writing the loop yourself as it will optimise the locks which 1524 * need to be taken if the folios are freed. 1525 * 1526 * Context: May be called in process or interrupt context, but not in NMI 1527 * context. May be called while holding a spinlock. 1528 */ 1529 static inline void folios_put(struct folio **folios, unsigned int nr) 1530 { 1531 release_pages(folios, nr); 1532 } 1533 1534 static inline void put_page(struct page *page) 1535 { 1536 struct folio *folio = page_folio(page); 1537 1538 /* 1539 * For some devmap managed pages we need to catch refcount transition 1540 * from 2 to 1: 1541 */ 1542 if (put_devmap_managed_page(&folio->page)) 1543 return; 1544 folio_put(folio); 1545 } 1546 1547 /* 1548 * GUP_PIN_COUNTING_BIAS, and the associated functions that use it, overload 1549 * the page's refcount so that two separate items are tracked: the original page 1550 * reference count, and also a new count of how many pin_user_pages() calls were 1551 * made against the page. ("gup-pinned" is another term for the latter). 1552 * 1553 * With this scheme, pin_user_pages() becomes special: such pages are marked as 1554 * distinct from normal pages. As such, the unpin_user_page() call (and its 1555 * variants) must be used in order to release gup-pinned pages. 1556 * 1557 * Choice of value: 1558 * 1559 * By making GUP_PIN_COUNTING_BIAS a power of two, debugging of page reference 1560 * counts with respect to pin_user_pages() and unpin_user_page() becomes 1561 * simpler, due to the fact that adding an even power of two to the page 1562 * refcount has the effect of using only the upper N bits, for the code that 1563 * counts up using the bias value. This means that the lower bits are left for 1564 * the exclusive use of the original code that increments and decrements by one 1565 * (or at least, by much smaller values than the bias value). 1566 * 1567 * Of course, once the lower bits overflow into the upper bits (and this is 1568 * OK, because subtraction recovers the original values), then visual inspection 1569 * no longer suffices to directly view the separate counts. However, for normal 1570 * applications that don't have huge page reference counts, this won't be an 1571 * issue. 1572 * 1573 * Locking: the lockless algorithm described in folio_try_get_rcu() 1574 * provides safe operation for get_user_pages(), page_mkclean() and 1575 * other calls that race to set up page table entries. 1576 */ 1577 #define GUP_PIN_COUNTING_BIAS (1U << 10) 1578 1579 void unpin_user_page(struct page *page); 1580 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages, 1581 bool make_dirty); 1582 void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages, 1583 bool make_dirty); 1584 void unpin_user_pages(struct page **pages, unsigned long npages); 1585 1586 static inline bool is_cow_mapping(vm_flags_t flags) 1587 { 1588 return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; 1589 } 1590 1591 #ifndef CONFIG_MMU 1592 static inline bool is_nommu_shared_mapping(vm_flags_t flags) 1593 { 1594 /* 1595 * NOMMU shared mappings are ordinary MAP_SHARED mappings and selected 1596 * R/O MAP_PRIVATE file mappings that are an effective R/O overlay of 1597 * a file mapping. R/O MAP_PRIVATE mappings might still modify 1598 * underlying memory if ptrace is active, so this is only possible if 1599 * ptrace does not apply. Note that there is no mprotect() to upgrade 1600 * write permissions later. 1601 */ 1602 return flags & (VM_MAYSHARE | VM_MAYOVERLAY); 1603 } 1604 #endif 1605 1606 #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) 1607 #define SECTION_IN_PAGE_FLAGS 1608 #endif 1609 1610 /* 1611 * The identification function is mainly used by the buddy allocator for 1612 * determining if two pages could be buddies. We are not really identifying 1613 * the zone since we could be using the section number id if we do not have 1614 * node id available in page flags. 1615 * We only guarantee that it will return the same value for two combinable 1616 * pages in a zone. 1617 */ 1618 static inline int page_zone_id(struct page *page) 1619 { 1620 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK; 1621 } 1622 1623 #ifdef NODE_NOT_IN_PAGE_FLAGS 1624 extern int page_to_nid(const struct page *page); 1625 #else 1626 static inline int page_to_nid(const struct page *page) 1627 { 1628 struct page *p = (struct page *)page; 1629 1630 return (PF_POISONED_CHECK(p)->flags >> NODES_PGSHIFT) & NODES_MASK; 1631 } 1632 #endif 1633 1634 static inline int folio_nid(const struct folio *folio) 1635 { 1636 return page_to_nid(&folio->page); 1637 } 1638 1639 #ifdef CONFIG_NUMA_BALANCING 1640 /* page access time bits needs to hold at least 4 seconds */ 1641 #define PAGE_ACCESS_TIME_MIN_BITS 12 1642 #if LAST_CPUPID_SHIFT < PAGE_ACCESS_TIME_MIN_BITS 1643 #define PAGE_ACCESS_TIME_BUCKETS \ 1644 (PAGE_ACCESS_TIME_MIN_BITS - LAST_CPUPID_SHIFT) 1645 #else 1646 #define PAGE_ACCESS_TIME_BUCKETS 0 1647 #endif 1648 1649 #define PAGE_ACCESS_TIME_MASK \ 1650 (LAST_CPUPID_MASK << PAGE_ACCESS_TIME_BUCKETS) 1651 1652 static inline int cpu_pid_to_cpupid(int cpu, int pid) 1653 { 1654 return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK); 1655 } 1656 1657 static inline int cpupid_to_pid(int cpupid) 1658 { 1659 return cpupid & LAST__PID_MASK; 1660 } 1661 1662 static inline int cpupid_to_cpu(int cpupid) 1663 { 1664 return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK; 1665 } 1666 1667 static inline int cpupid_to_nid(int cpupid) 1668 { 1669 return cpu_to_node(cpupid_to_cpu(cpupid)); 1670 } 1671 1672 static inline bool cpupid_pid_unset(int cpupid) 1673 { 1674 return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK); 1675 } 1676 1677 static inline bool cpupid_cpu_unset(int cpupid) 1678 { 1679 return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK); 1680 } 1681 1682 static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid) 1683 { 1684 return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid); 1685 } 1686 1687 #define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid) 1688 #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS 1689 static inline int page_cpupid_xchg_last(struct page *page, int cpupid) 1690 { 1691 return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK); 1692 } 1693 1694 static inline int page_cpupid_last(struct page *page) 1695 { 1696 return page->_last_cpupid; 1697 } 1698 static inline void page_cpupid_reset_last(struct page *page) 1699 { 1700 page->_last_cpupid = -1 & LAST_CPUPID_MASK; 1701 } 1702 #else 1703 static inline int page_cpupid_last(struct page *page) 1704 { 1705 return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK; 1706 } 1707 1708 extern int page_cpupid_xchg_last(struct page *page, int cpupid); 1709 1710 static inline void page_cpupid_reset_last(struct page *page) 1711 { 1712 page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT; 1713 } 1714 #endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */ 1715 1716 static inline int xchg_page_access_time(struct page *page, int time) 1717 { 1718 int last_time; 1719 1720 last_time = page_cpupid_xchg_last(page, time >> PAGE_ACCESS_TIME_BUCKETS); 1721 return last_time << PAGE_ACCESS_TIME_BUCKETS; 1722 } 1723 1724 static inline void vma_set_access_pid_bit(struct vm_area_struct *vma) 1725 { 1726 unsigned int pid_bit; 1727 1728 pid_bit = hash_32(current->pid, ilog2(BITS_PER_LONG)); 1729 if (vma->numab_state && !test_bit(pid_bit, &vma->numab_state->access_pids[1])) { 1730 __set_bit(pid_bit, &vma->numab_state->access_pids[1]); 1731 } 1732 } 1733 #else /* !CONFIG_NUMA_BALANCING */ 1734 static inline int page_cpupid_xchg_last(struct page *page, int cpupid) 1735 { 1736 return page_to_nid(page); /* XXX */ 1737 } 1738 1739 static inline int xchg_page_access_time(struct page *page, int time) 1740 { 1741 return 0; 1742 } 1743 1744 static inline int page_cpupid_last(struct page *page) 1745 { 1746 return page_to_nid(page); /* XXX */ 1747 } 1748 1749 static inline int cpupid_to_nid(int cpupid) 1750 { 1751 return -1; 1752 } 1753 1754 static inline int cpupid_to_pid(int cpupid) 1755 { 1756 return -1; 1757 } 1758 1759 static inline int cpupid_to_cpu(int cpupid) 1760 { 1761 return -1; 1762 } 1763 1764 static inline int cpu_pid_to_cpupid(int nid, int pid) 1765 { 1766 return -1; 1767 } 1768 1769 static inline bool cpupid_pid_unset(int cpupid) 1770 { 1771 return true; 1772 } 1773 1774 static inline void page_cpupid_reset_last(struct page *page) 1775 { 1776 } 1777 1778 static inline bool cpupid_match_pid(struct task_struct *task, int cpupid) 1779 { 1780 return false; 1781 } 1782 1783 static inline void vma_set_access_pid_bit(struct vm_area_struct *vma) 1784 { 1785 } 1786 #endif /* CONFIG_NUMA_BALANCING */ 1787 1788 #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) 1789 1790 /* 1791 * KASAN per-page tags are stored xor'ed with 0xff. This allows to avoid 1792 * setting tags for all pages to native kernel tag value 0xff, as the default 1793 * value 0x00 maps to 0xff. 1794 */ 1795 1796 static inline u8 page_kasan_tag(const struct page *page) 1797 { 1798 u8 tag = 0xff; 1799 1800 if (kasan_enabled()) { 1801 tag = (page->flags >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK; 1802 tag ^= 0xff; 1803 } 1804 1805 return tag; 1806 } 1807 1808 static inline void page_kasan_tag_set(struct page *page, u8 tag) 1809 { 1810 unsigned long old_flags, flags; 1811 1812 if (!kasan_enabled()) 1813 return; 1814 1815 tag ^= 0xff; 1816 old_flags = READ_ONCE(page->flags); 1817 do { 1818 flags = old_flags; 1819 flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT); 1820 flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT; 1821 } while (unlikely(!try_cmpxchg(&page->flags, &old_flags, flags))); 1822 } 1823 1824 static inline void page_kasan_tag_reset(struct page *page) 1825 { 1826 if (kasan_enabled()) 1827 page_kasan_tag_set(page, 0xff); 1828 } 1829 1830 #else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */ 1831 1832 static inline u8 page_kasan_tag(const struct page *page) 1833 { 1834 return 0xff; 1835 } 1836 1837 static inline void page_kasan_tag_set(struct page *page, u8 tag) { } 1838 static inline void page_kasan_tag_reset(struct page *page) { } 1839 1840 #endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */ 1841 1842 static inline struct zone *page_zone(const struct page *page) 1843 { 1844 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)]; 1845 } 1846 1847 static inline pg_data_t *page_pgdat(const struct page *page) 1848 { 1849 return NODE_DATA(page_to_nid(page)); 1850 } 1851 1852 static inline struct zone *folio_zone(const struct folio *folio) 1853 { 1854 return page_zone(&folio->page); 1855 } 1856 1857 static inline pg_data_t *folio_pgdat(const struct folio *folio) 1858 { 1859 return page_pgdat(&folio->page); 1860 } 1861 1862 #ifdef SECTION_IN_PAGE_FLAGS 1863 static inline void set_page_section(struct page *page, unsigned long section) 1864 { 1865 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT); 1866 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT; 1867 } 1868 1869 static inline unsigned long page_to_section(const struct page *page) 1870 { 1871 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK; 1872 } 1873 #endif 1874 1875 /** 1876 * folio_pfn - Return the Page Frame Number of a folio. 1877 * @folio: The folio. 1878 * 1879 * A folio may contain multiple pages. The pages have consecutive 1880 * Page Frame Numbers. 1881 * 1882 * Return: The Page Frame Number of the first page in the folio. 1883 */ 1884 static inline unsigned long folio_pfn(struct folio *folio) 1885 { 1886 return page_to_pfn(&folio->page); 1887 } 1888 1889 static inline struct folio *pfn_folio(unsigned long pfn) 1890 { 1891 return page_folio(pfn_to_page(pfn)); 1892 } 1893 1894 /** 1895 * folio_maybe_dma_pinned - Report if a folio may be pinned for DMA. 1896 * @folio: The folio. 1897 * 1898 * This function checks if a folio has been pinned via a call to 1899 * a function in the pin_user_pages() family. 1900 * 1901 * For small folios, the return value is partially fuzzy: false is not fuzzy, 1902 * because it means "definitely not pinned for DMA", but true means "probably 1903 * pinned for DMA, but possibly a false positive due to having at least 1904 * GUP_PIN_COUNTING_BIAS worth of normal folio references". 1905 * 1906 * False positives are OK, because: a) it's unlikely for a folio to 1907 * get that many refcounts, and b) all the callers of this routine are 1908 * expected to be able to deal gracefully with a false positive. 1909 * 1910 * For large folios, the result will be exactly correct. That's because 1911 * we have more tracking data available: the _pincount field is used 1912 * instead of the GUP_PIN_COUNTING_BIAS scheme. 1913 * 1914 * For more information, please see Documentation/core-api/pin_user_pages.rst. 1915 * 1916 * Return: True, if it is likely that the page has been "dma-pinned". 1917 * False, if the page is definitely not dma-pinned. 1918 */ 1919 static inline bool folio_maybe_dma_pinned(struct folio *folio) 1920 { 1921 if (folio_test_large(folio)) 1922 return atomic_read(&folio->_pincount) > 0; 1923 1924 /* 1925 * folio_ref_count() is signed. If that refcount overflows, then 1926 * folio_ref_count() returns a negative value, and callers will avoid 1927 * further incrementing the refcount. 1928 * 1929 * Here, for that overflow case, use the sign bit to count a little 1930 * bit higher via unsigned math, and thus still get an accurate result. 1931 */ 1932 return ((unsigned int)folio_ref_count(folio)) >= 1933 GUP_PIN_COUNTING_BIAS; 1934 } 1935 1936 static inline bool page_maybe_dma_pinned(struct page *page) 1937 { 1938 return folio_maybe_dma_pinned(page_folio(page)); 1939 } 1940 1941 /* 1942 * This should most likely only be called during fork() to see whether we 1943 * should break the cow immediately for an anon page on the src mm. 1944 * 1945 * The caller has to hold the PT lock and the vma->vm_mm->->write_protect_seq. 1946 */ 1947 static inline bool page_needs_cow_for_dma(struct vm_area_struct *vma, 1948 struct page *page) 1949 { 1950 VM_BUG_ON(!(raw_read_seqcount(&vma->vm_mm->write_protect_seq) & 1)); 1951 1952 if (!test_bit(MMF_HAS_PINNED, &vma->vm_mm->flags)) 1953 return false; 1954 1955 return page_maybe_dma_pinned(page); 1956 } 1957 1958 /** 1959 * is_zero_page - Query if a page is a zero page 1960 * @page: The page to query 1961 * 1962 * This returns true if @page is one of the permanent zero pages. 1963 */ 1964 static inline bool is_zero_page(const struct page *page) 1965 { 1966 return is_zero_pfn(page_to_pfn(page)); 1967 } 1968 1969 /** 1970 * is_zero_folio - Query if a folio is a zero page 1971 * @folio: The folio to query 1972 * 1973 * This returns true if @folio is one of the permanent zero pages. 1974 */ 1975 static inline bool is_zero_folio(const struct folio *folio) 1976 { 1977 return is_zero_page(&folio->page); 1978 } 1979 1980 /* MIGRATE_CMA and ZONE_MOVABLE do not allow pin folios */ 1981 #ifdef CONFIG_MIGRATION 1982 static inline bool folio_is_longterm_pinnable(struct folio *folio) 1983 { 1984 #ifdef CONFIG_CMA 1985 int mt = folio_migratetype(folio); 1986 1987 if (mt == MIGRATE_CMA || mt == MIGRATE_ISOLATE) 1988 return false; 1989 #endif 1990 /* The zero page can be "pinned" but gets special handling. */ 1991 if (is_zero_folio(folio)) 1992 return true; 1993 1994 /* Coherent device memory must always allow eviction. */ 1995 if (folio_is_device_coherent(folio)) 1996 return false; 1997 1998 /* Otherwise, non-movable zone folios can be pinned. */ 1999 return !folio_is_zone_movable(folio); 2000 2001 } 2002 #else 2003 static inline bool folio_is_longterm_pinnable(struct folio *folio) 2004 { 2005 return true; 2006 } 2007 #endif 2008 2009 static inline void set_page_zone(struct page *page, enum zone_type zone) 2010 { 2011 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT); 2012 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT; 2013 } 2014 2015 static inline void set_page_node(struct page *page, unsigned long node) 2016 { 2017 page->flags &= ~(NODES_MASK << NODES_PGSHIFT); 2018 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT; 2019 } 2020 2021 static inline void set_page_links(struct page *page, enum zone_type zone, 2022 unsigned long node, unsigned long pfn) 2023 { 2024 set_page_zone(page, zone); 2025 set_page_node(page, node); 2026 #ifdef SECTION_IN_PAGE_FLAGS 2027 set_page_section(page, pfn_to_section_nr(pfn)); 2028 #endif 2029 } 2030 2031 /** 2032 * folio_nr_pages - The number of pages in the folio. 2033 * @folio: The folio. 2034 * 2035 * Return: A positive power of two. 2036 */ 2037 static inline long folio_nr_pages(struct folio *folio) 2038 { 2039 if (!folio_test_large(folio)) 2040 return 1; 2041 #ifdef CONFIG_64BIT 2042 return folio->_folio_nr_pages; 2043 #else 2044 return 1L << (folio->_flags_1 & 0xff); 2045 #endif 2046 } 2047 2048 /* 2049 * compound_nr() returns the number of pages in this potentially compound 2050 * page. compound_nr() can be called on a tail page, and is defined to 2051 * return 1 in that case. 2052 */ 2053 static inline unsigned long compound_nr(struct page *page) 2054 { 2055 struct folio *folio = (struct folio *)page; 2056 2057 if (!test_bit(PG_head, &folio->flags)) 2058 return 1; 2059 #ifdef CONFIG_64BIT 2060 return folio->_folio_nr_pages; 2061 #else 2062 return 1L << (folio->_flags_1 & 0xff); 2063 #endif 2064 } 2065 2066 /** 2067 * thp_nr_pages - The number of regular pages in this huge page. 2068 * @page: The head page of a huge page. 2069 */ 2070 static inline int thp_nr_pages(struct page *page) 2071 { 2072 return folio_nr_pages((struct folio *)page); 2073 } 2074 2075 /** 2076 * folio_next - Move to the next physical folio. 2077 * @folio: The folio we're currently operating on. 2078 * 2079 * If you have physically contiguous memory which may span more than 2080 * one folio (eg a &struct bio_vec), use this function to move from one 2081 * folio to the next. Do not use it if the memory is only virtually 2082 * contiguous as the folios are almost certainly not adjacent to each 2083 * other. This is the folio equivalent to writing ``page++``. 2084 * 2085 * Context: We assume that the folios are refcounted and/or locked at a 2086 * higher level and do not adjust the reference counts. 2087 * Return: The next struct folio. 2088 */ 2089 static inline struct folio *folio_next(struct folio *folio) 2090 { 2091 return (struct folio *)folio_page(folio, folio_nr_pages(folio)); 2092 } 2093 2094 /** 2095 * folio_shift - The size of the memory described by this folio. 2096 * @folio: The folio. 2097 * 2098 * A folio represents a number of bytes which is a power-of-two in size. 2099 * This function tells you which power-of-two the folio is. See also 2100 * folio_size() and folio_order(). 2101 * 2102 * Context: The caller should have a reference on the folio to prevent 2103 * it from being split. It is not necessary for the folio to be locked. 2104 * Return: The base-2 logarithm of the size of this folio. 2105 */ 2106 static inline unsigned int folio_shift(struct folio *folio) 2107 { 2108 return PAGE_SHIFT + folio_order(folio); 2109 } 2110 2111 /** 2112 * folio_size - The number of bytes in a folio. 2113 * @folio: The folio. 2114 * 2115 * Context: The caller should have a reference on the folio to prevent 2116 * it from being split. It is not necessary for the folio to be locked. 2117 * Return: The number of bytes in this folio. 2118 */ 2119 static inline size_t folio_size(struct folio *folio) 2120 { 2121 return PAGE_SIZE << folio_order(folio); 2122 } 2123 2124 /** 2125 * folio_estimated_sharers - Estimate the number of sharers of a folio. 2126 * @folio: The folio. 2127 * 2128 * folio_estimated_sharers() aims to serve as a function to efficiently 2129 * estimate the number of processes sharing a folio. This is done by 2130 * looking at the precise mapcount of the first subpage in the folio, and 2131 * assuming the other subpages are the same. This may not be true for large 2132 * folios. If you want exact mapcounts for exact calculations, look at 2133 * page_mapcount() or folio_total_mapcount(). 2134 * 2135 * Return: The estimated number of processes sharing a folio. 2136 */ 2137 static inline int folio_estimated_sharers(struct folio *folio) 2138 { 2139 return page_mapcount(folio_page(folio, 0)); 2140 } 2141 2142 #ifndef HAVE_ARCH_MAKE_PAGE_ACCESSIBLE 2143 static inline int arch_make_page_accessible(struct page *page) 2144 { 2145 return 0; 2146 } 2147 #endif 2148 2149 #ifndef HAVE_ARCH_MAKE_FOLIO_ACCESSIBLE 2150 static inline int arch_make_folio_accessible(struct folio *folio) 2151 { 2152 int ret; 2153 long i, nr = folio_nr_pages(folio); 2154 2155 for (i = 0; i < nr; i++) { 2156 ret = arch_make_page_accessible(folio_page(folio, i)); 2157 if (ret) 2158 break; 2159 } 2160 2161 return ret; 2162 } 2163 #endif 2164 2165 /* 2166 * Some inline functions in vmstat.h depend on page_zone() 2167 */ 2168 #include <linux/vmstat.h> 2169 2170 static __always_inline void *lowmem_page_address(const struct page *page) 2171 { 2172 return page_to_virt(page); 2173 } 2174 2175 #if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL) 2176 #define HASHED_PAGE_VIRTUAL 2177 #endif 2178 2179 #if defined(WANT_PAGE_VIRTUAL) 2180 static inline void *page_address(const struct page *page) 2181 { 2182 return page->virtual; 2183 } 2184 static inline void set_page_address(struct page *page, void *address) 2185 { 2186 page->virtual = address; 2187 } 2188 #define page_address_init() do { } while(0) 2189 #endif 2190 2191 #if defined(HASHED_PAGE_VIRTUAL) 2192 void *page_address(const struct page *page); 2193 void set_page_address(struct page *page, void *virtual); 2194 void page_address_init(void); 2195 #endif 2196 2197 #if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL) 2198 #define page_address(page) lowmem_page_address(page) 2199 #define set_page_address(page, address) do { } while(0) 2200 #define page_address_init() do { } while(0) 2201 #endif 2202 2203 static inline void *folio_address(const struct folio *folio) 2204 { 2205 return page_address(&folio->page); 2206 } 2207 2208 extern pgoff_t __page_file_index(struct page *page); 2209 2210 /* 2211 * Return the pagecache index of the passed page. Regular pagecache pages 2212 * use ->index whereas swapcache pages use swp_offset(->private) 2213 */ 2214 static inline pgoff_t page_index(struct page *page) 2215 { 2216 if (unlikely(PageSwapCache(page))) 2217 return __page_file_index(page); 2218 return page->index; 2219 } 2220 2221 /* 2222 * Return true only if the page has been allocated with 2223 * ALLOC_NO_WATERMARKS and the low watermark was not 2224 * met implying that the system is under some pressure. 2225 */ 2226 static inline bool page_is_pfmemalloc(const struct page *page) 2227 { 2228 /* 2229 * lru.next has bit 1 set if the page is allocated from the 2230 * pfmemalloc reserves. Callers may simply overwrite it if 2231 * they do not need to preserve that information. 2232 */ 2233 return (uintptr_t)page->lru.next & BIT(1); 2234 } 2235 2236 /* 2237 * Return true only if the folio has been allocated with 2238 * ALLOC_NO_WATERMARKS and the low watermark was not 2239 * met implying that the system is under some pressure. 2240 */ 2241 static inline bool folio_is_pfmemalloc(const struct folio *folio) 2242 { 2243 /* 2244 * lru.next has bit 1 set if the page is allocated from the 2245 * pfmemalloc reserves. Callers may simply overwrite it if 2246 * they do not need to preserve that information. 2247 */ 2248 return (uintptr_t)folio->lru.next & BIT(1); 2249 } 2250 2251 /* 2252 * Only to be called by the page allocator on a freshly allocated 2253 * page. 2254 */ 2255 static inline void set_page_pfmemalloc(struct page *page) 2256 { 2257 page->lru.next = (void *)BIT(1); 2258 } 2259 2260 static inline void clear_page_pfmemalloc(struct page *page) 2261 { 2262 page->lru.next = NULL; 2263 } 2264 2265 /* 2266 * Can be called by the pagefault handler when it gets a VM_FAULT_OOM. 2267 */ 2268 extern void pagefault_out_of_memory(void); 2269 2270 #define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK) 2271 #define offset_in_thp(page, p) ((unsigned long)(p) & (thp_size(page) - 1)) 2272 #define offset_in_folio(folio, p) ((unsigned long)(p) & (folio_size(folio) - 1)) 2273 2274 /* 2275 * Parameter block passed down to zap_pte_range in exceptional cases. 2276 */ 2277 struct zap_details { 2278 struct folio *single_folio; /* Locked folio to be unmapped */ 2279 bool even_cows; /* Zap COWed private pages too? */ 2280 zap_flags_t zap_flags; /* Extra flags for zapping */ 2281 }; 2282 2283 /* 2284 * Whether to drop the pte markers, for example, the uffd-wp information for 2285 * file-backed memory. This should only be specified when we will completely 2286 * drop the page in the mm, either by truncation or unmapping of the vma. By 2287 * default, the flag is not set. 2288 */ 2289 #define ZAP_FLAG_DROP_MARKER ((__force zap_flags_t) BIT(0)) 2290 /* Set in unmap_vmas() to indicate a final unmap call. Only used by hugetlb */ 2291 #define ZAP_FLAG_UNMAP ((__force zap_flags_t) BIT(1)) 2292 2293 #ifdef CONFIG_SCHED_MM_CID 2294 void sched_mm_cid_before_execve(struct task_struct *t); 2295 void sched_mm_cid_after_execve(struct task_struct *t); 2296 void sched_mm_cid_fork(struct task_struct *t); 2297 void sched_mm_cid_exit_signals(struct task_struct *t); 2298 static inline int task_mm_cid(struct task_struct *t) 2299 { 2300 return t->mm_cid; 2301 } 2302 #else 2303 static inline void sched_mm_cid_before_execve(struct task_struct *t) { } 2304 static inline void sched_mm_cid_after_execve(struct task_struct *t) { } 2305 static inline void sched_mm_cid_fork(struct task_struct *t) { } 2306 static inline void sched_mm_cid_exit_signals(struct task_struct *t) { } 2307 static inline int task_mm_cid(struct task_struct *t) 2308 { 2309 /* 2310 * Use the processor id as a fall-back when the mm cid feature is 2311 * disabled. This provides functional per-cpu data structure accesses 2312 * in user-space, althrough it won't provide the memory usage benefits. 2313 */ 2314 return raw_smp_processor_id(); 2315 } 2316 #endif 2317 2318 #ifdef CONFIG_MMU 2319 extern bool can_do_mlock(void); 2320 #else 2321 static inline bool can_do_mlock(void) { return false; } 2322 #endif 2323 extern int user_shm_lock(size_t, struct ucounts *); 2324 extern void user_shm_unlock(size_t, struct ucounts *); 2325 2326 struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr, 2327 pte_t pte); 2328 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, 2329 pte_t pte); 2330 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr, 2331 pmd_t pmd); 2332 2333 void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, 2334 unsigned long size); 2335 void zap_page_range_single(struct vm_area_struct *vma, unsigned long address, 2336 unsigned long size, struct zap_details *details); 2337 static inline void zap_vma_pages(struct vm_area_struct *vma) 2338 { 2339 zap_page_range_single(vma, vma->vm_start, 2340 vma->vm_end - vma->vm_start, NULL); 2341 } 2342 void unmap_vmas(struct mmu_gather *tlb, struct ma_state *mas, 2343 struct vm_area_struct *start_vma, unsigned long start, 2344 unsigned long end, unsigned long tree_end, bool mm_wr_locked); 2345 2346 struct mmu_notifier_range; 2347 2348 void free_pgd_range(struct mmu_gather *tlb, unsigned long addr, 2349 unsigned long end, unsigned long floor, unsigned long ceiling); 2350 int 2351 copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma); 2352 int follow_pte(struct mm_struct *mm, unsigned long address, 2353 pte_t **ptepp, spinlock_t **ptlp); 2354 int follow_pfn(struct vm_area_struct *vma, unsigned long address, 2355 unsigned long *pfn); 2356 int follow_phys(struct vm_area_struct *vma, unsigned long address, 2357 unsigned int flags, unsigned long *prot, resource_size_t *phys); 2358 int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, 2359 void *buf, int len, int write); 2360 2361 extern void truncate_pagecache(struct inode *inode, loff_t new); 2362 extern void truncate_setsize(struct inode *inode, loff_t newsize); 2363 void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to); 2364 void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end); 2365 int generic_error_remove_page(struct address_space *mapping, struct page *page); 2366 2367 struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm, 2368 unsigned long address, struct pt_regs *regs); 2369 2370 #ifdef CONFIG_MMU 2371 extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma, 2372 unsigned long address, unsigned int flags, 2373 struct pt_regs *regs); 2374 extern int fixup_user_fault(struct mm_struct *mm, 2375 unsigned long address, unsigned int fault_flags, 2376 bool *unlocked); 2377 void unmap_mapping_pages(struct address_space *mapping, 2378 pgoff_t start, pgoff_t nr, bool even_cows); 2379 void unmap_mapping_range(struct address_space *mapping, 2380 loff_t const holebegin, loff_t const holelen, int even_cows); 2381 #else 2382 static inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma, 2383 unsigned long address, unsigned int flags, 2384 struct pt_regs *regs) 2385 { 2386 /* should never happen if there's no MMU */ 2387 BUG(); 2388 return VM_FAULT_SIGBUS; 2389 } 2390 static inline int fixup_user_fault(struct mm_struct *mm, unsigned long address, 2391 unsigned int fault_flags, bool *unlocked) 2392 { 2393 /* should never happen if there's no MMU */ 2394 BUG(); 2395 return -EFAULT; 2396 } 2397 static inline void unmap_mapping_pages(struct address_space *mapping, 2398 pgoff_t start, pgoff_t nr, bool even_cows) { } 2399 static inline void unmap_mapping_range(struct address_space *mapping, 2400 loff_t const holebegin, loff_t const holelen, int even_cows) { } 2401 #endif 2402 2403 static inline void unmap_shared_mapping_range(struct address_space *mapping, 2404 loff_t const holebegin, loff_t const holelen) 2405 { 2406 unmap_mapping_range(mapping, holebegin, holelen, 0); 2407 } 2408 2409 static inline struct vm_area_struct *vma_lookup(struct mm_struct *mm, 2410 unsigned long addr); 2411 2412 extern int access_process_vm(struct task_struct *tsk, unsigned long addr, 2413 void *buf, int len, unsigned int gup_flags); 2414 extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, 2415 void *buf, int len, unsigned int gup_flags); 2416 extern int __access_remote_vm(struct mm_struct *mm, unsigned long addr, 2417 void *buf, int len, unsigned int gup_flags); 2418 2419 long get_user_pages_remote(struct mm_struct *mm, 2420 unsigned long start, unsigned long nr_pages, 2421 unsigned int gup_flags, struct page **pages, 2422 int *locked); 2423 long pin_user_pages_remote(struct mm_struct *mm, 2424 unsigned long start, unsigned long nr_pages, 2425 unsigned int gup_flags, struct page **pages, 2426 int *locked); 2427 2428 static inline struct page *get_user_page_vma_remote(struct mm_struct *mm, 2429 unsigned long addr, 2430 int gup_flags, 2431 struct vm_area_struct **vmap) 2432 { 2433 struct page *page; 2434 struct vm_area_struct *vma; 2435 int got = get_user_pages_remote(mm, addr, 1, gup_flags, &page, NULL); 2436 2437 if (got < 0) 2438 return ERR_PTR(got); 2439 if (got == 0) 2440 return NULL; 2441 2442 vma = vma_lookup(mm, addr); 2443 if (WARN_ON_ONCE(!vma)) { 2444 put_page(page); 2445 return ERR_PTR(-EINVAL); 2446 } 2447 2448 *vmap = vma; 2449 return page; 2450 } 2451 2452 long get_user_pages(unsigned long start, unsigned long nr_pages, 2453 unsigned int gup_flags, struct page **pages); 2454 long pin_user_pages(unsigned long start, unsigned long nr_pages, 2455 unsigned int gup_flags, struct page **pages); 2456 long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, 2457 struct page **pages, unsigned int gup_flags); 2458 long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages, 2459 struct page **pages, unsigned int gup_flags); 2460 2461 int get_user_pages_fast(unsigned long start, int nr_pages, 2462 unsigned int gup_flags, struct page **pages); 2463 int pin_user_pages_fast(unsigned long start, int nr_pages, 2464 unsigned int gup_flags, struct page **pages); 2465 void folio_add_pin(struct folio *folio); 2466 2467 int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc); 2468 int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, 2469 struct task_struct *task, bool bypass_rlim); 2470 2471 struct kvec; 2472 struct page *get_dump_page(unsigned long addr); 2473 2474 bool folio_mark_dirty(struct folio *folio); 2475 bool set_page_dirty(struct page *page); 2476 int set_page_dirty_lock(struct page *page); 2477 2478 int get_cmdline(struct task_struct *task, char *buffer, int buflen); 2479 2480 extern unsigned long move_page_tables(struct vm_area_struct *vma, 2481 unsigned long old_addr, struct vm_area_struct *new_vma, 2482 unsigned long new_addr, unsigned long len, 2483 bool need_rmap_locks); 2484 2485 /* 2486 * Flags used by change_protection(). For now we make it a bitmap so 2487 * that we can pass in multiple flags just like parameters. However 2488 * for now all the callers are only use one of the flags at the same 2489 * time. 2490 */ 2491 /* 2492 * Whether we should manually check if we can map individual PTEs writable, 2493 * because something (e.g., COW, uffd-wp) blocks that from happening for all 2494 * PTEs automatically in a writable mapping. 2495 */ 2496 #define MM_CP_TRY_CHANGE_WRITABLE (1UL << 0) 2497 /* Whether this protection change is for NUMA hints */ 2498 #define MM_CP_PROT_NUMA (1UL << 1) 2499 /* Whether this change is for write protecting */ 2500 #define MM_CP_UFFD_WP (1UL << 2) /* do wp */ 2501 #define MM_CP_UFFD_WP_RESOLVE (1UL << 3) /* Resolve wp */ 2502 #define MM_CP_UFFD_WP_ALL (MM_CP_UFFD_WP | \ 2503 MM_CP_UFFD_WP_RESOLVE) 2504 2505 bool vma_needs_dirty_tracking(struct vm_area_struct *vma); 2506 int vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot); 2507 static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma) 2508 { 2509 /* 2510 * We want to check manually if we can change individual PTEs writable 2511 * if we can't do that automatically for all PTEs in a mapping. For 2512 * private mappings, that's always the case when we have write 2513 * permissions as we properly have to handle COW. 2514 */ 2515 if (vma->vm_flags & VM_SHARED) 2516 return vma_wants_writenotify(vma, vma->vm_page_prot); 2517 return !!(vma->vm_flags & VM_WRITE); 2518 2519 } 2520 bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, 2521 pte_t pte); 2522 extern long change_protection(struct mmu_gather *tlb, 2523 struct vm_area_struct *vma, unsigned long start, 2524 unsigned long end, unsigned long cp_flags); 2525 extern int mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, 2526 struct vm_area_struct *vma, struct vm_area_struct **pprev, 2527 unsigned long start, unsigned long end, unsigned long newflags); 2528 2529 /* 2530 * doesn't attempt to fault and will return short. 2531 */ 2532 int get_user_pages_fast_only(unsigned long start, int nr_pages, 2533 unsigned int gup_flags, struct page **pages); 2534 2535 static inline bool get_user_page_fast_only(unsigned long addr, 2536 unsigned int gup_flags, struct page **pagep) 2537 { 2538 return get_user_pages_fast_only(addr, 1, gup_flags, pagep) == 1; 2539 } 2540 /* 2541 * per-process(per-mm_struct) statistics. 2542 */ 2543 static inline unsigned long get_mm_counter(struct mm_struct *mm, int member) 2544 { 2545 return percpu_counter_read_positive(&mm->rss_stat[member]); 2546 } 2547 2548 void mm_trace_rss_stat(struct mm_struct *mm, int member); 2549 2550 static inline void add_mm_counter(struct mm_struct *mm, int member, long value) 2551 { 2552 percpu_counter_add(&mm->rss_stat[member], value); 2553 2554 mm_trace_rss_stat(mm, member); 2555 } 2556 2557 static inline void inc_mm_counter(struct mm_struct *mm, int member) 2558 { 2559 percpu_counter_inc(&mm->rss_stat[member]); 2560 2561 mm_trace_rss_stat(mm, member); 2562 } 2563 2564 static inline void dec_mm_counter(struct mm_struct *mm, int member) 2565 { 2566 percpu_counter_dec(&mm->rss_stat[member]); 2567 2568 mm_trace_rss_stat(mm, member); 2569 } 2570 2571 /* Optimized variant when page is already known not to be PageAnon */ 2572 static inline int mm_counter_file(struct page *page) 2573 { 2574 if (PageSwapBacked(page)) 2575 return MM_SHMEMPAGES; 2576 return MM_FILEPAGES; 2577 } 2578 2579 static inline int mm_counter(struct page *page) 2580 { 2581 if (PageAnon(page)) 2582 return MM_ANONPAGES; 2583 return mm_counter_file(page); 2584 } 2585 2586 static inline unsigned long get_mm_rss(struct mm_struct *mm) 2587 { 2588 return get_mm_counter(mm, MM_FILEPAGES) + 2589 get_mm_counter(mm, MM_ANONPAGES) + 2590 get_mm_counter(mm, MM_SHMEMPAGES); 2591 } 2592 2593 static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm) 2594 { 2595 return max(mm->hiwater_rss, get_mm_rss(mm)); 2596 } 2597 2598 static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm) 2599 { 2600 return max(mm->hiwater_vm, mm->total_vm); 2601 } 2602 2603 static inline void update_hiwater_rss(struct mm_struct *mm) 2604 { 2605 unsigned long _rss = get_mm_rss(mm); 2606 2607 if ((mm)->hiwater_rss < _rss) 2608 (mm)->hiwater_rss = _rss; 2609 } 2610 2611 static inline void update_hiwater_vm(struct mm_struct *mm) 2612 { 2613 if (mm->hiwater_vm < mm->total_vm) 2614 mm->hiwater_vm = mm->total_vm; 2615 } 2616 2617 static inline void reset_mm_hiwater_rss(struct mm_struct *mm) 2618 { 2619 mm->hiwater_rss = get_mm_rss(mm); 2620 } 2621 2622 static inline void setmax_mm_hiwater_rss(unsigned long *maxrss, 2623 struct mm_struct *mm) 2624 { 2625 unsigned long hiwater_rss = get_mm_hiwater_rss(mm); 2626 2627 if (*maxrss < hiwater_rss) 2628 *maxrss = hiwater_rss; 2629 } 2630 2631 #if defined(SPLIT_RSS_COUNTING) 2632 void sync_mm_rss(struct mm_struct *mm); 2633 #else 2634 static inline void sync_mm_rss(struct mm_struct *mm) 2635 { 2636 } 2637 #endif 2638 2639 #ifndef CONFIG_ARCH_HAS_PTE_SPECIAL 2640 static inline int pte_special(pte_t pte) 2641 { 2642 return 0; 2643 } 2644 2645 static inline pte_t pte_mkspecial(pte_t pte) 2646 { 2647 return pte; 2648 } 2649 #endif 2650 2651 #ifndef CONFIG_ARCH_HAS_PTE_DEVMAP 2652 static inline int pte_devmap(pte_t pte) 2653 { 2654 return 0; 2655 } 2656 #endif 2657 2658 extern pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr, 2659 spinlock_t **ptl); 2660 static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr, 2661 spinlock_t **ptl) 2662 { 2663 pte_t *ptep; 2664 __cond_lock(*ptl, ptep = __get_locked_pte(mm, addr, ptl)); 2665 return ptep; 2666 } 2667 2668 #ifdef __PAGETABLE_P4D_FOLDED 2669 static inline int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, 2670 unsigned long address) 2671 { 2672 return 0; 2673 } 2674 #else 2675 int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address); 2676 #endif 2677 2678 #if defined(__PAGETABLE_PUD_FOLDED) || !defined(CONFIG_MMU) 2679 static inline int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, 2680 unsigned long address) 2681 { 2682 return 0; 2683 } 2684 static inline void mm_inc_nr_puds(struct mm_struct *mm) {} 2685 static inline void mm_dec_nr_puds(struct mm_struct *mm) {} 2686 2687 #else 2688 int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address); 2689 2690 static inline void mm_inc_nr_puds(struct mm_struct *mm) 2691 { 2692 if (mm_pud_folded(mm)) 2693 return; 2694 atomic_long_add(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes); 2695 } 2696 2697 static inline void mm_dec_nr_puds(struct mm_struct *mm) 2698 { 2699 if (mm_pud_folded(mm)) 2700 return; 2701 atomic_long_sub(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes); 2702 } 2703 #endif 2704 2705 #if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU) 2706 static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud, 2707 unsigned long address) 2708 { 2709 return 0; 2710 } 2711 2712 static inline void mm_inc_nr_pmds(struct mm_struct *mm) {} 2713 static inline void mm_dec_nr_pmds(struct mm_struct *mm) {} 2714 2715 #else 2716 int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address); 2717 2718 static inline void mm_inc_nr_pmds(struct mm_struct *mm) 2719 { 2720 if (mm_pmd_folded(mm)) 2721 return; 2722 atomic_long_add(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes); 2723 } 2724 2725 static inline void mm_dec_nr_pmds(struct mm_struct *mm) 2726 { 2727 if (mm_pmd_folded(mm)) 2728 return; 2729 atomic_long_sub(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes); 2730 } 2731 #endif 2732 2733 #ifdef CONFIG_MMU 2734 static inline void mm_pgtables_bytes_init(struct mm_struct *mm) 2735 { 2736 atomic_long_set(&mm->pgtables_bytes, 0); 2737 } 2738 2739 static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm) 2740 { 2741 return atomic_long_read(&mm->pgtables_bytes); 2742 } 2743 2744 static inline void mm_inc_nr_ptes(struct mm_struct *mm) 2745 { 2746 atomic_long_add(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes); 2747 } 2748 2749 static inline void mm_dec_nr_ptes(struct mm_struct *mm) 2750 { 2751 atomic_long_sub(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes); 2752 } 2753 #else 2754 2755 static inline void mm_pgtables_bytes_init(struct mm_struct *mm) {} 2756 static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm) 2757 { 2758 return 0; 2759 } 2760 2761 static inline void mm_inc_nr_ptes(struct mm_struct *mm) {} 2762 static inline void mm_dec_nr_ptes(struct mm_struct *mm) {} 2763 #endif 2764 2765 int __pte_alloc(struct mm_struct *mm, pmd_t *pmd); 2766 int __pte_alloc_kernel(pmd_t *pmd); 2767 2768 #if defined(CONFIG_MMU) 2769 2770 static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd, 2771 unsigned long address) 2772 { 2773 return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ? 2774 NULL : p4d_offset(pgd, address); 2775 } 2776 2777 static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d, 2778 unsigned long address) 2779 { 2780 return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ? 2781 NULL : pud_offset(p4d, address); 2782 } 2783 2784 static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) 2785 { 2786 return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))? 2787 NULL: pmd_offset(pud, address); 2788 } 2789 #endif /* CONFIG_MMU */ 2790 2791 static inline struct ptdesc *virt_to_ptdesc(const void *x) 2792 { 2793 return page_ptdesc(virt_to_page(x)); 2794 } 2795 2796 static inline void *ptdesc_to_virt(const struct ptdesc *pt) 2797 { 2798 return page_to_virt(ptdesc_page(pt)); 2799 } 2800 2801 static inline void *ptdesc_address(const struct ptdesc *pt) 2802 { 2803 return folio_address(ptdesc_folio(pt)); 2804 } 2805 2806 static inline bool pagetable_is_reserved(struct ptdesc *pt) 2807 { 2808 return folio_test_reserved(ptdesc_folio(pt)); 2809 } 2810 2811 /** 2812 * pagetable_alloc - Allocate pagetables 2813 * @gfp: GFP flags 2814 * @order: desired pagetable order 2815 * 2816 * pagetable_alloc allocates memory for page tables as well as a page table 2817 * descriptor to describe that memory. 2818 * 2819 * Return: The ptdesc describing the allocated page tables. 2820 */ 2821 static inline struct ptdesc *pagetable_alloc(gfp_t gfp, unsigned int order) 2822 { 2823 struct page *page = alloc_pages(gfp | __GFP_COMP, order); 2824 2825 return page_ptdesc(page); 2826 } 2827 2828 /** 2829 * pagetable_free - Free pagetables 2830 * @pt: The page table descriptor 2831 * 2832 * pagetable_free frees the memory of all page tables described by a page 2833 * table descriptor and the memory for the descriptor itself. 2834 */ 2835 static inline void pagetable_free(struct ptdesc *pt) 2836 { 2837 struct page *page = ptdesc_page(pt); 2838 2839 __free_pages(page, compound_order(page)); 2840 } 2841 2842 #if USE_SPLIT_PTE_PTLOCKS 2843 #if ALLOC_SPLIT_PTLOCKS 2844 void __init ptlock_cache_init(void); 2845 bool ptlock_alloc(struct ptdesc *ptdesc); 2846 void ptlock_free(struct ptdesc *ptdesc); 2847 2848 static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc) 2849 { 2850 return ptdesc->ptl; 2851 } 2852 #else /* ALLOC_SPLIT_PTLOCKS */ 2853 static inline void ptlock_cache_init(void) 2854 { 2855 } 2856 2857 static inline bool ptlock_alloc(struct ptdesc *ptdesc) 2858 { 2859 return true; 2860 } 2861 2862 static inline void ptlock_free(struct ptdesc *ptdesc) 2863 { 2864 } 2865 2866 static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc) 2867 { 2868 return &ptdesc->ptl; 2869 } 2870 #endif /* ALLOC_SPLIT_PTLOCKS */ 2871 2872 static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd) 2873 { 2874 return ptlock_ptr(page_ptdesc(pmd_page(*pmd))); 2875 } 2876 2877 static inline bool ptlock_init(struct ptdesc *ptdesc) 2878 { 2879 /* 2880 * prep_new_page() initialize page->private (and therefore page->ptl) 2881 * with 0. Make sure nobody took it in use in between. 2882 * 2883 * It can happen if arch try to use slab for page table allocation: 2884 * slab code uses page->slab_cache, which share storage with page->ptl. 2885 */ 2886 VM_BUG_ON_PAGE(*(unsigned long *)&ptdesc->ptl, ptdesc_page(ptdesc)); 2887 if (!ptlock_alloc(ptdesc)) 2888 return false; 2889 spin_lock_init(ptlock_ptr(ptdesc)); 2890 return true; 2891 } 2892 2893 #else /* !USE_SPLIT_PTE_PTLOCKS */ 2894 /* 2895 * We use mm->page_table_lock to guard all pagetable pages of the mm. 2896 */ 2897 static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd) 2898 { 2899 return &mm->page_table_lock; 2900 } 2901 static inline void ptlock_cache_init(void) {} 2902 static inline bool ptlock_init(struct ptdesc *ptdesc) { return true; } 2903 static inline void ptlock_free(struct ptdesc *ptdesc) {} 2904 #endif /* USE_SPLIT_PTE_PTLOCKS */ 2905 2906 static inline bool pagetable_pte_ctor(struct ptdesc *ptdesc) 2907 { 2908 struct folio *folio = ptdesc_folio(ptdesc); 2909 2910 if (!ptlock_init(ptdesc)) 2911 return false; 2912 __folio_set_pgtable(folio); 2913 lruvec_stat_add_folio(folio, NR_PAGETABLE); 2914 return true; 2915 } 2916 2917 static inline void pagetable_pte_dtor(struct ptdesc *ptdesc) 2918 { 2919 struct folio *folio = ptdesc_folio(ptdesc); 2920 2921 ptlock_free(ptdesc); 2922 __folio_clear_pgtable(folio); 2923 lruvec_stat_sub_folio(folio, NR_PAGETABLE); 2924 } 2925 2926 pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp); 2927 static inline pte_t *pte_offset_map(pmd_t *pmd, unsigned long addr) 2928 { 2929 return __pte_offset_map(pmd, addr, NULL); 2930 } 2931 2932 pte_t *__pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, 2933 unsigned long addr, spinlock_t **ptlp); 2934 static inline pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, 2935 unsigned long addr, spinlock_t **ptlp) 2936 { 2937 pte_t *pte; 2938 2939 __cond_lock(*ptlp, pte = __pte_offset_map_lock(mm, pmd, addr, ptlp)); 2940 return pte; 2941 } 2942 2943 pte_t *pte_offset_map_nolock(struct mm_struct *mm, pmd_t *pmd, 2944 unsigned long addr, spinlock_t **ptlp); 2945 2946 #define pte_unmap_unlock(pte, ptl) do { \ 2947 spin_unlock(ptl); \ 2948 pte_unmap(pte); \ 2949 } while (0) 2950 2951 #define pte_alloc(mm, pmd) (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd)) 2952 2953 #define pte_alloc_map(mm, pmd, address) \ 2954 (pte_alloc(mm, pmd) ? NULL : pte_offset_map(pmd, address)) 2955 2956 #define pte_alloc_map_lock(mm, pmd, address, ptlp) \ 2957 (pte_alloc(mm, pmd) ? \ 2958 NULL : pte_offset_map_lock(mm, pmd, address, ptlp)) 2959 2960 #define pte_alloc_kernel(pmd, address) \ 2961 ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd))? \ 2962 NULL: pte_offset_kernel(pmd, address)) 2963 2964 #if USE_SPLIT_PMD_PTLOCKS 2965 2966 static inline struct page *pmd_pgtable_page(pmd_t *pmd) 2967 { 2968 unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1); 2969 return virt_to_page((void *)((unsigned long) pmd & mask)); 2970 } 2971 2972 static inline struct ptdesc *pmd_ptdesc(pmd_t *pmd) 2973 { 2974 return page_ptdesc(pmd_pgtable_page(pmd)); 2975 } 2976 2977 static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd) 2978 { 2979 return ptlock_ptr(pmd_ptdesc(pmd)); 2980 } 2981 2982 static inline bool pmd_ptlock_init(struct ptdesc *ptdesc) 2983 { 2984 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2985 ptdesc->pmd_huge_pte = NULL; 2986 #endif 2987 return ptlock_init(ptdesc); 2988 } 2989 2990 static inline void pmd_ptlock_free(struct ptdesc *ptdesc) 2991 { 2992 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2993 VM_BUG_ON_PAGE(ptdesc->pmd_huge_pte, ptdesc_page(ptdesc)); 2994 #endif 2995 ptlock_free(ptdesc); 2996 } 2997 2998 #define pmd_huge_pte(mm, pmd) (pmd_ptdesc(pmd)->pmd_huge_pte) 2999 3000 #else 3001 3002 static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd) 3003 { 3004 return &mm->page_table_lock; 3005 } 3006 3007 static inline bool pmd_ptlock_init(struct ptdesc *ptdesc) { return true; } 3008 static inline void pmd_ptlock_free(struct ptdesc *ptdesc) {} 3009 3010 #define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte) 3011 3012 #endif 3013 3014 static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd) 3015 { 3016 spinlock_t *ptl = pmd_lockptr(mm, pmd); 3017 spin_lock(ptl); 3018 return ptl; 3019 } 3020 3021 static inline bool pagetable_pmd_ctor(struct ptdesc *ptdesc) 3022 { 3023 struct folio *folio = ptdesc_folio(ptdesc); 3024 3025 if (!pmd_ptlock_init(ptdesc)) 3026 return false; 3027 __folio_set_pgtable(folio); 3028 lruvec_stat_add_folio(folio, NR_PAGETABLE); 3029 return true; 3030 } 3031 3032 static inline void pagetable_pmd_dtor(struct ptdesc *ptdesc) 3033 { 3034 struct folio *folio = ptdesc_folio(ptdesc); 3035 3036 pmd_ptlock_free(ptdesc); 3037 __folio_clear_pgtable(folio); 3038 lruvec_stat_sub_folio(folio, NR_PAGETABLE); 3039 } 3040 3041 /* 3042 * No scalability reason to split PUD locks yet, but follow the same pattern 3043 * as the PMD locks to make it easier if we decide to. The VM should not be 3044 * considered ready to switch to split PUD locks yet; there may be places 3045 * which need to be converted from page_table_lock. 3046 */ 3047 static inline spinlock_t *pud_lockptr(struct mm_struct *mm, pud_t *pud) 3048 { 3049 return &mm->page_table_lock; 3050 } 3051 3052 static inline spinlock_t *pud_lock(struct mm_struct *mm, pud_t *pud) 3053 { 3054 spinlock_t *ptl = pud_lockptr(mm, pud); 3055 3056 spin_lock(ptl); 3057 return ptl; 3058 } 3059 3060 extern void __init pagecache_init(void); 3061 extern void free_initmem(void); 3062 3063 /* 3064 * Free reserved pages within range [PAGE_ALIGN(start), end & PAGE_MASK) 3065 * into the buddy system. The freed pages will be poisoned with pattern 3066 * "poison" if it's within range [0, UCHAR_MAX]. 3067 * Return pages freed into the buddy system. 3068 */ 3069 extern unsigned long free_reserved_area(void *start, void *end, 3070 int poison, const char *s); 3071 3072 extern void adjust_managed_page_count(struct page *page, long count); 3073 3074 extern void reserve_bootmem_region(phys_addr_t start, 3075 phys_addr_t end, int nid); 3076 3077 /* Free the reserved page into the buddy system, so it gets managed. */ 3078 static inline void free_reserved_page(struct page *page) 3079 { 3080 ClearPageReserved(page); 3081 init_page_count(page); 3082 __free_page(page); 3083 adjust_managed_page_count(page, 1); 3084 } 3085 #define free_highmem_page(page) free_reserved_page(page) 3086 3087 static inline void mark_page_reserved(struct page *page) 3088 { 3089 SetPageReserved(page); 3090 adjust_managed_page_count(page, -1); 3091 } 3092 3093 static inline void free_reserved_ptdesc(struct ptdesc *pt) 3094 { 3095 free_reserved_page(ptdesc_page(pt)); 3096 } 3097 3098 /* 3099 * Default method to free all the __init memory into the buddy system. 3100 * The freed pages will be poisoned with pattern "poison" if it's within 3101 * range [0, UCHAR_MAX]. 3102 * Return pages freed into the buddy system. 3103 */ 3104 static inline unsigned long free_initmem_default(int poison) 3105 { 3106 extern char __init_begin[], __init_end[]; 3107 3108 return free_reserved_area(&__init_begin, &__init_end, 3109 poison, "unused kernel image (initmem)"); 3110 } 3111 3112 static inline unsigned long get_num_physpages(void) 3113 { 3114 int nid; 3115 unsigned long phys_pages = 0; 3116 3117 for_each_online_node(nid) 3118 phys_pages += node_present_pages(nid); 3119 3120 return phys_pages; 3121 } 3122 3123 /* 3124 * Using memblock node mappings, an architecture may initialise its 3125 * zones, allocate the backing mem_map and account for memory holes in an 3126 * architecture independent manner. 3127 * 3128 * An architecture is expected to register range of page frames backed by 3129 * physical memory with memblock_add[_node]() before calling 3130 * free_area_init() passing in the PFN each zone ends at. At a basic 3131 * usage, an architecture is expected to do something like 3132 * 3133 * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn, 3134 * max_highmem_pfn}; 3135 * for_each_valid_physical_page_range() 3136 * memblock_add_node(base, size, nid, MEMBLOCK_NONE) 3137 * free_area_init(max_zone_pfns); 3138 */ 3139 void free_area_init(unsigned long *max_zone_pfn); 3140 unsigned long node_map_pfn_alignment(void); 3141 unsigned long __absent_pages_in_range(int nid, unsigned long start_pfn, 3142 unsigned long end_pfn); 3143 extern unsigned long absent_pages_in_range(unsigned long start_pfn, 3144 unsigned long end_pfn); 3145 extern void get_pfn_range_for_nid(unsigned int nid, 3146 unsigned long *start_pfn, unsigned long *end_pfn); 3147 3148 #ifndef CONFIG_NUMA 3149 static inline int early_pfn_to_nid(unsigned long pfn) 3150 { 3151 return 0; 3152 } 3153 #else 3154 /* please see mm/page_alloc.c */ 3155 extern int __meminit early_pfn_to_nid(unsigned long pfn); 3156 #endif 3157 3158 extern void set_dma_reserve(unsigned long new_dma_reserve); 3159 extern void mem_init(void); 3160 extern void __init mmap_init(void); 3161 3162 extern void __show_mem(unsigned int flags, nodemask_t *nodemask, int max_zone_idx); 3163 static inline void show_mem(void) 3164 { 3165 __show_mem(0, NULL, MAX_NR_ZONES - 1); 3166 } 3167 extern long si_mem_available(void); 3168 extern void si_meminfo(struct sysinfo * val); 3169 extern void si_meminfo_node(struct sysinfo *val, int nid); 3170 #ifdef __HAVE_ARCH_RESERVED_KERNEL_PAGES 3171 extern unsigned long arch_reserved_kernel_pages(void); 3172 #endif 3173 3174 extern __printf(3, 4) 3175 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...); 3176 3177 extern void setup_per_cpu_pageset(void); 3178 3179 /* nommu.c */ 3180 extern atomic_long_t mmap_pages_allocated; 3181 extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t); 3182 3183 /* interval_tree.c */ 3184 void vma_interval_tree_insert(struct vm_area_struct *node, 3185 struct rb_root_cached *root); 3186 void vma_interval_tree_insert_after(struct vm_area_struct *node, 3187 struct vm_area_struct *prev, 3188 struct rb_root_cached *root); 3189 void vma_interval_tree_remove(struct vm_area_struct *node, 3190 struct rb_root_cached *root); 3191 struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root_cached *root, 3192 unsigned long start, unsigned long last); 3193 struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node, 3194 unsigned long start, unsigned long last); 3195 3196 #define vma_interval_tree_foreach(vma, root, start, last) \ 3197 for (vma = vma_interval_tree_iter_first(root, start, last); \ 3198 vma; vma = vma_interval_tree_iter_next(vma, start, last)) 3199 3200 void anon_vma_interval_tree_insert(struct anon_vma_chain *node, 3201 struct rb_root_cached *root); 3202 void anon_vma_interval_tree_remove(struct anon_vma_chain *node, 3203 struct rb_root_cached *root); 3204 struct anon_vma_chain * 3205 anon_vma_interval_tree_iter_first(struct rb_root_cached *root, 3206 unsigned long start, unsigned long last); 3207 struct anon_vma_chain *anon_vma_interval_tree_iter_next( 3208 struct anon_vma_chain *node, unsigned long start, unsigned long last); 3209 #ifdef CONFIG_DEBUG_VM_RB 3210 void anon_vma_interval_tree_verify(struct anon_vma_chain *node); 3211 #endif 3212 3213 #define anon_vma_interval_tree_foreach(avc, root, start, last) \ 3214 for (avc = anon_vma_interval_tree_iter_first(root, start, last); \ 3215 avc; avc = anon_vma_interval_tree_iter_next(avc, start, last)) 3216 3217 /* mmap.c */ 3218 extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin); 3219 extern int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma, 3220 unsigned long start, unsigned long end, pgoff_t pgoff, 3221 struct vm_area_struct *next); 3222 extern int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, 3223 unsigned long start, unsigned long end, pgoff_t pgoff); 3224 extern struct vm_area_struct *vma_merge(struct vma_iterator *vmi, 3225 struct mm_struct *, struct vm_area_struct *prev, unsigned long addr, 3226 unsigned long end, unsigned long vm_flags, struct anon_vma *, 3227 struct file *, pgoff_t, struct mempolicy *, struct vm_userfaultfd_ctx, 3228 struct anon_vma_name *); 3229 extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *); 3230 extern int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *, 3231 unsigned long addr, int new_below); 3232 extern int split_vma(struct vma_iterator *vmi, struct vm_area_struct *, 3233 unsigned long addr, int new_below); 3234 extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *); 3235 extern void unlink_file_vma(struct vm_area_struct *); 3236 extern struct vm_area_struct *copy_vma(struct vm_area_struct **, 3237 unsigned long addr, unsigned long len, pgoff_t pgoff, 3238 bool *need_rmap_locks); 3239 extern void exit_mmap(struct mm_struct *); 3240 3241 static inline int check_data_rlimit(unsigned long rlim, 3242 unsigned long new, 3243 unsigned long start, 3244 unsigned long end_data, 3245 unsigned long start_data) 3246 { 3247 if (rlim < RLIM_INFINITY) { 3248 if (((new - start) + (end_data - start_data)) > rlim) 3249 return -ENOSPC; 3250 } 3251 3252 return 0; 3253 } 3254 3255 extern int mm_take_all_locks(struct mm_struct *mm); 3256 extern void mm_drop_all_locks(struct mm_struct *mm); 3257 3258 extern int set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file); 3259 extern int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file); 3260 extern struct file *get_mm_exe_file(struct mm_struct *mm); 3261 extern struct file *get_task_exe_file(struct task_struct *task); 3262 3263 extern bool may_expand_vm(struct mm_struct *, vm_flags_t, unsigned long npages); 3264 extern void vm_stat_account(struct mm_struct *, vm_flags_t, long npages); 3265 3266 extern bool vma_is_special_mapping(const struct vm_area_struct *vma, 3267 const struct vm_special_mapping *sm); 3268 extern struct vm_area_struct *_install_special_mapping(struct mm_struct *mm, 3269 unsigned long addr, unsigned long len, 3270 unsigned long flags, 3271 const struct vm_special_mapping *spec); 3272 /* This is an obsolete alternative to _install_special_mapping. */ 3273 extern int install_special_mapping(struct mm_struct *mm, 3274 unsigned long addr, unsigned long len, 3275 unsigned long flags, struct page **pages); 3276 3277 unsigned long randomize_stack_top(unsigned long stack_top); 3278 unsigned long randomize_page(unsigned long start, unsigned long range); 3279 3280 extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); 3281 3282 extern unsigned long mmap_region(struct file *file, unsigned long addr, 3283 unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, 3284 struct list_head *uf); 3285 extern unsigned long do_mmap(struct file *file, unsigned long addr, 3286 unsigned long len, unsigned long prot, unsigned long flags, 3287 vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, 3288 struct list_head *uf); 3289 extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, 3290 unsigned long start, size_t len, struct list_head *uf, 3291 bool unlock); 3292 extern int do_munmap(struct mm_struct *, unsigned long, size_t, 3293 struct list_head *uf); 3294 extern int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior); 3295 3296 #ifdef CONFIG_MMU 3297 extern int do_vma_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, 3298 unsigned long start, unsigned long end, 3299 struct list_head *uf, bool unlock); 3300 extern int __mm_populate(unsigned long addr, unsigned long len, 3301 int ignore_errors); 3302 static inline void mm_populate(unsigned long addr, unsigned long len) 3303 { 3304 /* Ignore errors */ 3305 (void) __mm_populate(addr, len, 1); 3306 } 3307 #else 3308 static inline void mm_populate(unsigned long addr, unsigned long len) {} 3309 #endif 3310 3311 /* These take the mm semaphore themselves */ 3312 extern int __must_check vm_brk(unsigned long, unsigned long); 3313 extern int __must_check vm_brk_flags(unsigned long, unsigned long, unsigned long); 3314 extern int vm_munmap(unsigned long, size_t); 3315 extern unsigned long __must_check vm_mmap(struct file *, unsigned long, 3316 unsigned long, unsigned long, 3317 unsigned long, unsigned long); 3318 3319 struct vm_unmapped_area_info { 3320 #define VM_UNMAPPED_AREA_TOPDOWN 1 3321 unsigned long flags; 3322 unsigned long length; 3323 unsigned long low_limit; 3324 unsigned long high_limit; 3325 unsigned long align_mask; 3326 unsigned long align_offset; 3327 }; 3328 3329 extern unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info); 3330 3331 /* truncate.c */ 3332 extern void truncate_inode_pages(struct address_space *, loff_t); 3333 extern void truncate_inode_pages_range(struct address_space *, 3334 loff_t lstart, loff_t lend); 3335 extern void truncate_inode_pages_final(struct address_space *); 3336 3337 /* generic vm_area_ops exported for stackable file systems */ 3338 extern vm_fault_t filemap_fault(struct vm_fault *vmf); 3339 extern vm_fault_t filemap_map_pages(struct vm_fault *vmf, 3340 pgoff_t start_pgoff, pgoff_t end_pgoff); 3341 extern vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf); 3342 3343 extern unsigned long stack_guard_gap; 3344 /* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */ 3345 int expand_stack_locked(struct vm_area_struct *vma, unsigned long address); 3346 struct vm_area_struct *expand_stack(struct mm_struct * mm, unsigned long addr); 3347 3348 /* CONFIG_STACK_GROWSUP still needs to grow downwards at some places */ 3349 int expand_downwards(struct vm_area_struct *vma, unsigned long address); 3350 3351 /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */ 3352 extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr); 3353 extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr, 3354 struct vm_area_struct **pprev); 3355 3356 /* 3357 * Look up the first VMA which intersects the interval [start_addr, end_addr) 3358 * NULL if none. Assume start_addr < end_addr. 3359 */ 3360 struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, 3361 unsigned long start_addr, unsigned long end_addr); 3362 3363 /** 3364 * vma_lookup() - Find a VMA at a specific address 3365 * @mm: The process address space. 3366 * @addr: The user address. 3367 * 3368 * Return: The vm_area_struct at the given address, %NULL otherwise. 3369 */ 3370 static inline 3371 struct vm_area_struct *vma_lookup(struct mm_struct *mm, unsigned long addr) 3372 { 3373 return mtree_load(&mm->mm_mt, addr); 3374 } 3375 3376 static inline unsigned long stack_guard_start_gap(struct vm_area_struct *vma) 3377 { 3378 if (vma->vm_flags & VM_GROWSDOWN) 3379 return stack_guard_gap; 3380 3381 /* See reasoning around the VM_SHADOW_STACK definition */ 3382 if (vma->vm_flags & VM_SHADOW_STACK) 3383 return PAGE_SIZE; 3384 3385 return 0; 3386 } 3387 3388 static inline unsigned long vm_start_gap(struct vm_area_struct *vma) 3389 { 3390 unsigned long gap = stack_guard_start_gap(vma); 3391 unsigned long vm_start = vma->vm_start; 3392 3393 vm_start -= gap; 3394 if (vm_start > vma->vm_start) 3395 vm_start = 0; 3396 return vm_start; 3397 } 3398 3399 static inline unsigned long vm_end_gap(struct vm_area_struct *vma) 3400 { 3401 unsigned long vm_end = vma->vm_end; 3402 3403 if (vma->vm_flags & VM_GROWSUP) { 3404 vm_end += stack_guard_gap; 3405 if (vm_end < vma->vm_end) 3406 vm_end = -PAGE_SIZE; 3407 } 3408 return vm_end; 3409 } 3410 3411 static inline unsigned long vma_pages(struct vm_area_struct *vma) 3412 { 3413 return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; 3414 } 3415 3416 /* Look up the first VMA which exactly match the interval vm_start ... vm_end */ 3417 static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm, 3418 unsigned long vm_start, unsigned long vm_end) 3419 { 3420 struct vm_area_struct *vma = vma_lookup(mm, vm_start); 3421 3422 if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end)) 3423 vma = NULL; 3424 3425 return vma; 3426 } 3427 3428 static inline bool range_in_vma(struct vm_area_struct *vma, 3429 unsigned long start, unsigned long end) 3430 { 3431 return (vma && vma->vm_start <= start && end <= vma->vm_end); 3432 } 3433 3434 #ifdef CONFIG_MMU 3435 pgprot_t vm_get_page_prot(unsigned long vm_flags); 3436 void vma_set_page_prot(struct vm_area_struct *vma); 3437 #else 3438 static inline pgprot_t vm_get_page_prot(unsigned long vm_flags) 3439 { 3440 return __pgprot(0); 3441 } 3442 static inline void vma_set_page_prot(struct vm_area_struct *vma) 3443 { 3444 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); 3445 } 3446 #endif 3447 3448 void vma_set_file(struct vm_area_struct *vma, struct file *file); 3449 3450 #ifdef CONFIG_NUMA_BALANCING 3451 unsigned long change_prot_numa(struct vm_area_struct *vma, 3452 unsigned long start, unsigned long end); 3453 #endif 3454 3455 struct vm_area_struct *find_extend_vma_locked(struct mm_struct *, 3456 unsigned long addr); 3457 int remap_pfn_range(struct vm_area_struct *, unsigned long addr, 3458 unsigned long pfn, unsigned long size, pgprot_t); 3459 int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr, 3460 unsigned long pfn, unsigned long size, pgprot_t prot); 3461 int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *); 3462 int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, 3463 struct page **pages, unsigned long *num); 3464 int vm_map_pages(struct vm_area_struct *vma, struct page **pages, 3465 unsigned long num); 3466 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, 3467 unsigned long num); 3468 vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr, 3469 unsigned long pfn); 3470 vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, 3471 unsigned long pfn, pgprot_t pgprot); 3472 vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr, 3473 pfn_t pfn); 3474 vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma, 3475 unsigned long addr, pfn_t pfn); 3476 int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len); 3477 3478 static inline vm_fault_t vmf_insert_page(struct vm_area_struct *vma, 3479 unsigned long addr, struct page *page) 3480 { 3481 int err = vm_insert_page(vma, addr, page); 3482 3483 if (err == -ENOMEM) 3484 return VM_FAULT_OOM; 3485 if (err < 0 && err != -EBUSY) 3486 return VM_FAULT_SIGBUS; 3487 3488 return VM_FAULT_NOPAGE; 3489 } 3490 3491 #ifndef io_remap_pfn_range 3492 static inline int io_remap_pfn_range(struct vm_area_struct *vma, 3493 unsigned long addr, unsigned long pfn, 3494 unsigned long size, pgprot_t prot) 3495 { 3496 return remap_pfn_range(vma, addr, pfn, size, pgprot_decrypted(prot)); 3497 } 3498 #endif 3499 3500 static inline vm_fault_t vmf_error(int err) 3501 { 3502 if (err == -ENOMEM) 3503 return VM_FAULT_OOM; 3504 else if (err == -EHWPOISON) 3505 return VM_FAULT_HWPOISON; 3506 return VM_FAULT_SIGBUS; 3507 } 3508 3509 /* 3510 * Convert errno to return value for ->page_mkwrite() calls. 3511 * 3512 * This should eventually be merged with vmf_error() above, but will need a 3513 * careful audit of all vmf_error() callers. 3514 */ 3515 static inline vm_fault_t vmf_fs_error(int err) 3516 { 3517 if (err == 0) 3518 return VM_FAULT_LOCKED; 3519 if (err == -EFAULT || err == -EAGAIN) 3520 return VM_FAULT_NOPAGE; 3521 if (err == -ENOMEM) 3522 return VM_FAULT_OOM; 3523 /* -ENOSPC, -EDQUOT, -EIO ... */ 3524 return VM_FAULT_SIGBUS; 3525 } 3526 3527 struct page *follow_page(struct vm_area_struct *vma, unsigned long address, 3528 unsigned int foll_flags); 3529 3530 static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags) 3531 { 3532 if (vm_fault & VM_FAULT_OOM) 3533 return -ENOMEM; 3534 if (vm_fault & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE)) 3535 return (foll_flags & FOLL_HWPOISON) ? -EHWPOISON : -EFAULT; 3536 if (vm_fault & (VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV)) 3537 return -EFAULT; 3538 return 0; 3539 } 3540 3541 /* 3542 * Indicates whether GUP can follow a PROT_NONE mapped page, or whether 3543 * a (NUMA hinting) fault is required. 3544 */ 3545 static inline bool gup_can_follow_protnone(struct vm_area_struct *vma, 3546 unsigned int flags) 3547 { 3548 /* 3549 * If callers don't want to honor NUMA hinting faults, no need to 3550 * determine if we would actually have to trigger a NUMA hinting fault. 3551 */ 3552 if (!(flags & FOLL_HONOR_NUMA_FAULT)) 3553 return true; 3554 3555 /* 3556 * NUMA hinting faults don't apply in inaccessible (PROT_NONE) VMAs. 3557 * 3558 * Requiring a fault here even for inaccessible VMAs would mean that 3559 * FOLL_FORCE cannot make any progress, because handle_mm_fault() 3560 * refuses to process NUMA hinting faults in inaccessible VMAs. 3561 */ 3562 return !vma_is_accessible(vma); 3563 } 3564 3565 typedef int (*pte_fn_t)(pte_t *pte, unsigned long addr, void *data); 3566 extern int apply_to_page_range(struct mm_struct *mm, unsigned long address, 3567 unsigned long size, pte_fn_t fn, void *data); 3568 extern int apply_to_existing_page_range(struct mm_struct *mm, 3569 unsigned long address, unsigned long size, 3570 pte_fn_t fn, void *data); 3571 3572 #ifdef CONFIG_PAGE_POISONING 3573 extern void __kernel_poison_pages(struct page *page, int numpages); 3574 extern void __kernel_unpoison_pages(struct page *page, int numpages); 3575 extern bool _page_poisoning_enabled_early; 3576 DECLARE_STATIC_KEY_FALSE(_page_poisoning_enabled); 3577 static inline bool page_poisoning_enabled(void) 3578 { 3579 return _page_poisoning_enabled_early; 3580 } 3581 /* 3582 * For use in fast paths after init_mem_debugging() has run, or when a 3583 * false negative result is not harmful when called too early. 3584 */ 3585 static inline bool page_poisoning_enabled_static(void) 3586 { 3587 return static_branch_unlikely(&_page_poisoning_enabled); 3588 } 3589 static inline void kernel_poison_pages(struct page *page, int numpages) 3590 { 3591 if (page_poisoning_enabled_static()) 3592 __kernel_poison_pages(page, numpages); 3593 } 3594 static inline void kernel_unpoison_pages(struct page *page, int numpages) 3595 { 3596 if (page_poisoning_enabled_static()) 3597 __kernel_unpoison_pages(page, numpages); 3598 } 3599 #else 3600 static inline bool page_poisoning_enabled(void) { return false; } 3601 static inline bool page_poisoning_enabled_static(void) { return false; } 3602 static inline void __kernel_poison_pages(struct page *page, int nunmpages) { } 3603 static inline void kernel_poison_pages(struct page *page, int numpages) { } 3604 static inline void kernel_unpoison_pages(struct page *page, int numpages) { } 3605 #endif 3606 3607 DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc); 3608 static inline bool want_init_on_alloc(gfp_t flags) 3609 { 3610 if (static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, 3611 &init_on_alloc)) 3612 return true; 3613 return flags & __GFP_ZERO; 3614 } 3615 3616 DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free); 3617 static inline bool want_init_on_free(void) 3618 { 3619 return static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON, 3620 &init_on_free); 3621 } 3622 3623 extern bool _debug_pagealloc_enabled_early; 3624 DECLARE_STATIC_KEY_FALSE(_debug_pagealloc_enabled); 3625 3626 static inline bool debug_pagealloc_enabled(void) 3627 { 3628 return IS_ENABLED(CONFIG_DEBUG_PAGEALLOC) && 3629 _debug_pagealloc_enabled_early; 3630 } 3631 3632 /* 3633 * For use in fast paths after mem_debugging_and_hardening_init() has run, 3634 * or when a false negative result is not harmful when called too early. 3635 */ 3636 static inline bool debug_pagealloc_enabled_static(void) 3637 { 3638 if (!IS_ENABLED(CONFIG_DEBUG_PAGEALLOC)) 3639 return false; 3640 3641 return static_branch_unlikely(&_debug_pagealloc_enabled); 3642 } 3643 3644 /* 3645 * To support DEBUG_PAGEALLOC architecture must ensure that 3646 * __kernel_map_pages() never fails 3647 */ 3648 extern void __kernel_map_pages(struct page *page, int numpages, int enable); 3649 #ifdef CONFIG_DEBUG_PAGEALLOC 3650 static inline void debug_pagealloc_map_pages(struct page *page, int numpages) 3651 { 3652 if (debug_pagealloc_enabled_static()) 3653 __kernel_map_pages(page, numpages, 1); 3654 } 3655 3656 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) 3657 { 3658 if (debug_pagealloc_enabled_static()) 3659 __kernel_map_pages(page, numpages, 0); 3660 } 3661 3662 extern unsigned int _debug_guardpage_minorder; 3663 DECLARE_STATIC_KEY_FALSE(_debug_guardpage_enabled); 3664 3665 static inline unsigned int debug_guardpage_minorder(void) 3666 { 3667 return _debug_guardpage_minorder; 3668 } 3669 3670 static inline bool debug_guardpage_enabled(void) 3671 { 3672 return static_branch_unlikely(&_debug_guardpage_enabled); 3673 } 3674 3675 static inline bool page_is_guard(struct page *page) 3676 { 3677 if (!debug_guardpage_enabled()) 3678 return false; 3679 3680 return PageGuard(page); 3681 } 3682 3683 bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order, 3684 int migratetype); 3685 static inline bool set_page_guard(struct zone *zone, struct page *page, 3686 unsigned int order, int migratetype) 3687 { 3688 if (!debug_guardpage_enabled()) 3689 return false; 3690 return __set_page_guard(zone, page, order, migratetype); 3691 } 3692 3693 void __clear_page_guard(struct zone *zone, struct page *page, unsigned int order, 3694 int migratetype); 3695 static inline void clear_page_guard(struct zone *zone, struct page *page, 3696 unsigned int order, int migratetype) 3697 { 3698 if (!debug_guardpage_enabled()) 3699 return; 3700 __clear_page_guard(zone, page, order, migratetype); 3701 } 3702 3703 #else /* CONFIG_DEBUG_PAGEALLOC */ 3704 static inline void debug_pagealloc_map_pages(struct page *page, int numpages) {} 3705 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) {} 3706 static inline unsigned int debug_guardpage_minorder(void) { return 0; } 3707 static inline bool debug_guardpage_enabled(void) { return false; } 3708 static inline bool page_is_guard(struct page *page) { return false; } 3709 static inline bool set_page_guard(struct zone *zone, struct page *page, 3710 unsigned int order, int migratetype) { return false; } 3711 static inline void clear_page_guard(struct zone *zone, struct page *page, 3712 unsigned int order, int migratetype) {} 3713 #endif /* CONFIG_DEBUG_PAGEALLOC */ 3714 3715 #ifdef __HAVE_ARCH_GATE_AREA 3716 extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm); 3717 extern int in_gate_area_no_mm(unsigned long addr); 3718 extern int in_gate_area(struct mm_struct *mm, unsigned long addr); 3719 #else 3720 static inline struct vm_area_struct *get_gate_vma(struct mm_struct *mm) 3721 { 3722 return NULL; 3723 } 3724 static inline int in_gate_area_no_mm(unsigned long addr) { return 0; } 3725 static inline int in_gate_area(struct mm_struct *mm, unsigned long addr) 3726 { 3727 return 0; 3728 } 3729 #endif /* __HAVE_ARCH_GATE_AREA */ 3730 3731 extern bool process_shares_mm(struct task_struct *p, struct mm_struct *mm); 3732 3733 #ifdef CONFIG_SYSCTL 3734 extern int sysctl_drop_caches; 3735 int drop_caches_sysctl_handler(struct ctl_table *, int, void *, size_t *, 3736 loff_t *); 3737 #endif 3738 3739 void drop_slab(void); 3740 3741 #ifndef CONFIG_MMU 3742 #define randomize_va_space 0 3743 #else 3744 extern int randomize_va_space; 3745 #endif 3746 3747 const char * arch_vma_name(struct vm_area_struct *vma); 3748 #ifdef CONFIG_MMU 3749 void print_vma_addr(char *prefix, unsigned long rip); 3750 #else 3751 static inline void print_vma_addr(char *prefix, unsigned long rip) 3752 { 3753 } 3754 #endif 3755 3756 void *sparse_buffer_alloc(unsigned long size); 3757 struct page * __populate_section_memmap(unsigned long pfn, 3758 unsigned long nr_pages, int nid, struct vmem_altmap *altmap, 3759 struct dev_pagemap *pgmap); 3760 void pmd_init(void *addr); 3761 void pud_init(void *addr); 3762 pgd_t *vmemmap_pgd_populate(unsigned long addr, int node); 3763 p4d_t *vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node); 3764 pud_t *vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node); 3765 pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node); 3766 pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node, 3767 struct vmem_altmap *altmap, struct page *reuse); 3768 void *vmemmap_alloc_block(unsigned long size, int node); 3769 struct vmem_altmap; 3770 void *vmemmap_alloc_block_buf(unsigned long size, int node, 3771 struct vmem_altmap *altmap); 3772 void vmemmap_verify(pte_t *, int, unsigned long, unsigned long); 3773 void vmemmap_set_pmd(pmd_t *pmd, void *p, int node, 3774 unsigned long addr, unsigned long next); 3775 int vmemmap_check_pmd(pmd_t *pmd, int node, 3776 unsigned long addr, unsigned long next); 3777 int vmemmap_populate_basepages(unsigned long start, unsigned long end, 3778 int node, struct vmem_altmap *altmap); 3779 int vmemmap_populate_hugepages(unsigned long start, unsigned long end, 3780 int node, struct vmem_altmap *altmap); 3781 int vmemmap_populate(unsigned long start, unsigned long end, int node, 3782 struct vmem_altmap *altmap); 3783 void vmemmap_populate_print_last(void); 3784 #ifdef CONFIG_MEMORY_HOTPLUG 3785 void vmemmap_free(unsigned long start, unsigned long end, 3786 struct vmem_altmap *altmap); 3787 #endif 3788 3789 #define VMEMMAP_RESERVE_NR 2 3790 #ifdef CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAP 3791 static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap, 3792 struct dev_pagemap *pgmap) 3793 { 3794 unsigned long nr_pages; 3795 unsigned long nr_vmemmap_pages; 3796 3797 if (!pgmap || !is_power_of_2(sizeof(struct page))) 3798 return false; 3799 3800 nr_pages = pgmap_vmemmap_nr(pgmap); 3801 nr_vmemmap_pages = ((nr_pages * sizeof(struct page)) >> PAGE_SHIFT); 3802 /* 3803 * For vmemmap optimization with DAX we need minimum 2 vmemmap 3804 * pages. See layout diagram in Documentation/mm/vmemmap_dedup.rst 3805 */ 3806 return !altmap && (nr_vmemmap_pages > VMEMMAP_RESERVE_NR); 3807 } 3808 /* 3809 * If we don't have an architecture override, use the generic rule 3810 */ 3811 #ifndef vmemmap_can_optimize 3812 #define vmemmap_can_optimize __vmemmap_can_optimize 3813 #endif 3814 3815 #else 3816 static inline bool vmemmap_can_optimize(struct vmem_altmap *altmap, 3817 struct dev_pagemap *pgmap) 3818 { 3819 return false; 3820 } 3821 #endif 3822 3823 void register_page_bootmem_memmap(unsigned long section_nr, struct page *map, 3824 unsigned long nr_pages); 3825 3826 enum mf_flags { 3827 MF_COUNT_INCREASED = 1 << 0, 3828 MF_ACTION_REQUIRED = 1 << 1, 3829 MF_MUST_KILL = 1 << 2, 3830 MF_SOFT_OFFLINE = 1 << 3, 3831 MF_UNPOISON = 1 << 4, 3832 MF_SW_SIMULATED = 1 << 5, 3833 MF_NO_RETRY = 1 << 6, 3834 }; 3835 int mf_dax_kill_procs(struct address_space *mapping, pgoff_t index, 3836 unsigned long count, int mf_flags); 3837 extern int memory_failure(unsigned long pfn, int flags); 3838 extern void memory_failure_queue_kick(int cpu); 3839 extern int unpoison_memory(unsigned long pfn); 3840 extern void shake_page(struct page *p); 3841 extern atomic_long_t num_poisoned_pages __read_mostly; 3842 extern int soft_offline_page(unsigned long pfn, int flags); 3843 #ifdef CONFIG_MEMORY_FAILURE 3844 /* 3845 * Sysfs entries for memory failure handling statistics. 3846 */ 3847 extern const struct attribute_group memory_failure_attr_group; 3848 extern void memory_failure_queue(unsigned long pfn, int flags); 3849 extern int __get_huge_page_for_hwpoison(unsigned long pfn, int flags, 3850 bool *migratable_cleared); 3851 void num_poisoned_pages_inc(unsigned long pfn); 3852 void num_poisoned_pages_sub(unsigned long pfn, long i); 3853 struct task_struct *task_early_kill(struct task_struct *tsk, int force_early); 3854 #else 3855 static inline void memory_failure_queue(unsigned long pfn, int flags) 3856 { 3857 } 3858 3859 static inline int __get_huge_page_for_hwpoison(unsigned long pfn, int flags, 3860 bool *migratable_cleared) 3861 { 3862 return 0; 3863 } 3864 3865 static inline void num_poisoned_pages_inc(unsigned long pfn) 3866 { 3867 } 3868 3869 static inline void num_poisoned_pages_sub(unsigned long pfn, long i) 3870 { 3871 } 3872 #endif 3873 3874 #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_KSM) 3875 void add_to_kill_ksm(struct task_struct *tsk, struct page *p, 3876 struct vm_area_struct *vma, struct list_head *to_kill, 3877 unsigned long ksm_addr); 3878 #endif 3879 3880 #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG) 3881 extern void memblk_nr_poison_inc(unsigned long pfn); 3882 extern void memblk_nr_poison_sub(unsigned long pfn, long i); 3883 #else 3884 static inline void memblk_nr_poison_inc(unsigned long pfn) 3885 { 3886 } 3887 3888 static inline void memblk_nr_poison_sub(unsigned long pfn, long i) 3889 { 3890 } 3891 #endif 3892 3893 #ifndef arch_memory_failure 3894 static inline int arch_memory_failure(unsigned long pfn, int flags) 3895 { 3896 return -ENXIO; 3897 } 3898 #endif 3899 3900 #ifndef arch_is_platform_page 3901 static inline bool arch_is_platform_page(u64 paddr) 3902 { 3903 return false; 3904 } 3905 #endif 3906 3907 /* 3908 * Error handlers for various types of pages. 3909 */ 3910 enum mf_result { 3911 MF_IGNORED, /* Error: cannot be handled */ 3912 MF_FAILED, /* Error: handling failed */ 3913 MF_DELAYED, /* Will be handled later */ 3914 MF_RECOVERED, /* Successfully recovered */ 3915 }; 3916 3917 enum mf_action_page_type { 3918 MF_MSG_KERNEL, 3919 MF_MSG_KERNEL_HIGH_ORDER, 3920 MF_MSG_SLAB, 3921 MF_MSG_DIFFERENT_COMPOUND, 3922 MF_MSG_HUGE, 3923 MF_MSG_FREE_HUGE, 3924 MF_MSG_UNMAP_FAILED, 3925 MF_MSG_DIRTY_SWAPCACHE, 3926 MF_MSG_CLEAN_SWAPCACHE, 3927 MF_MSG_DIRTY_MLOCKED_LRU, 3928 MF_MSG_CLEAN_MLOCKED_LRU, 3929 MF_MSG_DIRTY_UNEVICTABLE_LRU, 3930 MF_MSG_CLEAN_UNEVICTABLE_LRU, 3931 MF_MSG_DIRTY_LRU, 3932 MF_MSG_CLEAN_LRU, 3933 MF_MSG_TRUNCATED_LRU, 3934 MF_MSG_BUDDY, 3935 MF_MSG_DAX, 3936 MF_MSG_UNSPLIT_THP, 3937 MF_MSG_UNKNOWN, 3938 }; 3939 3940 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS) 3941 extern void clear_huge_page(struct page *page, 3942 unsigned long addr_hint, 3943 unsigned int pages_per_huge_page); 3944 int copy_user_large_folio(struct folio *dst, struct folio *src, 3945 unsigned long addr_hint, 3946 struct vm_area_struct *vma); 3947 long copy_folio_from_user(struct folio *dst_folio, 3948 const void __user *usr_src, 3949 bool allow_pagefault); 3950 3951 /** 3952 * vma_is_special_huge - Are transhuge page-table entries considered special? 3953 * @vma: Pointer to the struct vm_area_struct to consider 3954 * 3955 * Whether transhuge page-table entries are considered "special" following 3956 * the definition in vm_normal_page(). 3957 * 3958 * Return: true if transhuge page-table entries should be considered special, 3959 * false otherwise. 3960 */ 3961 static inline bool vma_is_special_huge(const struct vm_area_struct *vma) 3962 { 3963 return vma_is_dax(vma) || (vma->vm_file && 3964 (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))); 3965 } 3966 3967 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */ 3968 3969 #if MAX_NUMNODES > 1 3970 void __init setup_nr_node_ids(void); 3971 #else 3972 static inline void setup_nr_node_ids(void) {} 3973 #endif 3974 3975 extern int memcmp_pages(struct page *page1, struct page *page2); 3976 3977 static inline int pages_identical(struct page *page1, struct page *page2) 3978 { 3979 return !memcmp_pages(page1, page2); 3980 } 3981 3982 #ifdef CONFIG_MAPPING_DIRTY_HELPERS 3983 unsigned long clean_record_shared_mapping_range(struct address_space *mapping, 3984 pgoff_t first_index, pgoff_t nr, 3985 pgoff_t bitmap_pgoff, 3986 unsigned long *bitmap, 3987 pgoff_t *start, 3988 pgoff_t *end); 3989 3990 unsigned long wp_shared_mapping_range(struct address_space *mapping, 3991 pgoff_t first_index, pgoff_t nr); 3992 #endif 3993 3994 extern int sysctl_nr_trim_pages; 3995 3996 #ifdef CONFIG_PRINTK 3997 void mem_dump_obj(void *object); 3998 #else 3999 static inline void mem_dump_obj(void *object) {} 4000 #endif 4001 4002 /** 4003 * seal_check_future_write - Check for F_SEAL_FUTURE_WRITE flag and handle it 4004 * @seals: the seals to check 4005 * @vma: the vma to operate on 4006 * 4007 * Check whether F_SEAL_FUTURE_WRITE is set; if so, do proper check/handling on 4008 * the vma flags. Return 0 if check pass, or <0 for errors. 4009 */ 4010 static inline int seal_check_future_write(int seals, struct vm_area_struct *vma) 4011 { 4012 if (seals & F_SEAL_FUTURE_WRITE) { 4013 /* 4014 * New PROT_WRITE and MAP_SHARED mmaps are not allowed when 4015 * "future write" seal active. 4016 */ 4017 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE)) 4018 return -EPERM; 4019 4020 /* 4021 * Since an F_SEAL_FUTURE_WRITE sealed memfd can be mapped as 4022 * MAP_SHARED and read-only, take care to not allow mprotect to 4023 * revert protections on such mappings. Do this only for shared 4024 * mappings. For private mappings, don't need to mask 4025 * VM_MAYWRITE as we still want them to be COW-writable. 4026 */ 4027 if (vma->vm_flags & VM_SHARED) 4028 vm_flags_clear(vma, VM_MAYWRITE); 4029 } 4030 4031 return 0; 4032 } 4033 4034 #ifdef CONFIG_ANON_VMA_NAME 4035 int madvise_set_anon_name(struct mm_struct *mm, unsigned long start, 4036 unsigned long len_in, 4037 struct anon_vma_name *anon_name); 4038 #else 4039 static inline int 4040 madvise_set_anon_name(struct mm_struct *mm, unsigned long start, 4041 unsigned long len_in, struct anon_vma_name *anon_name) { 4042 return 0; 4043 } 4044 #endif 4045 4046 #ifdef CONFIG_UNACCEPTED_MEMORY 4047 4048 bool range_contains_unaccepted_memory(phys_addr_t start, phys_addr_t end); 4049 void accept_memory(phys_addr_t start, phys_addr_t end); 4050 4051 #else 4052 4053 static inline bool range_contains_unaccepted_memory(phys_addr_t start, 4054 phys_addr_t end) 4055 { 4056 return false; 4057 } 4058 4059 static inline void accept_memory(phys_addr_t start, phys_addr_t end) 4060 { 4061 } 4062 4063 #endif 4064 4065 #endif /* _LINUX_MM_H */ 4066