1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * This kernel test validates architecture page table helpers and 4 * accessors and helps in verifying their continued compliance with 5 * expected generic MM semantics. 6 * 7 * Copyright (C) 2019 ARM Ltd. 8 * 9 * Author: Anshuman Khandual <anshuman.khandual@arm.com> 10 */ 11 #define pr_fmt(fmt) "debug_vm_pgtable: [%-25s]: " fmt, __func__ 12 13 #include <linux/gfp.h> 14 #include <linux/highmem.h> 15 #include <linux/hugetlb.h> 16 #include <linux/kernel.h> 17 #include <linux/kconfig.h> 18 #include <linux/mm.h> 19 #include <linux/mman.h> 20 #include <linux/mm_types.h> 21 #include <linux/module.h> 22 #include <linux/pfn_t.h> 23 #include <linux/printk.h> 24 #include <linux/pgtable.h> 25 #include <linux/random.h> 26 #include <linux/spinlock.h> 27 #include <linux/swap.h> 28 #include <linux/swapops.h> 29 #include <linux/start_kernel.h> 30 #include <linux/sched/mm.h> 31 #include <linux/io.h> 32 33 #include <asm/cacheflush.h> 34 #include <asm/pgalloc.h> 35 #include <asm/tlbflush.h> 36 37 /* 38 * Please refer Documentation/vm/arch_pgtable_helpers.rst for the semantics 39 * expectations that are being validated here. All future changes in here 40 * or the documentation need to be in sync. 41 */ 42 43 #define VMFLAGS (VM_READ|VM_WRITE|VM_EXEC) 44 45 /* 46 * On s390 platform, the lower 4 bits are used to identify given page table 47 * entry type. But these bits might affect the ability to clear entries with 48 * pxx_clear() because of how dynamic page table folding works on s390. So 49 * while loading up the entries do not change the lower 4 bits. It does not 50 * have affect any other platform. Also avoid the 62nd bit on ppc64 that is 51 * used to mark a pte entry. 52 */ 53 #define S390_SKIP_MASK GENMASK(3, 0) 54 #if __BITS_PER_LONG == 64 55 #define PPC64_SKIP_MASK GENMASK(62, 62) 56 #else 57 #define PPC64_SKIP_MASK 0x0 58 #endif 59 #define ARCH_SKIP_MASK (S390_SKIP_MASK | PPC64_SKIP_MASK) 60 #define RANDOM_ORVALUE (GENMASK(BITS_PER_LONG - 1, 0) & ~ARCH_SKIP_MASK) 61 #define RANDOM_NZVALUE GENMASK(7, 0) 62 63 struct pgtable_debug_args { 64 struct mm_struct *mm; 65 struct vm_area_struct *vma; 66 67 pgd_t *pgdp; 68 p4d_t *p4dp; 69 pud_t *pudp; 70 pmd_t *pmdp; 71 pte_t *ptep; 72 73 p4d_t *start_p4dp; 74 pud_t *start_pudp; 75 pmd_t *start_pmdp; 76 pgtable_t start_ptep; 77 78 unsigned long vaddr; 79 pgprot_t page_prot; 80 pgprot_t page_prot_none; 81 82 bool is_contiguous_page; 83 unsigned long pud_pfn; 84 unsigned long pmd_pfn; 85 unsigned long pte_pfn; 86 87 unsigned long fixed_pgd_pfn; 88 unsigned long fixed_p4d_pfn; 89 unsigned long fixed_pud_pfn; 90 unsigned long fixed_pmd_pfn; 91 unsigned long fixed_pte_pfn; 92 }; 93 94 static void __init pte_basic_tests(struct pgtable_debug_args *args, int idx) 95 { 96 pgprot_t prot = protection_map[idx]; 97 pte_t pte = pfn_pte(args->fixed_pte_pfn, prot); 98 unsigned long val = idx, *ptr = &val; 99 100 pr_debug("Validating PTE basic (%pGv)\n", ptr); 101 102 /* 103 * This test needs to be executed after the given page table entry 104 * is created with pfn_pte() to make sure that protection_map[idx] 105 * does not have the dirty bit enabled from the beginning. This is 106 * important for platforms like arm64 where (!PTE_RDONLY) indicate 107 * dirty bit being set. 108 */ 109 WARN_ON(pte_dirty(pte_wrprotect(pte))); 110 111 WARN_ON(!pte_same(pte, pte)); 112 WARN_ON(!pte_young(pte_mkyoung(pte_mkold(pte)))); 113 WARN_ON(!pte_dirty(pte_mkdirty(pte_mkclean(pte)))); 114 WARN_ON(!pte_write(pte_mkwrite(pte_wrprotect(pte)))); 115 WARN_ON(pte_young(pte_mkold(pte_mkyoung(pte)))); 116 WARN_ON(pte_dirty(pte_mkclean(pte_mkdirty(pte)))); 117 WARN_ON(pte_write(pte_wrprotect(pte_mkwrite(pte)))); 118 WARN_ON(pte_dirty(pte_wrprotect(pte_mkclean(pte)))); 119 WARN_ON(!pte_dirty(pte_wrprotect(pte_mkdirty(pte)))); 120 } 121 122 static void __init pte_advanced_tests(struct pgtable_debug_args *args) 123 { 124 struct page *page; 125 pte_t pte; 126 127 /* 128 * Architectures optimize set_pte_at by avoiding TLB flush. 129 * This requires set_pte_at to be not used to update an 130 * existing pte entry. Clear pte before we do set_pte_at 131 * 132 * flush_dcache_page() is called after set_pte_at() to clear 133 * PG_arch_1 for the page on ARM64. The page flag isn't cleared 134 * when it's released and page allocation check will fail when 135 * the page is allocated again. For architectures other than ARM64, 136 * the unexpected overhead of cache flushing is acceptable. 137 */ 138 page = (args->pte_pfn != ULONG_MAX) ? pfn_to_page(args->pte_pfn) : NULL; 139 if (!page) 140 return; 141 142 pr_debug("Validating PTE advanced\n"); 143 pte = pfn_pte(args->pte_pfn, args->page_prot); 144 set_pte_at(args->mm, args->vaddr, args->ptep, pte); 145 flush_dcache_page(page); 146 ptep_set_wrprotect(args->mm, args->vaddr, args->ptep); 147 pte = ptep_get(args->ptep); 148 WARN_ON(pte_write(pte)); 149 ptep_get_and_clear(args->mm, args->vaddr, args->ptep); 150 pte = ptep_get(args->ptep); 151 WARN_ON(!pte_none(pte)); 152 153 pte = pfn_pte(args->pte_pfn, args->page_prot); 154 pte = pte_wrprotect(pte); 155 pte = pte_mkclean(pte); 156 set_pte_at(args->mm, args->vaddr, args->ptep, pte); 157 flush_dcache_page(page); 158 pte = pte_mkwrite(pte); 159 pte = pte_mkdirty(pte); 160 ptep_set_access_flags(args->vma, args->vaddr, args->ptep, pte, 1); 161 pte = ptep_get(args->ptep); 162 WARN_ON(!(pte_write(pte) && pte_dirty(pte))); 163 ptep_get_and_clear_full(args->mm, args->vaddr, args->ptep, 1); 164 pte = ptep_get(args->ptep); 165 WARN_ON(!pte_none(pte)); 166 167 pte = pfn_pte(args->pte_pfn, args->page_prot); 168 pte = pte_mkyoung(pte); 169 set_pte_at(args->mm, args->vaddr, args->ptep, pte); 170 flush_dcache_page(page); 171 ptep_test_and_clear_young(args->vma, args->vaddr, args->ptep); 172 pte = ptep_get(args->ptep); 173 WARN_ON(pte_young(pte)); 174 } 175 176 static void __init pte_savedwrite_tests(struct pgtable_debug_args *args) 177 { 178 pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot_none); 179 180 if (!IS_ENABLED(CONFIG_NUMA_BALANCING)) 181 return; 182 183 pr_debug("Validating PTE saved write\n"); 184 WARN_ON(!pte_savedwrite(pte_mk_savedwrite(pte_clear_savedwrite(pte)))); 185 WARN_ON(pte_savedwrite(pte_clear_savedwrite(pte_mk_savedwrite(pte)))); 186 } 187 188 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 189 static void __init pmd_basic_tests(struct pgtable_debug_args *args, int idx) 190 { 191 pgprot_t prot = protection_map[idx]; 192 unsigned long val = idx, *ptr = &val; 193 pmd_t pmd; 194 195 if (!has_transparent_hugepage()) 196 return; 197 198 pr_debug("Validating PMD basic (%pGv)\n", ptr); 199 pmd = pfn_pmd(args->fixed_pmd_pfn, prot); 200 201 /* 202 * This test needs to be executed after the given page table entry 203 * is created with pfn_pmd() to make sure that protection_map[idx] 204 * does not have the dirty bit enabled from the beginning. This is 205 * important for platforms like arm64 where (!PTE_RDONLY) indicate 206 * dirty bit being set. 207 */ 208 WARN_ON(pmd_dirty(pmd_wrprotect(pmd))); 209 210 211 WARN_ON(!pmd_same(pmd, pmd)); 212 WARN_ON(!pmd_young(pmd_mkyoung(pmd_mkold(pmd)))); 213 WARN_ON(!pmd_dirty(pmd_mkdirty(pmd_mkclean(pmd)))); 214 WARN_ON(!pmd_write(pmd_mkwrite(pmd_wrprotect(pmd)))); 215 WARN_ON(pmd_young(pmd_mkold(pmd_mkyoung(pmd)))); 216 WARN_ON(pmd_dirty(pmd_mkclean(pmd_mkdirty(pmd)))); 217 WARN_ON(pmd_write(pmd_wrprotect(pmd_mkwrite(pmd)))); 218 WARN_ON(pmd_dirty(pmd_wrprotect(pmd_mkclean(pmd)))); 219 WARN_ON(!pmd_dirty(pmd_wrprotect(pmd_mkdirty(pmd)))); 220 /* 221 * A huge page does not point to next level page table 222 * entry. Hence this must qualify as pmd_bad(). 223 */ 224 WARN_ON(!pmd_bad(pmd_mkhuge(pmd))); 225 } 226 227 static void __init pmd_advanced_tests(struct pgtable_debug_args *args) 228 { 229 struct page *page; 230 pmd_t pmd; 231 unsigned long vaddr = args->vaddr; 232 233 if (!has_transparent_hugepage()) 234 return; 235 236 page = (args->pmd_pfn != ULONG_MAX) ? pfn_to_page(args->pmd_pfn) : NULL; 237 if (!page) 238 return; 239 240 /* 241 * flush_dcache_page() is called after set_pmd_at() to clear 242 * PG_arch_1 for the page on ARM64. The page flag isn't cleared 243 * when it's released and page allocation check will fail when 244 * the page is allocated again. For architectures other than ARM64, 245 * the unexpected overhead of cache flushing is acceptable. 246 */ 247 pr_debug("Validating PMD advanced\n"); 248 /* Align the address wrt HPAGE_PMD_SIZE */ 249 vaddr &= HPAGE_PMD_MASK; 250 251 pgtable_trans_huge_deposit(args->mm, args->pmdp, args->start_ptep); 252 253 pmd = pfn_pmd(args->pmd_pfn, args->page_prot); 254 set_pmd_at(args->mm, vaddr, args->pmdp, pmd); 255 flush_dcache_page(page); 256 pmdp_set_wrprotect(args->mm, vaddr, args->pmdp); 257 pmd = READ_ONCE(*args->pmdp); 258 WARN_ON(pmd_write(pmd)); 259 pmdp_huge_get_and_clear(args->mm, vaddr, args->pmdp); 260 pmd = READ_ONCE(*args->pmdp); 261 WARN_ON(!pmd_none(pmd)); 262 263 pmd = pfn_pmd(args->pmd_pfn, args->page_prot); 264 pmd = pmd_wrprotect(pmd); 265 pmd = pmd_mkclean(pmd); 266 set_pmd_at(args->mm, vaddr, args->pmdp, pmd); 267 flush_dcache_page(page); 268 pmd = pmd_mkwrite(pmd); 269 pmd = pmd_mkdirty(pmd); 270 pmdp_set_access_flags(args->vma, vaddr, args->pmdp, pmd, 1); 271 pmd = READ_ONCE(*args->pmdp); 272 WARN_ON(!(pmd_write(pmd) && pmd_dirty(pmd))); 273 pmdp_huge_get_and_clear_full(args->vma, vaddr, args->pmdp, 1); 274 pmd = READ_ONCE(*args->pmdp); 275 WARN_ON(!pmd_none(pmd)); 276 277 pmd = pmd_mkhuge(pfn_pmd(args->pmd_pfn, args->page_prot)); 278 pmd = pmd_mkyoung(pmd); 279 set_pmd_at(args->mm, vaddr, args->pmdp, pmd); 280 flush_dcache_page(page); 281 pmdp_test_and_clear_young(args->vma, vaddr, args->pmdp); 282 pmd = READ_ONCE(*args->pmdp); 283 WARN_ON(pmd_young(pmd)); 284 285 /* Clear the pte entries */ 286 pmdp_huge_get_and_clear(args->mm, vaddr, args->pmdp); 287 pgtable_trans_huge_withdraw(args->mm, args->pmdp); 288 } 289 290 static void __init pmd_leaf_tests(struct pgtable_debug_args *args) 291 { 292 pmd_t pmd; 293 294 if (!has_transparent_hugepage()) 295 return; 296 297 pr_debug("Validating PMD leaf\n"); 298 pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot); 299 300 /* 301 * PMD based THP is a leaf entry. 302 */ 303 pmd = pmd_mkhuge(pmd); 304 WARN_ON(!pmd_leaf(pmd)); 305 } 306 307 static void __init pmd_savedwrite_tests(struct pgtable_debug_args *args) 308 { 309 pmd_t pmd; 310 311 if (!IS_ENABLED(CONFIG_NUMA_BALANCING)) 312 return; 313 314 if (!has_transparent_hugepage()) 315 return; 316 317 pr_debug("Validating PMD saved write\n"); 318 pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot_none); 319 WARN_ON(!pmd_savedwrite(pmd_mk_savedwrite(pmd_clear_savedwrite(pmd)))); 320 WARN_ON(pmd_savedwrite(pmd_clear_savedwrite(pmd_mk_savedwrite(pmd)))); 321 } 322 323 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 324 static void __init pud_basic_tests(struct pgtable_debug_args *args, int idx) 325 { 326 pgprot_t prot = protection_map[idx]; 327 unsigned long val = idx, *ptr = &val; 328 pud_t pud; 329 330 if (!has_transparent_hugepage()) 331 return; 332 333 pr_debug("Validating PUD basic (%pGv)\n", ptr); 334 pud = pfn_pud(args->fixed_pud_pfn, prot); 335 336 /* 337 * This test needs to be executed after the given page table entry 338 * is created with pfn_pud() to make sure that protection_map[idx] 339 * does not have the dirty bit enabled from the beginning. This is 340 * important for platforms like arm64 where (!PTE_RDONLY) indicate 341 * dirty bit being set. 342 */ 343 WARN_ON(pud_dirty(pud_wrprotect(pud))); 344 345 WARN_ON(!pud_same(pud, pud)); 346 WARN_ON(!pud_young(pud_mkyoung(pud_mkold(pud)))); 347 WARN_ON(!pud_dirty(pud_mkdirty(pud_mkclean(pud)))); 348 WARN_ON(pud_dirty(pud_mkclean(pud_mkdirty(pud)))); 349 WARN_ON(!pud_write(pud_mkwrite(pud_wrprotect(pud)))); 350 WARN_ON(pud_write(pud_wrprotect(pud_mkwrite(pud)))); 351 WARN_ON(pud_young(pud_mkold(pud_mkyoung(pud)))); 352 WARN_ON(pud_dirty(pud_wrprotect(pud_mkclean(pud)))); 353 WARN_ON(!pud_dirty(pud_wrprotect(pud_mkdirty(pud)))); 354 355 if (mm_pmd_folded(args->mm)) 356 return; 357 358 /* 359 * A huge page does not point to next level page table 360 * entry. Hence this must qualify as pud_bad(). 361 */ 362 WARN_ON(!pud_bad(pud_mkhuge(pud))); 363 } 364 365 static void __init pud_advanced_tests(struct pgtable_debug_args *args) 366 { 367 struct page *page; 368 unsigned long vaddr = args->vaddr; 369 pud_t pud; 370 371 if (!has_transparent_hugepage()) 372 return; 373 374 page = (args->pud_pfn != ULONG_MAX) ? pfn_to_page(args->pud_pfn) : NULL; 375 if (!page) 376 return; 377 378 /* 379 * flush_dcache_page() is called after set_pud_at() to clear 380 * PG_arch_1 for the page on ARM64. The page flag isn't cleared 381 * when it's released and page allocation check will fail when 382 * the page is allocated again. For architectures other than ARM64, 383 * the unexpected overhead of cache flushing is acceptable. 384 */ 385 pr_debug("Validating PUD advanced\n"); 386 /* Align the address wrt HPAGE_PUD_SIZE */ 387 vaddr &= HPAGE_PUD_MASK; 388 389 pud = pfn_pud(args->pud_pfn, args->page_prot); 390 set_pud_at(args->mm, vaddr, args->pudp, pud); 391 flush_dcache_page(page); 392 pudp_set_wrprotect(args->mm, vaddr, args->pudp); 393 pud = READ_ONCE(*args->pudp); 394 WARN_ON(pud_write(pud)); 395 396 #ifndef __PAGETABLE_PMD_FOLDED 397 pudp_huge_get_and_clear(args->mm, vaddr, args->pudp); 398 pud = READ_ONCE(*args->pudp); 399 WARN_ON(!pud_none(pud)); 400 #endif /* __PAGETABLE_PMD_FOLDED */ 401 pud = pfn_pud(args->pud_pfn, args->page_prot); 402 pud = pud_wrprotect(pud); 403 pud = pud_mkclean(pud); 404 set_pud_at(args->mm, vaddr, args->pudp, pud); 405 flush_dcache_page(page); 406 pud = pud_mkwrite(pud); 407 pud = pud_mkdirty(pud); 408 pudp_set_access_flags(args->vma, vaddr, args->pudp, pud, 1); 409 pud = READ_ONCE(*args->pudp); 410 WARN_ON(!(pud_write(pud) && pud_dirty(pud))); 411 412 #ifndef __PAGETABLE_PMD_FOLDED 413 pudp_huge_get_and_clear_full(args->mm, vaddr, args->pudp, 1); 414 pud = READ_ONCE(*args->pudp); 415 WARN_ON(!pud_none(pud)); 416 #endif /* __PAGETABLE_PMD_FOLDED */ 417 418 pud = pfn_pud(args->pud_pfn, args->page_prot); 419 pud = pud_mkyoung(pud); 420 set_pud_at(args->mm, vaddr, args->pudp, pud); 421 flush_dcache_page(page); 422 pudp_test_and_clear_young(args->vma, vaddr, args->pudp); 423 pud = READ_ONCE(*args->pudp); 424 WARN_ON(pud_young(pud)); 425 426 pudp_huge_get_and_clear(args->mm, vaddr, args->pudp); 427 } 428 429 static void __init pud_leaf_tests(struct pgtable_debug_args *args) 430 { 431 pud_t pud; 432 433 if (!has_transparent_hugepage()) 434 return; 435 436 pr_debug("Validating PUD leaf\n"); 437 pud = pfn_pud(args->fixed_pud_pfn, args->page_prot); 438 /* 439 * PUD based THP is a leaf entry. 440 */ 441 pud = pud_mkhuge(pud); 442 WARN_ON(!pud_leaf(pud)); 443 } 444 #else /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 445 static void __init pud_basic_tests(struct pgtable_debug_args *args, int idx) { } 446 static void __init pud_advanced_tests(struct pgtable_debug_args *args) { } 447 static void __init pud_leaf_tests(struct pgtable_debug_args *args) { } 448 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 449 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */ 450 static void __init pmd_basic_tests(struct pgtable_debug_args *args, int idx) { } 451 static void __init pud_basic_tests(struct pgtable_debug_args *args, int idx) { } 452 static void __init pmd_advanced_tests(struct pgtable_debug_args *args) { } 453 static void __init pud_advanced_tests(struct pgtable_debug_args *args) { } 454 static void __init pmd_leaf_tests(struct pgtable_debug_args *args) { } 455 static void __init pud_leaf_tests(struct pgtable_debug_args *args) { } 456 static void __init pmd_savedwrite_tests(struct pgtable_debug_args *args) { } 457 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 458 459 #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP 460 static void __init pmd_huge_tests(struct pgtable_debug_args *args) 461 { 462 pmd_t pmd; 463 464 if (!arch_vmap_pmd_supported(args->page_prot)) 465 return; 466 467 pr_debug("Validating PMD huge\n"); 468 /* 469 * X86 defined pmd_set_huge() verifies that the given 470 * PMD is not a populated non-leaf entry. 471 */ 472 WRITE_ONCE(*args->pmdp, __pmd(0)); 473 WARN_ON(!pmd_set_huge(args->pmdp, __pfn_to_phys(args->fixed_pmd_pfn), args->page_prot)); 474 WARN_ON(!pmd_clear_huge(args->pmdp)); 475 pmd = READ_ONCE(*args->pmdp); 476 WARN_ON(!pmd_none(pmd)); 477 } 478 479 static void __init pud_huge_tests(struct pgtable_debug_args *args) 480 { 481 pud_t pud; 482 483 if (!arch_vmap_pud_supported(args->page_prot)) 484 return; 485 486 pr_debug("Validating PUD huge\n"); 487 /* 488 * X86 defined pud_set_huge() verifies that the given 489 * PUD is not a populated non-leaf entry. 490 */ 491 WRITE_ONCE(*args->pudp, __pud(0)); 492 WARN_ON(!pud_set_huge(args->pudp, __pfn_to_phys(args->fixed_pud_pfn), args->page_prot)); 493 WARN_ON(!pud_clear_huge(args->pudp)); 494 pud = READ_ONCE(*args->pudp); 495 WARN_ON(!pud_none(pud)); 496 } 497 #else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */ 498 static void __init pmd_huge_tests(struct pgtable_debug_args *args) { } 499 static void __init pud_huge_tests(struct pgtable_debug_args *args) { } 500 #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 501 502 static void __init p4d_basic_tests(struct pgtable_debug_args *args) 503 { 504 p4d_t p4d; 505 506 pr_debug("Validating P4D basic\n"); 507 memset(&p4d, RANDOM_NZVALUE, sizeof(p4d_t)); 508 WARN_ON(!p4d_same(p4d, p4d)); 509 } 510 511 static void __init pgd_basic_tests(struct pgtable_debug_args *args) 512 { 513 pgd_t pgd; 514 515 pr_debug("Validating PGD basic\n"); 516 memset(&pgd, RANDOM_NZVALUE, sizeof(pgd_t)); 517 WARN_ON(!pgd_same(pgd, pgd)); 518 } 519 520 #ifndef __PAGETABLE_PUD_FOLDED 521 static void __init pud_clear_tests(struct pgtable_debug_args *args) 522 { 523 pud_t pud = READ_ONCE(*args->pudp); 524 525 if (mm_pmd_folded(args->mm)) 526 return; 527 528 pr_debug("Validating PUD clear\n"); 529 pud = __pud(pud_val(pud) | RANDOM_ORVALUE); 530 WRITE_ONCE(*args->pudp, pud); 531 pud_clear(args->pudp); 532 pud = READ_ONCE(*args->pudp); 533 WARN_ON(!pud_none(pud)); 534 } 535 536 static void __init pud_populate_tests(struct pgtable_debug_args *args) 537 { 538 pud_t pud; 539 540 if (mm_pmd_folded(args->mm)) 541 return; 542 543 pr_debug("Validating PUD populate\n"); 544 /* 545 * This entry points to next level page table page. 546 * Hence this must not qualify as pud_bad(). 547 */ 548 pud_populate(args->mm, args->pudp, args->start_pmdp); 549 pud = READ_ONCE(*args->pudp); 550 WARN_ON(pud_bad(pud)); 551 } 552 #else /* !__PAGETABLE_PUD_FOLDED */ 553 static void __init pud_clear_tests(struct pgtable_debug_args *args) { } 554 static void __init pud_populate_tests(struct pgtable_debug_args *args) { } 555 #endif /* PAGETABLE_PUD_FOLDED */ 556 557 #ifndef __PAGETABLE_P4D_FOLDED 558 static void __init p4d_clear_tests(struct pgtable_debug_args *args) 559 { 560 p4d_t p4d = READ_ONCE(*args->p4dp); 561 562 if (mm_pud_folded(args->mm)) 563 return; 564 565 pr_debug("Validating P4D clear\n"); 566 p4d = __p4d(p4d_val(p4d) | RANDOM_ORVALUE); 567 WRITE_ONCE(*args->p4dp, p4d); 568 p4d_clear(args->p4dp); 569 p4d = READ_ONCE(*args->p4dp); 570 WARN_ON(!p4d_none(p4d)); 571 } 572 573 static void __init p4d_populate_tests(struct pgtable_debug_args *args) 574 { 575 p4d_t p4d; 576 577 if (mm_pud_folded(args->mm)) 578 return; 579 580 pr_debug("Validating P4D populate\n"); 581 /* 582 * This entry points to next level page table page. 583 * Hence this must not qualify as p4d_bad(). 584 */ 585 pud_clear(args->pudp); 586 p4d_clear(args->p4dp); 587 p4d_populate(args->mm, args->p4dp, args->start_pudp); 588 p4d = READ_ONCE(*args->p4dp); 589 WARN_ON(p4d_bad(p4d)); 590 } 591 592 static void __init pgd_clear_tests(struct pgtable_debug_args *args) 593 { 594 pgd_t pgd = READ_ONCE(*(args->pgdp)); 595 596 if (mm_p4d_folded(args->mm)) 597 return; 598 599 pr_debug("Validating PGD clear\n"); 600 pgd = __pgd(pgd_val(pgd) | RANDOM_ORVALUE); 601 WRITE_ONCE(*args->pgdp, pgd); 602 pgd_clear(args->pgdp); 603 pgd = READ_ONCE(*args->pgdp); 604 WARN_ON(!pgd_none(pgd)); 605 } 606 607 static void __init pgd_populate_tests(struct pgtable_debug_args *args) 608 { 609 pgd_t pgd; 610 611 if (mm_p4d_folded(args->mm)) 612 return; 613 614 pr_debug("Validating PGD populate\n"); 615 /* 616 * This entry points to next level page table page. 617 * Hence this must not qualify as pgd_bad(). 618 */ 619 p4d_clear(args->p4dp); 620 pgd_clear(args->pgdp); 621 pgd_populate(args->mm, args->pgdp, args->start_p4dp); 622 pgd = READ_ONCE(*args->pgdp); 623 WARN_ON(pgd_bad(pgd)); 624 } 625 #else /* !__PAGETABLE_P4D_FOLDED */ 626 static void __init p4d_clear_tests(struct pgtable_debug_args *args) { } 627 static void __init pgd_clear_tests(struct pgtable_debug_args *args) { } 628 static void __init p4d_populate_tests(struct pgtable_debug_args *args) { } 629 static void __init pgd_populate_tests(struct pgtable_debug_args *args) { } 630 #endif /* PAGETABLE_P4D_FOLDED */ 631 632 static void __init pte_clear_tests(struct pgtable_debug_args *args) 633 { 634 struct page *page; 635 pte_t pte = pfn_pte(args->pte_pfn, args->page_prot); 636 637 page = (args->pte_pfn != ULONG_MAX) ? pfn_to_page(args->pte_pfn) : NULL; 638 if (!page) 639 return; 640 641 /* 642 * flush_dcache_page() is called after set_pte_at() to clear 643 * PG_arch_1 for the page on ARM64. The page flag isn't cleared 644 * when it's released and page allocation check will fail when 645 * the page is allocated again. For architectures other than ARM64, 646 * the unexpected overhead of cache flushing is acceptable. 647 */ 648 pr_debug("Validating PTE clear\n"); 649 #ifndef CONFIG_RISCV 650 pte = __pte(pte_val(pte) | RANDOM_ORVALUE); 651 #endif 652 set_pte_at(args->mm, args->vaddr, args->ptep, pte); 653 flush_dcache_page(page); 654 barrier(); 655 pte_clear(args->mm, args->vaddr, args->ptep); 656 pte = ptep_get(args->ptep); 657 WARN_ON(!pte_none(pte)); 658 } 659 660 static void __init pmd_clear_tests(struct pgtable_debug_args *args) 661 { 662 pmd_t pmd = READ_ONCE(*args->pmdp); 663 664 pr_debug("Validating PMD clear\n"); 665 pmd = __pmd(pmd_val(pmd) | RANDOM_ORVALUE); 666 WRITE_ONCE(*args->pmdp, pmd); 667 pmd_clear(args->pmdp); 668 pmd = READ_ONCE(*args->pmdp); 669 WARN_ON(!pmd_none(pmd)); 670 } 671 672 static void __init pmd_populate_tests(struct pgtable_debug_args *args) 673 { 674 pmd_t pmd; 675 676 pr_debug("Validating PMD populate\n"); 677 /* 678 * This entry points to next level page table page. 679 * Hence this must not qualify as pmd_bad(). 680 */ 681 pmd_populate(args->mm, args->pmdp, args->start_ptep); 682 pmd = READ_ONCE(*args->pmdp); 683 WARN_ON(pmd_bad(pmd)); 684 } 685 686 static void __init pte_special_tests(struct pgtable_debug_args *args) 687 { 688 pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot); 689 690 if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL)) 691 return; 692 693 pr_debug("Validating PTE special\n"); 694 WARN_ON(!pte_special(pte_mkspecial(pte))); 695 } 696 697 static void __init pte_protnone_tests(struct pgtable_debug_args *args) 698 { 699 pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot_none); 700 701 if (!IS_ENABLED(CONFIG_NUMA_BALANCING)) 702 return; 703 704 pr_debug("Validating PTE protnone\n"); 705 WARN_ON(!pte_protnone(pte)); 706 WARN_ON(!pte_present(pte)); 707 } 708 709 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 710 static void __init pmd_protnone_tests(struct pgtable_debug_args *args) 711 { 712 pmd_t pmd; 713 714 if (!IS_ENABLED(CONFIG_NUMA_BALANCING)) 715 return; 716 717 if (!has_transparent_hugepage()) 718 return; 719 720 pr_debug("Validating PMD protnone\n"); 721 pmd = pmd_mkhuge(pfn_pmd(args->fixed_pmd_pfn, args->page_prot_none)); 722 WARN_ON(!pmd_protnone(pmd)); 723 WARN_ON(!pmd_present(pmd)); 724 } 725 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */ 726 static void __init pmd_protnone_tests(struct pgtable_debug_args *args) { } 727 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 728 729 #ifdef CONFIG_ARCH_HAS_PTE_DEVMAP 730 static void __init pte_devmap_tests(struct pgtable_debug_args *args) 731 { 732 pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot); 733 734 pr_debug("Validating PTE devmap\n"); 735 WARN_ON(!pte_devmap(pte_mkdevmap(pte))); 736 } 737 738 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 739 static void __init pmd_devmap_tests(struct pgtable_debug_args *args) 740 { 741 pmd_t pmd; 742 743 if (!has_transparent_hugepage()) 744 return; 745 746 pr_debug("Validating PMD devmap\n"); 747 pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot); 748 WARN_ON(!pmd_devmap(pmd_mkdevmap(pmd))); 749 } 750 751 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 752 static void __init pud_devmap_tests(struct pgtable_debug_args *args) 753 { 754 pud_t pud; 755 756 if (!has_transparent_hugepage()) 757 return; 758 759 pr_debug("Validating PUD devmap\n"); 760 pud = pfn_pud(args->fixed_pud_pfn, args->page_prot); 761 WARN_ON(!pud_devmap(pud_mkdevmap(pud))); 762 } 763 #else /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 764 static void __init pud_devmap_tests(struct pgtable_debug_args *args) { } 765 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 766 #else /* CONFIG_TRANSPARENT_HUGEPAGE */ 767 static void __init pmd_devmap_tests(struct pgtable_debug_args *args) { } 768 static void __init pud_devmap_tests(struct pgtable_debug_args *args) { } 769 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 770 #else 771 static void __init pte_devmap_tests(struct pgtable_debug_args *args) { } 772 static void __init pmd_devmap_tests(struct pgtable_debug_args *args) { } 773 static void __init pud_devmap_tests(struct pgtable_debug_args *args) { } 774 #endif /* CONFIG_ARCH_HAS_PTE_DEVMAP */ 775 776 static void __init pte_soft_dirty_tests(struct pgtable_debug_args *args) 777 { 778 pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot); 779 780 if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY)) 781 return; 782 783 pr_debug("Validating PTE soft dirty\n"); 784 WARN_ON(!pte_soft_dirty(pte_mksoft_dirty(pte))); 785 WARN_ON(pte_soft_dirty(pte_clear_soft_dirty(pte))); 786 } 787 788 static void __init pte_swap_soft_dirty_tests(struct pgtable_debug_args *args) 789 { 790 pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot); 791 792 if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY)) 793 return; 794 795 pr_debug("Validating PTE swap soft dirty\n"); 796 WARN_ON(!pte_swp_soft_dirty(pte_swp_mksoft_dirty(pte))); 797 WARN_ON(pte_swp_soft_dirty(pte_swp_clear_soft_dirty(pte))); 798 } 799 800 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 801 static void __init pmd_soft_dirty_tests(struct pgtable_debug_args *args) 802 { 803 pmd_t pmd; 804 805 if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY)) 806 return; 807 808 if (!has_transparent_hugepage()) 809 return; 810 811 pr_debug("Validating PMD soft dirty\n"); 812 pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot); 813 WARN_ON(!pmd_soft_dirty(pmd_mksoft_dirty(pmd))); 814 WARN_ON(pmd_soft_dirty(pmd_clear_soft_dirty(pmd))); 815 } 816 817 static void __init pmd_swap_soft_dirty_tests(struct pgtable_debug_args *args) 818 { 819 pmd_t pmd; 820 821 if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) || 822 !IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION)) 823 return; 824 825 if (!has_transparent_hugepage()) 826 return; 827 828 pr_debug("Validating PMD swap soft dirty\n"); 829 pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot); 830 WARN_ON(!pmd_swp_soft_dirty(pmd_swp_mksoft_dirty(pmd))); 831 WARN_ON(pmd_swp_soft_dirty(pmd_swp_clear_soft_dirty(pmd))); 832 } 833 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */ 834 static void __init pmd_soft_dirty_tests(struct pgtable_debug_args *args) { } 835 static void __init pmd_swap_soft_dirty_tests(struct pgtable_debug_args *args) { } 836 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 837 838 static void __init pte_swap_tests(struct pgtable_debug_args *args) 839 { 840 swp_entry_t swp; 841 pte_t pte; 842 843 pr_debug("Validating PTE swap\n"); 844 pte = pfn_pte(args->fixed_pte_pfn, args->page_prot); 845 swp = __pte_to_swp_entry(pte); 846 pte = __swp_entry_to_pte(swp); 847 WARN_ON(args->fixed_pte_pfn != pte_pfn(pte)); 848 } 849 850 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 851 static void __init pmd_swap_tests(struct pgtable_debug_args *args) 852 { 853 swp_entry_t swp; 854 pmd_t pmd; 855 856 if (!has_transparent_hugepage()) 857 return; 858 859 pr_debug("Validating PMD swap\n"); 860 pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot); 861 swp = __pmd_to_swp_entry(pmd); 862 pmd = __swp_entry_to_pmd(swp); 863 WARN_ON(args->fixed_pmd_pfn != pmd_pfn(pmd)); 864 } 865 #else /* !CONFIG_ARCH_ENABLE_THP_MIGRATION */ 866 static void __init pmd_swap_tests(struct pgtable_debug_args *args) { } 867 #endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ 868 869 static void __init swap_migration_tests(struct pgtable_debug_args *args) 870 { 871 struct page *page; 872 swp_entry_t swp; 873 874 if (!IS_ENABLED(CONFIG_MIGRATION)) 875 return; 876 877 /* 878 * swap_migration_tests() requires a dedicated page as it needs to 879 * be locked before creating a migration entry from it. Locking the 880 * page that actually maps kernel text ('start_kernel') can be real 881 * problematic. Lets use the allocated page explicitly for this 882 * purpose. 883 */ 884 page = (args->pte_pfn != ULONG_MAX) ? pfn_to_page(args->pte_pfn) : NULL; 885 if (!page) 886 return; 887 888 pr_debug("Validating swap migration\n"); 889 890 /* 891 * make_migration_entry() expects given page to be 892 * locked, otherwise it stumbles upon a BUG_ON(). 893 */ 894 __SetPageLocked(page); 895 swp = make_writable_migration_entry(page_to_pfn(page)); 896 WARN_ON(!is_migration_entry(swp)); 897 WARN_ON(!is_writable_migration_entry(swp)); 898 899 swp = make_readable_migration_entry(swp_offset(swp)); 900 WARN_ON(!is_migration_entry(swp)); 901 WARN_ON(is_writable_migration_entry(swp)); 902 903 swp = make_readable_migration_entry(page_to_pfn(page)); 904 WARN_ON(!is_migration_entry(swp)); 905 WARN_ON(is_writable_migration_entry(swp)); 906 __ClearPageLocked(page); 907 } 908 909 #ifdef CONFIG_HUGETLB_PAGE 910 static void __init hugetlb_basic_tests(struct pgtable_debug_args *args) 911 { 912 struct page *page; 913 pte_t pte; 914 915 pr_debug("Validating HugeTLB basic\n"); 916 /* 917 * Accessing the page associated with the pfn is safe here, 918 * as it was previously derived from a real kernel symbol. 919 */ 920 page = pfn_to_page(args->fixed_pmd_pfn); 921 pte = mk_huge_pte(page, args->page_prot); 922 923 WARN_ON(!huge_pte_dirty(huge_pte_mkdirty(pte))); 924 WARN_ON(!huge_pte_write(huge_pte_mkwrite(huge_pte_wrprotect(pte)))); 925 WARN_ON(huge_pte_write(huge_pte_wrprotect(huge_pte_mkwrite(pte)))); 926 927 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB 928 pte = pfn_pte(args->fixed_pmd_pfn, args->page_prot); 929 930 WARN_ON(!pte_huge(pte_mkhuge(pte))); 931 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */ 932 } 933 #else /* !CONFIG_HUGETLB_PAGE */ 934 static void __init hugetlb_basic_tests(struct pgtable_debug_args *args) { } 935 #endif /* CONFIG_HUGETLB_PAGE */ 936 937 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 938 static void __init pmd_thp_tests(struct pgtable_debug_args *args) 939 { 940 pmd_t pmd; 941 942 if (!has_transparent_hugepage()) 943 return; 944 945 pr_debug("Validating PMD based THP\n"); 946 /* 947 * pmd_trans_huge() and pmd_present() must return positive after 948 * MMU invalidation with pmd_mkinvalid(). This behavior is an 949 * optimization for transparent huge page. pmd_trans_huge() must 950 * be true if pmd_page() returns a valid THP to avoid taking the 951 * pmd_lock when others walk over non transhuge pmds (i.e. there 952 * are no THP allocated). Especially when splitting a THP and 953 * removing the present bit from the pmd, pmd_trans_huge() still 954 * needs to return true. pmd_present() should be true whenever 955 * pmd_trans_huge() returns true. 956 */ 957 pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot); 958 WARN_ON(!pmd_trans_huge(pmd_mkhuge(pmd))); 959 960 #ifndef __HAVE_ARCH_PMDP_INVALIDATE 961 WARN_ON(!pmd_trans_huge(pmd_mkinvalid(pmd_mkhuge(pmd)))); 962 WARN_ON(!pmd_present(pmd_mkinvalid(pmd_mkhuge(pmd)))); 963 #endif /* __HAVE_ARCH_PMDP_INVALIDATE */ 964 } 965 966 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 967 static void __init pud_thp_tests(struct pgtable_debug_args *args) 968 { 969 pud_t pud; 970 971 if (!has_transparent_hugepage()) 972 return; 973 974 pr_debug("Validating PUD based THP\n"); 975 pud = pfn_pud(args->fixed_pud_pfn, args->page_prot); 976 WARN_ON(!pud_trans_huge(pud_mkhuge(pud))); 977 978 /* 979 * pud_mkinvalid() has been dropped for now. Enable back 980 * these tests when it comes back with a modified pud_present(). 981 * 982 * WARN_ON(!pud_trans_huge(pud_mkinvalid(pud_mkhuge(pud)))); 983 * WARN_ON(!pud_present(pud_mkinvalid(pud_mkhuge(pud)))); 984 */ 985 } 986 #else /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 987 static void __init pud_thp_tests(struct pgtable_debug_args *args) { } 988 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 989 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */ 990 static void __init pmd_thp_tests(struct pgtable_debug_args *args) { } 991 static void __init pud_thp_tests(struct pgtable_debug_args *args) { } 992 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 993 994 static unsigned long __init get_random_vaddr(void) 995 { 996 unsigned long random_vaddr, random_pages, total_user_pages; 997 998 total_user_pages = (TASK_SIZE - FIRST_USER_ADDRESS) / PAGE_SIZE; 999 1000 random_pages = get_random_long() % total_user_pages; 1001 random_vaddr = FIRST_USER_ADDRESS + random_pages * PAGE_SIZE; 1002 1003 return random_vaddr; 1004 } 1005 1006 static void __init destroy_args(struct pgtable_debug_args *args) 1007 { 1008 struct page *page = NULL; 1009 1010 /* Free (huge) page */ 1011 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && 1012 IS_ENABLED(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) && 1013 has_transparent_hugepage() && 1014 args->pud_pfn != ULONG_MAX) { 1015 if (args->is_contiguous_page) { 1016 free_contig_range(args->pud_pfn, 1017 (1 << (HPAGE_PUD_SHIFT - PAGE_SHIFT))); 1018 } else { 1019 page = pfn_to_page(args->pud_pfn); 1020 __free_pages(page, HPAGE_PUD_SHIFT - PAGE_SHIFT); 1021 } 1022 1023 args->pud_pfn = ULONG_MAX; 1024 args->pmd_pfn = ULONG_MAX; 1025 args->pte_pfn = ULONG_MAX; 1026 } 1027 1028 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && 1029 has_transparent_hugepage() && 1030 args->pmd_pfn != ULONG_MAX) { 1031 if (args->is_contiguous_page) { 1032 free_contig_range(args->pmd_pfn, (1 << HPAGE_PMD_ORDER)); 1033 } else { 1034 page = pfn_to_page(args->pmd_pfn); 1035 __free_pages(page, HPAGE_PMD_ORDER); 1036 } 1037 1038 args->pmd_pfn = ULONG_MAX; 1039 args->pte_pfn = ULONG_MAX; 1040 } 1041 1042 if (args->pte_pfn != ULONG_MAX) { 1043 page = pfn_to_page(args->pte_pfn); 1044 __free_pages(page, 0); 1045 1046 args->pte_pfn = ULONG_MAX; 1047 } 1048 1049 /* Free page table entries */ 1050 if (args->start_ptep) { 1051 pte_free(args->mm, args->start_ptep); 1052 mm_dec_nr_ptes(args->mm); 1053 } 1054 1055 if (args->start_pmdp) { 1056 pmd_free(args->mm, args->start_pmdp); 1057 mm_dec_nr_pmds(args->mm); 1058 } 1059 1060 if (args->start_pudp) { 1061 pud_free(args->mm, args->start_pudp); 1062 mm_dec_nr_puds(args->mm); 1063 } 1064 1065 if (args->start_p4dp) 1066 p4d_free(args->mm, args->start_p4dp); 1067 1068 /* Free vma and mm struct */ 1069 if (args->vma) 1070 vm_area_free(args->vma); 1071 1072 if (args->mm) 1073 mmdrop(args->mm); 1074 } 1075 1076 static struct page * __init 1077 debug_vm_pgtable_alloc_huge_page(struct pgtable_debug_args *args, int order) 1078 { 1079 struct page *page = NULL; 1080 1081 #ifdef CONFIG_CONTIG_ALLOC 1082 if (order >= MAX_ORDER) { 1083 page = alloc_contig_pages((1 << order), GFP_KERNEL, 1084 first_online_node, NULL); 1085 if (page) { 1086 args->is_contiguous_page = true; 1087 return page; 1088 } 1089 } 1090 #endif 1091 1092 if (order < MAX_ORDER) 1093 page = alloc_pages(GFP_KERNEL, order); 1094 1095 return page; 1096 } 1097 1098 static int __init init_args(struct pgtable_debug_args *args) 1099 { 1100 struct page *page = NULL; 1101 phys_addr_t phys; 1102 int ret = 0; 1103 1104 /* 1105 * Initialize the debugging data. 1106 * 1107 * protection_map[0] (or even protection_map[8]) will help create 1108 * page table entries with PROT_NONE permission as required for 1109 * pxx_protnone_tests(). 1110 */ 1111 memset(args, 0, sizeof(*args)); 1112 args->vaddr = get_random_vaddr(); 1113 args->page_prot = vm_get_page_prot(VMFLAGS); 1114 args->page_prot_none = protection_map[0]; 1115 args->is_contiguous_page = false; 1116 args->pud_pfn = ULONG_MAX; 1117 args->pmd_pfn = ULONG_MAX; 1118 args->pte_pfn = ULONG_MAX; 1119 args->fixed_pgd_pfn = ULONG_MAX; 1120 args->fixed_p4d_pfn = ULONG_MAX; 1121 args->fixed_pud_pfn = ULONG_MAX; 1122 args->fixed_pmd_pfn = ULONG_MAX; 1123 args->fixed_pte_pfn = ULONG_MAX; 1124 1125 /* Allocate mm and vma */ 1126 args->mm = mm_alloc(); 1127 if (!args->mm) { 1128 pr_err("Failed to allocate mm struct\n"); 1129 ret = -ENOMEM; 1130 goto error; 1131 } 1132 1133 args->vma = vm_area_alloc(args->mm); 1134 if (!args->vma) { 1135 pr_err("Failed to allocate vma\n"); 1136 ret = -ENOMEM; 1137 goto error; 1138 } 1139 1140 /* 1141 * Allocate page table entries. They will be modified in the tests. 1142 * Lets save the page table entries so that they can be released 1143 * when the tests are completed. 1144 */ 1145 args->pgdp = pgd_offset(args->mm, args->vaddr); 1146 args->p4dp = p4d_alloc(args->mm, args->pgdp, args->vaddr); 1147 if (!args->p4dp) { 1148 pr_err("Failed to allocate p4d entries\n"); 1149 ret = -ENOMEM; 1150 goto error; 1151 } 1152 args->start_p4dp = p4d_offset(args->pgdp, 0UL); 1153 WARN_ON(!args->start_p4dp); 1154 1155 args->pudp = pud_alloc(args->mm, args->p4dp, args->vaddr); 1156 if (!args->pudp) { 1157 pr_err("Failed to allocate pud entries\n"); 1158 ret = -ENOMEM; 1159 goto error; 1160 } 1161 args->start_pudp = pud_offset(args->p4dp, 0UL); 1162 WARN_ON(!args->start_pudp); 1163 1164 args->pmdp = pmd_alloc(args->mm, args->pudp, args->vaddr); 1165 if (!args->pmdp) { 1166 pr_err("Failed to allocate pmd entries\n"); 1167 ret = -ENOMEM; 1168 goto error; 1169 } 1170 args->start_pmdp = pmd_offset(args->pudp, 0UL); 1171 WARN_ON(!args->start_pmdp); 1172 1173 if (pte_alloc(args->mm, args->pmdp)) { 1174 pr_err("Failed to allocate pte entries\n"); 1175 ret = -ENOMEM; 1176 goto error; 1177 } 1178 args->start_ptep = pmd_pgtable(READ_ONCE(*args->pmdp)); 1179 WARN_ON(!args->start_ptep); 1180 1181 /* 1182 * PFN for mapping at PTE level is determined from a standard kernel 1183 * text symbol. But pfns for higher page table levels are derived by 1184 * masking lower bits of this real pfn. These derived pfns might not 1185 * exist on the platform but that does not really matter as pfn_pxx() 1186 * helpers will still create appropriate entries for the test. This 1187 * helps avoid large memory block allocations to be used for mapping 1188 * at higher page table levels in some of the tests. 1189 */ 1190 phys = __pa_symbol(&start_kernel); 1191 args->fixed_pgd_pfn = __phys_to_pfn(phys & PGDIR_MASK); 1192 args->fixed_p4d_pfn = __phys_to_pfn(phys & P4D_MASK); 1193 args->fixed_pud_pfn = __phys_to_pfn(phys & PUD_MASK); 1194 args->fixed_pmd_pfn = __phys_to_pfn(phys & PMD_MASK); 1195 args->fixed_pte_pfn = __phys_to_pfn(phys & PAGE_MASK); 1196 WARN_ON(!pfn_valid(args->fixed_pte_pfn)); 1197 1198 /* 1199 * Allocate (huge) pages because some of the tests need to access 1200 * the data in the pages. The corresponding tests will be skipped 1201 * if we fail to allocate (huge) pages. 1202 */ 1203 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && 1204 IS_ENABLED(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) && 1205 has_transparent_hugepage()) { 1206 page = debug_vm_pgtable_alloc_huge_page(args, 1207 HPAGE_PUD_SHIFT - PAGE_SHIFT); 1208 if (page) { 1209 args->pud_pfn = page_to_pfn(page); 1210 args->pmd_pfn = args->pud_pfn; 1211 args->pte_pfn = args->pud_pfn; 1212 return 0; 1213 } 1214 } 1215 1216 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && 1217 has_transparent_hugepage()) { 1218 page = debug_vm_pgtable_alloc_huge_page(args, HPAGE_PMD_ORDER); 1219 if (page) { 1220 args->pmd_pfn = page_to_pfn(page); 1221 args->pte_pfn = args->pmd_pfn; 1222 return 0; 1223 } 1224 } 1225 1226 page = alloc_pages(GFP_KERNEL, 0); 1227 if (page) 1228 args->pte_pfn = page_to_pfn(page); 1229 1230 return 0; 1231 1232 error: 1233 destroy_args(args); 1234 return ret; 1235 } 1236 1237 static int __init debug_vm_pgtable(void) 1238 { 1239 struct pgtable_debug_args args; 1240 spinlock_t *ptl = NULL; 1241 int idx, ret; 1242 1243 pr_info("Validating architecture page table helpers\n"); 1244 ret = init_args(&args); 1245 if (ret) 1246 return ret; 1247 1248 /* 1249 * Iterate over the protection_map[] to make sure that all 1250 * the basic page table transformation validations just hold 1251 * true irrespective of the starting protection value for a 1252 * given page table entry. 1253 */ 1254 for (idx = 0; idx < ARRAY_SIZE(protection_map); idx++) { 1255 pte_basic_tests(&args, idx); 1256 pmd_basic_tests(&args, idx); 1257 pud_basic_tests(&args, idx); 1258 } 1259 1260 /* 1261 * Both P4D and PGD level tests are very basic which do not 1262 * involve creating page table entries from the protection 1263 * value and the given pfn. Hence just keep them out from 1264 * the above iteration for now to save some test execution 1265 * time. 1266 */ 1267 p4d_basic_tests(&args); 1268 pgd_basic_tests(&args); 1269 1270 pmd_leaf_tests(&args); 1271 pud_leaf_tests(&args); 1272 1273 pte_savedwrite_tests(&args); 1274 pmd_savedwrite_tests(&args); 1275 1276 pte_special_tests(&args); 1277 pte_protnone_tests(&args); 1278 pmd_protnone_tests(&args); 1279 1280 pte_devmap_tests(&args); 1281 pmd_devmap_tests(&args); 1282 pud_devmap_tests(&args); 1283 1284 pte_soft_dirty_tests(&args); 1285 pmd_soft_dirty_tests(&args); 1286 pte_swap_soft_dirty_tests(&args); 1287 pmd_swap_soft_dirty_tests(&args); 1288 1289 pte_swap_tests(&args); 1290 pmd_swap_tests(&args); 1291 1292 swap_migration_tests(&args); 1293 1294 pmd_thp_tests(&args); 1295 pud_thp_tests(&args); 1296 1297 hugetlb_basic_tests(&args); 1298 1299 /* 1300 * Page table modifying tests. They need to hold 1301 * proper page table lock. 1302 */ 1303 1304 args.ptep = pte_offset_map_lock(args.mm, args.pmdp, args.vaddr, &ptl); 1305 pte_clear_tests(&args); 1306 pte_advanced_tests(&args); 1307 pte_unmap_unlock(args.ptep, ptl); 1308 1309 ptl = pmd_lock(args.mm, args.pmdp); 1310 pmd_clear_tests(&args); 1311 pmd_advanced_tests(&args); 1312 pmd_huge_tests(&args); 1313 pmd_populate_tests(&args); 1314 spin_unlock(ptl); 1315 1316 ptl = pud_lock(args.mm, args.pudp); 1317 pud_clear_tests(&args); 1318 pud_advanced_tests(&args); 1319 pud_huge_tests(&args); 1320 pud_populate_tests(&args); 1321 spin_unlock(ptl); 1322 1323 spin_lock(&(args.mm->page_table_lock)); 1324 p4d_clear_tests(&args); 1325 pgd_clear_tests(&args); 1326 p4d_populate_tests(&args); 1327 pgd_populate_tests(&args); 1328 spin_unlock(&(args.mm->page_table_lock)); 1329 1330 destroy_args(&args); 1331 return 0; 1332 } 1333 late_initcall(debug_vm_pgtable); 1334