1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright IBM Corp. 2006 4 */ 5 6 #include <linux/memory_hotplug.h> 7 #include <linux/memblock.h> 8 #include <linux/kasan.h> 9 #include <linux/pfn.h> 10 #include <linux/mm.h> 11 #include <linux/init.h> 12 #include <linux/list.h> 13 #include <linux/hugetlb.h> 14 #include <linux/slab.h> 15 #include <linux/sort.h> 16 #include <asm/cacheflush.h> 17 #include <asm/nospec-branch.h> 18 #include <asm/pgalloc.h> 19 #include <asm/setup.h> 20 #include <asm/tlbflush.h> 21 #include <asm/sections.h> 22 #include <asm/set_memory.h> 23 24 static DEFINE_MUTEX(vmem_mutex); 25 26 static void __ref *vmem_alloc_pages(unsigned int order) 27 { 28 unsigned long size = PAGE_SIZE << order; 29 30 if (slab_is_available()) 31 return (void *)__get_free_pages(GFP_KERNEL, order); 32 return memblock_alloc(size, size); 33 } 34 35 static void vmem_free_pages(unsigned long addr, int order) 36 { 37 /* We don't expect boot memory to be removed ever. */ 38 if (!slab_is_available() || 39 WARN_ON_ONCE(PageReserved(virt_to_page(addr)))) 40 return; 41 free_pages(addr, order); 42 } 43 44 void *vmem_crst_alloc(unsigned long val) 45 { 46 unsigned long *table; 47 48 table = vmem_alloc_pages(CRST_ALLOC_ORDER); 49 if (table) 50 crst_table_init(table, val); 51 return table; 52 } 53 54 pte_t __ref *vmem_pte_alloc(void) 55 { 56 unsigned long size = PTRS_PER_PTE * sizeof(pte_t); 57 pte_t *pte; 58 59 if (slab_is_available()) 60 pte = (pte_t *) page_table_alloc(&init_mm); 61 else 62 pte = (pte_t *) memblock_alloc(size, size); 63 if (!pte) 64 return NULL; 65 memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE); 66 return pte; 67 } 68 69 static void vmem_pte_free(unsigned long *table) 70 { 71 /* We don't expect boot memory to be removed ever. */ 72 if (!slab_is_available() || 73 WARN_ON_ONCE(PageReserved(virt_to_page(table)))) 74 return; 75 page_table_free(&init_mm, table); 76 } 77 78 #define PAGE_UNUSED 0xFD 79 80 /* 81 * The unused vmemmap range, which was not yet memset(PAGE_UNUSED) ranges 82 * from unused_sub_pmd_start to next PMD_SIZE boundary. 83 */ 84 static unsigned long unused_sub_pmd_start; 85 86 static void vmemmap_flush_unused_sub_pmd(void) 87 { 88 if (!unused_sub_pmd_start) 89 return; 90 memset((void *)unused_sub_pmd_start, PAGE_UNUSED, 91 ALIGN(unused_sub_pmd_start, PMD_SIZE) - unused_sub_pmd_start); 92 unused_sub_pmd_start = 0; 93 } 94 95 static void vmemmap_mark_sub_pmd_used(unsigned long start, unsigned long end) 96 { 97 /* 98 * As we expect to add in the same granularity as we remove, it's 99 * sufficient to mark only some piece used to block the memmap page from 100 * getting removed (just in case the memmap never gets initialized, 101 * e.g., because the memory block never gets onlined). 102 */ 103 memset((void *)start, 0, sizeof(struct page)); 104 } 105 106 static void vmemmap_use_sub_pmd(unsigned long start, unsigned long end) 107 { 108 /* 109 * We only optimize if the new used range directly follows the 110 * previously unused range (esp., when populating consecutive sections). 111 */ 112 if (unused_sub_pmd_start == start) { 113 unused_sub_pmd_start = end; 114 if (likely(IS_ALIGNED(unused_sub_pmd_start, PMD_SIZE))) 115 unused_sub_pmd_start = 0; 116 return; 117 } 118 vmemmap_flush_unused_sub_pmd(); 119 vmemmap_mark_sub_pmd_used(start, end); 120 } 121 122 static void vmemmap_use_new_sub_pmd(unsigned long start, unsigned long end) 123 { 124 unsigned long page = ALIGN_DOWN(start, PMD_SIZE); 125 126 vmemmap_flush_unused_sub_pmd(); 127 128 /* Could be our memmap page is filled with PAGE_UNUSED already ... */ 129 vmemmap_mark_sub_pmd_used(start, end); 130 131 /* Mark the unused parts of the new memmap page PAGE_UNUSED. */ 132 if (!IS_ALIGNED(start, PMD_SIZE)) 133 memset((void *)page, PAGE_UNUSED, start - page); 134 /* 135 * We want to avoid memset(PAGE_UNUSED) when populating the vmemmap of 136 * consecutive sections. Remember for the last added PMD the last 137 * unused range in the populated PMD. 138 */ 139 if (!IS_ALIGNED(end, PMD_SIZE)) 140 unused_sub_pmd_start = end; 141 } 142 143 /* Returns true if the PMD is completely unused and can be freed. */ 144 static bool vmemmap_unuse_sub_pmd(unsigned long start, unsigned long end) 145 { 146 unsigned long page = ALIGN_DOWN(start, PMD_SIZE); 147 148 vmemmap_flush_unused_sub_pmd(); 149 memset((void *)start, PAGE_UNUSED, end - start); 150 return !memchr_inv((void *)page, PAGE_UNUSED, PMD_SIZE); 151 } 152 153 /* __ref: we'll only call vmemmap_alloc_block() via vmemmap_populate() */ 154 static int __ref modify_pte_table(pmd_t *pmd, unsigned long addr, 155 unsigned long end, bool add, bool direct) 156 { 157 unsigned long prot, pages = 0; 158 int ret = -ENOMEM; 159 pte_t *pte; 160 161 prot = pgprot_val(PAGE_KERNEL); 162 if (!MACHINE_HAS_NX) 163 prot &= ~_PAGE_NOEXEC; 164 165 pte = pte_offset_kernel(pmd, addr); 166 for (; addr < end; addr += PAGE_SIZE, pte++) { 167 if (!add) { 168 if (pte_none(*pte)) 169 continue; 170 if (!direct) 171 vmem_free_pages((unsigned long) pfn_to_virt(pte_pfn(*pte)), 0); 172 pte_clear(&init_mm, addr, pte); 173 } else if (pte_none(*pte)) { 174 if (!direct) { 175 void *new_page = vmemmap_alloc_block(PAGE_SIZE, NUMA_NO_NODE); 176 177 if (!new_page) 178 goto out; 179 set_pte(pte, __pte(__pa(new_page) | prot)); 180 } else { 181 set_pte(pte, __pte(__pa(addr) | prot)); 182 } 183 } else { 184 continue; 185 } 186 pages++; 187 } 188 ret = 0; 189 out: 190 if (direct) 191 update_page_count(PG_DIRECT_MAP_4K, add ? pages : -pages); 192 return ret; 193 } 194 195 static void try_free_pte_table(pmd_t *pmd, unsigned long start) 196 { 197 pte_t *pte; 198 int i; 199 200 /* We can safely assume this is fully in 1:1 mapping & vmemmap area */ 201 pte = pte_offset_kernel(pmd, start); 202 for (i = 0; i < PTRS_PER_PTE; i++, pte++) { 203 if (!pte_none(*pte)) 204 return; 205 } 206 vmem_pte_free((unsigned long *) pmd_deref(*pmd)); 207 pmd_clear(pmd); 208 } 209 210 /* __ref: we'll only call vmemmap_alloc_block() via vmemmap_populate() */ 211 static int __ref modify_pmd_table(pud_t *pud, unsigned long addr, 212 unsigned long end, bool add, bool direct) 213 { 214 unsigned long next, prot, pages = 0; 215 int ret = -ENOMEM; 216 pmd_t *pmd; 217 pte_t *pte; 218 219 prot = pgprot_val(SEGMENT_KERNEL); 220 if (!MACHINE_HAS_NX) 221 prot &= ~_SEGMENT_ENTRY_NOEXEC; 222 223 pmd = pmd_offset(pud, addr); 224 for (; addr < end; addr = next, pmd++) { 225 next = pmd_addr_end(addr, end); 226 if (!add) { 227 if (pmd_none(*pmd)) 228 continue; 229 if (pmd_large(*pmd)) { 230 if (IS_ALIGNED(addr, PMD_SIZE) && 231 IS_ALIGNED(next, PMD_SIZE)) { 232 if (!direct) 233 vmem_free_pages(pmd_deref(*pmd), get_order(PMD_SIZE)); 234 pmd_clear(pmd); 235 pages++; 236 } else if (!direct && vmemmap_unuse_sub_pmd(addr, next)) { 237 vmem_free_pages(pmd_deref(*pmd), get_order(PMD_SIZE)); 238 pmd_clear(pmd); 239 } 240 continue; 241 } 242 } else if (pmd_none(*pmd)) { 243 if (IS_ALIGNED(addr, PMD_SIZE) && 244 IS_ALIGNED(next, PMD_SIZE) && 245 MACHINE_HAS_EDAT1 && direct && 246 !debug_pagealloc_enabled()) { 247 set_pmd(pmd, __pmd(__pa(addr) | prot)); 248 pages++; 249 continue; 250 } else if (!direct && MACHINE_HAS_EDAT1) { 251 void *new_page; 252 253 /* 254 * Use 1MB frames for vmemmap if available. We 255 * always use large frames even if they are only 256 * partially used. Otherwise we would have also 257 * page tables since vmemmap_populate gets 258 * called for each section separately. 259 */ 260 new_page = vmemmap_alloc_block(PMD_SIZE, NUMA_NO_NODE); 261 if (new_page) { 262 set_pmd(pmd, __pmd(__pa(new_page) | prot)); 263 if (!IS_ALIGNED(addr, PMD_SIZE) || 264 !IS_ALIGNED(next, PMD_SIZE)) { 265 vmemmap_use_new_sub_pmd(addr, next); 266 } 267 continue; 268 } 269 } 270 pte = vmem_pte_alloc(); 271 if (!pte) 272 goto out; 273 pmd_populate(&init_mm, pmd, pte); 274 } else if (pmd_large(*pmd)) { 275 if (!direct) 276 vmemmap_use_sub_pmd(addr, next); 277 continue; 278 } 279 ret = modify_pte_table(pmd, addr, next, add, direct); 280 if (ret) 281 goto out; 282 if (!add) 283 try_free_pte_table(pmd, addr & PMD_MASK); 284 } 285 ret = 0; 286 out: 287 if (direct) 288 update_page_count(PG_DIRECT_MAP_1M, add ? pages : -pages); 289 return ret; 290 } 291 292 static void try_free_pmd_table(pud_t *pud, unsigned long start) 293 { 294 const unsigned long end = start + PUD_SIZE; 295 pmd_t *pmd; 296 int i; 297 298 /* Don't mess with any tables not fully in 1:1 mapping & vmemmap area */ 299 if (end > VMALLOC_START) 300 return; 301 302 pmd = pmd_offset(pud, start); 303 for (i = 0; i < PTRS_PER_PMD; i++, pmd++) 304 if (!pmd_none(*pmd)) 305 return; 306 vmem_free_pages(pud_deref(*pud), CRST_ALLOC_ORDER); 307 pud_clear(pud); 308 } 309 310 static int modify_pud_table(p4d_t *p4d, unsigned long addr, unsigned long end, 311 bool add, bool direct) 312 { 313 unsigned long next, prot, pages = 0; 314 int ret = -ENOMEM; 315 pud_t *pud; 316 pmd_t *pmd; 317 318 prot = pgprot_val(REGION3_KERNEL); 319 if (!MACHINE_HAS_NX) 320 prot &= ~_REGION_ENTRY_NOEXEC; 321 pud = pud_offset(p4d, addr); 322 for (; addr < end; addr = next, pud++) { 323 next = pud_addr_end(addr, end); 324 if (!add) { 325 if (pud_none(*pud)) 326 continue; 327 if (pud_large(*pud)) { 328 if (IS_ALIGNED(addr, PUD_SIZE) && 329 IS_ALIGNED(next, PUD_SIZE)) { 330 pud_clear(pud); 331 pages++; 332 } 333 continue; 334 } 335 } else if (pud_none(*pud)) { 336 if (IS_ALIGNED(addr, PUD_SIZE) && 337 IS_ALIGNED(next, PUD_SIZE) && 338 MACHINE_HAS_EDAT2 && direct && 339 !debug_pagealloc_enabled()) { 340 set_pud(pud, __pud(__pa(addr) | prot)); 341 pages++; 342 continue; 343 } 344 pmd = vmem_crst_alloc(_SEGMENT_ENTRY_EMPTY); 345 if (!pmd) 346 goto out; 347 pud_populate(&init_mm, pud, pmd); 348 } else if (pud_large(*pud)) { 349 continue; 350 } 351 ret = modify_pmd_table(pud, addr, next, add, direct); 352 if (ret) 353 goto out; 354 if (!add) 355 try_free_pmd_table(pud, addr & PUD_MASK); 356 } 357 ret = 0; 358 out: 359 if (direct) 360 update_page_count(PG_DIRECT_MAP_2G, add ? pages : -pages); 361 return ret; 362 } 363 364 static void try_free_pud_table(p4d_t *p4d, unsigned long start) 365 { 366 const unsigned long end = start + P4D_SIZE; 367 pud_t *pud; 368 int i; 369 370 /* Don't mess with any tables not fully in 1:1 mapping & vmemmap area */ 371 if (end > VMALLOC_START) 372 return; 373 374 pud = pud_offset(p4d, start); 375 for (i = 0; i < PTRS_PER_PUD; i++, pud++) { 376 if (!pud_none(*pud)) 377 return; 378 } 379 vmem_free_pages(p4d_deref(*p4d), CRST_ALLOC_ORDER); 380 p4d_clear(p4d); 381 } 382 383 static int modify_p4d_table(pgd_t *pgd, unsigned long addr, unsigned long end, 384 bool add, bool direct) 385 { 386 unsigned long next; 387 int ret = -ENOMEM; 388 p4d_t *p4d; 389 pud_t *pud; 390 391 p4d = p4d_offset(pgd, addr); 392 for (; addr < end; addr = next, p4d++) { 393 next = p4d_addr_end(addr, end); 394 if (!add) { 395 if (p4d_none(*p4d)) 396 continue; 397 } else if (p4d_none(*p4d)) { 398 pud = vmem_crst_alloc(_REGION3_ENTRY_EMPTY); 399 if (!pud) 400 goto out; 401 p4d_populate(&init_mm, p4d, pud); 402 } 403 ret = modify_pud_table(p4d, addr, next, add, direct); 404 if (ret) 405 goto out; 406 if (!add) 407 try_free_pud_table(p4d, addr & P4D_MASK); 408 } 409 ret = 0; 410 out: 411 return ret; 412 } 413 414 static void try_free_p4d_table(pgd_t *pgd, unsigned long start) 415 { 416 const unsigned long end = start + PGDIR_SIZE; 417 p4d_t *p4d; 418 int i; 419 420 /* Don't mess with any tables not fully in 1:1 mapping & vmemmap area */ 421 if (end > VMALLOC_START) 422 return; 423 424 p4d = p4d_offset(pgd, start); 425 for (i = 0; i < PTRS_PER_P4D; i++, p4d++) { 426 if (!p4d_none(*p4d)) 427 return; 428 } 429 vmem_free_pages(pgd_deref(*pgd), CRST_ALLOC_ORDER); 430 pgd_clear(pgd); 431 } 432 433 static int modify_pagetable(unsigned long start, unsigned long end, bool add, 434 bool direct) 435 { 436 unsigned long addr, next; 437 int ret = -ENOMEM; 438 pgd_t *pgd; 439 p4d_t *p4d; 440 441 if (WARN_ON_ONCE(!PAGE_ALIGNED(start | end))) 442 return -EINVAL; 443 for (addr = start; addr < end; addr = next) { 444 next = pgd_addr_end(addr, end); 445 pgd = pgd_offset_k(addr); 446 447 if (!add) { 448 if (pgd_none(*pgd)) 449 continue; 450 } else if (pgd_none(*pgd)) { 451 p4d = vmem_crst_alloc(_REGION2_ENTRY_EMPTY); 452 if (!p4d) 453 goto out; 454 pgd_populate(&init_mm, pgd, p4d); 455 } 456 ret = modify_p4d_table(pgd, addr, next, add, direct); 457 if (ret) 458 goto out; 459 if (!add) 460 try_free_p4d_table(pgd, addr & PGDIR_MASK); 461 } 462 ret = 0; 463 out: 464 if (!add) 465 flush_tlb_kernel_range(start, end); 466 return ret; 467 } 468 469 static int add_pagetable(unsigned long start, unsigned long end, bool direct) 470 { 471 return modify_pagetable(start, end, true, direct); 472 } 473 474 static int remove_pagetable(unsigned long start, unsigned long end, bool direct) 475 { 476 return modify_pagetable(start, end, false, direct); 477 } 478 479 /* 480 * Add a physical memory range to the 1:1 mapping. 481 */ 482 static int vmem_add_range(unsigned long start, unsigned long size) 483 { 484 return add_pagetable(start, start + size, true); 485 } 486 487 /* 488 * Remove a physical memory range from the 1:1 mapping. 489 */ 490 static void vmem_remove_range(unsigned long start, unsigned long size) 491 { 492 remove_pagetable(start, start + size, true); 493 } 494 495 /* 496 * Add a backed mem_map array to the virtual mem_map array. 497 */ 498 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, 499 struct vmem_altmap *altmap) 500 { 501 int ret; 502 503 mutex_lock(&vmem_mutex); 504 /* We don't care about the node, just use NUMA_NO_NODE on allocations */ 505 ret = add_pagetable(start, end, false); 506 if (ret) 507 remove_pagetable(start, end, false); 508 mutex_unlock(&vmem_mutex); 509 return ret; 510 } 511 512 void vmemmap_free(unsigned long start, unsigned long end, 513 struct vmem_altmap *altmap) 514 { 515 mutex_lock(&vmem_mutex); 516 remove_pagetable(start, end, false); 517 mutex_unlock(&vmem_mutex); 518 } 519 520 void vmem_remove_mapping(unsigned long start, unsigned long size) 521 { 522 mutex_lock(&vmem_mutex); 523 vmem_remove_range(start, size); 524 mutex_unlock(&vmem_mutex); 525 } 526 527 struct range arch_get_mappable_range(void) 528 { 529 struct range mhp_range; 530 531 mhp_range.start = 0; 532 mhp_range.end = VMEM_MAX_PHYS - 1; 533 return mhp_range; 534 } 535 536 int vmem_add_mapping(unsigned long start, unsigned long size) 537 { 538 struct range range = arch_get_mappable_range(); 539 int ret; 540 541 if (start < range.start || 542 start + size > range.end + 1 || 543 start + size < start) 544 return -ERANGE; 545 546 mutex_lock(&vmem_mutex); 547 ret = vmem_add_range(start, size); 548 if (ret) 549 vmem_remove_range(start, size); 550 mutex_unlock(&vmem_mutex); 551 return ret; 552 } 553 554 /* 555 * Allocate new or return existing page-table entry, but do not map it 556 * to any physical address. If missing, allocate segment- and region- 557 * table entries along. Meeting a large segment- or region-table entry 558 * while traversing is an error, since the function is expected to be 559 * called against virtual regions reserverd for 4KB mappings only. 560 */ 561 pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc) 562 { 563 pte_t *ptep = NULL; 564 pgd_t *pgd; 565 p4d_t *p4d; 566 pud_t *pud; 567 pmd_t *pmd; 568 pte_t *pte; 569 570 pgd = pgd_offset_k(addr); 571 if (pgd_none(*pgd)) { 572 if (!alloc) 573 goto out; 574 p4d = vmem_crst_alloc(_REGION2_ENTRY_EMPTY); 575 if (!p4d) 576 goto out; 577 pgd_populate(&init_mm, pgd, p4d); 578 } 579 p4d = p4d_offset(pgd, addr); 580 if (p4d_none(*p4d)) { 581 if (!alloc) 582 goto out; 583 pud = vmem_crst_alloc(_REGION3_ENTRY_EMPTY); 584 if (!pud) 585 goto out; 586 p4d_populate(&init_mm, p4d, pud); 587 } 588 pud = pud_offset(p4d, addr); 589 if (pud_none(*pud)) { 590 if (!alloc) 591 goto out; 592 pmd = vmem_crst_alloc(_SEGMENT_ENTRY_EMPTY); 593 if (!pmd) 594 goto out; 595 pud_populate(&init_mm, pud, pmd); 596 } else if (WARN_ON_ONCE(pud_large(*pud))) { 597 goto out; 598 } 599 pmd = pmd_offset(pud, addr); 600 if (pmd_none(*pmd)) { 601 if (!alloc) 602 goto out; 603 pte = vmem_pte_alloc(); 604 if (!pte) 605 goto out; 606 pmd_populate(&init_mm, pmd, pte); 607 } else if (WARN_ON_ONCE(pmd_large(*pmd))) { 608 goto out; 609 } 610 ptep = pte_offset_kernel(pmd, addr); 611 out: 612 return ptep; 613 } 614 615 int __vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot, bool alloc) 616 { 617 pte_t *ptep, pte; 618 619 if (!IS_ALIGNED(addr, PAGE_SIZE)) 620 return -EINVAL; 621 ptep = vmem_get_alloc_pte(addr, alloc); 622 if (!ptep) 623 return -ENOMEM; 624 __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); 625 pte = mk_pte_phys(phys, prot); 626 set_pte(ptep, pte); 627 return 0; 628 } 629 630 int vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot) 631 { 632 int rc; 633 634 mutex_lock(&vmem_mutex); 635 rc = __vmem_map_4k_page(addr, phys, prot, true); 636 mutex_unlock(&vmem_mutex); 637 return rc; 638 } 639 640 void vmem_unmap_4k_page(unsigned long addr) 641 { 642 pte_t *ptep; 643 644 mutex_lock(&vmem_mutex); 645 ptep = virt_to_kpte(addr); 646 __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); 647 pte_clear(&init_mm, addr, ptep); 648 mutex_unlock(&vmem_mutex); 649 } 650 651 static int __init memblock_region_cmp(const void *a, const void *b) 652 { 653 const struct memblock_region *r1 = a; 654 const struct memblock_region *r2 = b; 655 656 if (r1->base < r2->base) 657 return -1; 658 if (r1->base > r2->base) 659 return 1; 660 return 0; 661 } 662 663 static void __init memblock_region_swap(void *a, void *b, int size) 664 { 665 swap(*(struct memblock_region *)a, *(struct memblock_region *)b); 666 } 667 668 #ifdef CONFIG_KASAN 669 #define __sha(x) ((unsigned long)kasan_mem_to_shadow((void *)x)) 670 #endif 671 /* 672 * map whole physical memory to virtual memory (identity mapping) 673 * we reserve enough space in the vmalloc area for vmemmap to hotplug 674 * additional memory segments. 675 */ 676 void __init vmem_map_init(void) 677 { 678 struct memblock_region memory_rwx_regions[] = { 679 { 680 .base = 0, 681 .size = sizeof(struct lowcore), 682 .flags = MEMBLOCK_NONE, 683 #ifdef CONFIG_NUMA 684 .nid = NUMA_NO_NODE, 685 #endif 686 }, 687 { 688 .base = __pa(_stext), 689 .size = _etext - _stext, 690 .flags = MEMBLOCK_NONE, 691 #ifdef CONFIG_NUMA 692 .nid = NUMA_NO_NODE, 693 #endif 694 }, 695 { 696 .base = __pa(_sinittext), 697 .size = _einittext - _sinittext, 698 .flags = MEMBLOCK_NONE, 699 #ifdef CONFIG_NUMA 700 .nid = NUMA_NO_NODE, 701 #endif 702 }, 703 { 704 .base = __stext_amode31, 705 .size = __etext_amode31 - __stext_amode31, 706 .flags = MEMBLOCK_NONE, 707 #ifdef CONFIG_NUMA 708 .nid = NUMA_NO_NODE, 709 #endif 710 }, 711 }; 712 struct memblock_type memory_rwx = { 713 .regions = memory_rwx_regions, 714 .cnt = ARRAY_SIZE(memory_rwx_regions), 715 .max = ARRAY_SIZE(memory_rwx_regions), 716 }; 717 phys_addr_t base, end; 718 u64 i; 719 720 /* 721 * Set RW+NX attribute on all memory, except regions enumerated with 722 * memory_rwx exclude type. These regions need different attributes, 723 * which are enforced afterwards. 724 * 725 * __for_each_mem_range() iterate and exclude types should be sorted. 726 * The relative location of _stext and _sinittext is hardcoded in the 727 * linker script. However a location of __stext_amode31 and the kernel 728 * image itself are chosen dynamically. Thus, sort the exclude type. 729 */ 730 sort(&memory_rwx_regions, 731 ARRAY_SIZE(memory_rwx_regions), sizeof(memory_rwx_regions[0]), 732 memblock_region_cmp, memblock_region_swap); 733 __for_each_mem_range(i, &memblock.memory, &memory_rwx, 734 NUMA_NO_NODE, MEMBLOCK_NONE, &base, &end, NULL) { 735 set_memory_rwnx((unsigned long)__va(base), 736 (end - base) >> PAGE_SHIFT); 737 } 738 739 #ifdef CONFIG_KASAN 740 for_each_mem_range(i, &base, &end) { 741 set_memory_rwnx(__sha(base), 742 (__sha(end) - __sha(base)) >> PAGE_SHIFT); 743 } 744 #endif 745 set_memory_rox((unsigned long)_stext, 746 (unsigned long)(_etext - _stext) >> PAGE_SHIFT); 747 set_memory_ro((unsigned long)_etext, 748 (unsigned long)(__end_rodata - _etext) >> PAGE_SHIFT); 749 set_memory_rox((unsigned long)_sinittext, 750 (unsigned long)(_einittext - _sinittext) >> PAGE_SHIFT); 751 set_memory_rox(__stext_amode31, 752 (__etext_amode31 - __stext_amode31) >> PAGE_SHIFT); 753 754 /* lowcore must be executable for LPSWE */ 755 if (static_key_enabled(&cpu_has_bear)) 756 set_memory_nx(0, 1); 757 set_memory_nx(PAGE_SIZE, 1); 758 759 pr_info("Write protected kernel read-only data: %luk\n", 760 (unsigned long)(__end_rodata - _stext) >> 10); 761 } 762