1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (C) 2019 Andes Technology Corporation 3 4 #include <linux/pfn.h> 5 #include <linux/init_task.h> 6 #include <linux/kasan.h> 7 #include <linux/kernel.h> 8 #include <linux/memblock.h> 9 #include <linux/pgtable.h> 10 #include <asm/tlbflush.h> 11 #include <asm/fixmap.h> 12 #include <asm/pgalloc.h> 13 14 /* 15 * Kasan shadow region must lie at a fixed address across sv39, sv48 and sv57 16 * which is right before the kernel. 17 * 18 * For sv39, the region is aligned on PGDIR_SIZE so we only need to populate 19 * the page global directory with kasan_early_shadow_pmd. 20 * 21 * For sv48 and sv57, the region is not aligned on PGDIR_SIZE so the mapping 22 * must be divided as follows: 23 * - the first PGD entry, although incomplete, is populated with 24 * kasan_early_shadow_pud/p4d 25 * - the PGD entries in the middle are populated with kasan_early_shadow_pud/p4d 26 * - the last PGD entry is shared with the kernel mapping so populated at the 27 * lower levels pud/p4d 28 * 29 * In addition, when shallow populating a kasan region (for example vmalloc), 30 * this region may also not be aligned on PGDIR size, so we must go down to the 31 * pud level too. 32 */ 33 34 extern pgd_t early_pg_dir[PTRS_PER_PGD]; 35 36 static void __init kasan_populate_pte(pmd_t *pmd, unsigned long vaddr, unsigned long end) 37 { 38 phys_addr_t phys_addr; 39 pte_t *ptep, *base_pte; 40 41 if (pmd_none(*pmd)) 42 base_pte = memblock_alloc(PTRS_PER_PTE * sizeof(pte_t), PAGE_SIZE); 43 else 44 base_pte = (pte_t *)pmd_page_vaddr(*pmd); 45 46 ptep = base_pte + pte_index(vaddr); 47 48 do { 49 if (pte_none(*ptep)) { 50 phys_addr = memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 51 set_pte(ptep, pfn_pte(PFN_DOWN(phys_addr), PAGE_KERNEL)); 52 } 53 } while (ptep++, vaddr += PAGE_SIZE, vaddr != end); 54 55 set_pmd(pmd, pfn_pmd(PFN_DOWN(__pa(base_pte)), PAGE_TABLE)); 56 } 57 58 static void __init kasan_populate_pmd(pud_t *pud, unsigned long vaddr, unsigned long end) 59 { 60 phys_addr_t phys_addr; 61 pmd_t *pmdp, *base_pmd; 62 unsigned long next; 63 64 if (pud_none(*pud)) { 65 base_pmd = memblock_alloc(PTRS_PER_PMD * sizeof(pmd_t), PAGE_SIZE); 66 } else { 67 base_pmd = (pmd_t *)pud_pgtable(*pud); 68 if (base_pmd == lm_alias(kasan_early_shadow_pmd)) 69 base_pmd = memblock_alloc(PTRS_PER_PMD * sizeof(pmd_t), PAGE_SIZE); 70 } 71 72 pmdp = base_pmd + pmd_index(vaddr); 73 74 do { 75 next = pmd_addr_end(vaddr, end); 76 77 if (pmd_none(*pmdp) && IS_ALIGNED(vaddr, PMD_SIZE) && (next - vaddr) >= PMD_SIZE) { 78 phys_addr = memblock_phys_alloc(PMD_SIZE, PMD_SIZE); 79 if (phys_addr) { 80 set_pmd(pmdp, pfn_pmd(PFN_DOWN(phys_addr), PAGE_KERNEL)); 81 continue; 82 } 83 } 84 85 kasan_populate_pte(pmdp, vaddr, next); 86 } while (pmdp++, vaddr = next, vaddr != end); 87 88 /* 89 * Wait for the whole PGD to be populated before setting the PGD in 90 * the page table, otherwise, if we did set the PGD before populating 91 * it entirely, memblock could allocate a page at a physical address 92 * where KASAN is not populated yet and then we'd get a page fault. 93 */ 94 set_pud(pud, pfn_pud(PFN_DOWN(__pa(base_pmd)), PAGE_TABLE)); 95 } 96 97 static void __init kasan_populate_pud(pgd_t *pgd, 98 unsigned long vaddr, unsigned long end, 99 bool early) 100 { 101 phys_addr_t phys_addr; 102 pud_t *pudp, *base_pud; 103 unsigned long next; 104 105 if (early) { 106 /* 107 * We can't use pgd_page_vaddr here as it would return a linear 108 * mapping address but it is not mapped yet, but when populating 109 * early_pg_dir, we need the physical address and when populating 110 * swapper_pg_dir, we need the kernel virtual address so use 111 * pt_ops facility. 112 */ 113 base_pud = pt_ops.get_pud_virt(pfn_to_phys(_pgd_pfn(*pgd))); 114 } else if (pgd_none(*pgd)) { 115 base_pud = memblock_alloc(PTRS_PER_PUD * sizeof(pud_t), PAGE_SIZE); 116 } else { 117 base_pud = (pud_t *)pgd_page_vaddr(*pgd); 118 if (base_pud == lm_alias(kasan_early_shadow_pud)) { 119 base_pud = memblock_alloc(PTRS_PER_PUD * sizeof(pud_t), PAGE_SIZE); 120 memcpy(base_pud, (void *)kasan_early_shadow_pud, 121 sizeof(pud_t) * PTRS_PER_PUD); 122 } 123 } 124 125 pudp = base_pud + pud_index(vaddr); 126 127 do { 128 next = pud_addr_end(vaddr, end); 129 130 if (pud_none(*pudp) && IS_ALIGNED(vaddr, PUD_SIZE) && (next - vaddr) >= PUD_SIZE) { 131 if (early) { 132 phys_addr = __pa(((uintptr_t)kasan_early_shadow_pmd)); 133 set_pud(pudp, pfn_pud(PFN_DOWN(phys_addr), PAGE_TABLE)); 134 continue; 135 } else { 136 phys_addr = memblock_phys_alloc(PUD_SIZE, PUD_SIZE); 137 if (phys_addr) { 138 set_pud(pudp, pfn_pud(PFN_DOWN(phys_addr), PAGE_KERNEL)); 139 continue; 140 } 141 } 142 } 143 144 kasan_populate_pmd(pudp, vaddr, next); 145 } while (pudp++, vaddr = next, vaddr != end); 146 147 /* 148 * Wait for the whole PGD to be populated before setting the PGD in 149 * the page table, otherwise, if we did set the PGD before populating 150 * it entirely, memblock could allocate a page at a physical address 151 * where KASAN is not populated yet and then we'd get a page fault. 152 */ 153 if (!early) 154 set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(base_pud)), PAGE_TABLE)); 155 } 156 157 static void __init kasan_populate_p4d(pgd_t *pgd, 158 unsigned long vaddr, unsigned long end, 159 bool early) 160 { 161 phys_addr_t phys_addr; 162 p4d_t *p4dp, *base_p4d; 163 unsigned long next; 164 165 if (early) { 166 /* 167 * We can't use pgd_page_vaddr here as it would return a linear 168 * mapping address but it is not mapped yet, but when populating 169 * early_pg_dir, we need the physical address and when populating 170 * swapper_pg_dir, we need the kernel virtual address so use 171 * pt_ops facility. 172 */ 173 base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(*pgd))); 174 } else { 175 base_p4d = (p4d_t *)pgd_page_vaddr(*pgd); 176 if (base_p4d == lm_alias(kasan_early_shadow_p4d)) 177 base_p4d = memblock_alloc(PTRS_PER_PUD * sizeof(p4d_t), PAGE_SIZE); 178 } 179 180 p4dp = base_p4d + p4d_index(vaddr); 181 182 do { 183 next = p4d_addr_end(vaddr, end); 184 185 if (p4d_none(*p4dp) && IS_ALIGNED(vaddr, P4D_SIZE) && (next - vaddr) >= P4D_SIZE) { 186 if (early) { 187 phys_addr = __pa(((uintptr_t)kasan_early_shadow_pud)); 188 set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_TABLE)); 189 continue; 190 } else { 191 phys_addr = memblock_phys_alloc(P4D_SIZE, P4D_SIZE); 192 if (phys_addr) { 193 set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_KERNEL)); 194 continue; 195 } 196 } 197 } 198 199 kasan_populate_pud((pgd_t *)p4dp, vaddr, next, early); 200 } while (p4dp++, vaddr = next, vaddr != end); 201 202 /* 203 * Wait for the whole P4D to be populated before setting the P4D in 204 * the page table, otherwise, if we did set the P4D before populating 205 * it entirely, memblock could allocate a page at a physical address 206 * where KASAN is not populated yet and then we'd get a page fault. 207 */ 208 if (!early) 209 set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(base_p4d)), PAGE_TABLE)); 210 } 211 212 #define kasan_early_shadow_pgd_next (pgtable_l5_enabled ? \ 213 (uintptr_t)kasan_early_shadow_p4d : \ 214 (pgtable_l4_enabled ? \ 215 (uintptr_t)kasan_early_shadow_pud : \ 216 (uintptr_t)kasan_early_shadow_pmd)) 217 #define kasan_populate_pgd_next(pgdp, vaddr, next, early) \ 218 (pgtable_l5_enabled ? \ 219 kasan_populate_p4d(pgdp, vaddr, next, early) : \ 220 (pgtable_l4_enabled ? \ 221 kasan_populate_pud(pgdp, vaddr, next, early) : \ 222 kasan_populate_pmd((pud_t *)pgdp, vaddr, next))) 223 224 static void __init kasan_populate_pgd(pgd_t *pgdp, 225 unsigned long vaddr, unsigned long end, 226 bool early) 227 { 228 phys_addr_t phys_addr; 229 unsigned long next; 230 231 do { 232 next = pgd_addr_end(vaddr, end); 233 234 if (IS_ALIGNED(vaddr, PGDIR_SIZE) && (next - vaddr) >= PGDIR_SIZE) { 235 if (early) { 236 phys_addr = __pa((uintptr_t)kasan_early_shadow_pgd_next); 237 set_pgd(pgdp, pfn_pgd(PFN_DOWN(phys_addr), PAGE_TABLE)); 238 continue; 239 } else if (pgd_page_vaddr(*pgdp) == 240 (unsigned long)lm_alias(kasan_early_shadow_pgd_next)) { 241 /* 242 * pgdp can't be none since kasan_early_init 243 * initialized all KASAN shadow region with 244 * kasan_early_shadow_pud: if this is still the 245 * case, that means we can try to allocate a 246 * hugepage as a replacement. 247 */ 248 phys_addr = memblock_phys_alloc(PGDIR_SIZE, PGDIR_SIZE); 249 if (phys_addr) { 250 set_pgd(pgdp, pfn_pgd(PFN_DOWN(phys_addr), PAGE_KERNEL)); 251 continue; 252 } 253 } 254 } 255 256 kasan_populate_pgd_next(pgdp, vaddr, next, early); 257 } while (pgdp++, vaddr = next, vaddr != end); 258 } 259 260 asmlinkage void __init kasan_early_init(void) 261 { 262 uintptr_t i; 263 264 BUILD_BUG_ON(KASAN_SHADOW_OFFSET != 265 KASAN_SHADOW_END - (1UL << (64 - KASAN_SHADOW_SCALE_SHIFT))); 266 267 for (i = 0; i < PTRS_PER_PTE; ++i) 268 set_pte(kasan_early_shadow_pte + i, 269 pfn_pte(virt_to_pfn(kasan_early_shadow_page), PAGE_KERNEL)); 270 271 for (i = 0; i < PTRS_PER_PMD; ++i) 272 set_pmd(kasan_early_shadow_pmd + i, 273 pfn_pmd(PFN_DOWN 274 (__pa((uintptr_t)kasan_early_shadow_pte)), 275 PAGE_TABLE)); 276 277 if (pgtable_l4_enabled) { 278 for (i = 0; i < PTRS_PER_PUD; ++i) 279 set_pud(kasan_early_shadow_pud + i, 280 pfn_pud(PFN_DOWN 281 (__pa(((uintptr_t)kasan_early_shadow_pmd))), 282 PAGE_TABLE)); 283 } 284 285 if (pgtable_l5_enabled) { 286 for (i = 0; i < PTRS_PER_P4D; ++i) 287 set_p4d(kasan_early_shadow_p4d + i, 288 pfn_p4d(PFN_DOWN 289 (__pa(((uintptr_t)kasan_early_shadow_pud))), 290 PAGE_TABLE)); 291 } 292 293 kasan_populate_pgd(early_pg_dir + pgd_index(KASAN_SHADOW_START), 294 KASAN_SHADOW_START, KASAN_SHADOW_END, true); 295 296 local_flush_tlb_all(); 297 } 298 299 void __init kasan_swapper_init(void) 300 { 301 kasan_populate_pgd(pgd_offset_k(KASAN_SHADOW_START), 302 KASAN_SHADOW_START, KASAN_SHADOW_END, true); 303 304 local_flush_tlb_all(); 305 } 306 307 static void __init kasan_populate(void *start, void *end) 308 { 309 unsigned long vaddr = (unsigned long)start & PAGE_MASK; 310 unsigned long vend = PAGE_ALIGN((unsigned long)end); 311 312 kasan_populate_pgd(pgd_offset_k(vaddr), vaddr, vend, false); 313 314 local_flush_tlb_all(); 315 memset(start, KASAN_SHADOW_INIT, end - start); 316 } 317 318 static void __init kasan_shallow_populate_pmd(pgd_t *pgdp, 319 unsigned long vaddr, unsigned long end) 320 { 321 unsigned long next; 322 pmd_t *pmdp, *base_pmd; 323 bool is_kasan_pte; 324 325 base_pmd = (pmd_t *)pgd_page_vaddr(*pgdp); 326 pmdp = base_pmd + pmd_index(vaddr); 327 328 do { 329 next = pmd_addr_end(vaddr, end); 330 is_kasan_pte = (pmd_pgtable(*pmdp) == lm_alias(kasan_early_shadow_pte)); 331 332 if (is_kasan_pte) 333 pmd_clear(pmdp); 334 } while (pmdp++, vaddr = next, vaddr != end); 335 } 336 337 static void __init kasan_shallow_populate_pud(pgd_t *pgdp, 338 unsigned long vaddr, unsigned long end) 339 { 340 unsigned long next; 341 pud_t *pudp, *base_pud; 342 pmd_t *base_pmd; 343 bool is_kasan_pmd; 344 345 base_pud = (pud_t *)pgd_page_vaddr(*pgdp); 346 pudp = base_pud + pud_index(vaddr); 347 348 do { 349 next = pud_addr_end(vaddr, end); 350 is_kasan_pmd = (pud_pgtable(*pudp) == lm_alias(kasan_early_shadow_pmd)); 351 352 if (!is_kasan_pmd) 353 continue; 354 355 base_pmd = memblock_alloc(PAGE_SIZE, PAGE_SIZE); 356 set_pud(pudp, pfn_pud(PFN_DOWN(__pa(base_pmd)), PAGE_TABLE)); 357 358 if (IS_ALIGNED(vaddr, PUD_SIZE) && (next - vaddr) >= PUD_SIZE) 359 continue; 360 361 memcpy(base_pmd, (void *)kasan_early_shadow_pmd, PAGE_SIZE); 362 kasan_shallow_populate_pmd((pgd_t *)pudp, vaddr, next); 363 } while (pudp++, vaddr = next, vaddr != end); 364 } 365 366 static void __init kasan_shallow_populate_p4d(pgd_t *pgdp, 367 unsigned long vaddr, unsigned long end) 368 { 369 unsigned long next; 370 p4d_t *p4dp, *base_p4d; 371 pud_t *base_pud; 372 bool is_kasan_pud; 373 374 base_p4d = (p4d_t *)pgd_page_vaddr(*pgdp); 375 p4dp = base_p4d + p4d_index(vaddr); 376 377 do { 378 next = p4d_addr_end(vaddr, end); 379 is_kasan_pud = (p4d_pgtable(*p4dp) == lm_alias(kasan_early_shadow_pud)); 380 381 if (!is_kasan_pud) 382 continue; 383 384 base_pud = memblock_alloc(PAGE_SIZE, PAGE_SIZE); 385 set_p4d(p4dp, pfn_p4d(PFN_DOWN(__pa(base_pud)), PAGE_TABLE)); 386 387 if (IS_ALIGNED(vaddr, P4D_SIZE) && (next - vaddr) >= P4D_SIZE) 388 continue; 389 390 memcpy(base_pud, (void *)kasan_early_shadow_pud, PAGE_SIZE); 391 kasan_shallow_populate_pud((pgd_t *)p4dp, vaddr, next); 392 } while (p4dp++, vaddr = next, vaddr != end); 393 } 394 395 #define kasan_shallow_populate_pgd_next(pgdp, vaddr, next) \ 396 (pgtable_l5_enabled ? \ 397 kasan_shallow_populate_p4d(pgdp, vaddr, next) : \ 398 (pgtable_l4_enabled ? \ 399 kasan_shallow_populate_pud(pgdp, vaddr, next) : \ 400 kasan_shallow_populate_pmd(pgdp, vaddr, next))) 401 402 static void __init kasan_shallow_populate_pgd(unsigned long vaddr, unsigned long end) 403 { 404 unsigned long next; 405 void *p; 406 pgd_t *pgd_k = pgd_offset_k(vaddr); 407 bool is_kasan_pgd_next; 408 409 do { 410 next = pgd_addr_end(vaddr, end); 411 is_kasan_pgd_next = (pgd_page_vaddr(*pgd_k) == 412 (unsigned long)lm_alias(kasan_early_shadow_pgd_next)); 413 414 if (is_kasan_pgd_next) { 415 p = memblock_alloc(PAGE_SIZE, PAGE_SIZE); 416 set_pgd(pgd_k, pfn_pgd(PFN_DOWN(__pa(p)), PAGE_TABLE)); 417 } 418 419 if (IS_ALIGNED(vaddr, PGDIR_SIZE) && (next - vaddr) >= PGDIR_SIZE) 420 continue; 421 422 memcpy(p, (void *)kasan_early_shadow_pgd_next, PAGE_SIZE); 423 kasan_shallow_populate_pgd_next(pgd_k, vaddr, next); 424 } while (pgd_k++, vaddr = next, vaddr != end); 425 } 426 427 static void __init kasan_shallow_populate(void *start, void *end) 428 { 429 unsigned long vaddr = (unsigned long)start & PAGE_MASK; 430 unsigned long vend = PAGE_ALIGN((unsigned long)end); 431 432 kasan_shallow_populate_pgd(vaddr, vend); 433 local_flush_tlb_all(); 434 } 435 436 void __init kasan_init(void) 437 { 438 phys_addr_t p_start, p_end; 439 u64 i; 440 441 if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) 442 kasan_shallow_populate( 443 (void *)kasan_mem_to_shadow((void *)VMALLOC_START), 444 (void *)kasan_mem_to_shadow((void *)VMALLOC_END)); 445 446 /* Populate the linear mapping */ 447 for_each_mem_range(i, &p_start, &p_end) { 448 void *start = (void *)__va(p_start); 449 void *end = (void *)__va(p_end); 450 451 if (start >= end) 452 break; 453 454 kasan_populate(kasan_mem_to_shadow(start), kasan_mem_to_shadow(end)); 455 } 456 457 /* Populate kernel, BPF, modules mapping */ 458 kasan_populate(kasan_mem_to_shadow((const void *)MODULES_VADDR), 459 kasan_mem_to_shadow((const void *)MODULES_VADDR + SZ_2G)); 460 461 for (i = 0; i < PTRS_PER_PTE; i++) 462 set_pte(&kasan_early_shadow_pte[i], 463 mk_pte(virt_to_page(kasan_early_shadow_page), 464 __pgprot(_PAGE_PRESENT | _PAGE_READ | 465 _PAGE_ACCESSED))); 466 467 memset(kasan_early_shadow_page, KASAN_SHADOW_INIT, PAGE_SIZE); 468 init_task.kasan_depth = 0; 469 } 470