185155229SAndrey Ryabinin #define pr_fmt(fmt) "kasan: " fmt 2ef7f0d6aSAndrey Ryabinin #include <linux/bootmem.h> 3ef7f0d6aSAndrey Ryabinin #include <linux/kasan.h> 4ef7f0d6aSAndrey Ryabinin #include <linux/kdebug.h> 5ef7f0d6aSAndrey Ryabinin #include <linux/mm.h> 6ef7f0d6aSAndrey Ryabinin #include <linux/sched.h> 7ef7f0d6aSAndrey Ryabinin #include <linux/vmalloc.h> 8ef7f0d6aSAndrey Ryabinin 9ef7f0d6aSAndrey Ryabinin #include <asm/tlbflush.h> 10ef7f0d6aSAndrey Ryabinin #include <asm/sections.h> 11ef7f0d6aSAndrey Ryabinin 12ef7f0d6aSAndrey Ryabinin extern pgd_t early_level4_pgt[PTRS_PER_PGD]; 13ef7f0d6aSAndrey Ryabinin extern struct range pfn_mapped[E820_X_MAX]; 14ef7f0d6aSAndrey Ryabinin 15ef7f0d6aSAndrey Ryabinin static int __init map_range(struct range *range) 16ef7f0d6aSAndrey Ryabinin { 17ef7f0d6aSAndrey Ryabinin unsigned long start; 18ef7f0d6aSAndrey Ryabinin unsigned long end; 19ef7f0d6aSAndrey Ryabinin 20ef7f0d6aSAndrey Ryabinin start = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->start)); 21ef7f0d6aSAndrey Ryabinin end = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->end)); 22ef7f0d6aSAndrey Ryabinin 23ef7f0d6aSAndrey Ryabinin /* 24ef7f0d6aSAndrey Ryabinin * end + 1 here is intentional. We check several shadow bytes in advance 25ef7f0d6aSAndrey Ryabinin * to slightly speed up fastpath. In some rare cases we could cross 26ef7f0d6aSAndrey Ryabinin * boundary of mapped shadow, so we just map some more here. 27ef7f0d6aSAndrey Ryabinin */ 28ef7f0d6aSAndrey Ryabinin return vmemmap_populate(start, end + 1, NUMA_NO_NODE); 29ef7f0d6aSAndrey Ryabinin } 30ef7f0d6aSAndrey Ryabinin 31ef7f0d6aSAndrey Ryabinin static void __init clear_pgds(unsigned long start, 32ef7f0d6aSAndrey Ryabinin unsigned long end) 33ef7f0d6aSAndrey Ryabinin { 34ef7f0d6aSAndrey Ryabinin for (; start < end; start += PGDIR_SIZE) 35ef7f0d6aSAndrey Ryabinin pgd_clear(pgd_offset_k(start)); 36ef7f0d6aSAndrey Ryabinin } 37ef7f0d6aSAndrey Ryabinin 385d5aa3cfSAlexander Popov static void __init kasan_map_early_shadow(pgd_t *pgd) 39ef7f0d6aSAndrey Ryabinin { 40ef7f0d6aSAndrey Ryabinin int i; 41ef7f0d6aSAndrey Ryabinin unsigned long start = KASAN_SHADOW_START; 42ef7f0d6aSAndrey Ryabinin unsigned long end = KASAN_SHADOW_END; 43ef7f0d6aSAndrey Ryabinin 44ef7f0d6aSAndrey Ryabinin for (i = pgd_index(start); start < end; i++) { 45ef7f0d6aSAndrey Ryabinin pgd[i] = __pgd(__pa_nodebug(kasan_zero_pud) 46ef7f0d6aSAndrey Ryabinin | _KERNPG_TABLE); 47ef7f0d6aSAndrey Ryabinin start += PGDIR_SIZE; 48ef7f0d6aSAndrey Ryabinin } 49ef7f0d6aSAndrey Ryabinin } 50ef7f0d6aSAndrey Ryabinin 51ef7f0d6aSAndrey Ryabinin #ifdef CONFIG_KASAN_INLINE 52ef7f0d6aSAndrey Ryabinin static int kasan_die_handler(struct notifier_block *self, 53ef7f0d6aSAndrey Ryabinin unsigned long val, 54ef7f0d6aSAndrey Ryabinin void *data) 55ef7f0d6aSAndrey Ryabinin { 56ef7f0d6aSAndrey Ryabinin if (val == DIE_GPF) { 57ef7f0d6aSAndrey Ryabinin pr_emerg("CONFIG_KASAN_INLINE enabled"); 58ef7f0d6aSAndrey Ryabinin pr_emerg("GPF could be caused by NULL-ptr deref or user memory access"); 59ef7f0d6aSAndrey Ryabinin } 60ef7f0d6aSAndrey Ryabinin return NOTIFY_OK; 61ef7f0d6aSAndrey Ryabinin } 62ef7f0d6aSAndrey Ryabinin 63ef7f0d6aSAndrey Ryabinin static struct notifier_block kasan_die_notifier = { 64ef7f0d6aSAndrey Ryabinin .notifier_call = kasan_die_handler, 65ef7f0d6aSAndrey Ryabinin }; 66ef7f0d6aSAndrey Ryabinin #endif 67ef7f0d6aSAndrey Ryabinin 685d5aa3cfSAlexander Popov void __init kasan_early_init(void) 695d5aa3cfSAlexander Popov { 705d5aa3cfSAlexander Popov int i; 715d5aa3cfSAlexander Popov pteval_t pte_val = __pa_nodebug(kasan_zero_page) | __PAGE_KERNEL; 725d5aa3cfSAlexander Popov pmdval_t pmd_val = __pa_nodebug(kasan_zero_pte) | _KERNPG_TABLE; 735d5aa3cfSAlexander Popov pudval_t pud_val = __pa_nodebug(kasan_zero_pmd) | _KERNPG_TABLE; 745d5aa3cfSAlexander Popov 755d5aa3cfSAlexander Popov for (i = 0; i < PTRS_PER_PTE; i++) 765d5aa3cfSAlexander Popov kasan_zero_pte[i] = __pte(pte_val); 775d5aa3cfSAlexander Popov 785d5aa3cfSAlexander Popov for (i = 0; i < PTRS_PER_PMD; i++) 795d5aa3cfSAlexander Popov kasan_zero_pmd[i] = __pmd(pmd_val); 805d5aa3cfSAlexander Popov 815d5aa3cfSAlexander Popov for (i = 0; i < PTRS_PER_PUD; i++) 825d5aa3cfSAlexander Popov kasan_zero_pud[i] = __pud(pud_val); 835d5aa3cfSAlexander Popov 845d5aa3cfSAlexander Popov kasan_map_early_shadow(early_level4_pgt); 855d5aa3cfSAlexander Popov kasan_map_early_shadow(init_level4_pgt); 865d5aa3cfSAlexander Popov } 875d5aa3cfSAlexander Popov 88ef7f0d6aSAndrey Ryabinin void __init kasan_init(void) 89ef7f0d6aSAndrey Ryabinin { 90ef7f0d6aSAndrey Ryabinin int i; 91ef7f0d6aSAndrey Ryabinin 92ef7f0d6aSAndrey Ryabinin #ifdef CONFIG_KASAN_INLINE 93ef7f0d6aSAndrey Ryabinin register_die_notifier(&kasan_die_notifier); 94ef7f0d6aSAndrey Ryabinin #endif 95ef7f0d6aSAndrey Ryabinin 96ef7f0d6aSAndrey Ryabinin memcpy(early_level4_pgt, init_level4_pgt, sizeof(early_level4_pgt)); 97ef7f0d6aSAndrey Ryabinin load_cr3(early_level4_pgt); 98241d2c54SAndrey Ryabinin __flush_tlb_all(); 99ef7f0d6aSAndrey Ryabinin 100ef7f0d6aSAndrey Ryabinin clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END); 101ef7f0d6aSAndrey Ryabinin 10269786cdbSAndrey Ryabinin kasan_populate_zero_shadow((void *)KASAN_SHADOW_START, 103ef7f0d6aSAndrey Ryabinin kasan_mem_to_shadow((void *)PAGE_OFFSET)); 104ef7f0d6aSAndrey Ryabinin 105ef7f0d6aSAndrey Ryabinin for (i = 0; i < E820_X_MAX; i++) { 106ef7f0d6aSAndrey Ryabinin if (pfn_mapped[i].end == 0) 107ef7f0d6aSAndrey Ryabinin break; 108ef7f0d6aSAndrey Ryabinin 109ef7f0d6aSAndrey Ryabinin if (map_range(&pfn_mapped[i])) 110ef7f0d6aSAndrey Ryabinin panic("kasan: unable to allocate shadow!"); 111ef7f0d6aSAndrey Ryabinin } 11269786cdbSAndrey Ryabinin kasan_populate_zero_shadow( 11369786cdbSAndrey Ryabinin kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM), 114c420f167SAndrey Ryabinin kasan_mem_to_shadow((void *)__START_KERNEL_map)); 115c420f167SAndrey Ryabinin 116c420f167SAndrey Ryabinin vmemmap_populate((unsigned long)kasan_mem_to_shadow(_stext), 117c420f167SAndrey Ryabinin (unsigned long)kasan_mem_to_shadow(_end), 118c420f167SAndrey Ryabinin NUMA_NO_NODE); 119c420f167SAndrey Ryabinin 12069786cdbSAndrey Ryabinin kasan_populate_zero_shadow(kasan_mem_to_shadow((void *)MODULES_END), 121ef7f0d6aSAndrey Ryabinin (void *)KASAN_SHADOW_END); 122ef7f0d6aSAndrey Ryabinin 123ef7f0d6aSAndrey Ryabinin memset(kasan_zero_page, 0, PAGE_SIZE); 124ef7f0d6aSAndrey Ryabinin 125ef7f0d6aSAndrey Ryabinin load_cr3(init_level4_pgt); 126241d2c54SAndrey Ryabinin __flush_tlb_all(); 127c420f167SAndrey Ryabinin init_task.kasan_depth = 0; 12885155229SAndrey Ryabinin 12925add7ecSAndrey Konovalov pr_info("KernelAddressSanitizer initialized\n"); 130ef7f0d6aSAndrey Ryabinin } 131