1 #define DISABLE_BRANCH_PROFILING 2 #define pr_fmt(fmt) "kasan: " fmt 3 #include <linux/bootmem.h> 4 #include <linux/kasan.h> 5 #include <linux/kdebug.h> 6 #include <linux/mm.h> 7 #include <linux/sched.h> 8 #include <linux/sched/task.h> 9 #include <linux/vmalloc.h> 10 11 #include <asm/tlbflush.h> 12 #include <asm/sections.h> 13 14 extern pgd_t early_level4_pgt[PTRS_PER_PGD]; 15 extern struct range pfn_mapped[E820_X_MAX]; 16 17 static int __init map_range(struct range *range) 18 { 19 unsigned long start; 20 unsigned long end; 21 22 start = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->start)); 23 end = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->end)); 24 25 /* 26 * end + 1 here is intentional. We check several shadow bytes in advance 27 * to slightly speed up fastpath. In some rare cases we could cross 28 * boundary of mapped shadow, so we just map some more here. 29 */ 30 return vmemmap_populate(start, end + 1, NUMA_NO_NODE); 31 } 32 33 static void __init clear_pgds(unsigned long start, 34 unsigned long end) 35 { 36 for (; start < end; start += PGDIR_SIZE) 37 pgd_clear(pgd_offset_k(start)); 38 } 39 40 static void __init kasan_map_early_shadow(pgd_t *pgd) 41 { 42 int i; 43 unsigned long start = KASAN_SHADOW_START; 44 unsigned long end = KASAN_SHADOW_END; 45 46 for (i = pgd_index(start); start < end; i++) { 47 pgd[i] = __pgd(__pa_nodebug(kasan_zero_pud) 48 | _KERNPG_TABLE); 49 start += PGDIR_SIZE; 50 } 51 } 52 53 #ifdef CONFIG_KASAN_INLINE 54 static int kasan_die_handler(struct notifier_block *self, 55 unsigned long val, 56 void *data) 57 { 58 if (val == DIE_GPF) { 59 pr_emerg("CONFIG_KASAN_INLINE enabled\n"); 60 pr_emerg("GPF could be caused by NULL-ptr deref or user memory access\n"); 61 } 62 return NOTIFY_OK; 63 } 64 65 static struct notifier_block kasan_die_notifier = { 66 .notifier_call = kasan_die_handler, 67 }; 68 #endif 69 70 void __init kasan_early_init(void) 71 { 72 int i; 73 pteval_t pte_val = __pa_nodebug(kasan_zero_page) | __PAGE_KERNEL; 74 pmdval_t pmd_val = __pa_nodebug(kasan_zero_pte) | _KERNPG_TABLE; 75 pudval_t pud_val = __pa_nodebug(kasan_zero_pmd) | _KERNPG_TABLE; 76 77 for (i = 0; i < PTRS_PER_PTE; i++) 78 kasan_zero_pte[i] = __pte(pte_val); 79 80 for (i = 0; i < PTRS_PER_PMD; i++) 81 kasan_zero_pmd[i] = __pmd(pmd_val); 82 83 for (i = 0; i < PTRS_PER_PUD; i++) 84 kasan_zero_pud[i] = __pud(pud_val); 85 86 kasan_map_early_shadow(early_level4_pgt); 87 kasan_map_early_shadow(init_level4_pgt); 88 } 89 90 void __init kasan_init(void) 91 { 92 int i; 93 94 #ifdef CONFIG_KASAN_INLINE 95 register_die_notifier(&kasan_die_notifier); 96 #endif 97 98 memcpy(early_level4_pgt, init_level4_pgt, sizeof(early_level4_pgt)); 99 load_cr3(early_level4_pgt); 100 __flush_tlb_all(); 101 102 clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END); 103 104 kasan_populate_zero_shadow((void *)KASAN_SHADOW_START, 105 kasan_mem_to_shadow((void *)PAGE_OFFSET)); 106 107 for (i = 0; i < E820_X_MAX; i++) { 108 if (pfn_mapped[i].end == 0) 109 break; 110 111 if (map_range(&pfn_mapped[i])) 112 panic("kasan: unable to allocate shadow!"); 113 } 114 kasan_populate_zero_shadow( 115 kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM), 116 kasan_mem_to_shadow((void *)__START_KERNEL_map)); 117 118 vmemmap_populate((unsigned long)kasan_mem_to_shadow(_stext), 119 (unsigned long)kasan_mem_to_shadow(_end), 120 NUMA_NO_NODE); 121 122 kasan_populate_zero_shadow(kasan_mem_to_shadow((void *)MODULES_END), 123 (void *)KASAN_SHADOW_END); 124 125 load_cr3(init_level4_pgt); 126 __flush_tlb_all(); 127 128 /* 129 * kasan_zero_page has been used as early shadow memory, thus it may 130 * contain some garbage. Now we can clear and write protect it, since 131 * after the TLB flush no one should write to it. 132 */ 133 memset(kasan_zero_page, 0, PAGE_SIZE); 134 for (i = 0; i < PTRS_PER_PTE; i++) { 135 pte_t pte = __pte(__pa(kasan_zero_page) | __PAGE_KERNEL_RO); 136 set_pte(&kasan_zero_pte[i], pte); 137 } 138 /* Flush TLBs again to be sure that write protection applied. */ 139 __flush_tlb_all(); 140 141 init_task.kasan_depth = 0; 142 pr_info("KernelAddressSanitizer initialized\n"); 143 } 144