1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/string.h> 4 #include <linux/mm.h> 5 #include <linux/highmem.h> 6 #include <linux/page_ext.h> 7 #include <linux/poison.h> 8 #include <linux/ratelimit.h> 9 #include <linux/kasan.h> 10 11 static DEFINE_STATIC_KEY_FALSE_RO(want_page_poisoning); 12 13 static int __init early_page_poison_param(char *buf) 14 { 15 int ret; 16 bool tmp; 17 18 ret = strtobool(buf, &tmp); 19 if (ret) 20 return ret; 21 22 if (tmp) 23 static_branch_enable(&want_page_poisoning); 24 else 25 static_branch_disable(&want_page_poisoning); 26 27 return 0; 28 } 29 early_param("page_poison", early_page_poison_param); 30 31 /** 32 * page_poisoning_enabled - check if page poisoning is enabled 33 * 34 * Return true if page poisoning is enabled, or false if not. 35 */ 36 bool page_poisoning_enabled(void) 37 { 38 /* 39 * Assumes that debug_pagealloc_enabled is set before 40 * memblock_free_all. 41 * Page poisoning is debug page alloc for some arches. If 42 * either of those options are enabled, enable poisoning. 43 */ 44 return (static_branch_unlikely(&want_page_poisoning) || 45 (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) && 46 debug_pagealloc_enabled())); 47 } 48 EXPORT_SYMBOL_GPL(page_poisoning_enabled); 49 50 static void poison_page(struct page *page) 51 { 52 void *addr = kmap_atomic(page); 53 54 /* KASAN still think the page is in-use, so skip it. */ 55 kasan_disable_current(); 56 memset(addr, PAGE_POISON, PAGE_SIZE); 57 kasan_enable_current(); 58 kunmap_atomic(addr); 59 } 60 61 static void poison_pages(struct page *page, int n) 62 { 63 int i; 64 65 for (i = 0; i < n; i++) 66 poison_page(page + i); 67 } 68 69 static bool single_bit_flip(unsigned char a, unsigned char b) 70 { 71 unsigned char error = a ^ b; 72 73 return error && !(error & (error - 1)); 74 } 75 76 static void check_poison_mem(unsigned char *mem, size_t bytes) 77 { 78 static DEFINE_RATELIMIT_STATE(ratelimit, 5 * HZ, 10); 79 unsigned char *start; 80 unsigned char *end; 81 82 if (IS_ENABLED(CONFIG_PAGE_POISONING_NO_SANITY)) 83 return; 84 85 start = memchr_inv(mem, PAGE_POISON, bytes); 86 if (!start) 87 return; 88 89 for (end = mem + bytes - 1; end > start; end--) { 90 if (*end != PAGE_POISON) 91 break; 92 } 93 94 if (!__ratelimit(&ratelimit)) 95 return; 96 else if (start == end && single_bit_flip(*start, PAGE_POISON)) 97 pr_err("pagealloc: single bit error\n"); 98 else 99 pr_err("pagealloc: memory corruption\n"); 100 101 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_ADDRESS, 16, 1, start, 102 end - start + 1, 1); 103 dump_stack(); 104 } 105 106 static void unpoison_page(struct page *page) 107 { 108 void *addr; 109 110 addr = kmap_atomic(page); 111 /* 112 * Page poisoning when enabled poisons each and every page 113 * that is freed to buddy. Thus no extra check is done to 114 * see if a page was poisoned. 115 */ 116 check_poison_mem(addr, PAGE_SIZE); 117 kunmap_atomic(addr); 118 } 119 120 static void unpoison_pages(struct page *page, int n) 121 { 122 int i; 123 124 for (i = 0; i < n; i++) 125 unpoison_page(page + i); 126 } 127 128 void kernel_poison_pages(struct page *page, int numpages, int enable) 129 { 130 if (!page_poisoning_enabled()) 131 return; 132 133 if (enable) 134 unpoison_pages(page, numpages); 135 else 136 poison_pages(page, numpages); 137 } 138 139 #ifndef CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC 140 void __kernel_map_pages(struct page *page, int numpages, int enable) 141 { 142 /* This function does nothing, all work is done via poison pages */ 143 } 144 #endif 145