1 #include <linux/kernel.h> 2 #include <linux/string.h> 3 #include <linux/mm.h> 4 #include <linux/highmem.h> 5 #include <linux/page_ext.h> 6 #include <linux/poison.h> 7 #include <linux/ratelimit.h> 8 9 static bool __page_poisoning_enabled __read_mostly; 10 static bool want_page_poisoning __read_mostly; 11 12 static int early_page_poison_param(char *buf) 13 { 14 if (!buf) 15 return -EINVAL; 16 17 if (strcmp(buf, "on") == 0) 18 want_page_poisoning = true; 19 else if (strcmp(buf, "off") == 0) 20 want_page_poisoning = false; 21 22 return 0; 23 } 24 early_param("page_poison", early_page_poison_param); 25 26 bool page_poisoning_enabled(void) 27 { 28 return __page_poisoning_enabled; 29 } 30 31 static bool need_page_poisoning(void) 32 { 33 return want_page_poisoning; 34 } 35 36 static void init_page_poisoning(void) 37 { 38 /* 39 * page poisoning is debug page alloc for some arches. If either 40 * of those options are enabled, enable poisoning 41 */ 42 if (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC)) { 43 if (!want_page_poisoning && !debug_pagealloc_enabled()) 44 return; 45 } else { 46 if (!want_page_poisoning) 47 return; 48 } 49 50 __page_poisoning_enabled = true; 51 } 52 53 struct page_ext_operations page_poisoning_ops = { 54 .need = need_page_poisoning, 55 .init = init_page_poisoning, 56 }; 57 58 static inline void set_page_poison(struct page *page) 59 { 60 struct page_ext *page_ext; 61 62 page_ext = lookup_page_ext(page); 63 __set_bit(PAGE_EXT_DEBUG_POISON, &page_ext->flags); 64 } 65 66 static inline void clear_page_poison(struct page *page) 67 { 68 struct page_ext *page_ext; 69 70 page_ext = lookup_page_ext(page); 71 __clear_bit(PAGE_EXT_DEBUG_POISON, &page_ext->flags); 72 } 73 74 bool page_is_poisoned(struct page *page) 75 { 76 struct page_ext *page_ext; 77 78 page_ext = lookup_page_ext(page); 79 if (!page_ext) 80 return false; 81 82 return test_bit(PAGE_EXT_DEBUG_POISON, &page_ext->flags); 83 } 84 85 static void poison_page(struct page *page) 86 { 87 void *addr = kmap_atomic(page); 88 89 set_page_poison(page); 90 memset(addr, PAGE_POISON, PAGE_SIZE); 91 kunmap_atomic(addr); 92 } 93 94 static void poison_pages(struct page *page, int n) 95 { 96 int i; 97 98 for (i = 0; i < n; i++) 99 poison_page(page + i); 100 } 101 102 static bool single_bit_flip(unsigned char a, unsigned char b) 103 { 104 unsigned char error = a ^ b; 105 106 return error && !(error & (error - 1)); 107 } 108 109 static void check_poison_mem(unsigned char *mem, size_t bytes) 110 { 111 static DEFINE_RATELIMIT_STATE(ratelimit, 5 * HZ, 10); 112 unsigned char *start; 113 unsigned char *end; 114 115 if (IS_ENABLED(CONFIG_PAGE_POISONING_NO_SANITY)) 116 return; 117 118 start = memchr_inv(mem, PAGE_POISON, bytes); 119 if (!start) 120 return; 121 122 for (end = mem + bytes - 1; end > start; end--) { 123 if (*end != PAGE_POISON) 124 break; 125 } 126 127 if (!__ratelimit(&ratelimit)) 128 return; 129 else if (start == end && single_bit_flip(*start, PAGE_POISON)) 130 pr_err("pagealloc: single bit error\n"); 131 else 132 pr_err("pagealloc: memory corruption\n"); 133 134 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_ADDRESS, 16, 1, start, 135 end - start + 1, 1); 136 dump_stack(); 137 } 138 139 static void unpoison_page(struct page *page) 140 { 141 void *addr; 142 143 if (!page_is_poisoned(page)) 144 return; 145 146 addr = kmap_atomic(page); 147 check_poison_mem(addr, PAGE_SIZE); 148 clear_page_poison(page); 149 kunmap_atomic(addr); 150 } 151 152 static void unpoison_pages(struct page *page, int n) 153 { 154 int i; 155 156 for (i = 0; i < n; i++) 157 unpoison_page(page + i); 158 } 159 160 void kernel_poison_pages(struct page *page, int numpages, int enable) 161 { 162 if (!page_poisoning_enabled()) 163 return; 164 165 if (enable) 166 unpoison_pages(page, numpages); 167 else 168 poison_pages(page, numpages); 169 } 170 171 #ifndef CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC 172 void __kernel_map_pages(struct page *page, int numpages, int enable) 173 { 174 /* This function does nothing, all work is done via poison pages */ 175 } 176 #endif 177