1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * KFENCE reporting. 4 * 5 * Copyright (C) 2020, Google LLC. 6 */ 7 8 #include <linux/stdarg.h> 9 10 #include <linux/kernel.h> 11 #include <linux/lockdep.h> 12 #include <linux/math.h> 13 #include <linux/printk.h> 14 #include <linux/sched/debug.h> 15 #include <linux/seq_file.h> 16 #include <linux/stacktrace.h> 17 #include <linux/string.h> 18 #include <trace/events/error_report.h> 19 20 #include <asm/kfence.h> 21 22 #include "kfence.h" 23 24 /* May be overridden by <asm/kfence.h>. */ 25 #ifndef ARCH_FUNC_PREFIX 26 #define ARCH_FUNC_PREFIX "" 27 #endif 28 29 extern bool no_hash_pointers; 30 31 /* Helper function to either print to a seq_file or to console. */ 32 __printf(2, 3) 33 static void seq_con_printf(struct seq_file *seq, const char *fmt, ...) 34 { 35 va_list args; 36 37 va_start(args, fmt); 38 if (seq) 39 seq_vprintf(seq, fmt, args); 40 else 41 vprintk(fmt, args); 42 va_end(args); 43 } 44 45 /* 46 * Get the number of stack entries to skip to get out of MM internals. @type is 47 * optional, and if set to NULL, assumes an allocation or free stack. 48 */ 49 static int get_stack_skipnr(const unsigned long stack_entries[], int num_entries, 50 const enum kfence_error_type *type) 51 { 52 char buf[64]; 53 int skipnr, fallback = 0; 54 55 if (type) { 56 /* Depending on error type, find different stack entries. */ 57 switch (*type) { 58 case KFENCE_ERROR_UAF: 59 case KFENCE_ERROR_OOB: 60 case KFENCE_ERROR_INVALID: 61 /* 62 * kfence_handle_page_fault() may be called with pt_regs 63 * set to NULL; in that case we'll simply show the full 64 * stack trace. 65 */ 66 return 0; 67 case KFENCE_ERROR_CORRUPTION: 68 case KFENCE_ERROR_INVALID_FREE: 69 break; 70 } 71 } 72 73 for (skipnr = 0; skipnr < num_entries; skipnr++) { 74 int len = scnprintf(buf, sizeof(buf), "%ps", (void *)stack_entries[skipnr]); 75 76 if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfence_") || 77 str_has_prefix(buf, ARCH_FUNC_PREFIX "__kfence_") || 78 !strncmp(buf, ARCH_FUNC_PREFIX "__slab_free", len)) { 79 /* 80 * In case of tail calls from any of the below 81 * to any of the above. 82 */ 83 fallback = skipnr + 1; 84 } 85 86 /* Also the *_bulk() variants by only checking prefixes. */ 87 if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfree") || 88 str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_free") || 89 str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmalloc") || 90 str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_alloc")) 91 goto found; 92 } 93 if (fallback < num_entries) 94 return fallback; 95 found: 96 skipnr++; 97 return skipnr < num_entries ? skipnr : 0; 98 } 99 100 static void kfence_print_stack(struct seq_file *seq, const struct kfence_metadata *meta, 101 bool show_alloc) 102 { 103 const struct kfence_track *track = show_alloc ? &meta->alloc_track : &meta->free_track; 104 u64 ts_sec = track->ts_nsec; 105 unsigned long rem_nsec = do_div(ts_sec, NSEC_PER_SEC); 106 107 /* Timestamp matches printk timestamp format. */ 108 seq_con_printf(seq, "%s by task %d on cpu %d at %lu.%06lus:\n", 109 show_alloc ? "allocated" : "freed", track->pid, 110 track->cpu, (unsigned long)ts_sec, rem_nsec / 1000); 111 112 if (track->num_stack_entries) { 113 /* Skip allocation/free internals stack. */ 114 int i = get_stack_skipnr(track->stack_entries, track->num_stack_entries, NULL); 115 116 /* stack_trace_seq_print() does not exist; open code our own. */ 117 for (; i < track->num_stack_entries; i++) 118 seq_con_printf(seq, " %pS\n", (void *)track->stack_entries[i]); 119 } else { 120 seq_con_printf(seq, " no %s stack\n", show_alloc ? "allocation" : "deallocation"); 121 } 122 } 123 124 void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta) 125 { 126 const int size = abs(meta->size); 127 const unsigned long start = meta->addr; 128 const struct kmem_cache *const cache = meta->cache; 129 130 lockdep_assert_held(&meta->lock); 131 132 if (meta->state == KFENCE_OBJECT_UNUSED) { 133 seq_con_printf(seq, "kfence-#%td unused\n", meta - kfence_metadata); 134 return; 135 } 136 137 seq_con_printf(seq, "kfence-#%td: 0x%p-0x%p, size=%d, cache=%s\n\n", 138 meta - kfence_metadata, (void *)start, (void *)(start + size - 1), 139 size, (cache && cache->name) ? cache->name : "<destroyed>"); 140 141 kfence_print_stack(seq, meta, true); 142 143 if (meta->state == KFENCE_OBJECT_FREED) { 144 seq_con_printf(seq, "\n"); 145 kfence_print_stack(seq, meta, false); 146 } 147 } 148 149 /* 150 * Show bytes at @addr that are different from the expected canary values, up to 151 * @max_bytes. 152 */ 153 static void print_diff_canary(unsigned long address, size_t bytes_to_show, 154 const struct kfence_metadata *meta) 155 { 156 const unsigned long show_until_addr = address + bytes_to_show; 157 const u8 *cur, *end; 158 159 /* Do not show contents of object nor read into following guard page. */ 160 end = (const u8 *)(address < meta->addr ? min(show_until_addr, meta->addr) 161 : min(show_until_addr, PAGE_ALIGN(address))); 162 163 pr_cont("["); 164 for (cur = (const u8 *)address; cur < end; cur++) { 165 if (*cur == KFENCE_CANARY_PATTERN(cur)) 166 pr_cont(" ."); 167 else if (no_hash_pointers) 168 pr_cont(" 0x%02x", *cur); 169 else /* Do not leak kernel memory in non-debug builds. */ 170 pr_cont(" !"); 171 } 172 pr_cont(" ]"); 173 } 174 175 static const char *get_access_type(bool is_write) 176 { 177 return is_write ? "write" : "read"; 178 } 179 180 void kfence_report_error(unsigned long address, bool is_write, struct pt_regs *regs, 181 const struct kfence_metadata *meta, enum kfence_error_type type) 182 { 183 unsigned long stack_entries[KFENCE_STACK_DEPTH] = { 0 }; 184 const ptrdiff_t object_index = meta ? meta - kfence_metadata : -1; 185 int num_stack_entries; 186 int skipnr = 0; 187 188 if (regs) { 189 num_stack_entries = stack_trace_save_regs(regs, stack_entries, KFENCE_STACK_DEPTH, 0); 190 } else { 191 num_stack_entries = stack_trace_save(stack_entries, KFENCE_STACK_DEPTH, 1); 192 skipnr = get_stack_skipnr(stack_entries, num_stack_entries, &type); 193 } 194 195 /* Require non-NULL meta, except if KFENCE_ERROR_INVALID. */ 196 if (WARN_ON(type != KFENCE_ERROR_INVALID && !meta)) 197 return; 198 199 if (meta) 200 lockdep_assert_held(&meta->lock); 201 /* 202 * Because we may generate reports in printk-unfriendly parts of the 203 * kernel, such as scheduler code, the use of printk() could deadlock. 204 * Until such time that all printing code here is safe in all parts of 205 * the kernel, accept the risk, and just get our message out (given the 206 * system might already behave unpredictably due to the memory error). 207 * As such, also disable lockdep to hide warnings, and avoid disabling 208 * lockdep for the rest of the kernel. 209 */ 210 lockdep_off(); 211 212 pr_err("==================================================================\n"); 213 /* Print report header. */ 214 switch (type) { 215 case KFENCE_ERROR_OOB: { 216 const bool left_of_object = address < meta->addr; 217 218 pr_err("BUG: KFENCE: out-of-bounds %s in %pS\n\n", get_access_type(is_write), 219 (void *)stack_entries[skipnr]); 220 pr_err("Out-of-bounds %s at 0x%p (%luB %s of kfence-#%td):\n", 221 get_access_type(is_write), (void *)address, 222 left_of_object ? meta->addr - address : address - meta->addr, 223 left_of_object ? "left" : "right", object_index); 224 break; 225 } 226 case KFENCE_ERROR_UAF: 227 pr_err("BUG: KFENCE: use-after-free %s in %pS\n\n", get_access_type(is_write), 228 (void *)stack_entries[skipnr]); 229 pr_err("Use-after-free %s at 0x%p (in kfence-#%td):\n", 230 get_access_type(is_write), (void *)address, object_index); 231 break; 232 case KFENCE_ERROR_CORRUPTION: 233 pr_err("BUG: KFENCE: memory corruption in %pS\n\n", (void *)stack_entries[skipnr]); 234 pr_err("Corrupted memory at 0x%p ", (void *)address); 235 print_diff_canary(address, 16, meta); 236 pr_cont(" (in kfence-#%td):\n", object_index); 237 break; 238 case KFENCE_ERROR_INVALID: 239 pr_err("BUG: KFENCE: invalid %s in %pS\n\n", get_access_type(is_write), 240 (void *)stack_entries[skipnr]); 241 pr_err("Invalid %s at 0x%p:\n", get_access_type(is_write), 242 (void *)address); 243 break; 244 case KFENCE_ERROR_INVALID_FREE: 245 pr_err("BUG: KFENCE: invalid free in %pS\n\n", (void *)stack_entries[skipnr]); 246 pr_err("Invalid free of 0x%p (in kfence-#%td):\n", (void *)address, 247 object_index); 248 break; 249 } 250 251 /* Print stack trace and object info. */ 252 stack_trace_print(stack_entries + skipnr, num_stack_entries - skipnr, 0); 253 254 if (meta) { 255 pr_err("\n"); 256 kfence_print_object(NULL, meta); 257 } 258 259 /* Print report footer. */ 260 pr_err("\n"); 261 if (no_hash_pointers && regs) 262 show_regs(regs); 263 else 264 dump_stack_print_info(KERN_ERR); 265 trace_error_report_end(ERROR_DETECTOR_KFENCE, address); 266 pr_err("==================================================================\n"); 267 268 lockdep_on(); 269 270 if (panic_on_warn) 271 panic("panic_on_warn set ...\n"); 272 273 /* We encountered a memory safety error, taint the kernel! */ 274 add_taint(TAINT_BAD_PAGE, LOCKDEP_STILL_OK); 275 } 276 277 #ifdef CONFIG_PRINTK 278 static void kfence_to_kp_stack(const struct kfence_track *track, void **kp_stack) 279 { 280 int i, j; 281 282 i = get_stack_skipnr(track->stack_entries, track->num_stack_entries, NULL); 283 for (j = 0; i < track->num_stack_entries && j < KS_ADDRS_COUNT; ++i, ++j) 284 kp_stack[j] = (void *)track->stack_entries[i]; 285 if (j < KS_ADDRS_COUNT) 286 kp_stack[j] = NULL; 287 } 288 289 bool __kfence_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) 290 { 291 struct kfence_metadata *meta = addr_to_metadata((unsigned long)object); 292 unsigned long flags; 293 294 if (!meta) 295 return false; 296 297 /* 298 * If state is UNUSED at least show the pointer requested; the rest 299 * would be garbage data. 300 */ 301 kpp->kp_ptr = object; 302 303 /* Requesting info an a never-used object is almost certainly a bug. */ 304 if (WARN_ON(meta->state == KFENCE_OBJECT_UNUSED)) 305 return true; 306 307 raw_spin_lock_irqsave(&meta->lock, flags); 308 309 kpp->kp_slab = slab; 310 kpp->kp_slab_cache = meta->cache; 311 kpp->kp_objp = (void *)meta->addr; 312 kfence_to_kp_stack(&meta->alloc_track, kpp->kp_stack); 313 if (meta->state == KFENCE_OBJECT_FREED) 314 kfence_to_kp_stack(&meta->free_track, kpp->kp_free_stack); 315 /* get_stack_skipnr() ensures the first entry is outside allocator. */ 316 kpp->kp_ret = kpp->kp_stack[0]; 317 318 raw_spin_unlock_irqrestore(&meta->lock, flags); 319 320 return true; 321 } 322 #endif 323