xref: /openbmc/linux/mm/kasan/kasan.h (revision 5b4fc395)
1  /* SPDX-License-Identifier: GPL-2.0 */
2  #ifndef __MM_KASAN_KASAN_H
3  #define __MM_KASAN_KASAN_H
4  
5  #include <linux/kasan.h>
6  #include <linux/kasan-tags.h>
7  #include <linux/kfence.h>
8  #include <linux/stackdepot.h>
9  
10  #ifdef CONFIG_KASAN_HW_TAGS
11  
12  #include <linux/static_key.h>
13  #include "../slab.h"
14  
15  DECLARE_STATIC_KEY_TRUE(kasan_flag_vmalloc);
16  DECLARE_STATIC_KEY_TRUE(kasan_flag_stacktrace);
17  
18  enum kasan_mode {
19  	KASAN_MODE_SYNC,
20  	KASAN_MODE_ASYNC,
21  	KASAN_MODE_ASYMM,
22  };
23  
24  extern enum kasan_mode kasan_mode __ro_after_init;
25  
26  static inline bool kasan_vmalloc_enabled(void)
27  {
28  	return static_branch_likely(&kasan_flag_vmalloc);
29  }
30  
31  static inline bool kasan_stack_collection_enabled(void)
32  {
33  	return static_branch_unlikely(&kasan_flag_stacktrace);
34  }
35  
36  static inline bool kasan_async_fault_possible(void)
37  {
38  	return kasan_mode == KASAN_MODE_ASYNC || kasan_mode == KASAN_MODE_ASYMM;
39  }
40  
41  static inline bool kasan_sync_fault_possible(void)
42  {
43  	return kasan_mode == KASAN_MODE_SYNC || kasan_mode == KASAN_MODE_ASYMM;
44  }
45  
46  #else
47  
48  static inline bool kasan_stack_collection_enabled(void)
49  {
50  	return true;
51  }
52  
53  static inline bool kasan_async_fault_possible(void)
54  {
55  	return false;
56  }
57  
58  static inline bool kasan_sync_fault_possible(void)
59  {
60  	return true;
61  }
62  
63  #endif
64  
65  #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
66  #define KASAN_GRANULE_SIZE	(1UL << KASAN_SHADOW_SCALE_SHIFT)
67  #else
68  #include <asm/mte-kasan.h>
69  #define KASAN_GRANULE_SIZE	MTE_GRANULE_SIZE
70  #endif
71  
72  #define KASAN_GRANULE_MASK	(KASAN_GRANULE_SIZE - 1)
73  
74  #define KASAN_MEMORY_PER_SHADOW_PAGE	(KASAN_GRANULE_SIZE << PAGE_SHIFT)
75  
76  #ifdef CONFIG_KASAN_GENERIC
77  #define KASAN_PAGE_FREE		0xFF  /* freed page */
78  #define KASAN_PAGE_REDZONE	0xFE  /* redzone for kmalloc_large allocation */
79  #define KASAN_SLAB_REDZONE	0xFC  /* redzone for slab object */
80  #define KASAN_SLAB_FREE		0xFB  /* freed slab object */
81  #define KASAN_VMALLOC_INVALID	0xF8  /* inaccessible space in vmap area */
82  #else
83  #define KASAN_PAGE_FREE		KASAN_TAG_INVALID
84  #define KASAN_PAGE_REDZONE	KASAN_TAG_INVALID
85  #define KASAN_SLAB_REDZONE	KASAN_TAG_INVALID
86  #define KASAN_SLAB_FREE		KASAN_TAG_INVALID
87  #define KASAN_VMALLOC_INVALID	KASAN_TAG_INVALID /* only used for SW_TAGS */
88  #endif
89  
90  #ifdef CONFIG_KASAN_GENERIC
91  
92  #define KASAN_SLAB_FREETRACK	0xFA  /* freed slab object with free track */
93  #define KASAN_GLOBAL_REDZONE	0xF9  /* redzone for global variable */
94  
95  /* Stack redzone shadow values. Compiler ABI, do not change. */
96  #define KASAN_STACK_LEFT	0xF1
97  #define KASAN_STACK_MID		0xF2
98  #define KASAN_STACK_RIGHT	0xF3
99  #define KASAN_STACK_PARTIAL	0xF4
100  
101  /* alloca redzone shadow values. */
102  #define KASAN_ALLOCA_LEFT	0xCA
103  #define KASAN_ALLOCA_RIGHT	0xCB
104  
105  /* alloca redzone size. Compiler ABI, do not change. */
106  #define KASAN_ALLOCA_REDZONE_SIZE	32
107  
108  /* Stack frame marker. Compiler ABI, do not change. */
109  #define KASAN_CURRENT_STACK_FRAME_MAGIC 0x41B58AB3
110  
111  /* Dummy value to avoid breaking randconfig/all*config builds. */
112  #ifndef KASAN_ABI_VERSION
113  #define KASAN_ABI_VERSION 1
114  #endif
115  
116  #endif /* CONFIG_KASAN_GENERIC */
117  
118  /* Metadata layout customization. */
119  #define META_BYTES_PER_BLOCK 1
120  #define META_BLOCKS_PER_ROW 16
121  #define META_BYTES_PER_ROW (META_BLOCKS_PER_ROW * META_BYTES_PER_BLOCK)
122  #define META_MEM_BYTES_PER_ROW (META_BYTES_PER_ROW * KASAN_GRANULE_SIZE)
123  #define META_ROWS_AROUND_ADDR 2
124  
125  enum kasan_report_type {
126  	KASAN_REPORT_ACCESS,
127  	KASAN_REPORT_INVALID_FREE,
128  	KASAN_REPORT_DOUBLE_FREE,
129  };
130  
131  struct kasan_report_info {
132  	enum kasan_report_type type;
133  	void *access_addr;
134  	void *first_bad_addr;
135  	size_t access_size;
136  	bool is_write;
137  	unsigned long ip;
138  };
139  
140  /* Do not change the struct layout: compiler ABI. */
141  struct kasan_source_location {
142  	const char *filename;
143  	int line_no;
144  	int column_no;
145  };
146  
147  /* Do not change the struct layout: compiler ABI. */
148  struct kasan_global {
149  	const void *beg;		/* Address of the beginning of the global variable. */
150  	size_t size;			/* Size of the global variable. */
151  	size_t size_with_redzone;	/* Size of the variable + size of the redzone. 32 bytes aligned. */
152  	const void *name;
153  	const void *module_name;	/* Name of the module where the global variable is declared. */
154  	unsigned long has_dynamic_init;	/* This is needed for C++. */
155  #if KASAN_ABI_VERSION >= 4
156  	struct kasan_source_location *location;
157  #endif
158  #if KASAN_ABI_VERSION >= 5
159  	char *odr_indicator;
160  #endif
161  };
162  
163  /* Structures for keeping alloc and free tracks. */
164  
165  #define KASAN_STACK_DEPTH 64
166  
167  struct kasan_track {
168  	u32 pid;
169  	depot_stack_handle_t stack;
170  };
171  
172  #if defined(CONFIG_KASAN_TAGS_IDENTIFY) && defined(CONFIG_KASAN_SW_TAGS)
173  #define KASAN_NR_FREE_STACKS 5
174  #else
175  #define KASAN_NR_FREE_STACKS 1
176  #endif
177  
178  struct kasan_alloc_meta {
179  	struct kasan_track alloc_track;
180  	/* Generic mode stores free track in kasan_free_meta. */
181  #ifdef CONFIG_KASAN_GENERIC
182  	depot_stack_handle_t aux_stack[2];
183  #else
184  	struct kasan_track free_track[KASAN_NR_FREE_STACKS];
185  #endif
186  #ifdef CONFIG_KASAN_TAGS_IDENTIFY
187  	u8 free_pointer_tag[KASAN_NR_FREE_STACKS];
188  	u8 free_track_idx;
189  #endif
190  };
191  
192  struct qlist_node {
193  	struct qlist_node *next;
194  };
195  
196  /*
197   * Free meta is stored either in the object itself or in the redzone after the
198   * object. In the former case, free meta offset is 0. In the latter case, the
199   * offset is between 0 and INT_MAX. INT_MAX marks that free meta is not present.
200   */
201  #define KASAN_NO_FREE_META INT_MAX
202  
203  /*
204   * Free meta is only used by Generic mode while the object is in quarantine.
205   * After that, slab allocator stores the freelist pointer in the object.
206   */
207  struct kasan_free_meta {
208  #ifdef CONFIG_KASAN_GENERIC
209  	struct qlist_node quarantine_link;
210  	struct kasan_track free_track;
211  #endif
212  };
213  
214  #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
215  /* Used in KUnit-compatible KASAN tests. */
216  struct kunit_kasan_status {
217  	bool report_found;
218  	bool sync_fault;
219  };
220  #endif
221  
222  struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache,
223  						const void *object);
224  #ifdef CONFIG_KASAN_GENERIC
225  struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache,
226  						const void *object);
227  #endif
228  
229  #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
230  
231  static inline const void *kasan_shadow_to_mem(const void *shadow_addr)
232  {
233  	return (void *)(((unsigned long)shadow_addr - KASAN_SHADOW_OFFSET)
234  		<< KASAN_SHADOW_SCALE_SHIFT);
235  }
236  
237  static inline bool addr_has_metadata(const void *addr)
238  {
239  	return (kasan_reset_tag(addr) >=
240  		kasan_shadow_to_mem((void *)KASAN_SHADOW_START));
241  }
242  
243  /**
244   * kasan_check_range - Check memory region, and report if invalid access.
245   * @addr: the accessed address
246   * @size: the accessed size
247   * @write: true if access is a write access
248   * @ret_ip: return address
249   * @return: true if access was valid, false if invalid
250   */
251  bool kasan_check_range(unsigned long addr, size_t size, bool write,
252  				unsigned long ret_ip);
253  
254  #else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
255  
256  static inline bool addr_has_metadata(const void *addr)
257  {
258  	return (is_vmalloc_addr(addr) || virt_addr_valid(addr));
259  }
260  
261  #endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
262  
263  #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
264  void kasan_print_tags(u8 addr_tag, const void *addr);
265  #else
266  static inline void kasan_print_tags(u8 addr_tag, const void *addr) { }
267  #endif
268  
269  void *kasan_find_first_bad_addr(void *addr, size_t size);
270  const char *kasan_get_bug_type(struct kasan_report_info *info);
271  void kasan_metadata_fetch_row(char *buffer, void *row);
272  
273  #if defined(CONFIG_KASAN_STACK)
274  void kasan_print_address_stack_frame(const void *addr);
275  #else
276  static inline void kasan_print_address_stack_frame(const void *addr) { }
277  #endif
278  
279  bool kasan_report(unsigned long addr, size_t size,
280  		bool is_write, unsigned long ip);
281  void kasan_report_invalid_free(void *object, unsigned long ip, enum kasan_report_type type);
282  
283  struct page *kasan_addr_to_page(const void *addr);
284  struct slab *kasan_addr_to_slab(const void *addr);
285  
286  depot_stack_handle_t kasan_save_stack(gfp_t flags, bool can_alloc);
287  void kasan_set_track(struct kasan_track *track, gfp_t flags);
288  void kasan_set_free_info(struct kmem_cache *cache, void *object, u8 tag);
289  struct kasan_track *kasan_get_free_track(struct kmem_cache *cache,
290  				void *object, u8 tag);
291  
292  #if defined(CONFIG_KASAN_GENERIC) && \
293  	(defined(CONFIG_SLAB) || defined(CONFIG_SLUB))
294  bool kasan_quarantine_put(struct kmem_cache *cache, void *object);
295  void kasan_quarantine_reduce(void);
296  void kasan_quarantine_remove_cache(struct kmem_cache *cache);
297  #else
298  static inline bool kasan_quarantine_put(struct kmem_cache *cache, void *object) { return false; }
299  static inline void kasan_quarantine_reduce(void) { }
300  static inline void kasan_quarantine_remove_cache(struct kmem_cache *cache) { }
301  #endif
302  
303  #ifndef arch_kasan_set_tag
304  static inline const void *arch_kasan_set_tag(const void *addr, u8 tag)
305  {
306  	return addr;
307  }
308  #endif
309  #ifndef arch_kasan_get_tag
310  #define arch_kasan_get_tag(addr)	0
311  #endif
312  
313  #define set_tag(addr, tag)	((void *)arch_kasan_set_tag((addr), (tag)))
314  #define get_tag(addr)		arch_kasan_get_tag(addr)
315  
316  #ifdef CONFIG_KASAN_HW_TAGS
317  
318  #ifndef arch_enable_tagging_sync
319  #define arch_enable_tagging_sync()
320  #endif
321  #ifndef arch_enable_tagging_async
322  #define arch_enable_tagging_async()
323  #endif
324  #ifndef arch_enable_tagging_asymm
325  #define arch_enable_tagging_asymm()
326  #endif
327  #ifndef arch_force_async_tag_fault
328  #define arch_force_async_tag_fault()
329  #endif
330  #ifndef arch_get_random_tag
331  #define arch_get_random_tag()	(0xFF)
332  #endif
333  #ifndef arch_get_mem_tag
334  #define arch_get_mem_tag(addr)	(0xFF)
335  #endif
336  #ifndef arch_set_mem_tag_range
337  #define arch_set_mem_tag_range(addr, size, tag, init) ((void *)(addr))
338  #endif
339  
340  #define hw_enable_tagging_sync()		arch_enable_tagging_sync()
341  #define hw_enable_tagging_async()		arch_enable_tagging_async()
342  #define hw_enable_tagging_asymm()		arch_enable_tagging_asymm()
343  #define hw_force_async_tag_fault()		arch_force_async_tag_fault()
344  #define hw_get_random_tag()			arch_get_random_tag()
345  #define hw_get_mem_tag(addr)			arch_get_mem_tag(addr)
346  #define hw_set_mem_tag_range(addr, size, tag, init) \
347  			arch_set_mem_tag_range((addr), (size), (tag), (init))
348  
349  void kasan_enable_tagging(void);
350  
351  #else /* CONFIG_KASAN_HW_TAGS */
352  
353  #define hw_enable_tagging_sync()
354  #define hw_enable_tagging_async()
355  #define hw_enable_tagging_asymm()
356  
357  static inline void kasan_enable_tagging(void) { }
358  
359  #endif /* CONFIG_KASAN_HW_TAGS */
360  
361  #if defined(CONFIG_KASAN_HW_TAGS) && IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
362  
363  void kasan_force_async_fault(void);
364  
365  #else /* CONFIG_KASAN_HW_TAGS && CONFIG_KASAN_KUNIT_TEST */
366  
367  static inline void kasan_force_async_fault(void) { }
368  
369  #endif /* CONFIG_KASAN_HW_TAGS && CONFIG_KASAN_KUNIT_TEST */
370  
371  #ifdef CONFIG_KASAN_SW_TAGS
372  u8 kasan_random_tag(void);
373  #elif defined(CONFIG_KASAN_HW_TAGS)
374  static inline u8 kasan_random_tag(void) { return hw_get_random_tag(); }
375  #else
376  static inline u8 kasan_random_tag(void) { return 0; }
377  #endif
378  
379  #ifdef CONFIG_KASAN_HW_TAGS
380  
381  static inline void kasan_poison(const void *addr, size_t size, u8 value, bool init)
382  {
383  	addr = kasan_reset_tag(addr);
384  
385  	/* Skip KFENCE memory if called explicitly outside of sl*b. */
386  	if (is_kfence_address(addr))
387  		return;
388  
389  	if (WARN_ON((unsigned long)addr & KASAN_GRANULE_MASK))
390  		return;
391  	if (WARN_ON(size & KASAN_GRANULE_MASK))
392  		return;
393  
394  	hw_set_mem_tag_range((void *)addr, size, value, init);
395  }
396  
397  static inline void kasan_unpoison(const void *addr, size_t size, bool init)
398  {
399  	u8 tag = get_tag(addr);
400  
401  	addr = kasan_reset_tag(addr);
402  
403  	/* Skip KFENCE memory if called explicitly outside of sl*b. */
404  	if (is_kfence_address(addr))
405  		return;
406  
407  	if (WARN_ON((unsigned long)addr & KASAN_GRANULE_MASK))
408  		return;
409  	/*
410  	 * Explicitly initialize the memory with the precise object size to
411  	 * avoid overwriting the slab redzone. This disables initialization in
412  	 * the arch code and may thus lead to performance penalty. This penalty
413  	 * does not affect production builds, as slab redzones are not enabled
414  	 * there.
415  	 */
416  	if (__slub_debug_enabled() &&
417  	    init && ((unsigned long)size & KASAN_GRANULE_MASK)) {
418  		init = false;
419  		memzero_explicit((void *)addr, size);
420  	}
421  	size = round_up(size, KASAN_GRANULE_SIZE);
422  
423  	hw_set_mem_tag_range((void *)addr, size, tag, init);
424  }
425  
426  static inline bool kasan_byte_accessible(const void *addr)
427  {
428  	u8 ptr_tag = get_tag(addr);
429  	u8 mem_tag = hw_get_mem_tag((void *)addr);
430  
431  	return ptr_tag == KASAN_TAG_KERNEL || ptr_tag == mem_tag;
432  }
433  
434  #else /* CONFIG_KASAN_HW_TAGS */
435  
436  /**
437   * kasan_poison - mark the memory range as inaccessible
438   * @addr - range start address, must be aligned to KASAN_GRANULE_SIZE
439   * @size - range size, must be aligned to KASAN_GRANULE_SIZE
440   * @value - value that's written to metadata for the range
441   * @init - whether to initialize the memory range (only for hardware tag-based)
442   *
443   * The size gets aligned to KASAN_GRANULE_SIZE before marking the range.
444   */
445  void kasan_poison(const void *addr, size_t size, u8 value, bool init);
446  
447  /**
448   * kasan_unpoison - mark the memory range as accessible
449   * @addr - range start address, must be aligned to KASAN_GRANULE_SIZE
450   * @size - range size, can be unaligned
451   * @init - whether to initialize the memory range (only for hardware tag-based)
452   *
453   * For the tag-based modes, the @size gets aligned to KASAN_GRANULE_SIZE before
454   * marking the range.
455   * For the generic mode, the last granule of the memory range gets partially
456   * unpoisoned based on the @size.
457   */
458  void kasan_unpoison(const void *addr, size_t size, bool init);
459  
460  bool kasan_byte_accessible(const void *addr);
461  
462  #endif /* CONFIG_KASAN_HW_TAGS */
463  
464  #ifdef CONFIG_KASAN_GENERIC
465  
466  /**
467   * kasan_poison_last_granule - mark the last granule of the memory range as
468   * inaccessible
469   * @addr - range start address, must be aligned to KASAN_GRANULE_SIZE
470   * @size - range size
471   *
472   * This function is only available for the generic mode, as it's the only mode
473   * that has partially poisoned memory granules.
474   */
475  void kasan_poison_last_granule(const void *address, size_t size);
476  
477  #else /* CONFIG_KASAN_GENERIC */
478  
479  static inline void kasan_poison_last_granule(const void *address, size_t size) { }
480  
481  #endif /* CONFIG_KASAN_GENERIC */
482  
483  #ifndef kasan_arch_is_ready
484  static inline bool kasan_arch_is_ready(void)	{ return true; }
485  #elif !defined(CONFIG_KASAN_GENERIC) || !defined(CONFIG_KASAN_OUTLINE)
486  #error kasan_arch_is_ready only works in KASAN generic outline mode!
487  #endif
488  
489  #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST) || IS_ENABLED(CONFIG_KASAN_MODULE_TEST)
490  
491  bool kasan_save_enable_multi_shot(void);
492  void kasan_restore_multi_shot(bool enabled);
493  
494  #endif
495  
496  /*
497   * Exported functions for interfaces called from assembly or from generated
498   * code. Declared here to avoid warnings about missing declarations.
499   */
500  
501  asmlinkage void kasan_unpoison_task_stack_below(const void *watermark);
502  void __asan_register_globals(struct kasan_global *globals, size_t size);
503  void __asan_unregister_globals(struct kasan_global *globals, size_t size);
504  void __asan_handle_no_return(void);
505  void __asan_alloca_poison(unsigned long addr, size_t size);
506  void __asan_allocas_unpoison(const void *stack_top, const void *stack_bottom);
507  
508  void __asan_load1(unsigned long addr);
509  void __asan_store1(unsigned long addr);
510  void __asan_load2(unsigned long addr);
511  void __asan_store2(unsigned long addr);
512  void __asan_load4(unsigned long addr);
513  void __asan_store4(unsigned long addr);
514  void __asan_load8(unsigned long addr);
515  void __asan_store8(unsigned long addr);
516  void __asan_load16(unsigned long addr);
517  void __asan_store16(unsigned long addr);
518  void __asan_loadN(unsigned long addr, size_t size);
519  void __asan_storeN(unsigned long addr, size_t size);
520  
521  void __asan_load1_noabort(unsigned long addr);
522  void __asan_store1_noabort(unsigned long addr);
523  void __asan_load2_noabort(unsigned long addr);
524  void __asan_store2_noabort(unsigned long addr);
525  void __asan_load4_noabort(unsigned long addr);
526  void __asan_store4_noabort(unsigned long addr);
527  void __asan_load8_noabort(unsigned long addr);
528  void __asan_store8_noabort(unsigned long addr);
529  void __asan_load16_noabort(unsigned long addr);
530  void __asan_store16_noabort(unsigned long addr);
531  void __asan_loadN_noabort(unsigned long addr, size_t size);
532  void __asan_storeN_noabort(unsigned long addr, size_t size);
533  
534  void __asan_report_load1_noabort(unsigned long addr);
535  void __asan_report_store1_noabort(unsigned long addr);
536  void __asan_report_load2_noabort(unsigned long addr);
537  void __asan_report_store2_noabort(unsigned long addr);
538  void __asan_report_load4_noabort(unsigned long addr);
539  void __asan_report_store4_noabort(unsigned long addr);
540  void __asan_report_load8_noabort(unsigned long addr);
541  void __asan_report_store8_noabort(unsigned long addr);
542  void __asan_report_load16_noabort(unsigned long addr);
543  void __asan_report_store16_noabort(unsigned long addr);
544  void __asan_report_load_n_noabort(unsigned long addr, size_t size);
545  void __asan_report_store_n_noabort(unsigned long addr, size_t size);
546  
547  void __asan_set_shadow_00(const void *addr, size_t size);
548  void __asan_set_shadow_f1(const void *addr, size_t size);
549  void __asan_set_shadow_f2(const void *addr, size_t size);
550  void __asan_set_shadow_f3(const void *addr, size_t size);
551  void __asan_set_shadow_f5(const void *addr, size_t size);
552  void __asan_set_shadow_f8(const void *addr, size_t size);
553  
554  void __hwasan_load1_noabort(unsigned long addr);
555  void __hwasan_store1_noabort(unsigned long addr);
556  void __hwasan_load2_noabort(unsigned long addr);
557  void __hwasan_store2_noabort(unsigned long addr);
558  void __hwasan_load4_noabort(unsigned long addr);
559  void __hwasan_store4_noabort(unsigned long addr);
560  void __hwasan_load8_noabort(unsigned long addr);
561  void __hwasan_store8_noabort(unsigned long addr);
562  void __hwasan_load16_noabort(unsigned long addr);
563  void __hwasan_store16_noabort(unsigned long addr);
564  void __hwasan_loadN_noabort(unsigned long addr, size_t size);
565  void __hwasan_storeN_noabort(unsigned long addr, size_t size);
566  
567  void __hwasan_tag_memory(unsigned long addr, u8 tag, unsigned long size);
568  
569  #endif /* __MM_KASAN_KASAN_H */
570