xref: /openbmc/linux/mm/kmsan/hooks.c (revision 4852a805)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * KMSAN hooks for kernel subsystems.
4  *
5  * These functions handle creation of KMSAN metadata for memory allocations.
6  *
7  * Copyright (C) 2018-2022 Google LLC
8  * Author: Alexander Potapenko <glider@google.com>
9  *
10  */
11 
12 #include <linux/cacheflush.h>
13 #include <linux/dma-direction.h>
14 #include <linux/gfp.h>
15 #include <linux/kmsan.h>
16 #include <linux/mm.h>
17 #include <linux/mm_types.h>
18 #include <linux/scatterlist.h>
19 #include <linux/slab.h>
20 #include <linux/uaccess.h>
21 #include <linux/usb.h>
22 
23 #include "../internal.h"
24 #include "../slab.h"
25 #include "kmsan.h"
26 
27 /*
28  * Instrumented functions shouldn't be called under
29  * kmsan_enter_runtime()/kmsan_leave_runtime(), because this will lead to
30  * skipping effects of functions like memset() inside instrumented code.
31  */
32 
kmsan_task_create(struct task_struct * task)33 void kmsan_task_create(struct task_struct *task)
34 {
35 	kmsan_enter_runtime();
36 	kmsan_internal_task_create(task);
37 	kmsan_leave_runtime();
38 }
39 
kmsan_task_exit(struct task_struct * task)40 void kmsan_task_exit(struct task_struct *task)
41 {
42 	struct kmsan_ctx *ctx = &task->kmsan_ctx;
43 
44 	if (!kmsan_enabled || kmsan_in_runtime())
45 		return;
46 
47 	ctx->allow_reporting = false;
48 }
49 
kmsan_slab_alloc(struct kmem_cache * s,void * object,gfp_t flags)50 void kmsan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags)
51 {
52 	if (unlikely(object == NULL))
53 		return;
54 	if (!kmsan_enabled || kmsan_in_runtime())
55 		return;
56 	/*
57 	 * There's a ctor or this is an RCU cache - do nothing. The memory
58 	 * status hasn't changed since last use.
59 	 */
60 	if (s->ctor || (s->flags & SLAB_TYPESAFE_BY_RCU))
61 		return;
62 
63 	kmsan_enter_runtime();
64 	if (flags & __GFP_ZERO)
65 		kmsan_internal_unpoison_memory(object, s->object_size,
66 					       KMSAN_POISON_CHECK);
67 	else
68 		kmsan_internal_poison_memory(object, s->object_size, flags,
69 					     KMSAN_POISON_CHECK);
70 	kmsan_leave_runtime();
71 }
72 
kmsan_slab_free(struct kmem_cache * s,void * object)73 void kmsan_slab_free(struct kmem_cache *s, void *object)
74 {
75 	if (!kmsan_enabled || kmsan_in_runtime())
76 		return;
77 
78 	/* RCU slabs could be legally used after free within the RCU period */
79 	if (unlikely(s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)))
80 		return;
81 	/*
82 	 * If there's a constructor, freed memory must remain in the same state
83 	 * until the next allocation. We cannot save its state to detect
84 	 * use-after-free bugs, instead we just keep it unpoisoned.
85 	 */
86 	if (s->ctor)
87 		return;
88 	kmsan_enter_runtime();
89 	kmsan_internal_poison_memory(object, s->object_size, GFP_KERNEL,
90 				     KMSAN_POISON_CHECK | KMSAN_POISON_FREE);
91 	kmsan_leave_runtime();
92 }
93 
kmsan_kmalloc_large(const void * ptr,size_t size,gfp_t flags)94 void kmsan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
95 {
96 	if (unlikely(ptr == NULL))
97 		return;
98 	if (!kmsan_enabled || kmsan_in_runtime())
99 		return;
100 	kmsan_enter_runtime();
101 	if (flags & __GFP_ZERO)
102 		kmsan_internal_unpoison_memory((void *)ptr, size,
103 					       /*checked*/ true);
104 	else
105 		kmsan_internal_poison_memory((void *)ptr, size, flags,
106 					     KMSAN_POISON_CHECK);
107 	kmsan_leave_runtime();
108 }
109 
kmsan_kfree_large(const void * ptr)110 void kmsan_kfree_large(const void *ptr)
111 {
112 	struct page *page;
113 
114 	if (!kmsan_enabled || kmsan_in_runtime())
115 		return;
116 	kmsan_enter_runtime();
117 	page = virt_to_head_page((void *)ptr);
118 	KMSAN_WARN_ON(ptr != page_address(page));
119 	kmsan_internal_poison_memory((void *)ptr,
120 				     page_size(page),
121 				     GFP_KERNEL,
122 				     KMSAN_POISON_CHECK | KMSAN_POISON_FREE);
123 	kmsan_leave_runtime();
124 }
125 
vmalloc_shadow(unsigned long addr)126 static unsigned long vmalloc_shadow(unsigned long addr)
127 {
128 	return (unsigned long)kmsan_get_metadata((void *)addr,
129 						 KMSAN_META_SHADOW);
130 }
131 
vmalloc_origin(unsigned long addr)132 static unsigned long vmalloc_origin(unsigned long addr)
133 {
134 	return (unsigned long)kmsan_get_metadata((void *)addr,
135 						 KMSAN_META_ORIGIN);
136 }
137 
kmsan_vunmap_range_noflush(unsigned long start,unsigned long end)138 void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end)
139 {
140 	__vunmap_range_noflush(vmalloc_shadow(start), vmalloc_shadow(end));
141 	__vunmap_range_noflush(vmalloc_origin(start), vmalloc_origin(end));
142 	flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
143 	flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
144 }
145 
146 /*
147  * This function creates new shadow/origin pages for the physical pages mapped
148  * into the virtual memory. If those physical pages already had shadow/origin,
149  * those are ignored.
150  */
kmsan_ioremap_page_range(unsigned long start,unsigned long end,phys_addr_t phys_addr,pgprot_t prot,unsigned int page_shift)151 int kmsan_ioremap_page_range(unsigned long start, unsigned long end,
152 			     phys_addr_t phys_addr, pgprot_t prot,
153 			     unsigned int page_shift)
154 {
155 	gfp_t gfp_mask = GFP_KERNEL | __GFP_ZERO;
156 	struct page *shadow, *origin;
157 	unsigned long off = 0;
158 	int nr, err = 0, clean = 0, mapped;
159 
160 	if (!kmsan_enabled || kmsan_in_runtime())
161 		return 0;
162 
163 	nr = (end - start) / PAGE_SIZE;
164 	kmsan_enter_runtime();
165 	for (int i = 0; i < nr; i++, off += PAGE_SIZE, clean = i) {
166 		shadow = alloc_pages(gfp_mask, 1);
167 		origin = alloc_pages(gfp_mask, 1);
168 		if (!shadow || !origin) {
169 			err = -ENOMEM;
170 			goto ret;
171 		}
172 		mapped = __vmap_pages_range_noflush(
173 			vmalloc_shadow(start + off),
174 			vmalloc_shadow(start + off + PAGE_SIZE), prot, &shadow,
175 			PAGE_SHIFT);
176 		if (mapped) {
177 			err = mapped;
178 			goto ret;
179 		}
180 		shadow = NULL;
181 		mapped = __vmap_pages_range_noflush(
182 			vmalloc_origin(start + off),
183 			vmalloc_origin(start + off + PAGE_SIZE), prot, &origin,
184 			PAGE_SHIFT);
185 		if (mapped) {
186 			__vunmap_range_noflush(
187 				vmalloc_shadow(start + off),
188 				vmalloc_shadow(start + off + PAGE_SIZE));
189 			err = mapped;
190 			goto ret;
191 		}
192 		origin = NULL;
193 	}
194 	/* Page mapping loop finished normally, nothing to clean up. */
195 	clean = 0;
196 
197 ret:
198 	if (clean > 0) {
199 		/*
200 		 * Something went wrong. Clean up shadow/origin pages allocated
201 		 * on the last loop iteration, then delete mappings created
202 		 * during the previous iterations.
203 		 */
204 		if (shadow)
205 			__free_pages(shadow, 1);
206 		if (origin)
207 			__free_pages(origin, 1);
208 		__vunmap_range_noflush(
209 			vmalloc_shadow(start),
210 			vmalloc_shadow(start + clean * PAGE_SIZE));
211 		__vunmap_range_noflush(
212 			vmalloc_origin(start),
213 			vmalloc_origin(start + clean * PAGE_SIZE));
214 	}
215 	flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
216 	flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
217 	kmsan_leave_runtime();
218 	return err;
219 }
220 
kmsan_iounmap_page_range(unsigned long start,unsigned long end)221 void kmsan_iounmap_page_range(unsigned long start, unsigned long end)
222 {
223 	unsigned long v_shadow, v_origin;
224 	struct page *shadow, *origin;
225 	int nr;
226 
227 	if (!kmsan_enabled || kmsan_in_runtime())
228 		return;
229 
230 	nr = (end - start) / PAGE_SIZE;
231 	kmsan_enter_runtime();
232 	v_shadow = (unsigned long)vmalloc_shadow(start);
233 	v_origin = (unsigned long)vmalloc_origin(start);
234 	for (int i = 0; i < nr;
235 	     i++, v_shadow += PAGE_SIZE, v_origin += PAGE_SIZE) {
236 		shadow = kmsan_vmalloc_to_page_or_null((void *)v_shadow);
237 		origin = kmsan_vmalloc_to_page_or_null((void *)v_origin);
238 		__vunmap_range_noflush(v_shadow, vmalloc_shadow(end));
239 		__vunmap_range_noflush(v_origin, vmalloc_origin(end));
240 		if (shadow)
241 			__free_pages(shadow, 1);
242 		if (origin)
243 			__free_pages(origin, 1);
244 	}
245 	flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
246 	flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
247 	kmsan_leave_runtime();
248 }
249 
kmsan_copy_to_user(void __user * to,const void * from,size_t to_copy,size_t left)250 void kmsan_copy_to_user(void __user *to, const void *from, size_t to_copy,
251 			size_t left)
252 {
253 	unsigned long ua_flags;
254 
255 	if (!kmsan_enabled || kmsan_in_runtime())
256 		return;
257 	/*
258 	 * At this point we've copied the memory already. It's hard to check it
259 	 * before copying, as the size of actually copied buffer is unknown.
260 	 */
261 
262 	/* copy_to_user() may copy zero bytes. No need to check. */
263 	if (!to_copy)
264 		return;
265 	/* Or maybe copy_to_user() failed to copy anything. */
266 	if (to_copy <= left)
267 		return;
268 
269 	ua_flags = user_access_save();
270 	if ((u64)to < TASK_SIZE) {
271 		/* This is a user memory access, check it. */
272 		kmsan_internal_check_memory((void *)from, to_copy - left, to,
273 					    REASON_COPY_TO_USER);
274 	} else {
275 		/* Otherwise this is a kernel memory access. This happens when a
276 		 * compat syscall passes an argument allocated on the kernel
277 		 * stack to a real syscall.
278 		 * Don't check anything, just copy the shadow of the copied
279 		 * bytes.
280 		 */
281 		kmsan_internal_memmove_metadata((void *)to, (void *)from,
282 						to_copy - left);
283 	}
284 	user_access_restore(ua_flags);
285 }
286 EXPORT_SYMBOL(kmsan_copy_to_user);
287 
288 /* Helper function to check an URB. */
kmsan_handle_urb(const struct urb * urb,bool is_out)289 void kmsan_handle_urb(const struct urb *urb, bool is_out)
290 {
291 	if (!urb)
292 		return;
293 	if (is_out)
294 		kmsan_internal_check_memory(urb->transfer_buffer,
295 					    urb->transfer_buffer_length,
296 					    /*user_addr*/ 0, REASON_SUBMIT_URB);
297 	else
298 		kmsan_internal_unpoison_memory(urb->transfer_buffer,
299 					       urb->transfer_buffer_length,
300 					       /*checked*/ false);
301 }
302 EXPORT_SYMBOL_GPL(kmsan_handle_urb);
303 
kmsan_handle_dma_page(const void * addr,size_t size,enum dma_data_direction dir)304 static void kmsan_handle_dma_page(const void *addr, size_t size,
305 				  enum dma_data_direction dir)
306 {
307 	switch (dir) {
308 	case DMA_BIDIRECTIONAL:
309 		kmsan_internal_check_memory((void *)addr, size, /*user_addr*/ 0,
310 					    REASON_ANY);
311 		kmsan_internal_unpoison_memory((void *)addr, size,
312 					       /*checked*/ false);
313 		break;
314 	case DMA_TO_DEVICE:
315 		kmsan_internal_check_memory((void *)addr, size, /*user_addr*/ 0,
316 					    REASON_ANY);
317 		break;
318 	case DMA_FROM_DEVICE:
319 		kmsan_internal_unpoison_memory((void *)addr, size,
320 					       /*checked*/ false);
321 		break;
322 	case DMA_NONE:
323 		break;
324 	}
325 }
326 
327 /* Helper function to handle DMA data transfers. */
kmsan_handle_dma(struct page * page,size_t offset,size_t size,enum dma_data_direction dir)328 void kmsan_handle_dma(struct page *page, size_t offset, size_t size,
329 		      enum dma_data_direction dir)
330 {
331 	u64 page_offset, to_go, addr;
332 
333 	if (PageHighMem(page))
334 		return;
335 	addr = (u64)page_address(page) + offset;
336 	/*
337 	 * The kernel may occasionally give us adjacent DMA pages not belonging
338 	 * to the same allocation. Process them separately to avoid triggering
339 	 * internal KMSAN checks.
340 	 */
341 	while (size > 0) {
342 		page_offset = offset_in_page(addr);
343 		to_go = min(PAGE_SIZE - page_offset, (u64)size);
344 		kmsan_handle_dma_page((void *)addr, to_go, dir);
345 		addr += to_go;
346 		size -= to_go;
347 	}
348 }
349 
kmsan_handle_dma_sg(struct scatterlist * sg,int nents,enum dma_data_direction dir)350 void kmsan_handle_dma_sg(struct scatterlist *sg, int nents,
351 			 enum dma_data_direction dir)
352 {
353 	struct scatterlist *item;
354 	int i;
355 
356 	for_each_sg(sg, item, nents, i)
357 		kmsan_handle_dma(sg_page(item), item->offset, item->length,
358 				 dir);
359 }
360 
361 /* Functions from kmsan-checks.h follow. */
kmsan_poison_memory(const void * address,size_t size,gfp_t flags)362 void kmsan_poison_memory(const void *address, size_t size, gfp_t flags)
363 {
364 	if (!kmsan_enabled || kmsan_in_runtime())
365 		return;
366 	kmsan_enter_runtime();
367 	/* The users may want to poison/unpoison random memory. */
368 	kmsan_internal_poison_memory((void *)address, size, flags,
369 				     KMSAN_POISON_NOCHECK);
370 	kmsan_leave_runtime();
371 }
372 EXPORT_SYMBOL(kmsan_poison_memory);
373 
kmsan_unpoison_memory(const void * address,size_t size)374 void kmsan_unpoison_memory(const void *address, size_t size)
375 {
376 	unsigned long ua_flags;
377 
378 	if (!kmsan_enabled || kmsan_in_runtime())
379 		return;
380 
381 	ua_flags = user_access_save();
382 	kmsan_enter_runtime();
383 	/* The users may want to poison/unpoison random memory. */
384 	kmsan_internal_unpoison_memory((void *)address, size,
385 				       KMSAN_POISON_NOCHECK);
386 	kmsan_leave_runtime();
387 	user_access_restore(ua_flags);
388 }
389 EXPORT_SYMBOL(kmsan_unpoison_memory);
390 
391 /*
392  * Version of kmsan_unpoison_memory() that can be called from within the KMSAN
393  * runtime.
394  *
395  * Non-instrumented IRQ entry functions receive struct pt_regs from assembly
396  * code. Those regs need to be unpoisoned, otherwise using them will result in
397  * false positives.
398  * Using kmsan_unpoison_memory() is not an option in entry code, because the
399  * return value of in_task() is inconsistent - as a result, certain calls to
400  * kmsan_unpoison_memory() are ignored. kmsan_unpoison_entry_regs() ensures that
401  * the registers are unpoisoned even if kmsan_in_runtime() is true in the early
402  * entry code.
403  */
kmsan_unpoison_entry_regs(const struct pt_regs * regs)404 void kmsan_unpoison_entry_regs(const struct pt_regs *regs)
405 {
406 	unsigned long ua_flags;
407 
408 	if (!kmsan_enabled)
409 		return;
410 
411 	ua_flags = user_access_save();
412 	kmsan_internal_unpoison_memory((void *)regs, sizeof(*regs),
413 				       KMSAN_POISON_NOCHECK);
414 	user_access_restore(ua_flags);
415 }
416 
kmsan_check_memory(const void * addr,size_t size)417 void kmsan_check_memory(const void *addr, size_t size)
418 {
419 	if (!kmsan_enabled)
420 		return;
421 	return kmsan_internal_check_memory((void *)addr, size, /*user_addr*/ 0,
422 					   REASON_ANY);
423 }
424 EXPORT_SYMBOL(kmsan_check_memory);
425