1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * arch/arm/kernel/kprobes-test.c
4  *
5  * Copyright (C) 2011 Jon Medhurst <tixy@yxit.co.uk>.
6  */
7 
8 /*
9  * This file contains test code for ARM kprobes.
10  *
11  * The top level function run_all_tests() executes tests for all of the
12  * supported instruction sets: ARM, 16-bit Thumb, and 32-bit Thumb. These tests
13  * fall into two categories; run_api_tests() checks basic functionality of the
14  * kprobes API, and run_test_cases() is a comprehensive test for kprobes
15  * instruction decoding and simulation.
16  *
17  * run_test_cases() first checks the kprobes decoding table for self consistency
18  * (using table_test()) then executes a series of test cases for each of the CPU
19  * instruction forms. coverage_start() and coverage_end() are used to verify
20  * that these test cases cover all of the possible combinations of instructions
21  * described by the kprobes decoding tables.
22  *
23  * The individual test cases are in kprobes-test-arm.c and kprobes-test-thumb.c
24  * which use the macros defined in kprobes-test.h. The rest of this
25  * documentation will describe the operation of the framework used by these
26  * test cases.
27  */
28 
29 /*
30  * TESTING METHODOLOGY
31  * -------------------
32  *
33  * The methodology used to test an ARM instruction 'test_insn' is to use
34  * inline assembler like:
35  *
36  * test_before: nop
37  * test_case:	test_insn
38  * test_after:	nop
39  *
40  * When the test case is run a kprobe is placed of each nop. The
41  * post-handler of the test_before probe is used to modify the saved CPU
42  * register context to that which we require for the test case. The
43  * pre-handler of the of the test_after probe saves a copy of the CPU
44  * register context. In this way we can execute test_insn with a specific
45  * register context and see the results afterwards.
46  *
47  * To actually test the kprobes instruction emulation we perform the above
48  * step a second time but with an additional kprobe on the test_case
49  * instruction itself. If the emulation is accurate then the results seen
50  * by the test_after probe will be identical to the first run which didn't
51  * have a probe on test_case.
52  *
53  * Each test case is run several times with a variety of variations in the
54  * flags value of stored in CPSR, and for Thumb code, different ITState.
55  *
56  * For instructions which can modify PC, a second test_after probe is used
57  * like this:
58  *
59  * test_before: nop
60  * test_case:	test_insn
61  * test_after:	nop
62  *		b test_done
63  * test_after2: nop
64  * test_done:
65  *
66  * The test case is constructed such that test_insn branches to
67  * test_after2, or, if testing a conditional instruction, it may just
68  * continue to test_after. The probes inserted at both locations let us
69  * determine which happened. A similar approach is used for testing
70  * backwards branches...
71  *
72  *		b test_before
73  *		b test_done  @ helps to cope with off by 1 branches
74  * test_after2: nop
75  *		b test_done
76  * test_before: nop
77  * test_case:	test_insn
78  * test_after:	nop
79  * test_done:
80  *
81  * The macros used to generate the assembler instructions describe above
82  * are TEST_INSTRUCTION, TEST_BRANCH_F (branch forwards) and TEST_BRANCH_B
83  * (branch backwards). In these, the local variables numbered 1, 50, 2 and
84  * 99 represent: test_before, test_case, test_after2 and test_done.
85  *
86  * FRAMEWORK
87  * ---------
88  *
89  * Each test case is wrapped between the pair of macros TESTCASE_START and
90  * TESTCASE_END. As well as performing the inline assembler boilerplate,
91  * these call out to the kprobes_test_case_start() and
92  * kprobes_test_case_end() functions which drive the execution of the test
93  * case. The specific arguments to use for each test case are stored as
94  * inline data constructed using the various TEST_ARG_* macros. Putting
95  * this all together, a simple test case may look like:
96  *
97  *	TESTCASE_START("Testing mov r0, r7")
98  *	TEST_ARG_REG(7, 0x12345678) // Set r7=0x12345678
99  *	TEST_ARG_END("")
100  *	TEST_INSTRUCTION("mov r0, r7")
101  *	TESTCASE_END
102  *
103  * Note, in practice the single convenience macro TEST_R would be used for this
104  * instead.
105  *
106  * The above would expand to assembler looking something like:
107  *
108  *	@ TESTCASE_START
109  *	bl	__kprobes_test_case_start
110  *	.pushsection .rodata
111  *	"10:
112  *	.ascii "mov r0, r7"	@ text title for test case
113  *	.byte	0
114  *	.popsection
115  *	@ start of inline data...
116  *	.word	10b		@ pointer to title in .rodata section
117  *
118  *	@ TEST_ARG_REG
119  *	.byte	ARG_TYPE_REG
120  *	.byte	7
121  *	.short	0
122  *	.word	0x1234567
123  *
124  *	@ TEST_ARG_END
125  *	.byte	ARG_TYPE_END
126  *	.byte	TEST_ISA	@ flags, including ISA being tested
127  *	.short	50f-0f		@ offset of 'test_before'
128  *	.short	2f-0f		@ offset of 'test_after2' (if relevent)
129  *	.short	99f-0f		@ offset of 'test_done'
130  *	@ start of test case code...
131  *	0:
132  *	.code	TEST_ISA	@ switch to ISA being tested
133  *
134  *	@ TEST_INSTRUCTION
135  *	50:	nop		@ location for 'test_before' probe
136  *	1:	mov r0, r7	@ the test case instruction 'test_insn'
137  *		nop		@ location for 'test_after' probe
138  *
139  *	// TESTCASE_END
140  *	2:
141  *	99:	bl __kprobes_test_case_end_##TEST_ISA
142  *	.code	NONMAL_ISA
143  *
144  * When the above is execute the following happens...
145  *
146  * __kprobes_test_case_start() is an assembler wrapper which sets up space
147  * for a stack buffer and calls the C function kprobes_test_case_start().
148  * This C function will do some initial processing of the inline data and
149  * setup some global state. It then inserts the test_before and test_after
150  * kprobes and returns a value which causes the assembler wrapper to jump
151  * to the start of the test case code, (local label '0').
152  *
153  * When the test case code executes, the test_before probe will be hit and
154  * test_before_post_handler will call setup_test_context(). This fills the
155  * stack buffer and CPU registers with a test pattern and then processes
156  * the test case arguments. In our example there is one TEST_ARG_REG which
157  * indicates that R7 should be loaded with the value 0x12345678.
158  *
159  * When the test_before probe ends, the test case continues and executes
160  * the "mov r0, r7" instruction. It then hits the test_after probe and the
161  * pre-handler for this (test_after_pre_handler) will save a copy of the
162  * CPU register context. This should now have R0 holding the same value as
163  * R7.
164  *
165  * Finally we get to the call to __kprobes_test_case_end_{32,16}. This is
166  * an assembler wrapper which switches back to the ISA used by the test
167  * code and calls the C function kprobes_test_case_end().
168  *
169  * For each run through the test case, test_case_run_count is incremented
170  * by one. For even runs, kprobes_test_case_end() saves a copy of the
171  * register and stack buffer contents from the test case just run. It then
172  * inserts a kprobe on the test case instruction 'test_insn' and returns a
173  * value to cause the test case code to be re-run.
174  *
175  * For odd numbered runs, kprobes_test_case_end() compares the register and
176  * stack buffer contents to those that were saved on the previous even
177  * numbered run (the one without the kprobe on test_insn). These should be
178  * the same if the kprobe instruction simulation routine is correct.
179  *
180  * The pair of test case runs is repeated with different combinations of
181  * flag values in CPSR and, for Thumb, different ITState. This is
182  * controlled by test_context_cpsr().
183  *
184  * BUILDING TEST CASES
185  * -------------------
186  *
187  *
188  * As an aid to building test cases, the stack buffer is initialised with
189  * some special values:
190  *
191  *   [SP+13*4]	Contains SP+120. This can be used to test instructions
192  *		which load a value into SP.
193  *
194  *   [SP+15*4]	When testing branching instructions using TEST_BRANCH_{F,B},
195  *		this holds the target address of the branch, 'test_after2'.
196  *		This can be used to test instructions which load a PC value
197  *		from memory.
198  */
199 
200 #include <linux/kernel.h>
201 #include <linux/module.h>
202 #include <linux/slab.h>
203 #include <linux/sched/clock.h>
204 #include <linux/kprobes.h>
205 #include <linux/errno.h>
206 #include <linux/stddef.h>
207 #include <linux/bug.h>
208 #include <asm/opcodes.h>
209 
210 #include "core.h"
211 #include "test-core.h"
212 #include "../decode-arm.h"
213 #include "../decode-thumb.h"
214 
215 
216 #define BENCHMARKING	1
217 
218 
219 /*
220  * Test basic API
221  */
222 
223 static bool test_regs_ok;
224 static int test_func_instance;
225 static int pre_handler_called;
226 static int post_handler_called;
227 static int kretprobe_handler_called;
228 static int tests_failed;
229 
230 #define FUNC_ARG1 0x12345678
231 #define FUNC_ARG2 0xabcdef
232 
233 
234 #ifndef CONFIG_THUMB2_KERNEL
235 
236 #define RET(reg)	"mov	pc, "#reg
237 
238 long arm_func(long r0, long r1);
239 
240 static void __used __naked __arm_kprobes_test_func(void)
241 {
242 	__asm__ __volatile__ (
243 		".arm					\n\t"
244 		".type arm_func, %%function		\n\t"
245 		"arm_func:				\n\t"
246 		"adds	r0, r0, r1			\n\t"
247 		"mov	pc, lr				\n\t"
248 		".code "NORMAL_ISA	 /* Back to Thumb if necessary */
249 		: : : "r0", "r1", "cc"
250 	);
251 }
252 
253 #else /* CONFIG_THUMB2_KERNEL */
254 
255 #define RET(reg)	"bx	"#reg
256 
257 long thumb16_func(long r0, long r1);
258 long thumb32even_func(long r0, long r1);
259 long thumb32odd_func(long r0, long r1);
260 
261 static void __used __naked __thumb_kprobes_test_funcs(void)
262 {
263 	__asm__ __volatile__ (
264 		".type thumb16_func, %%function		\n\t"
265 		"thumb16_func:				\n\t"
266 		"adds.n	r0, r0, r1			\n\t"
267 		"bx	lr				\n\t"
268 
269 		".align					\n\t"
270 		".type thumb32even_func, %%function	\n\t"
271 		"thumb32even_func:			\n\t"
272 		"adds.w	r0, r0, r1			\n\t"
273 		"bx	lr				\n\t"
274 
275 		".align					\n\t"
276 		"nop.n					\n\t"
277 		".type thumb32odd_func, %%function	\n\t"
278 		"thumb32odd_func:			\n\t"
279 		"adds.w	r0, r0, r1			\n\t"
280 		"bx	lr				\n\t"
281 
282 		: : : "r0", "r1", "cc"
283 	);
284 }
285 
286 #endif /* CONFIG_THUMB2_KERNEL */
287 
288 
289 static int call_test_func(long (*func)(long, long), bool check_test_regs)
290 {
291 	long ret;
292 
293 	++test_func_instance;
294 	test_regs_ok = false;
295 
296 	ret = (*func)(FUNC_ARG1, FUNC_ARG2);
297 	if (ret != FUNC_ARG1 + FUNC_ARG2) {
298 		pr_err("FAIL: call_test_func: func returned %lx\n", ret);
299 		return false;
300 	}
301 
302 	if (check_test_regs && !test_regs_ok) {
303 		pr_err("FAIL: test regs not OK\n");
304 		return false;
305 	}
306 
307 	return true;
308 }
309 
310 static int __kprobes pre_handler(struct kprobe *p, struct pt_regs *regs)
311 {
312 	pre_handler_called = test_func_instance;
313 	if (regs->ARM_r0 == FUNC_ARG1 && regs->ARM_r1 == FUNC_ARG2)
314 		test_regs_ok = true;
315 	return 0;
316 }
317 
318 static void __kprobes post_handler(struct kprobe *p, struct pt_regs *regs,
319 				unsigned long flags)
320 {
321 	post_handler_called = test_func_instance;
322 	if (regs->ARM_r0 != FUNC_ARG1 + FUNC_ARG2 || regs->ARM_r1 != FUNC_ARG2)
323 		test_regs_ok = false;
324 }
325 
326 static struct kprobe the_kprobe = {
327 	.addr		= 0,
328 	.pre_handler	= pre_handler,
329 	.post_handler	= post_handler
330 };
331 
332 static int test_kprobe(long (*func)(long, long))
333 {
334 	int ret;
335 
336 	the_kprobe.addr = (kprobe_opcode_t *)func;
337 	ret = register_kprobe(&the_kprobe);
338 	if (ret < 0) {
339 		pr_err("FAIL: register_kprobe failed with %d\n", ret);
340 		return ret;
341 	}
342 
343 	ret = call_test_func(func, true);
344 
345 	unregister_kprobe(&the_kprobe);
346 	the_kprobe.flags = 0; /* Clear disable flag to allow reuse */
347 
348 	if (!ret)
349 		return -EINVAL;
350 	if (pre_handler_called != test_func_instance) {
351 		pr_err("FAIL: kprobe pre_handler not called\n");
352 		return -EINVAL;
353 	}
354 	if (post_handler_called != test_func_instance) {
355 		pr_err("FAIL: kprobe post_handler not called\n");
356 		return -EINVAL;
357 	}
358 	if (!call_test_func(func, false))
359 		return -EINVAL;
360 	if (pre_handler_called == test_func_instance ||
361 				post_handler_called == test_func_instance) {
362 		pr_err("FAIL: probe called after unregistering\n");
363 		return -EINVAL;
364 	}
365 
366 	return 0;
367 }
368 
369 static int __kprobes
370 kretprobe_handler(struct kretprobe_instance *ri, struct pt_regs *regs)
371 {
372 	kretprobe_handler_called = test_func_instance;
373 	if (regs_return_value(regs) == FUNC_ARG1 + FUNC_ARG2)
374 		test_regs_ok = true;
375 	return 0;
376 }
377 
378 static struct kretprobe the_kretprobe = {
379 	.handler	= kretprobe_handler,
380 };
381 
382 static int test_kretprobe(long (*func)(long, long))
383 {
384 	int ret;
385 
386 	the_kretprobe.kp.addr = (kprobe_opcode_t *)func;
387 	ret = register_kretprobe(&the_kretprobe);
388 	if (ret < 0) {
389 		pr_err("FAIL: register_kretprobe failed with %d\n", ret);
390 		return ret;
391 	}
392 
393 	ret = call_test_func(func, true);
394 
395 	unregister_kretprobe(&the_kretprobe);
396 	the_kretprobe.kp.flags = 0; /* Clear disable flag to allow reuse */
397 
398 	if (!ret)
399 		return -EINVAL;
400 	if (kretprobe_handler_called != test_func_instance) {
401 		pr_err("FAIL: kretprobe handler not called\n");
402 		return -EINVAL;
403 	}
404 	if (!call_test_func(func, false))
405 		return -EINVAL;
406 	if (kretprobe_handler_called == test_func_instance) {
407 		pr_err("FAIL: kretprobe called after unregistering\n");
408 		return -EINVAL;
409 	}
410 
411 	return 0;
412 }
413 
414 static int run_api_tests(long (*func)(long, long))
415 {
416 	int ret;
417 
418 	pr_info("    kprobe\n");
419 	ret = test_kprobe(func);
420 	if (ret < 0)
421 		return ret;
422 
423 	pr_info("    kretprobe\n");
424 	ret = test_kretprobe(func);
425 	if (ret < 0)
426 		return ret;
427 
428 	return 0;
429 }
430 
431 
432 /*
433  * Benchmarking
434  */
435 
436 #if BENCHMARKING
437 
438 static void __naked benchmark_nop(void)
439 {
440 	__asm__ __volatile__ (
441 		"nop		\n\t"
442 		RET(lr)"	\n\t"
443 	);
444 }
445 
446 #ifdef CONFIG_THUMB2_KERNEL
447 #define wide ".w"
448 #else
449 #define wide
450 #endif
451 
452 static void __naked benchmark_pushpop1(void)
453 {
454 	__asm__ __volatile__ (
455 		"stmdb"wide"	sp!, {r3-r11,lr}  \n\t"
456 		"ldmia"wide"	sp!, {r3-r11,pc}"
457 	);
458 }
459 
460 static void __naked benchmark_pushpop2(void)
461 {
462 	__asm__ __volatile__ (
463 		"stmdb"wide"	sp!, {r0-r8,lr}  \n\t"
464 		"ldmia"wide"	sp!, {r0-r8,pc}"
465 	);
466 }
467 
468 static void __naked benchmark_pushpop3(void)
469 {
470 	__asm__ __volatile__ (
471 		"stmdb"wide"	sp!, {r4,lr}  \n\t"
472 		"ldmia"wide"	sp!, {r4,pc}"
473 	);
474 }
475 
476 static void __naked benchmark_pushpop4(void)
477 {
478 	__asm__ __volatile__ (
479 		"stmdb"wide"	sp!, {r0,lr}  \n\t"
480 		"ldmia"wide"	sp!, {r0,pc}"
481 	);
482 }
483 
484 
485 #ifdef CONFIG_THUMB2_KERNEL
486 
487 static void __naked benchmark_pushpop_thumb(void)
488 {
489 	__asm__ __volatile__ (
490 		"push.n	{r0-r7,lr}  \n\t"
491 		"pop.n	{r0-r7,pc}"
492 	);
493 }
494 
495 #endif
496 
497 static int __kprobes
498 benchmark_pre_handler(struct kprobe *p, struct pt_regs *regs)
499 {
500 	return 0;
501 }
502 
503 static int benchmark(void(*fn)(void))
504 {
505 	unsigned n, i, t, t0;
506 
507 	for (n = 1000; ; n *= 2) {
508 		t0 = sched_clock();
509 		for (i = n; i > 0; --i)
510 			fn();
511 		t = sched_clock() - t0;
512 		if (t >= 250000000)
513 			break; /* Stop once we took more than 0.25 seconds */
514 	}
515 	return t / n; /* Time for one iteration in nanoseconds */
516 };
517 
518 static int kprobe_benchmark(void(*fn)(void), unsigned offset)
519 {
520 	struct kprobe k = {
521 		.addr		= (kprobe_opcode_t *)((uintptr_t)fn + offset),
522 		.pre_handler	= benchmark_pre_handler,
523 	};
524 
525 	int ret = register_kprobe(&k);
526 	if (ret < 0) {
527 		pr_err("FAIL: register_kprobe failed with %d\n", ret);
528 		return ret;
529 	}
530 
531 	ret = benchmark(fn);
532 
533 	unregister_kprobe(&k);
534 	return ret;
535 };
536 
537 struct benchmarks {
538 	void		(*fn)(void);
539 	unsigned	offset;
540 	const char	*title;
541 };
542 
543 static int run_benchmarks(void)
544 {
545 	int ret;
546 	struct benchmarks list[] = {
547 		{&benchmark_nop, 0, "nop"},
548 		/*
549 		 * benchmark_pushpop{1,3} will have the optimised
550 		 * instruction emulation, whilst benchmark_pushpop{2,4} will
551 		 * be the equivalent unoptimised instructions.
552 		 */
553 		{&benchmark_pushpop1, 0, "stmdb	sp!, {r3-r11,lr}"},
554 		{&benchmark_pushpop1, 4, "ldmia	sp!, {r3-r11,pc}"},
555 		{&benchmark_pushpop2, 0, "stmdb	sp!, {r0-r8,lr}"},
556 		{&benchmark_pushpop2, 4, "ldmia	sp!, {r0-r8,pc}"},
557 		{&benchmark_pushpop3, 0, "stmdb	sp!, {r4,lr}"},
558 		{&benchmark_pushpop3, 4, "ldmia	sp!, {r4,pc}"},
559 		{&benchmark_pushpop4, 0, "stmdb	sp!, {r0,lr}"},
560 		{&benchmark_pushpop4, 4, "ldmia	sp!, {r0,pc}"},
561 #ifdef CONFIG_THUMB2_KERNEL
562 		{&benchmark_pushpop_thumb, 0, "push.n	{r0-r7,lr}"},
563 		{&benchmark_pushpop_thumb, 2, "pop.n	{r0-r7,pc}"},
564 #endif
565 		{0}
566 	};
567 
568 	struct benchmarks *b;
569 	for (b = list; b->fn; ++b) {
570 		ret = kprobe_benchmark(b->fn, b->offset);
571 		if (ret < 0)
572 			return ret;
573 		pr_info("    %dns for kprobe %s\n", ret, b->title);
574 	}
575 
576 	pr_info("\n");
577 	return 0;
578 }
579 
580 #endif /* BENCHMARKING */
581 
582 
583 /*
584  * Decoding table self-consistency tests
585  */
586 
587 static const int decode_struct_sizes[NUM_DECODE_TYPES] = {
588 	[DECODE_TYPE_TABLE]	= sizeof(struct decode_table),
589 	[DECODE_TYPE_CUSTOM]	= sizeof(struct decode_custom),
590 	[DECODE_TYPE_SIMULATE]	= sizeof(struct decode_simulate),
591 	[DECODE_TYPE_EMULATE]	= sizeof(struct decode_emulate),
592 	[DECODE_TYPE_OR]	= sizeof(struct decode_or),
593 	[DECODE_TYPE_REJECT]	= sizeof(struct decode_reject)
594 };
595 
596 static int table_iter(const union decode_item *table,
597 			int (*fn)(const struct decode_header *, void *),
598 			void *args)
599 {
600 	const struct decode_header *h = (struct decode_header *)table;
601 	int result;
602 
603 	for (;;) {
604 		enum decode_type type = h->type_regs.bits & DECODE_TYPE_MASK;
605 
606 		if (type == DECODE_TYPE_END)
607 			return 0;
608 
609 		result = fn(h, args);
610 		if (result)
611 			return result;
612 
613 		h = (struct decode_header *)
614 			((uintptr_t)h + decode_struct_sizes[type]);
615 
616 	}
617 }
618 
619 static int table_test_fail(const struct decode_header *h, const char* message)
620 {
621 
622 	pr_err("FAIL: kprobes test failure \"%s\" (mask %08x, value %08x)\n",
623 					message, h->mask.bits, h->value.bits);
624 	return -EINVAL;
625 }
626 
627 struct table_test_args {
628 	const union decode_item *root_table;
629 	u32			parent_mask;
630 	u32			parent_value;
631 };
632 
633 static int table_test_fn(const struct decode_header *h, void *args)
634 {
635 	struct table_test_args *a = (struct table_test_args *)args;
636 	enum decode_type type = h->type_regs.bits & DECODE_TYPE_MASK;
637 
638 	if (h->value.bits & ~h->mask.bits)
639 		return table_test_fail(h, "Match value has bits not in mask");
640 
641 	if ((h->mask.bits & a->parent_mask) != a->parent_mask)
642 		return table_test_fail(h, "Mask has bits not in parent mask");
643 
644 	if ((h->value.bits ^ a->parent_value) & a->parent_mask)
645 		return table_test_fail(h, "Value is inconsistent with parent");
646 
647 	if (type == DECODE_TYPE_TABLE) {
648 		struct decode_table *d = (struct decode_table *)h;
649 		struct table_test_args args2 = *a;
650 		args2.parent_mask = h->mask.bits;
651 		args2.parent_value = h->value.bits;
652 		return table_iter(d->table.table, table_test_fn, &args2);
653 	}
654 
655 	return 0;
656 }
657 
658 static int table_test(const union decode_item *table)
659 {
660 	struct table_test_args args = {
661 		.root_table	= table,
662 		.parent_mask	= 0,
663 		.parent_value	= 0
664 	};
665 	return table_iter(args.root_table, table_test_fn, &args);
666 }
667 
668 
669 /*
670  * Decoding table test coverage analysis
671  *
672  * coverage_start() builds a coverage_table which contains a list of
673  * coverage_entry's to match each entry in the specified kprobes instruction
674  * decoding table.
675  *
676  * When test cases are run, coverage_add() is called to process each case.
677  * This looks up the corresponding entry in the coverage_table and sets it as
678  * being matched, as well as clearing the regs flag appropriate for the test.
679  *
680  * After all test cases have been run, coverage_end() is called to check that
681  * all entries in coverage_table have been matched and that all regs flags are
682  * cleared. I.e. that all possible combinations of instructions described by
683  * the kprobes decoding tables have had a test case executed for them.
684  */
685 
686 bool coverage_fail;
687 
688 #define MAX_COVERAGE_ENTRIES 256
689 
690 struct coverage_entry {
691 	const struct decode_header	*header;
692 	unsigned			regs;
693 	unsigned			nesting;
694 	char				matched;
695 };
696 
697 struct coverage_table {
698 	struct coverage_entry	*base;
699 	unsigned		num_entries;
700 	unsigned		nesting;
701 };
702 
703 struct coverage_table coverage;
704 
705 #define COVERAGE_ANY_REG	(1<<0)
706 #define COVERAGE_SP		(1<<1)
707 #define COVERAGE_PC		(1<<2)
708 #define COVERAGE_PCWB		(1<<3)
709 
710 static const char coverage_register_lookup[16] = {
711 	[REG_TYPE_ANY]		= COVERAGE_ANY_REG | COVERAGE_SP | COVERAGE_PC,
712 	[REG_TYPE_SAMEAS16]	= COVERAGE_ANY_REG,
713 	[REG_TYPE_SP]		= COVERAGE_SP,
714 	[REG_TYPE_PC]		= COVERAGE_PC,
715 	[REG_TYPE_NOSP]		= COVERAGE_ANY_REG | COVERAGE_SP,
716 	[REG_TYPE_NOSPPC]	= COVERAGE_ANY_REG | COVERAGE_SP | COVERAGE_PC,
717 	[REG_TYPE_NOPC]		= COVERAGE_ANY_REG | COVERAGE_PC,
718 	[REG_TYPE_NOPCWB]	= COVERAGE_ANY_REG | COVERAGE_PC | COVERAGE_PCWB,
719 	[REG_TYPE_NOPCX]	= COVERAGE_ANY_REG,
720 	[REG_TYPE_NOSPPCX]	= COVERAGE_ANY_REG | COVERAGE_SP,
721 };
722 
723 unsigned coverage_start_registers(const struct decode_header *h)
724 {
725 	unsigned regs = 0;
726 	int i;
727 	for (i = 0; i < 20; i += 4) {
728 		int r = (h->type_regs.bits >> (DECODE_TYPE_BITS + i)) & 0xf;
729 		regs |= coverage_register_lookup[r] << i;
730 	}
731 	return regs;
732 }
733 
734 static int coverage_start_fn(const struct decode_header *h, void *args)
735 {
736 	struct coverage_table *coverage = (struct coverage_table *)args;
737 	enum decode_type type = h->type_regs.bits & DECODE_TYPE_MASK;
738 	struct coverage_entry *entry = coverage->base + coverage->num_entries;
739 
740 	if (coverage->num_entries == MAX_COVERAGE_ENTRIES - 1) {
741 		pr_err("FAIL: Out of space for test coverage data");
742 		return -ENOMEM;
743 	}
744 
745 	++coverage->num_entries;
746 
747 	entry->header = h;
748 	entry->regs = coverage_start_registers(h);
749 	entry->nesting = coverage->nesting;
750 	entry->matched = false;
751 
752 	if (type == DECODE_TYPE_TABLE) {
753 		struct decode_table *d = (struct decode_table *)h;
754 		int ret;
755 		++coverage->nesting;
756 		ret = table_iter(d->table.table, coverage_start_fn, coverage);
757 		--coverage->nesting;
758 		return ret;
759 	}
760 
761 	return 0;
762 }
763 
764 static int coverage_start(const union decode_item *table)
765 {
766 	coverage.base = kmalloc_array(MAX_COVERAGE_ENTRIES,
767 				      sizeof(struct coverage_entry),
768 				      GFP_KERNEL);
769 	coverage.num_entries = 0;
770 	coverage.nesting = 0;
771 	return table_iter(table, coverage_start_fn, &coverage);
772 }
773 
774 static void
775 coverage_add_registers(struct coverage_entry *entry, kprobe_opcode_t insn)
776 {
777 	int regs = entry->header->type_regs.bits >> DECODE_TYPE_BITS;
778 	int i;
779 	for (i = 0; i < 20; i += 4) {
780 		enum decode_reg_type reg_type = (regs >> i) & 0xf;
781 		int reg = (insn >> i) & 0xf;
782 		int flag;
783 
784 		if (!reg_type)
785 			continue;
786 
787 		if (reg == 13)
788 			flag = COVERAGE_SP;
789 		else if (reg == 15)
790 			flag = COVERAGE_PC;
791 		else
792 			flag = COVERAGE_ANY_REG;
793 		entry->regs &= ~(flag << i);
794 
795 		switch (reg_type) {
796 
797 		case REG_TYPE_NONE:
798 		case REG_TYPE_ANY:
799 		case REG_TYPE_SAMEAS16:
800 			break;
801 
802 		case REG_TYPE_SP:
803 			if (reg != 13)
804 				return;
805 			break;
806 
807 		case REG_TYPE_PC:
808 			if (reg != 15)
809 				return;
810 			break;
811 
812 		case REG_TYPE_NOSP:
813 			if (reg == 13)
814 				return;
815 			break;
816 
817 		case REG_TYPE_NOSPPC:
818 		case REG_TYPE_NOSPPCX:
819 			if (reg == 13 || reg == 15)
820 				return;
821 			break;
822 
823 		case REG_TYPE_NOPCWB:
824 			if (!is_writeback(insn))
825 				break;
826 			if (reg == 15) {
827 				entry->regs &= ~(COVERAGE_PCWB << i);
828 				return;
829 			}
830 			break;
831 
832 		case REG_TYPE_NOPC:
833 		case REG_TYPE_NOPCX:
834 			if (reg == 15)
835 				return;
836 			break;
837 		}
838 
839 	}
840 }
841 
842 static void coverage_add(kprobe_opcode_t insn)
843 {
844 	struct coverage_entry *entry = coverage.base;
845 	struct coverage_entry *end = coverage.base + coverage.num_entries;
846 	bool matched = false;
847 	unsigned nesting = 0;
848 
849 	for (; entry < end; ++entry) {
850 		const struct decode_header *h = entry->header;
851 		enum decode_type type = h->type_regs.bits & DECODE_TYPE_MASK;
852 
853 		if (entry->nesting > nesting)
854 			continue; /* Skip sub-table we didn't match */
855 
856 		if (entry->nesting < nesting)
857 			break; /* End of sub-table we were scanning */
858 
859 		if (!matched) {
860 			if ((insn & h->mask.bits) != h->value.bits)
861 				continue;
862 			entry->matched = true;
863 		}
864 
865 		switch (type) {
866 
867 		case DECODE_TYPE_TABLE:
868 			++nesting;
869 			break;
870 
871 		case DECODE_TYPE_CUSTOM:
872 		case DECODE_TYPE_SIMULATE:
873 		case DECODE_TYPE_EMULATE:
874 			coverage_add_registers(entry, insn);
875 			return;
876 
877 		case DECODE_TYPE_OR:
878 			matched = true;
879 			break;
880 
881 		case DECODE_TYPE_REJECT:
882 		default:
883 			return;
884 		}
885 
886 	}
887 }
888 
889 static void coverage_end(void)
890 {
891 	struct coverage_entry *entry = coverage.base;
892 	struct coverage_entry *end = coverage.base + coverage.num_entries;
893 
894 	for (; entry < end; ++entry) {
895 		u32 mask = entry->header->mask.bits;
896 		u32 value = entry->header->value.bits;
897 
898 		if (entry->regs) {
899 			pr_err("FAIL: Register test coverage missing for %08x %08x (%05x)\n",
900 				mask, value, entry->regs);
901 			coverage_fail = true;
902 		}
903 		if (!entry->matched) {
904 			pr_err("FAIL: Test coverage entry missing for %08x %08x\n",
905 				mask, value);
906 			coverage_fail = true;
907 		}
908 	}
909 
910 	kfree(coverage.base);
911 }
912 
913 
914 /*
915  * Framework for instruction set test cases
916  */
917 
918 void __naked __kprobes_test_case_start(void)
919 {
920 	__asm__ __volatile__ (
921 		"mov	r2, sp					\n\t"
922 		"bic	r3, r2, #7				\n\t"
923 		"mov	sp, r3					\n\t"
924 		"stmdb	sp!, {r2-r11}				\n\t"
925 		"sub	sp, sp, #"__stringify(TEST_MEMORY_SIZE)"\n\t"
926 		"bic	r0, lr, #1  @ r0 = inline data		\n\t"
927 		"mov	r1, sp					\n\t"
928 		"bl	kprobes_test_case_start			\n\t"
929 		RET(r0)"					\n\t"
930 	);
931 }
932 
933 #ifndef CONFIG_THUMB2_KERNEL
934 
935 void __naked __kprobes_test_case_end_32(void)
936 {
937 	__asm__ __volatile__ (
938 		"mov	r4, lr					\n\t"
939 		"bl	kprobes_test_case_end			\n\t"
940 		"cmp	r0, #0					\n\t"
941 		"movne	pc, r0					\n\t"
942 		"mov	r0, r4					\n\t"
943 		"add	sp, sp, #"__stringify(TEST_MEMORY_SIZE)"\n\t"
944 		"ldmia	sp!, {r2-r11}				\n\t"
945 		"mov	sp, r2					\n\t"
946 		"mov	pc, r0					\n\t"
947 	);
948 }
949 
950 #else /* CONFIG_THUMB2_KERNEL */
951 
952 void __naked __kprobes_test_case_end_16(void)
953 {
954 	__asm__ __volatile__ (
955 		"mov	r4, lr					\n\t"
956 		"bl	kprobes_test_case_end			\n\t"
957 		"cmp	r0, #0					\n\t"
958 		"bxne	r0					\n\t"
959 		"mov	r0, r4					\n\t"
960 		"add	sp, sp, #"__stringify(TEST_MEMORY_SIZE)"\n\t"
961 		"ldmia	sp!, {r2-r11}				\n\t"
962 		"mov	sp, r2					\n\t"
963 		"bx	r0					\n\t"
964 	);
965 }
966 
967 void __naked __kprobes_test_case_end_32(void)
968 {
969 	__asm__ __volatile__ (
970 		".arm						\n\t"
971 		"orr	lr, lr, #1  @ will return to Thumb code	\n\t"
972 		"ldr	pc, 1f					\n\t"
973 		"1:						\n\t"
974 		".word	__kprobes_test_case_end_16		\n\t"
975 	);
976 }
977 
978 #endif
979 
980 
981 int kprobe_test_flags;
982 int kprobe_test_cc_position;
983 
984 static int test_try_count;
985 static int test_pass_count;
986 static int test_fail_count;
987 
988 static struct pt_regs initial_regs;
989 static struct pt_regs expected_regs;
990 static struct pt_regs result_regs;
991 
992 static u32 expected_memory[TEST_MEMORY_SIZE/sizeof(u32)];
993 
994 static const char *current_title;
995 static struct test_arg *current_args;
996 static u32 *current_stack;
997 static uintptr_t current_branch_target;
998 
999 static uintptr_t current_code_start;
1000 static kprobe_opcode_t current_instruction;
1001 
1002 
1003 #define TEST_CASE_PASSED -1
1004 #define TEST_CASE_FAILED -2
1005 
1006 static int test_case_run_count;
1007 static bool test_case_is_thumb;
1008 static int test_instance;
1009 
1010 static unsigned long test_check_cc(int cc, unsigned long cpsr)
1011 {
1012 	int ret = arm_check_condition(cc << 28, cpsr);
1013 
1014 	return (ret != ARM_OPCODE_CONDTEST_FAIL);
1015 }
1016 
1017 static int is_last_scenario;
1018 static int probe_should_run; /* 0 = no, 1 = yes, -1 = unknown */
1019 static int memory_needs_checking;
1020 
1021 static unsigned long test_context_cpsr(int scenario)
1022 {
1023 	unsigned long cpsr;
1024 
1025 	probe_should_run = 1;
1026 
1027 	/* Default case is that we cycle through 16 combinations of flags */
1028 	cpsr  = (scenario & 0xf) << 28; /* N,Z,C,V flags */
1029 	cpsr |= (scenario & 0xf) << 16; /* GE flags */
1030 	cpsr |= (scenario & 0x1) << 27; /* Toggle Q flag */
1031 
1032 	if (!test_case_is_thumb) {
1033 		/* Testing ARM code */
1034 		int cc = current_instruction >> 28;
1035 
1036 		probe_should_run = test_check_cc(cc, cpsr) != 0;
1037 		if (scenario == 15)
1038 			is_last_scenario = true;
1039 
1040 	} else if (kprobe_test_flags & TEST_FLAG_NO_ITBLOCK) {
1041 		/* Testing Thumb code without setting ITSTATE */
1042 		if (kprobe_test_cc_position) {
1043 			int cc = (current_instruction >> kprobe_test_cc_position) & 0xf;
1044 			probe_should_run = test_check_cc(cc, cpsr) != 0;
1045 		}
1046 
1047 		if (scenario == 15)
1048 			is_last_scenario = true;
1049 
1050 	} else if (kprobe_test_flags & TEST_FLAG_FULL_ITBLOCK) {
1051 		/* Testing Thumb code with all combinations of ITSTATE */
1052 		unsigned x = (scenario >> 4);
1053 		unsigned cond_base = x % 7; /* ITSTATE<7:5> */
1054 		unsigned mask = x / 7 + 2;  /* ITSTATE<4:0>, bits reversed */
1055 
1056 		if (mask > 0x1f) {
1057 			/* Finish by testing state from instruction 'itt al' */
1058 			cond_base = 7;
1059 			mask = 0x4;
1060 			if ((scenario & 0xf) == 0xf)
1061 				is_last_scenario = true;
1062 		}
1063 
1064 		cpsr |= cond_base << 13;	/* ITSTATE<7:5> */
1065 		cpsr |= (mask & 0x1) << 12;	/* ITSTATE<4> */
1066 		cpsr |= (mask & 0x2) << 10;	/* ITSTATE<3> */
1067 		cpsr |= (mask & 0x4) << 8;	/* ITSTATE<2> */
1068 		cpsr |= (mask & 0x8) << 23;	/* ITSTATE<1> */
1069 		cpsr |= (mask & 0x10) << 21;	/* ITSTATE<0> */
1070 
1071 		probe_should_run = test_check_cc((cpsr >> 12) & 0xf, cpsr) != 0;
1072 
1073 	} else {
1074 		/* Testing Thumb code with several combinations of ITSTATE */
1075 		switch (scenario) {
1076 		case 16: /* Clear NZCV flags and 'it eq' state (false as Z=0) */
1077 			cpsr = 0x00000800;
1078 			probe_should_run = 0;
1079 			break;
1080 		case 17: /* Set NZCV flags and 'it vc' state (false as V=1) */
1081 			cpsr = 0xf0007800;
1082 			probe_should_run = 0;
1083 			break;
1084 		case 18: /* Clear NZCV flags and 'it ls' state (true as C=0) */
1085 			cpsr = 0x00009800;
1086 			break;
1087 		case 19: /* Set NZCV flags and 'it cs' state (true as C=1) */
1088 			cpsr = 0xf0002800;
1089 			is_last_scenario = true;
1090 			break;
1091 		}
1092 	}
1093 
1094 	return cpsr;
1095 }
1096 
1097 static void setup_test_context(struct pt_regs *regs)
1098 {
1099 	int scenario = test_case_run_count>>1;
1100 	unsigned long val;
1101 	struct test_arg *args;
1102 	int i;
1103 
1104 	is_last_scenario = false;
1105 	memory_needs_checking = false;
1106 
1107 	/* Initialise test memory on stack */
1108 	val = (scenario & 1) ? VALM : ~VALM;
1109 	for (i = 0; i < TEST_MEMORY_SIZE / sizeof(current_stack[0]); ++i)
1110 		current_stack[i] = val + (i << 8);
1111 	/* Put target of branch on stack for tests which load PC from memory */
1112 	if (current_branch_target)
1113 		current_stack[15] = current_branch_target;
1114 	/* Put a value for SP on stack for tests which load SP from memory */
1115 	current_stack[13] = (u32)current_stack + 120;
1116 
1117 	/* Initialise register values to their default state */
1118 	val = (scenario & 2) ? VALR : ~VALR;
1119 	for (i = 0; i < 13; ++i)
1120 		regs->uregs[i] = val ^ (i << 8);
1121 	regs->ARM_lr = val ^ (14 << 8);
1122 	regs->ARM_cpsr &= ~(APSR_MASK | PSR_IT_MASK);
1123 	regs->ARM_cpsr |= test_context_cpsr(scenario);
1124 
1125 	/* Perform testcase specific register setup  */
1126 	args = current_args;
1127 	for (; args[0].type != ARG_TYPE_END; ++args)
1128 		switch (args[0].type) {
1129 		case ARG_TYPE_REG: {
1130 			struct test_arg_regptr *arg =
1131 				(struct test_arg_regptr *)args;
1132 			regs->uregs[arg->reg] = arg->val;
1133 			break;
1134 		}
1135 		case ARG_TYPE_PTR: {
1136 			struct test_arg_regptr *arg =
1137 				(struct test_arg_regptr *)args;
1138 			regs->uregs[arg->reg] =
1139 				(unsigned long)current_stack + arg->val;
1140 			memory_needs_checking = true;
1141 			/*
1142 			 * Test memory at an address below SP is in danger of
1143 			 * being altered by an interrupt occurring and pushing
1144 			 * data onto the stack. Disable interrupts to stop this.
1145 			 */
1146 			if (arg->reg == 13)
1147 				regs->ARM_cpsr |= PSR_I_BIT;
1148 			break;
1149 		}
1150 		case ARG_TYPE_MEM: {
1151 			struct test_arg_mem *arg = (struct test_arg_mem *)args;
1152 			current_stack[arg->index] = arg->val;
1153 			break;
1154 		}
1155 		default:
1156 			break;
1157 		}
1158 }
1159 
1160 struct test_probe {
1161 	struct kprobe	kprobe;
1162 	bool		registered;
1163 	int		hit;
1164 };
1165 
1166 static void unregister_test_probe(struct test_probe *probe)
1167 {
1168 	if (probe->registered) {
1169 		unregister_kprobe(&probe->kprobe);
1170 		probe->kprobe.flags = 0; /* Clear disable flag to allow reuse */
1171 	}
1172 	probe->registered = false;
1173 }
1174 
1175 static int register_test_probe(struct test_probe *probe)
1176 {
1177 	int ret;
1178 
1179 	if (probe->registered)
1180 		BUG();
1181 
1182 	ret = register_kprobe(&probe->kprobe);
1183 	if (ret >= 0) {
1184 		probe->registered = true;
1185 		probe->hit = -1;
1186 	}
1187 	return ret;
1188 }
1189 
1190 static int __kprobes
1191 test_before_pre_handler(struct kprobe *p, struct pt_regs *regs)
1192 {
1193 	container_of(p, struct test_probe, kprobe)->hit = test_instance;
1194 	return 0;
1195 }
1196 
1197 static void __kprobes
1198 test_before_post_handler(struct kprobe *p, struct pt_regs *regs,
1199 							unsigned long flags)
1200 {
1201 	setup_test_context(regs);
1202 	initial_regs = *regs;
1203 	initial_regs.ARM_cpsr &= ~PSR_IGNORE_BITS;
1204 }
1205 
1206 static int __kprobes
1207 test_case_pre_handler(struct kprobe *p, struct pt_regs *regs)
1208 {
1209 	container_of(p, struct test_probe, kprobe)->hit = test_instance;
1210 	return 0;
1211 }
1212 
1213 static int __kprobes
1214 test_after_pre_handler(struct kprobe *p, struct pt_regs *regs)
1215 {
1216 	struct test_arg *args;
1217 
1218 	if (container_of(p, struct test_probe, kprobe)->hit == test_instance)
1219 		return 0; /* Already run for this test instance */
1220 
1221 	result_regs = *regs;
1222 
1223 	/* Mask out results which are indeterminate */
1224 	result_regs.ARM_cpsr &= ~PSR_IGNORE_BITS;
1225 	for (args = current_args; args[0].type != ARG_TYPE_END; ++args)
1226 		if (args[0].type == ARG_TYPE_REG_MASKED) {
1227 			struct test_arg_regptr *arg =
1228 				(struct test_arg_regptr *)args;
1229 			result_regs.uregs[arg->reg] &= arg->val;
1230 		}
1231 
1232 	/* Undo any changes done to SP by the test case */
1233 	regs->ARM_sp = (unsigned long)current_stack;
1234 	/* Enable interrupts in case setup_test_context disabled them */
1235 	regs->ARM_cpsr &= ~PSR_I_BIT;
1236 
1237 	container_of(p, struct test_probe, kprobe)->hit = test_instance;
1238 	return 0;
1239 }
1240 
1241 static struct test_probe test_before_probe = {
1242 	.kprobe.pre_handler	= test_before_pre_handler,
1243 	.kprobe.post_handler	= test_before_post_handler,
1244 };
1245 
1246 static struct test_probe test_case_probe = {
1247 	.kprobe.pre_handler	= test_case_pre_handler,
1248 };
1249 
1250 static struct test_probe test_after_probe = {
1251 	.kprobe.pre_handler	= test_after_pre_handler,
1252 };
1253 
1254 static struct test_probe test_after2_probe = {
1255 	.kprobe.pre_handler	= test_after_pre_handler,
1256 };
1257 
1258 static void test_case_cleanup(void)
1259 {
1260 	unregister_test_probe(&test_before_probe);
1261 	unregister_test_probe(&test_case_probe);
1262 	unregister_test_probe(&test_after_probe);
1263 	unregister_test_probe(&test_after2_probe);
1264 }
1265 
1266 static void print_registers(struct pt_regs *regs)
1267 {
1268 	pr_err("r0  %08lx | r1  %08lx | r2  %08lx | r3  %08lx\n",
1269 		regs->ARM_r0, regs->ARM_r1, regs->ARM_r2, regs->ARM_r3);
1270 	pr_err("r4  %08lx | r5  %08lx | r6  %08lx | r7  %08lx\n",
1271 		regs->ARM_r4, regs->ARM_r5, regs->ARM_r6, regs->ARM_r7);
1272 	pr_err("r8  %08lx | r9  %08lx | r10 %08lx | r11 %08lx\n",
1273 		regs->ARM_r8, regs->ARM_r9, regs->ARM_r10, regs->ARM_fp);
1274 	pr_err("r12 %08lx | sp  %08lx | lr  %08lx | pc  %08lx\n",
1275 		regs->ARM_ip, regs->ARM_sp, regs->ARM_lr, regs->ARM_pc);
1276 	pr_err("cpsr %08lx\n", regs->ARM_cpsr);
1277 }
1278 
1279 static void print_memory(u32 *mem, size_t size)
1280 {
1281 	int i;
1282 	for (i = 0; i < size / sizeof(u32); i += 4)
1283 		pr_err("%08x %08x %08x %08x\n", mem[i], mem[i+1],
1284 						mem[i+2], mem[i+3]);
1285 }
1286 
1287 static size_t expected_memory_size(u32 *sp)
1288 {
1289 	size_t size = sizeof(expected_memory);
1290 	int offset = (uintptr_t)sp - (uintptr_t)current_stack;
1291 	if (offset > 0)
1292 		size -= offset;
1293 	return size;
1294 }
1295 
1296 static void test_case_failed(const char *message)
1297 {
1298 	test_case_cleanup();
1299 
1300 	pr_err("FAIL: %s\n", message);
1301 	pr_err("FAIL: Test %s\n", current_title);
1302 	pr_err("FAIL: Scenario %d\n", test_case_run_count >> 1);
1303 }
1304 
1305 static unsigned long next_instruction(unsigned long pc)
1306 {
1307 #ifdef CONFIG_THUMB2_KERNEL
1308 	if ((pc & 1) &&
1309 	    !is_wide_instruction(__mem_to_opcode_thumb16(*(u16 *)(pc - 1))))
1310 		return pc + 2;
1311 	else
1312 #endif
1313 	return pc + 4;
1314 }
1315 
1316 static uintptr_t __used kprobes_test_case_start(const char **title, void *stack)
1317 {
1318 	struct test_arg *args;
1319 	struct test_arg_end *end_arg;
1320 	unsigned long test_code;
1321 
1322 	current_title = *title++;
1323 	args = (struct test_arg *)title;
1324 	current_args = args;
1325 	current_stack = stack;
1326 
1327 	++test_try_count;
1328 
1329 	while (args->type != ARG_TYPE_END)
1330 		++args;
1331 	end_arg = (struct test_arg_end *)args;
1332 
1333 	test_code = (unsigned long)(args + 1); /* Code starts after args */
1334 
1335 	test_case_is_thumb = end_arg->flags & ARG_FLAG_THUMB;
1336 	if (test_case_is_thumb)
1337 		test_code |= 1;
1338 
1339 	current_code_start = test_code;
1340 
1341 	current_branch_target = 0;
1342 	if (end_arg->branch_offset != end_arg->end_offset)
1343 		current_branch_target = test_code + end_arg->branch_offset;
1344 
1345 	test_code += end_arg->code_offset;
1346 	test_before_probe.kprobe.addr = (kprobe_opcode_t *)test_code;
1347 
1348 	test_code = next_instruction(test_code);
1349 	test_case_probe.kprobe.addr = (kprobe_opcode_t *)test_code;
1350 
1351 	if (test_case_is_thumb) {
1352 		u16 *p = (u16 *)(test_code & ~1);
1353 		current_instruction = __mem_to_opcode_thumb16(p[0]);
1354 		if (is_wide_instruction(current_instruction)) {
1355 			u16 instr2 = __mem_to_opcode_thumb16(p[1]);
1356 			current_instruction = __opcode_thumb32_compose(current_instruction, instr2);
1357 		}
1358 	} else {
1359 		current_instruction = __mem_to_opcode_arm(*(u32 *)test_code);
1360 	}
1361 
1362 	if (current_title[0] == '.')
1363 		verbose("%s\n", current_title);
1364 	else
1365 		verbose("%s\t@ %0*x\n", current_title,
1366 					test_case_is_thumb ? 4 : 8,
1367 					current_instruction);
1368 
1369 	test_code = next_instruction(test_code);
1370 	test_after_probe.kprobe.addr = (kprobe_opcode_t *)test_code;
1371 
1372 	if (kprobe_test_flags & TEST_FLAG_NARROW_INSTR) {
1373 		if (!test_case_is_thumb ||
1374 			is_wide_instruction(current_instruction)) {
1375 				test_case_failed("expected 16-bit instruction");
1376 				goto fail;
1377 		}
1378 	} else {
1379 		if (test_case_is_thumb &&
1380 			!is_wide_instruction(current_instruction)) {
1381 				test_case_failed("expected 32-bit instruction");
1382 				goto fail;
1383 		}
1384 	}
1385 
1386 	coverage_add(current_instruction);
1387 
1388 	if (end_arg->flags & ARG_FLAG_UNSUPPORTED) {
1389 		if (register_test_probe(&test_case_probe) < 0)
1390 			goto pass;
1391 		test_case_failed("registered probe for unsupported instruction");
1392 		goto fail;
1393 	}
1394 
1395 	if (end_arg->flags & ARG_FLAG_SUPPORTED) {
1396 		if (register_test_probe(&test_case_probe) >= 0)
1397 			goto pass;
1398 		test_case_failed("couldn't register probe for supported instruction");
1399 		goto fail;
1400 	}
1401 
1402 	if (register_test_probe(&test_before_probe) < 0) {
1403 		test_case_failed("register test_before_probe failed");
1404 		goto fail;
1405 	}
1406 	if (register_test_probe(&test_after_probe) < 0) {
1407 		test_case_failed("register test_after_probe failed");
1408 		goto fail;
1409 	}
1410 	if (current_branch_target) {
1411 		test_after2_probe.kprobe.addr =
1412 				(kprobe_opcode_t *)current_branch_target;
1413 		if (register_test_probe(&test_after2_probe) < 0) {
1414 			test_case_failed("register test_after2_probe failed");
1415 			goto fail;
1416 		}
1417 	}
1418 
1419 	/* Start first run of test case */
1420 	test_case_run_count = 0;
1421 	++test_instance;
1422 	return current_code_start;
1423 pass:
1424 	test_case_run_count = TEST_CASE_PASSED;
1425 	return (uintptr_t)test_after_probe.kprobe.addr;
1426 fail:
1427 	test_case_run_count = TEST_CASE_FAILED;
1428 	return (uintptr_t)test_after_probe.kprobe.addr;
1429 }
1430 
1431 static bool check_test_results(void)
1432 {
1433 	size_t mem_size = 0;
1434 	u32 *mem = 0;
1435 
1436 	if (memcmp(&expected_regs, &result_regs, sizeof(expected_regs))) {
1437 		test_case_failed("registers differ");
1438 		goto fail;
1439 	}
1440 
1441 	if (memory_needs_checking) {
1442 		mem = (u32 *)result_regs.ARM_sp;
1443 		mem_size = expected_memory_size(mem);
1444 		if (memcmp(expected_memory, mem, mem_size)) {
1445 			test_case_failed("test memory differs");
1446 			goto fail;
1447 		}
1448 	}
1449 
1450 	return true;
1451 
1452 fail:
1453 	pr_err("initial_regs:\n");
1454 	print_registers(&initial_regs);
1455 	pr_err("expected_regs:\n");
1456 	print_registers(&expected_regs);
1457 	pr_err("result_regs:\n");
1458 	print_registers(&result_regs);
1459 
1460 	if (mem) {
1461 		pr_err("expected_memory:\n");
1462 		print_memory(expected_memory, mem_size);
1463 		pr_err("result_memory:\n");
1464 		print_memory(mem, mem_size);
1465 	}
1466 
1467 	return false;
1468 }
1469 
1470 static uintptr_t __used kprobes_test_case_end(void)
1471 {
1472 	if (test_case_run_count < 0) {
1473 		if (test_case_run_count == TEST_CASE_PASSED)
1474 			/* kprobes_test_case_start did all the needed testing */
1475 			goto pass;
1476 		else
1477 			/* kprobes_test_case_start failed */
1478 			goto fail;
1479 	}
1480 
1481 	if (test_before_probe.hit != test_instance) {
1482 		test_case_failed("test_before_handler not run");
1483 		goto fail;
1484 	}
1485 
1486 	if (test_after_probe.hit != test_instance &&
1487 				test_after2_probe.hit != test_instance) {
1488 		test_case_failed("test_after_handler not run");
1489 		goto fail;
1490 	}
1491 
1492 	/*
1493 	 * Even numbered test runs ran without a probe on the test case so
1494 	 * we can gather reference results. The subsequent odd numbered run
1495 	 * will have the probe inserted.
1496 	*/
1497 	if ((test_case_run_count & 1) == 0) {
1498 		/* Save results from run without probe */
1499 		u32 *mem = (u32 *)result_regs.ARM_sp;
1500 		expected_regs = result_regs;
1501 		memcpy(expected_memory, mem, expected_memory_size(mem));
1502 
1503 		/* Insert probe onto test case instruction */
1504 		if (register_test_probe(&test_case_probe) < 0) {
1505 			test_case_failed("register test_case_probe failed");
1506 			goto fail;
1507 		}
1508 	} else {
1509 		/* Check probe ran as expected */
1510 		if (probe_should_run == 1) {
1511 			if (test_case_probe.hit != test_instance) {
1512 				test_case_failed("test_case_handler not run");
1513 				goto fail;
1514 			}
1515 		} else if (probe_should_run == 0) {
1516 			if (test_case_probe.hit == test_instance) {
1517 				test_case_failed("test_case_handler ran");
1518 				goto fail;
1519 			}
1520 		}
1521 
1522 		/* Remove probe for any subsequent reference run */
1523 		unregister_test_probe(&test_case_probe);
1524 
1525 		if (!check_test_results())
1526 			goto fail;
1527 
1528 		if (is_last_scenario)
1529 			goto pass;
1530 	}
1531 
1532 	/* Do next test run */
1533 	++test_case_run_count;
1534 	++test_instance;
1535 	return current_code_start;
1536 fail:
1537 	++test_fail_count;
1538 	goto end;
1539 pass:
1540 	++test_pass_count;
1541 end:
1542 	test_case_cleanup();
1543 	return 0;
1544 }
1545 
1546 
1547 /*
1548  * Top level test functions
1549  */
1550 
1551 static int run_test_cases(void (*tests)(void), const union decode_item *table)
1552 {
1553 	int ret;
1554 
1555 	pr_info("    Check decoding tables\n");
1556 	ret = table_test(table);
1557 	if (ret)
1558 		return ret;
1559 
1560 	pr_info("    Run test cases\n");
1561 	ret = coverage_start(table);
1562 	if (ret)
1563 		return ret;
1564 
1565 	tests();
1566 
1567 	coverage_end();
1568 	return 0;
1569 }
1570 
1571 
1572 static int __init run_all_tests(void)
1573 {
1574 	int ret = 0;
1575 
1576 	pr_info("Beginning kprobe tests...\n");
1577 
1578 #ifndef CONFIG_THUMB2_KERNEL
1579 
1580 	pr_info("Probe ARM code\n");
1581 	ret = run_api_tests(arm_func);
1582 	if (ret)
1583 		goto out;
1584 
1585 	pr_info("ARM instruction simulation\n");
1586 	ret = run_test_cases(kprobe_arm_test_cases, probes_decode_arm_table);
1587 	if (ret)
1588 		goto out;
1589 
1590 #else /* CONFIG_THUMB2_KERNEL */
1591 
1592 	pr_info("Probe 16-bit Thumb code\n");
1593 	ret = run_api_tests(thumb16_func);
1594 	if (ret)
1595 		goto out;
1596 
1597 	pr_info("Probe 32-bit Thumb code, even halfword\n");
1598 	ret = run_api_tests(thumb32even_func);
1599 	if (ret)
1600 		goto out;
1601 
1602 	pr_info("Probe 32-bit Thumb code, odd halfword\n");
1603 	ret = run_api_tests(thumb32odd_func);
1604 	if (ret)
1605 		goto out;
1606 
1607 	pr_info("16-bit Thumb instruction simulation\n");
1608 	ret = run_test_cases(kprobe_thumb16_test_cases,
1609 				probes_decode_thumb16_table);
1610 	if (ret)
1611 		goto out;
1612 
1613 	pr_info("32-bit Thumb instruction simulation\n");
1614 	ret = run_test_cases(kprobe_thumb32_test_cases,
1615 				probes_decode_thumb32_table);
1616 	if (ret)
1617 		goto out;
1618 #endif
1619 
1620 	pr_info("Total instruction simulation tests=%d, pass=%d fail=%d\n",
1621 		test_try_count, test_pass_count, test_fail_count);
1622 	if (test_fail_count) {
1623 		ret = -EINVAL;
1624 		goto out;
1625 	}
1626 
1627 #if BENCHMARKING
1628 	pr_info("Benchmarks\n");
1629 	ret = run_benchmarks();
1630 	if (ret)
1631 		goto out;
1632 #endif
1633 
1634 #if __LINUX_ARM_ARCH__ >= 7
1635 	/* We are able to run all test cases so coverage should be complete */
1636 	if (coverage_fail) {
1637 		pr_err("FAIL: Test coverage checks failed\n");
1638 		ret = -EINVAL;
1639 		goto out;
1640 	}
1641 #endif
1642 
1643 out:
1644 	if (ret == 0)
1645 		ret = tests_failed;
1646 	if (ret == 0)
1647 		pr_info("Finished kprobe tests OK\n");
1648 	else
1649 		pr_err("kprobe tests failed\n");
1650 
1651 	return ret;
1652 }
1653 
1654 
1655 /*
1656  * Module setup
1657  */
1658 
1659 #ifdef MODULE
1660 
1661 static void __exit kprobe_test_exit(void)
1662 {
1663 }
1664 
1665 module_init(run_all_tests)
1666 module_exit(kprobe_test_exit)
1667 MODULE_LICENSE("GPL");
1668 
1669 #else /* !MODULE */
1670 
1671 late_initcall(run_all_tests);
1672 
1673 #endif
1674