1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  Copyright (C) 2014 ARM Limited
4  */
5 
6 #include <linux/cpu.h>
7 #include <linux/init.h>
8 #include <linux/list.h>
9 #include <linux/perf_event.h>
10 #include <linux/sched.h>
11 #include <linux/slab.h>
12 #include <linux/sysctl.h>
13 #include <linux/uaccess.h>
14 
15 #include <asm/cpufeature.h>
16 #include <asm/insn.h>
17 #include <asm/sysreg.h>
18 #include <asm/system_misc.h>
19 #include <asm/traps.h>
20 
21 #define CREATE_TRACE_POINTS
22 #include "trace-events-emulation.h"
23 
24 /*
25  * The runtime support for deprecated instruction support can be in one of
26  * following three states -
27  *
28  * 0 = undef
29  * 1 = emulate (software emulation)
30  * 2 = hw (supported in hardware)
31  */
32 enum insn_emulation_mode {
33 	INSN_UNDEF,
34 	INSN_EMULATE,
35 	INSN_HW,
36 };
37 
38 enum legacy_insn_status {
39 	INSN_DEPRECATED,
40 	INSN_OBSOLETE,
41 	INSN_UNAVAILABLE,
42 };
43 
44 struct insn_emulation {
45 	const char			*name;
46 	enum legacy_insn_status		status;
47 	bool				(*try_emulate)(struct pt_regs *regs,
48 						       u32 insn);
49 	int				(*set_hw_mode)(bool enable);
50 
51 	int current_mode;
52 	int min;
53 	int max;
54 
55 	/*
56 	 * sysctl for this emulation + a sentinal entry.
57 	 */
58 	struct ctl_table sysctl[2];
59 };
60 
61 #define ARM_OPCODE_CONDTEST_FAIL   0
62 #define ARM_OPCODE_CONDTEST_PASS   1
63 #define ARM_OPCODE_CONDTEST_UNCOND 2
64 
65 #define	ARM_OPCODE_CONDITION_UNCOND	0xf
66 
67 static unsigned int __maybe_unused aarch32_check_condition(u32 opcode, u32 psr)
68 {
69 	u32 cc_bits  = opcode >> 28;
70 
71 	if (cc_bits != ARM_OPCODE_CONDITION_UNCOND) {
72 		if ((*aarch32_opcode_cond_checks[cc_bits])(psr))
73 			return ARM_OPCODE_CONDTEST_PASS;
74 		else
75 			return ARM_OPCODE_CONDTEST_FAIL;
76 	}
77 	return ARM_OPCODE_CONDTEST_UNCOND;
78 }
79 
80 #ifdef CONFIG_SWP_EMULATION
81 /*
82  *  Implement emulation of the SWP/SWPB instructions using load-exclusive and
83  *  store-exclusive.
84  *
85  *  Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>]
86  *  Where: Rt  = destination
87  *	   Rt2 = source
88  *	   Rn  = address
89  */
90 
91 /*
92  * Error-checking SWP macros implemented using ldxr{b}/stxr{b}
93  */
94 
95 /* Arbitrary constant to ensure forward-progress of the LL/SC loop */
96 #define __SWP_LL_SC_LOOPS	4
97 
98 #define __user_swpX_asm(data, addr, res, temp, temp2, B)	\
99 do {								\
100 	uaccess_enable_privileged();				\
101 	__asm__ __volatile__(					\
102 	"	mov		%w3, %w6\n"			\
103 	"0:	ldxr"B"		%w2, [%4]\n"			\
104 	"1:	stxr"B"		%w0, %w1, [%4]\n"		\
105 	"	cbz		%w0, 2f\n"			\
106 	"	sub		%w3, %w3, #1\n"			\
107 	"	cbnz		%w3, 0b\n"			\
108 	"	mov		%w0, %w5\n"			\
109 	"	b		3f\n"				\
110 	"2:\n"							\
111 	"	mov		%w1, %w2\n"			\
112 	"3:\n"							\
113 	_ASM_EXTABLE_UACCESS_ERR(0b, 3b, %w0)			\
114 	_ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w0)			\
115 	: "=&r" (res), "+r" (data), "=&r" (temp), "=&r" (temp2)	\
116 	: "r" ((unsigned long)addr), "i" (-EAGAIN),		\
117 	  "i" (__SWP_LL_SC_LOOPS)				\
118 	: "memory");						\
119 	uaccess_disable_privileged();				\
120 } while (0)
121 
122 #define __user_swp_asm(data, addr, res, temp, temp2) \
123 	__user_swpX_asm(data, addr, res, temp, temp2, "")
124 #define __user_swpb_asm(data, addr, res, temp, temp2) \
125 	__user_swpX_asm(data, addr, res, temp, temp2, "b")
126 
127 /*
128  * Bit 22 of the instruction encoding distinguishes between
129  * the SWP and SWPB variants (bit set means SWPB).
130  */
131 #define TYPE_SWPB (1 << 22)
132 
133 static int emulate_swpX(unsigned int address, unsigned int *data,
134 			unsigned int type)
135 {
136 	unsigned int res = 0;
137 
138 	if ((type != TYPE_SWPB) && (address & 0x3)) {
139 		/* SWP to unaligned address not permitted */
140 		pr_debug("SWP instruction on unaligned pointer!\n");
141 		return -EFAULT;
142 	}
143 
144 	while (1) {
145 		unsigned long temp, temp2;
146 
147 		if (type == TYPE_SWPB)
148 			__user_swpb_asm(*data, address, res, temp, temp2);
149 		else
150 			__user_swp_asm(*data, address, res, temp, temp2);
151 
152 		if (likely(res != -EAGAIN) || signal_pending(current))
153 			break;
154 
155 		cond_resched();
156 	}
157 
158 	return res;
159 }
160 
161 /*
162  * swp_handler logs the id of calling process, dissects the instruction, sanity
163  * checks the memory location, calls emulate_swpX for the actual operation and
164  * deals with fixup/error handling before returning
165  */
166 static int swp_handler(struct pt_regs *regs, u32 instr)
167 {
168 	u32 destreg, data, type, address = 0;
169 	const void __user *user_ptr;
170 	int rn, rt2, res = 0;
171 
172 	perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
173 
174 	type = instr & TYPE_SWPB;
175 
176 	switch (aarch32_check_condition(instr, regs->pstate)) {
177 	case ARM_OPCODE_CONDTEST_PASS:
178 		break;
179 	case ARM_OPCODE_CONDTEST_FAIL:
180 		/* Condition failed - return to next instruction */
181 		goto ret;
182 	case ARM_OPCODE_CONDTEST_UNCOND:
183 		/* If unconditional encoding - not a SWP, undef */
184 		return -EFAULT;
185 	default:
186 		return -EINVAL;
187 	}
188 
189 	rn = aarch32_insn_extract_reg_num(instr, A32_RN_OFFSET);
190 	rt2 = aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET);
191 
192 	address = (u32)regs->user_regs.regs[rn];
193 	data	= (u32)regs->user_regs.regs[rt2];
194 	destreg = aarch32_insn_extract_reg_num(instr, A32_RT_OFFSET);
195 
196 	pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n",
197 		rn, address, destreg,
198 		aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET), data);
199 
200 	/* Check access in reasonable access range for both SWP and SWPB */
201 	user_ptr = (const void __user *)(unsigned long)(address & ~3);
202 	if (!access_ok(user_ptr, 4)) {
203 		pr_debug("SWP{B} emulation: access to 0x%08x not allowed!\n",
204 			address);
205 		goto fault;
206 	}
207 
208 	res = emulate_swpX(address, &data, type);
209 	if (res == -EFAULT)
210 		goto fault;
211 	else if (res == 0)
212 		regs->user_regs.regs[destreg] = data;
213 
214 ret:
215 	if (type == TYPE_SWPB)
216 		trace_instruction_emulation("swpb", regs->pc);
217 	else
218 		trace_instruction_emulation("swp", regs->pc);
219 
220 	pr_warn_ratelimited("\"%s\" (%ld) uses obsolete SWP{B} instruction at 0x%llx\n",
221 			current->comm, (unsigned long)current->pid, regs->pc);
222 
223 	arm64_skip_faulting_instruction(regs, 4);
224 	return 0;
225 
226 fault:
227 	pr_debug("SWP{B} emulation: access caused memory abort!\n");
228 	arm64_notify_segfault(address);
229 
230 	return 0;
231 }
232 
233 static bool try_emulate_swp(struct pt_regs *regs, u32 insn)
234 {
235 	/* SWP{B} only exists in ARM state and does not exist in Thumb */
236 	if (!compat_user_mode(regs) || compat_thumb_mode(regs))
237 		return false;
238 
239 	if ((insn & 0x0fb00ff0) != 0x01000090)
240 		return false;
241 
242 	return swp_handler(regs, insn) == 0;
243 }
244 
245 static struct insn_emulation insn_swp = {
246 	.name = "swp",
247 	.status = INSN_OBSOLETE,
248 	.try_emulate = try_emulate_swp,
249 	.set_hw_mode = NULL,
250 };
251 #endif /* CONFIG_SWP_EMULATION */
252 
253 #ifdef CONFIG_CP15_BARRIER_EMULATION
254 static int cp15barrier_handler(struct pt_regs *regs, u32 instr)
255 {
256 	perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
257 
258 	switch (aarch32_check_condition(instr, regs->pstate)) {
259 	case ARM_OPCODE_CONDTEST_PASS:
260 		break;
261 	case ARM_OPCODE_CONDTEST_FAIL:
262 		/* Condition failed - return to next instruction */
263 		goto ret;
264 	case ARM_OPCODE_CONDTEST_UNCOND:
265 		/* If unconditional encoding - not a barrier instruction */
266 		return -EFAULT;
267 	default:
268 		return -EINVAL;
269 	}
270 
271 	switch (aarch32_insn_mcr_extract_crm(instr)) {
272 	case 10:
273 		/*
274 		 * dmb - mcr p15, 0, Rt, c7, c10, 5
275 		 * dsb - mcr p15, 0, Rt, c7, c10, 4
276 		 */
277 		if (aarch32_insn_mcr_extract_opc2(instr) == 5) {
278 			dmb(sy);
279 			trace_instruction_emulation(
280 				"mcr p15, 0, Rt, c7, c10, 5 ; dmb", regs->pc);
281 		} else {
282 			dsb(sy);
283 			trace_instruction_emulation(
284 				"mcr p15, 0, Rt, c7, c10, 4 ; dsb", regs->pc);
285 		}
286 		break;
287 	case 5:
288 		/*
289 		 * isb - mcr p15, 0, Rt, c7, c5, 4
290 		 *
291 		 * Taking an exception or returning from one acts as an
292 		 * instruction barrier. So no explicit barrier needed here.
293 		 */
294 		trace_instruction_emulation(
295 			"mcr p15, 0, Rt, c7, c5, 4 ; isb", regs->pc);
296 		break;
297 	}
298 
299 ret:
300 	pr_warn_ratelimited("\"%s\" (%ld) uses deprecated CP15 Barrier instruction at 0x%llx\n",
301 			current->comm, (unsigned long)current->pid, regs->pc);
302 
303 	arm64_skip_faulting_instruction(regs, 4);
304 	return 0;
305 }
306 
307 static int cp15_barrier_set_hw_mode(bool enable)
308 {
309 	if (enable)
310 		sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_CP15BEN);
311 	else
312 		sysreg_clear_set(sctlr_el1, SCTLR_EL1_CP15BEN, 0);
313 	return 0;
314 }
315 
316 static bool try_emulate_cp15_barrier(struct pt_regs *regs, u32 insn)
317 {
318 	if (!compat_user_mode(regs) || compat_thumb_mode(regs))
319 		return false;
320 
321 	if ((insn & 0x0fff0fdf) == 0x0e070f9a)
322 		return cp15barrier_handler(regs, insn) == 0;
323 
324 	if ((insn & 0x0fff0fff) == 0x0e070f95)
325 		return cp15barrier_handler(regs, insn) == 0;
326 
327 	return false;
328 }
329 
330 static struct insn_emulation insn_cp15_barrier = {
331 	.name = "cp15_barrier",
332 	.status = INSN_DEPRECATED,
333 	.try_emulate = try_emulate_cp15_barrier,
334 	.set_hw_mode = cp15_barrier_set_hw_mode,
335 };
336 #endif /* CONFIG_CP15_BARRIER_EMULATION */
337 
338 #ifdef CONFIG_SETEND_EMULATION
339 static int setend_set_hw_mode(bool enable)
340 {
341 	if (!cpu_supports_mixed_endian_el0())
342 		return -EINVAL;
343 
344 	if (enable)
345 		sysreg_clear_set(sctlr_el1, SCTLR_EL1_SED, 0);
346 	else
347 		sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_SED);
348 	return 0;
349 }
350 
351 static int compat_setend_handler(struct pt_regs *regs, u32 big_endian)
352 {
353 	char *insn;
354 
355 	perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
356 
357 	if (big_endian) {
358 		insn = "setend be";
359 		regs->pstate |= PSR_AA32_E_BIT;
360 	} else {
361 		insn = "setend le";
362 		regs->pstate &= ~PSR_AA32_E_BIT;
363 	}
364 
365 	trace_instruction_emulation(insn, regs->pc);
366 	pr_warn_ratelimited("\"%s\" (%ld) uses deprecated setend instruction at 0x%llx\n",
367 			current->comm, (unsigned long)current->pid, regs->pc);
368 
369 	return 0;
370 }
371 
372 static int a32_setend_handler(struct pt_regs *regs, u32 instr)
373 {
374 	int rc = compat_setend_handler(regs, (instr >> 9) & 1);
375 	arm64_skip_faulting_instruction(regs, 4);
376 	return rc;
377 }
378 
379 static int t16_setend_handler(struct pt_regs *regs, u32 instr)
380 {
381 	int rc = compat_setend_handler(regs, (instr >> 3) & 1);
382 	arm64_skip_faulting_instruction(regs, 2);
383 	return rc;
384 }
385 
386 static bool try_emulate_setend(struct pt_regs *regs, u32 insn)
387 {
388 	if (compat_thumb_mode(regs) &&
389 	    (insn & 0xfffffff7) == 0x0000b650)
390 		return t16_setend_handler(regs, insn) == 0;
391 
392 	if (compat_user_mode(regs) &&
393 	    (insn & 0xfffffdff) == 0xf1010000)
394 		return a32_setend_handler(regs, insn) == 0;
395 
396 	return false;
397 }
398 
399 static struct insn_emulation insn_setend = {
400 	.name = "setend",
401 	.status = INSN_DEPRECATED,
402 	.try_emulate = try_emulate_setend,
403 	.set_hw_mode = setend_set_hw_mode,
404 };
405 #endif /* CONFIG_SETEND_EMULATION */
406 
407 static struct insn_emulation *insn_emulations[] = {
408 #ifdef CONFIG_SWP_EMULATION
409 	&insn_swp,
410 #endif
411 #ifdef CONFIG_CP15_BARRIER_EMULATION
412 	&insn_cp15_barrier,
413 #endif
414 #ifdef CONFIG_SETEND_EMULATION
415 	&insn_setend,
416 #endif
417 };
418 
419 static DEFINE_MUTEX(insn_emulation_mutex);
420 
421 static void enable_insn_hw_mode(void *data)
422 {
423 	struct insn_emulation *insn = data;
424 	if (insn->set_hw_mode)
425 		insn->set_hw_mode(true);
426 }
427 
428 static void disable_insn_hw_mode(void *data)
429 {
430 	struct insn_emulation *insn = data;
431 	if (insn->set_hw_mode)
432 		insn->set_hw_mode(false);
433 }
434 
435 /* Run set_hw_mode(mode) on all active CPUs */
436 static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable)
437 {
438 	if (!insn->set_hw_mode)
439 		return -EINVAL;
440 	if (enable)
441 		on_each_cpu(enable_insn_hw_mode, (void *)insn, true);
442 	else
443 		on_each_cpu(disable_insn_hw_mode, (void *)insn, true);
444 	return 0;
445 }
446 
447 /*
448  * Run set_hw_mode for all insns on a starting CPU.
449  * Returns:
450  *  0 		- If all the hooks ran successfully.
451  * -EINVAL	- At least one hook is not supported by the CPU.
452  */
453 static int run_all_insn_set_hw_mode(unsigned int cpu)
454 {
455 	int rc = 0;
456 	unsigned long flags;
457 
458 	/*
459 	 * Disable IRQs to serialize against an IPI from
460 	 * run_all_cpu_set_hw_mode(), ensuring the HW is programmed to the most
461 	 * recent enablement state if the two race with one another.
462 	 */
463 	local_irq_save(flags);
464 	for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
465 		struct insn_emulation *insn = insn_emulations[i];
466 		bool enable = READ_ONCE(insn->current_mode) == INSN_HW;
467 		if (insn->set_hw_mode && insn->set_hw_mode(enable)) {
468 			pr_warn("CPU[%u] cannot support the emulation of %s",
469 				cpu, insn->name);
470 			rc = -EINVAL;
471 		}
472 	}
473 	local_irq_restore(flags);
474 
475 	return rc;
476 }
477 
478 static int update_insn_emulation_mode(struct insn_emulation *insn,
479 				       enum insn_emulation_mode prev)
480 {
481 	int ret = 0;
482 
483 	switch (prev) {
484 	case INSN_UNDEF: /* Nothing to be done */
485 		break;
486 	case INSN_EMULATE:
487 		break;
488 	case INSN_HW:
489 		if (!run_all_cpu_set_hw_mode(insn, false))
490 			pr_notice("Disabled %s support\n", insn->name);
491 		break;
492 	}
493 
494 	switch (insn->current_mode) {
495 	case INSN_UNDEF:
496 		break;
497 	case INSN_EMULATE:
498 		break;
499 	case INSN_HW:
500 		ret = run_all_cpu_set_hw_mode(insn, true);
501 		if (!ret)
502 			pr_notice("Enabled %s support\n", insn->name);
503 		break;
504 	}
505 
506 	return ret;
507 }
508 
509 static int emulation_proc_handler(struct ctl_table *table, int write,
510 				  void *buffer, size_t *lenp,
511 				  loff_t *ppos)
512 {
513 	int ret = 0;
514 	struct insn_emulation *insn = container_of(table->data, struct insn_emulation, current_mode);
515 	enum insn_emulation_mode prev_mode = insn->current_mode;
516 
517 	mutex_lock(&insn_emulation_mutex);
518 	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
519 
520 	if (ret || !write || prev_mode == insn->current_mode)
521 		goto ret;
522 
523 	ret = update_insn_emulation_mode(insn, prev_mode);
524 	if (ret) {
525 		/* Mode change failed, revert to previous mode. */
526 		WRITE_ONCE(insn->current_mode, prev_mode);
527 		update_insn_emulation_mode(insn, INSN_UNDEF);
528 	}
529 ret:
530 	mutex_unlock(&insn_emulation_mutex);
531 	return ret;
532 }
533 
534 static void __init register_insn_emulation(struct insn_emulation *insn)
535 {
536 	struct ctl_table *sysctl;
537 
538 	insn->min = INSN_UNDEF;
539 
540 	switch (insn->status) {
541 	case INSN_DEPRECATED:
542 		insn->current_mode = INSN_EMULATE;
543 		/* Disable the HW mode if it was turned on at early boot time */
544 		run_all_cpu_set_hw_mode(insn, false);
545 		insn->max = INSN_HW;
546 		break;
547 	case INSN_OBSOLETE:
548 		insn->current_mode = INSN_UNDEF;
549 		insn->max = INSN_EMULATE;
550 		break;
551 	case INSN_UNAVAILABLE:
552 		insn->current_mode = INSN_UNDEF;
553 		insn->max = INSN_UNDEF;
554 		break;
555 	}
556 
557 	/* Program the HW if required */
558 	update_insn_emulation_mode(insn, INSN_UNDEF);
559 
560 	if (insn->status != INSN_UNAVAILABLE) {
561 		sysctl = &insn->sysctl[0];
562 
563 		sysctl->mode = 0644;
564 		sysctl->maxlen = sizeof(int);
565 
566 		sysctl->procname = insn->name;
567 		sysctl->data = &insn->current_mode;
568 		sysctl->extra1 = &insn->min;
569 		sysctl->extra2 = &insn->max;
570 		sysctl->proc_handler = emulation_proc_handler;
571 
572 		register_sysctl_sz("abi", sysctl, 1);
573 	}
574 }
575 
576 bool try_emulate_armv8_deprecated(struct pt_regs *regs, u32 insn)
577 {
578 	for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
579 		struct insn_emulation *ie = insn_emulations[i];
580 
581 		if (ie->status == INSN_UNAVAILABLE)
582 			continue;
583 
584 		/*
585 		 * A trap may race with the mode being changed
586 		 * INSN_EMULATE<->INSN_HW. Try to emulate the instruction to
587 		 * avoid a spurious UNDEF.
588 		 */
589 		if (READ_ONCE(ie->current_mode) == INSN_UNDEF)
590 			continue;
591 
592 		if (ie->try_emulate(regs, insn))
593 			return true;
594 	}
595 
596 	return false;
597 }
598 
599 /*
600  * Invoked as core_initcall, which guarantees that the instruction
601  * emulation is ready for userspace.
602  */
603 static int __init armv8_deprecated_init(void)
604 {
605 #ifdef CONFIG_SETEND_EMULATION
606 	if (!system_supports_mixed_endian_el0()) {
607 		insn_setend.status = INSN_UNAVAILABLE;
608 		pr_info("setend instruction emulation is not supported on this system\n");
609 	}
610 
611 #endif
612 	for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
613 		struct insn_emulation *ie = insn_emulations[i];
614 
615 		if (ie->status == INSN_UNAVAILABLE)
616 			continue;
617 
618 		register_insn_emulation(ie);
619 	}
620 
621 	cpuhp_setup_state_nocalls(CPUHP_AP_ARM64_ISNDEP_STARTING,
622 				  "arm64/isndep:starting",
623 				  run_all_insn_set_hw_mode, NULL);
624 	return 0;
625 }
626 
627 core_initcall(armv8_deprecated_init);
628