1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2587064b6SPunit Agrawal /*
3587064b6SPunit Agrawal * Copyright (C) 2014 ARM Limited
4587064b6SPunit Agrawal */
5587064b6SPunit Agrawal
6c852f320SPunit Agrawal #include <linux/cpu.h>
7587064b6SPunit Agrawal #include <linux/init.h>
8587064b6SPunit Agrawal #include <linux/list.h>
9bd35a4adSPunit Agrawal #include <linux/perf_event.h>
10bd35a4adSPunit Agrawal #include <linux/sched.h>
11587064b6SPunit Agrawal #include <linux/slab.h>
12587064b6SPunit Agrawal #include <linux/sysctl.h>
1392faa7beSVincenzo Frascino #include <linux/uaccess.h>
14587064b6SPunit Agrawal
15338d4f49SJames Morse #include <asm/cpufeature.h>
16bd35a4adSPunit Agrawal #include <asm/insn.h>
17870828e5SJames Morse #include <asm/sysreg.h>
18bd35a4adSPunit Agrawal #include <asm/system_misc.h>
19587064b6SPunit Agrawal #include <asm/traps.h>
20587064b6SPunit Agrawal
21d784e298SPunit Agrawal #define CREATE_TRACE_POINTS
22d784e298SPunit Agrawal #include "trace-events-emulation.h"
23d784e298SPunit Agrawal
24587064b6SPunit Agrawal /*
25587064b6SPunit Agrawal * The runtime support for deprecated instruction support can be in one of
26587064b6SPunit Agrawal * following three states -
27587064b6SPunit Agrawal *
28587064b6SPunit Agrawal * 0 = undef
29587064b6SPunit Agrawal * 1 = emulate (software emulation)
30587064b6SPunit Agrawal * 2 = hw (supported in hardware)
31587064b6SPunit Agrawal */
32587064b6SPunit Agrawal enum insn_emulation_mode {
33587064b6SPunit Agrawal INSN_UNDEF,
34587064b6SPunit Agrawal INSN_EMULATE,
35587064b6SPunit Agrawal INSN_HW,
36587064b6SPunit Agrawal };
37587064b6SPunit Agrawal
38587064b6SPunit Agrawal enum legacy_insn_status {
39587064b6SPunit Agrawal INSN_DEPRECATED,
40587064b6SPunit Agrawal INSN_OBSOLETE,
41124c49b1SMark Rutland INSN_UNAVAILABLE,
42587064b6SPunit Agrawal };
43587064b6SPunit Agrawal
44b4453cc8SMark Rutland struct insn_emulation {
45587064b6SPunit Agrawal const char *name;
46587064b6SPunit Agrawal enum legacy_insn_status status;
47124c49b1SMark Rutland bool (*try_emulate)(struct pt_regs *regs,
48124c49b1SMark Rutland u32 insn);
49587064b6SPunit Agrawal int (*set_hw_mode)(bool enable);
50124c49b1SMark Rutland
51587064b6SPunit Agrawal int current_mode;
52587064b6SPunit Agrawal int min;
53587064b6SPunit Agrawal int max;
54124c49b1SMark Rutland
55124c49b1SMark Rutland /*
56124c49b1SMark Rutland * sysctl for this emulation + a sentinal entry.
57124c49b1SMark Rutland */
58124c49b1SMark Rutland struct ctl_table sysctl[2];
59587064b6SPunit Agrawal };
60587064b6SPunit Agrawal
610c5f4162SMark Rutland #define ARM_OPCODE_CONDTEST_FAIL 0
620c5f4162SMark Rutland #define ARM_OPCODE_CONDTEST_PASS 1
630c5f4162SMark Rutland #define ARM_OPCODE_CONDTEST_UNCOND 2
640c5f4162SMark Rutland
650c5f4162SMark Rutland #define ARM_OPCODE_CONDITION_UNCOND 0xf
660c5f4162SMark Rutland
aarch32_check_condition(u32 opcode,u32 psr)67223d3a0dSRen Zhijie static unsigned int __maybe_unused aarch32_check_condition(u32 opcode, u32 psr)
680c5f4162SMark Rutland {
690c5f4162SMark Rutland u32 cc_bits = opcode >> 28;
700c5f4162SMark Rutland
710c5f4162SMark Rutland if (cc_bits != ARM_OPCODE_CONDITION_UNCOND) {
720c5f4162SMark Rutland if ((*aarch32_opcode_cond_checks[cc_bits])(psr))
730c5f4162SMark Rutland return ARM_OPCODE_CONDTEST_PASS;
740c5f4162SMark Rutland else
750c5f4162SMark Rutland return ARM_OPCODE_CONDTEST_FAIL;
760c5f4162SMark Rutland }
770c5f4162SMark Rutland return ARM_OPCODE_CONDTEST_UNCOND;
780c5f4162SMark Rutland }
790c5f4162SMark Rutland
80124c49b1SMark Rutland #ifdef CONFIG_SWP_EMULATION
81587064b6SPunit Agrawal /*
82bd35a4adSPunit Agrawal * Implement emulation of the SWP/SWPB instructions using load-exclusive and
83bd35a4adSPunit Agrawal * store-exclusive.
84bd35a4adSPunit Agrawal *
85bd35a4adSPunit Agrawal * Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>]
86bd35a4adSPunit Agrawal * Where: Rt = destination
87bd35a4adSPunit Agrawal * Rt2 = source
88bd35a4adSPunit Agrawal * Rn = address
89bd35a4adSPunit Agrawal */
90bd35a4adSPunit Agrawal
91bd35a4adSPunit Agrawal /*
92bd35a4adSPunit Agrawal * Error-checking SWP macros implemented using ldxr{b}/stxr{b}
93bd35a4adSPunit Agrawal */
941c5b51dfSWill Deacon
951c5b51dfSWill Deacon /* Arbitrary constant to ensure forward-progress of the LL/SC loop */
961c5b51dfSWill Deacon #define __SWP_LL_SC_LOOPS 4
971c5b51dfSWill Deacon
981c5b51dfSWill Deacon #define __user_swpX_asm(data, addr, res, temp, temp2, B) \
99bd38967dSCatalin Marinas do { \
100923e1e7dSMark Rutland uaccess_enable_privileged(); \
101bd35a4adSPunit Agrawal __asm__ __volatile__( \
1022e77a62cSMark Rutland " mov %w3, %w6\n" \
1031c5b51dfSWill Deacon "0: ldxr"B" %w2, [%4]\n" \
1041c5b51dfSWill Deacon "1: stxr"B" %w0, %w1, [%4]\n" \
105bd35a4adSPunit Agrawal " cbz %w0, 2f\n" \
1061c5b51dfSWill Deacon " sub %w3, %w3, #1\n" \
1071c5b51dfSWill Deacon " cbnz %w3, 0b\n" \
1081c5b51dfSWill Deacon " mov %w0, %w5\n" \
109589cb22bSWill Deacon " b 3f\n" \
110bd35a4adSPunit Agrawal "2:\n" \
111589cb22bSWill Deacon " mov %w1, %w2\n" \
112589cb22bSWill Deacon "3:\n" \
1132e77a62cSMark Rutland _ASM_EXTABLE_UACCESS_ERR(0b, 3b, %w0) \
1142e77a62cSMark Rutland _ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w0) \
1151c5b51dfSWill Deacon : "=&r" (res), "+r" (data), "=&r" (temp), "=&r" (temp2) \
11655de49f9SMark Rutland : "r" ((unsigned long)addr), "i" (-EAGAIN), \
1171c5b51dfSWill Deacon "i" (__SWP_LL_SC_LOOPS) \
118bd38967dSCatalin Marinas : "memory"); \
119923e1e7dSMark Rutland uaccess_disable_privileged(); \
120bd38967dSCatalin Marinas } while (0)
121bd35a4adSPunit Agrawal
1221c5b51dfSWill Deacon #define __user_swp_asm(data, addr, res, temp, temp2) \
1231c5b51dfSWill Deacon __user_swpX_asm(data, addr, res, temp, temp2, "")
1241c5b51dfSWill Deacon #define __user_swpb_asm(data, addr, res, temp, temp2) \
1251c5b51dfSWill Deacon __user_swpX_asm(data, addr, res, temp, temp2, "b")
126bd35a4adSPunit Agrawal
127bd35a4adSPunit Agrawal /*
128bd35a4adSPunit Agrawal * Bit 22 of the instruction encoding distinguishes between
129bd35a4adSPunit Agrawal * the SWP and SWPB variants (bit set means SWPB).
130bd35a4adSPunit Agrawal */
131bd35a4adSPunit Agrawal #define TYPE_SWPB (1 << 22)
132bd35a4adSPunit Agrawal
emulate_swpX(unsigned int address,unsigned int * data,unsigned int type)133bd35a4adSPunit Agrawal static int emulate_swpX(unsigned int address, unsigned int *data,
134bd35a4adSPunit Agrawal unsigned int type)
135bd35a4adSPunit Agrawal {
136bd35a4adSPunit Agrawal unsigned int res = 0;
137bd35a4adSPunit Agrawal
138bd35a4adSPunit Agrawal if ((type != TYPE_SWPB) && (address & 0x3)) {
139bd35a4adSPunit Agrawal /* SWP to unaligned address not permitted */
140bd35a4adSPunit Agrawal pr_debug("SWP instruction on unaligned pointer!\n");
141bd35a4adSPunit Agrawal return -EFAULT;
142bd35a4adSPunit Agrawal }
143bd35a4adSPunit Agrawal
144bd35a4adSPunit Agrawal while (1) {
1451c5b51dfSWill Deacon unsigned long temp, temp2;
146bd35a4adSPunit Agrawal
147bd35a4adSPunit Agrawal if (type == TYPE_SWPB)
1481c5b51dfSWill Deacon __user_swpb_asm(*data, address, res, temp, temp2);
149bd35a4adSPunit Agrawal else
1501c5b51dfSWill Deacon __user_swp_asm(*data, address, res, temp, temp2);
151bd35a4adSPunit Agrawal
152bd35a4adSPunit Agrawal if (likely(res != -EAGAIN) || signal_pending(current))
153bd35a4adSPunit Agrawal break;
154bd35a4adSPunit Agrawal
155bd35a4adSPunit Agrawal cond_resched();
156bd35a4adSPunit Agrawal }
157bd35a4adSPunit Agrawal
158bd35a4adSPunit Agrawal return res;
159bd35a4adSPunit Agrawal }
160bd35a4adSPunit Agrawal
161bd35a4adSPunit Agrawal /*
162bd35a4adSPunit Agrawal * swp_handler logs the id of calling process, dissects the instruction, sanity
163bd35a4adSPunit Agrawal * checks the memory location, calls emulate_swpX for the actual operation and
164bd35a4adSPunit Agrawal * deals with fixup/error handling before returning
165bd35a4adSPunit Agrawal */
swp_handler(struct pt_regs * regs,u32 instr)166bd35a4adSPunit Agrawal static int swp_handler(struct pt_regs *regs, u32 instr)
167bd35a4adSPunit Agrawal {
168bd35a4adSPunit Agrawal u32 destreg, data, type, address = 0;
1699085b34dSRobin Murphy const void __user *user_ptr;
170bd35a4adSPunit Agrawal int rn, rt2, res = 0;
171bd35a4adSPunit Agrawal
172bd35a4adSPunit Agrawal perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
173bd35a4adSPunit Agrawal
174bd35a4adSPunit Agrawal type = instr & TYPE_SWPB;
175bd35a4adSPunit Agrawal
1762af3ec08SDavid A. Long switch (aarch32_check_condition(instr, regs->pstate)) {
177bd35a4adSPunit Agrawal case ARM_OPCODE_CONDTEST_PASS:
178bd35a4adSPunit Agrawal break;
179bd35a4adSPunit Agrawal case ARM_OPCODE_CONDTEST_FAIL:
180bd35a4adSPunit Agrawal /* Condition failed - return to next instruction */
181bd35a4adSPunit Agrawal goto ret;
182bd35a4adSPunit Agrawal case ARM_OPCODE_CONDTEST_UNCOND:
183bd35a4adSPunit Agrawal /* If unconditional encoding - not a SWP, undef */
184bd35a4adSPunit Agrawal return -EFAULT;
185bd35a4adSPunit Agrawal default:
186bd35a4adSPunit Agrawal return -EINVAL;
187bd35a4adSPunit Agrawal }
188bd35a4adSPunit Agrawal
189bd35a4adSPunit Agrawal rn = aarch32_insn_extract_reg_num(instr, A32_RN_OFFSET);
190bd35a4adSPunit Agrawal rt2 = aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET);
191bd35a4adSPunit Agrawal
192bd35a4adSPunit Agrawal address = (u32)regs->user_regs.regs[rn];
193bd35a4adSPunit Agrawal data = (u32)regs->user_regs.regs[rt2];
194bd35a4adSPunit Agrawal destreg = aarch32_insn_extract_reg_num(instr, A32_RT_OFFSET);
195bd35a4adSPunit Agrawal
196bd35a4adSPunit Agrawal pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n",
197bd35a4adSPunit Agrawal rn, address, destreg,
198bd35a4adSPunit Agrawal aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET), data);
199bd35a4adSPunit Agrawal
200bd35a4adSPunit Agrawal /* Check access in reasonable access range for both SWP and SWPB */
2019085b34dSRobin Murphy user_ptr = (const void __user *)(unsigned long)(address & ~3);
20296d4f267SLinus Torvalds if (!access_ok(user_ptr, 4)) {
203bd35a4adSPunit Agrawal pr_debug("SWP{B} emulation: access to 0x%08x not allowed!\n",
204bd35a4adSPunit Agrawal address);
205bd35a4adSPunit Agrawal goto fault;
206bd35a4adSPunit Agrawal }
207bd35a4adSPunit Agrawal
208bd35a4adSPunit Agrawal res = emulate_swpX(address, &data, type);
209bd35a4adSPunit Agrawal if (res == -EFAULT)
210bd35a4adSPunit Agrawal goto fault;
211bd35a4adSPunit Agrawal else if (res == 0)
212bd35a4adSPunit Agrawal regs->user_regs.regs[destreg] = data;
213bd35a4adSPunit Agrawal
214bd35a4adSPunit Agrawal ret:
215d784e298SPunit Agrawal if (type == TYPE_SWPB)
216d784e298SPunit Agrawal trace_instruction_emulation("swpb", regs->pc);
217d784e298SPunit Agrawal else
218d784e298SPunit Agrawal trace_instruction_emulation("swp", regs->pc);
219d784e298SPunit Agrawal
220bd35a4adSPunit Agrawal pr_warn_ratelimited("\"%s\" (%ld) uses obsolete SWP{B} instruction at 0x%llx\n",
221bd35a4adSPunit Agrawal current->comm, (unsigned long)current->pid, regs->pc);
222bd35a4adSPunit Agrawal
2236436beeeSJulien Thierry arm64_skip_faulting_instruction(regs, 4);
224bd35a4adSPunit Agrawal return 0;
225bd35a4adSPunit Agrawal
226bd35a4adSPunit Agrawal fault:
227390bf177SAndre Przywara pr_debug("SWP{B} emulation: access caused memory abort!\n");
2282c9120f3SWill Deacon arm64_notify_segfault(address);
229bd35a4adSPunit Agrawal
230bd35a4adSPunit Agrawal return 0;
231bd35a4adSPunit Agrawal }
232bd35a4adSPunit Agrawal
try_emulate_swp(struct pt_regs * regs,u32 insn)233124c49b1SMark Rutland static bool try_emulate_swp(struct pt_regs *regs, u32 insn)
234bd35a4adSPunit Agrawal {
235124c49b1SMark Rutland /* SWP{B} only exists in ARM state and does not exist in Thumb */
236124c49b1SMark Rutland if (!compat_user_mode(regs) || compat_thumb_mode(regs))
237124c49b1SMark Rutland return false;
238124c49b1SMark Rutland
239124c49b1SMark Rutland if ((insn & 0x0fb00ff0) != 0x01000090)
240124c49b1SMark Rutland return false;
241124c49b1SMark Rutland
242124c49b1SMark Rutland return swp_handler(regs, insn) == 0;
243124c49b1SMark Rutland }
244bd35a4adSPunit Agrawal
245b4453cc8SMark Rutland static struct insn_emulation insn_swp = {
246bd35a4adSPunit Agrawal .name = "swp",
247bd35a4adSPunit Agrawal .status = INSN_OBSOLETE,
248124c49b1SMark Rutland .try_emulate = try_emulate_swp,
249bd35a4adSPunit Agrawal .set_hw_mode = NULL,
250bd35a4adSPunit Agrawal };
251124c49b1SMark Rutland #endif /* CONFIG_SWP_EMULATION */
252bd35a4adSPunit Agrawal
253124c49b1SMark Rutland #ifdef CONFIG_CP15_BARRIER_EMULATION
cp15barrier_handler(struct pt_regs * regs,u32 instr)254c852f320SPunit Agrawal static int cp15barrier_handler(struct pt_regs *regs, u32 instr)
255c852f320SPunit Agrawal {
256c852f320SPunit Agrawal perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
257c852f320SPunit Agrawal
2582af3ec08SDavid A. Long switch (aarch32_check_condition(instr, regs->pstate)) {
259c852f320SPunit Agrawal case ARM_OPCODE_CONDTEST_PASS:
260c852f320SPunit Agrawal break;
261c852f320SPunit Agrawal case ARM_OPCODE_CONDTEST_FAIL:
262c852f320SPunit Agrawal /* Condition failed - return to next instruction */
263c852f320SPunit Agrawal goto ret;
264c852f320SPunit Agrawal case ARM_OPCODE_CONDTEST_UNCOND:
265c852f320SPunit Agrawal /* If unconditional encoding - not a barrier instruction */
266c852f320SPunit Agrawal return -EFAULT;
267c852f320SPunit Agrawal default:
268c852f320SPunit Agrawal return -EINVAL;
269c852f320SPunit Agrawal }
270c852f320SPunit Agrawal
271c852f320SPunit Agrawal switch (aarch32_insn_mcr_extract_crm(instr)) {
272c852f320SPunit Agrawal case 10:
273c852f320SPunit Agrawal /*
274c852f320SPunit Agrawal * dmb - mcr p15, 0, Rt, c7, c10, 5
275c852f320SPunit Agrawal * dsb - mcr p15, 0, Rt, c7, c10, 4
276c852f320SPunit Agrawal */
277d784e298SPunit Agrawal if (aarch32_insn_mcr_extract_opc2(instr) == 5) {
278c852f320SPunit Agrawal dmb(sy);
279d784e298SPunit Agrawal trace_instruction_emulation(
280d784e298SPunit Agrawal "mcr p15, 0, Rt, c7, c10, 5 ; dmb", regs->pc);
281d784e298SPunit Agrawal } else {
282c852f320SPunit Agrawal dsb(sy);
283d784e298SPunit Agrawal trace_instruction_emulation(
284d784e298SPunit Agrawal "mcr p15, 0, Rt, c7, c10, 4 ; dsb", regs->pc);
285d784e298SPunit Agrawal }
286c852f320SPunit Agrawal break;
287c852f320SPunit Agrawal case 5:
288c852f320SPunit Agrawal /*
289c852f320SPunit Agrawal * isb - mcr p15, 0, Rt, c7, c5, 4
290c852f320SPunit Agrawal *
291c852f320SPunit Agrawal * Taking an exception or returning from one acts as an
292c852f320SPunit Agrawal * instruction barrier. So no explicit barrier needed here.
293c852f320SPunit Agrawal */
294d784e298SPunit Agrawal trace_instruction_emulation(
295d784e298SPunit Agrawal "mcr p15, 0, Rt, c7, c5, 4 ; isb", regs->pc);
296c852f320SPunit Agrawal break;
297c852f320SPunit Agrawal }
298c852f320SPunit Agrawal
299c852f320SPunit Agrawal ret:
300c852f320SPunit Agrawal pr_warn_ratelimited("\"%s\" (%ld) uses deprecated CP15 Barrier instruction at 0x%llx\n",
301c852f320SPunit Agrawal current->comm, (unsigned long)current->pid, regs->pc);
302c852f320SPunit Agrawal
3036436beeeSJulien Thierry arm64_skip_faulting_instruction(regs, 4);
304c852f320SPunit Agrawal return 0;
305c852f320SPunit Agrawal }
306c852f320SPunit Agrawal
cp15_barrier_set_hw_mode(bool enable)307c852f320SPunit Agrawal static int cp15_barrier_set_hw_mode(bool enable)
308c852f320SPunit Agrawal {
309736d474fSSuzuki K. Poulose if (enable)
31025be597aSMark Rutland sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_CP15BEN);
311736d474fSSuzuki K. Poulose else
31225be597aSMark Rutland sysreg_clear_set(sctlr_el1, SCTLR_EL1_CP15BEN, 0);
313736d474fSSuzuki K. Poulose return 0;
314c852f320SPunit Agrawal }
315c852f320SPunit Agrawal
try_emulate_cp15_barrier(struct pt_regs * regs,u32 insn)316124c49b1SMark Rutland static bool try_emulate_cp15_barrier(struct pt_regs *regs, u32 insn)
317c852f320SPunit Agrawal {
318124c49b1SMark Rutland if (!compat_user_mode(regs) || compat_thumb_mode(regs))
319124c49b1SMark Rutland return false;
320124c49b1SMark Rutland
321124c49b1SMark Rutland if ((insn & 0x0fff0fdf) == 0x0e070f9a)
322124c49b1SMark Rutland return cp15barrier_handler(regs, insn) == 0;
323124c49b1SMark Rutland
324124c49b1SMark Rutland if ((insn & 0x0fff0fff) == 0x0e070f95)
325124c49b1SMark Rutland return cp15barrier_handler(regs, insn) == 0;
326124c49b1SMark Rutland
327124c49b1SMark Rutland return false;
328124c49b1SMark Rutland }
329c852f320SPunit Agrawal
330b4453cc8SMark Rutland static struct insn_emulation insn_cp15_barrier = {
331c852f320SPunit Agrawal .name = "cp15_barrier",
332c852f320SPunit Agrawal .status = INSN_DEPRECATED,
333124c49b1SMark Rutland .try_emulate = try_emulate_cp15_barrier,
334c852f320SPunit Agrawal .set_hw_mode = cp15_barrier_set_hw_mode,
335c852f320SPunit Agrawal };
336124c49b1SMark Rutland #endif /* CONFIG_CP15_BARRIER_EMULATION */
337c852f320SPunit Agrawal
338124c49b1SMark Rutland #ifdef CONFIG_SETEND_EMULATION
setend_set_hw_mode(bool enable)3392d888f48SSuzuki K. Poulose static int setend_set_hw_mode(bool enable)
3402d888f48SSuzuki K. Poulose {
3412d888f48SSuzuki K. Poulose if (!cpu_supports_mixed_endian_el0())
3422d888f48SSuzuki K. Poulose return -EINVAL;
3432d888f48SSuzuki K. Poulose
3442d888f48SSuzuki K. Poulose if (enable)
34525be597aSMark Rutland sysreg_clear_set(sctlr_el1, SCTLR_EL1_SED, 0);
3462d888f48SSuzuki K. Poulose else
34725be597aSMark Rutland sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_SED);
3482d888f48SSuzuki K. Poulose return 0;
3492d888f48SSuzuki K. Poulose }
3502d888f48SSuzuki K. Poulose
compat_setend_handler(struct pt_regs * regs,u32 big_endian)3512d888f48SSuzuki K. Poulose static int compat_setend_handler(struct pt_regs *regs, u32 big_endian)
3522d888f48SSuzuki K. Poulose {
3532d888f48SSuzuki K. Poulose char *insn;
3542d888f48SSuzuki K. Poulose
3552d888f48SSuzuki K. Poulose perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
3562d888f48SSuzuki K. Poulose
3572d888f48SSuzuki K. Poulose if (big_endian) {
3582d888f48SSuzuki K. Poulose insn = "setend be";
359d64567f6SMark Rutland regs->pstate |= PSR_AA32_E_BIT;
3602d888f48SSuzuki K. Poulose } else {
3612d888f48SSuzuki K. Poulose insn = "setend le";
362d64567f6SMark Rutland regs->pstate &= ~PSR_AA32_E_BIT;
3632d888f48SSuzuki K. Poulose }
3642d888f48SSuzuki K. Poulose
3652d888f48SSuzuki K. Poulose trace_instruction_emulation(insn, regs->pc);
3662d888f48SSuzuki K. Poulose pr_warn_ratelimited("\"%s\" (%ld) uses deprecated setend instruction at 0x%llx\n",
3672d888f48SSuzuki K. Poulose current->comm, (unsigned long)current->pid, regs->pc);
3682d888f48SSuzuki K. Poulose
3692d888f48SSuzuki K. Poulose return 0;
3702d888f48SSuzuki K. Poulose }
3712d888f48SSuzuki K. Poulose
a32_setend_handler(struct pt_regs * regs,u32 instr)3722d888f48SSuzuki K. Poulose static int a32_setend_handler(struct pt_regs *regs, u32 instr)
3732d888f48SSuzuki K. Poulose {
3742d888f48SSuzuki K. Poulose int rc = compat_setend_handler(regs, (instr >> 9) & 1);
3756436beeeSJulien Thierry arm64_skip_faulting_instruction(regs, 4);
3762d888f48SSuzuki K. Poulose return rc;
3772d888f48SSuzuki K. Poulose }
3782d888f48SSuzuki K. Poulose
t16_setend_handler(struct pt_regs * regs,u32 instr)3792d888f48SSuzuki K. Poulose static int t16_setend_handler(struct pt_regs *regs, u32 instr)
3802d888f48SSuzuki K. Poulose {
3812d888f48SSuzuki K. Poulose int rc = compat_setend_handler(regs, (instr >> 3) & 1);
3826436beeeSJulien Thierry arm64_skip_faulting_instruction(regs, 2);
3832d888f48SSuzuki K. Poulose return rc;
3842d888f48SSuzuki K. Poulose }
3852d888f48SSuzuki K. Poulose
try_emulate_setend(struct pt_regs * regs,u32 insn)386124c49b1SMark Rutland static bool try_emulate_setend(struct pt_regs *regs, u32 insn)
3872d888f48SSuzuki K. Poulose {
388124c49b1SMark Rutland if (compat_thumb_mode(regs) &&
389124c49b1SMark Rutland (insn & 0xfffffff7) == 0x0000b650)
390124c49b1SMark Rutland return t16_setend_handler(regs, insn) == 0;
391124c49b1SMark Rutland
392124c49b1SMark Rutland if (compat_user_mode(regs) &&
393124c49b1SMark Rutland (insn & 0xfffffdff) == 0xf1010000)
394124c49b1SMark Rutland return a32_setend_handler(regs, insn) == 0;
395124c49b1SMark Rutland
396124c49b1SMark Rutland return false;
397124c49b1SMark Rutland }
3982d888f48SSuzuki K. Poulose
399b4453cc8SMark Rutland static struct insn_emulation insn_setend = {
4002d888f48SSuzuki K. Poulose .name = "setend",
4012d888f48SSuzuki K. Poulose .status = INSN_DEPRECATED,
402124c49b1SMark Rutland .try_emulate = try_emulate_setend,
4032d888f48SSuzuki K. Poulose .set_hw_mode = setend_set_hw_mode,
4042d888f48SSuzuki K. Poulose };
405124c49b1SMark Rutland #endif /* CONFIG_SETEND_EMULATION */
4062d888f48SSuzuki K. Poulose
407124c49b1SMark Rutland static struct insn_emulation *insn_emulations[] = {
408124c49b1SMark Rutland #ifdef CONFIG_SWP_EMULATION
409124c49b1SMark Rutland &insn_swp,
410124c49b1SMark Rutland #endif
411124c49b1SMark Rutland #ifdef CONFIG_CP15_BARRIER_EMULATION
412124c49b1SMark Rutland &insn_cp15_barrier,
413124c49b1SMark Rutland #endif
414124c49b1SMark Rutland #ifdef CONFIG_SETEND_EMULATION
415124c49b1SMark Rutland &insn_setend,
416124c49b1SMark Rutland #endif
417124c49b1SMark Rutland };
418124c49b1SMark Rutland
41925eeac0cSMark Rutland static DEFINE_MUTEX(insn_emulation_mutex);
42025eeac0cSMark Rutland
enable_insn_hw_mode(void * data)42125eeac0cSMark Rutland static void enable_insn_hw_mode(void *data)
42225eeac0cSMark Rutland {
4230e2cb49eSYu Zhe struct insn_emulation *insn = data;
42425eeac0cSMark Rutland if (insn->set_hw_mode)
42525eeac0cSMark Rutland insn->set_hw_mode(true);
42625eeac0cSMark Rutland }
42725eeac0cSMark Rutland
disable_insn_hw_mode(void * data)42825eeac0cSMark Rutland static void disable_insn_hw_mode(void *data)
42925eeac0cSMark Rutland {
4300e2cb49eSYu Zhe struct insn_emulation *insn = data;
43125eeac0cSMark Rutland if (insn->set_hw_mode)
43225eeac0cSMark Rutland insn->set_hw_mode(false);
43325eeac0cSMark Rutland }
43425eeac0cSMark Rutland
43525eeac0cSMark Rutland /* Run set_hw_mode(mode) on all active CPUs */
run_all_cpu_set_hw_mode(struct insn_emulation * insn,bool enable)43625eeac0cSMark Rutland static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable)
43725eeac0cSMark Rutland {
43825eeac0cSMark Rutland if (!insn->set_hw_mode)
43925eeac0cSMark Rutland return -EINVAL;
44025eeac0cSMark Rutland if (enable)
44125eeac0cSMark Rutland on_each_cpu(enable_insn_hw_mode, (void *)insn, true);
44225eeac0cSMark Rutland else
44325eeac0cSMark Rutland on_each_cpu(disable_insn_hw_mode, (void *)insn, true);
44425eeac0cSMark Rutland return 0;
44525eeac0cSMark Rutland }
44625eeac0cSMark Rutland
44725eeac0cSMark Rutland /*
44825eeac0cSMark Rutland * Run set_hw_mode for all insns on a starting CPU.
44925eeac0cSMark Rutland * Returns:
45025eeac0cSMark Rutland * 0 - If all the hooks ran successfully.
45125eeac0cSMark Rutland * -EINVAL - At least one hook is not supported by the CPU.
45225eeac0cSMark Rutland */
run_all_insn_set_hw_mode(unsigned int cpu)45325eeac0cSMark Rutland static int run_all_insn_set_hw_mode(unsigned int cpu)
45425eeac0cSMark Rutland {
45525eeac0cSMark Rutland int rc = 0;
45625eeac0cSMark Rutland unsigned long flags;
45725eeac0cSMark Rutland
458124c49b1SMark Rutland /*
459124c49b1SMark Rutland * Disable IRQs to serialize against an IPI from
460124c49b1SMark Rutland * run_all_cpu_set_hw_mode(), ensuring the HW is programmed to the most
461124c49b1SMark Rutland * recent enablement state if the two race with one another.
462124c49b1SMark Rutland */
463124c49b1SMark Rutland local_irq_save(flags);
464124c49b1SMark Rutland for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
465124c49b1SMark Rutland struct insn_emulation *insn = insn_emulations[i];
466124c49b1SMark Rutland bool enable = READ_ONCE(insn->current_mode) == INSN_HW;
46725eeac0cSMark Rutland if (insn->set_hw_mode && insn->set_hw_mode(enable)) {
46825eeac0cSMark Rutland pr_warn("CPU[%u] cannot support the emulation of %s",
46925eeac0cSMark Rutland cpu, insn->name);
47025eeac0cSMark Rutland rc = -EINVAL;
47125eeac0cSMark Rutland }
47225eeac0cSMark Rutland }
473124c49b1SMark Rutland local_irq_restore(flags);
474124c49b1SMark Rutland
47525eeac0cSMark Rutland return rc;
47625eeac0cSMark Rutland }
47725eeac0cSMark Rutland
update_insn_emulation_mode(struct insn_emulation * insn,enum insn_emulation_mode prev)47825eeac0cSMark Rutland static int update_insn_emulation_mode(struct insn_emulation *insn,
47925eeac0cSMark Rutland enum insn_emulation_mode prev)
48025eeac0cSMark Rutland {
48125eeac0cSMark Rutland int ret = 0;
48225eeac0cSMark Rutland
48325eeac0cSMark Rutland switch (prev) {
48425eeac0cSMark Rutland case INSN_UNDEF: /* Nothing to be done */
48525eeac0cSMark Rutland break;
48625eeac0cSMark Rutland case INSN_EMULATE:
48725eeac0cSMark Rutland break;
48825eeac0cSMark Rutland case INSN_HW:
48925eeac0cSMark Rutland if (!run_all_cpu_set_hw_mode(insn, false))
49025eeac0cSMark Rutland pr_notice("Disabled %s support\n", insn->name);
49125eeac0cSMark Rutland break;
49225eeac0cSMark Rutland }
49325eeac0cSMark Rutland
49425eeac0cSMark Rutland switch (insn->current_mode) {
49525eeac0cSMark Rutland case INSN_UNDEF:
49625eeac0cSMark Rutland break;
49725eeac0cSMark Rutland case INSN_EMULATE:
49825eeac0cSMark Rutland break;
49925eeac0cSMark Rutland case INSN_HW:
50025eeac0cSMark Rutland ret = run_all_cpu_set_hw_mode(insn, true);
50125eeac0cSMark Rutland if (!ret)
50225eeac0cSMark Rutland pr_notice("Enabled %s support\n", insn->name);
50325eeac0cSMark Rutland break;
50425eeac0cSMark Rutland }
50525eeac0cSMark Rutland
50625eeac0cSMark Rutland return ret;
50725eeac0cSMark Rutland }
50825eeac0cSMark Rutland
emulation_proc_handler(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)50925eeac0cSMark Rutland static int emulation_proc_handler(struct ctl_table *table, int write,
51025eeac0cSMark Rutland void *buffer, size_t *lenp,
51125eeac0cSMark Rutland loff_t *ppos)
51225eeac0cSMark Rutland {
51325eeac0cSMark Rutland int ret = 0;
51425eeac0cSMark Rutland struct insn_emulation *insn = container_of(table->data, struct insn_emulation, current_mode);
51525eeac0cSMark Rutland enum insn_emulation_mode prev_mode = insn->current_mode;
51625eeac0cSMark Rutland
51725eeac0cSMark Rutland mutex_lock(&insn_emulation_mutex);
51825eeac0cSMark Rutland ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
51925eeac0cSMark Rutland
52025eeac0cSMark Rutland if (ret || !write || prev_mode == insn->current_mode)
52125eeac0cSMark Rutland goto ret;
52225eeac0cSMark Rutland
52325eeac0cSMark Rutland ret = update_insn_emulation_mode(insn, prev_mode);
52425eeac0cSMark Rutland if (ret) {
52525eeac0cSMark Rutland /* Mode change failed, revert to previous mode. */
526124c49b1SMark Rutland WRITE_ONCE(insn->current_mode, prev_mode);
52725eeac0cSMark Rutland update_insn_emulation_mode(insn, INSN_UNDEF);
52825eeac0cSMark Rutland }
52925eeac0cSMark Rutland ret:
53025eeac0cSMark Rutland mutex_unlock(&insn_emulation_mutex);
53125eeac0cSMark Rutland return ret;
53225eeac0cSMark Rutland }
53325eeac0cSMark Rutland
register_insn_emulation(struct insn_emulation * insn)534124c49b1SMark Rutland static void __init register_insn_emulation(struct insn_emulation *insn)
53525eeac0cSMark Rutland {
536124c49b1SMark Rutland struct ctl_table *sysctl;
53725eeac0cSMark Rutland
538124c49b1SMark Rutland insn->min = INSN_UNDEF;
53925eeac0cSMark Rutland
540124c49b1SMark Rutland switch (insn->status) {
541124c49b1SMark Rutland case INSN_DEPRECATED:
542124c49b1SMark Rutland insn->current_mode = INSN_EMULATE;
543124c49b1SMark Rutland /* Disable the HW mode if it was turned on at early boot time */
544124c49b1SMark Rutland run_all_cpu_set_hw_mode(insn, false);
545124c49b1SMark Rutland insn->max = INSN_HW;
546124c49b1SMark Rutland break;
547124c49b1SMark Rutland case INSN_OBSOLETE:
548124c49b1SMark Rutland insn->current_mode = INSN_UNDEF;
549124c49b1SMark Rutland insn->max = INSN_EMULATE;
550124c49b1SMark Rutland break;
551124c49b1SMark Rutland case INSN_UNAVAILABLE:
552124c49b1SMark Rutland insn->current_mode = INSN_UNDEF;
553124c49b1SMark Rutland insn->max = INSN_UNDEF;
554124c49b1SMark Rutland break;
555124c49b1SMark Rutland }
556124c49b1SMark Rutland
557124c49b1SMark Rutland /* Program the HW if required */
558124c49b1SMark Rutland update_insn_emulation_mode(insn, INSN_UNDEF);
559124c49b1SMark Rutland
560124c49b1SMark Rutland if (insn->status != INSN_UNAVAILABLE) {
561124c49b1SMark Rutland sysctl = &insn->sysctl[0];
56225eeac0cSMark Rutland
56325eeac0cSMark Rutland sysctl->mode = 0644;
56425eeac0cSMark Rutland sysctl->maxlen = sizeof(int);
56525eeac0cSMark Rutland
56625eeac0cSMark Rutland sysctl->procname = insn->name;
56725eeac0cSMark Rutland sysctl->data = &insn->current_mode;
56825eeac0cSMark Rutland sysctl->extra1 = &insn->min;
56925eeac0cSMark Rutland sysctl->extra2 = &insn->max;
57025eeac0cSMark Rutland sysctl->proc_handler = emulation_proc_handler;
57125eeac0cSMark Rutland
572*9edbfe92SJoel Granados register_sysctl_sz("abi", sysctl, 1);
573124c49b1SMark Rutland }
574124c49b1SMark Rutland }
575124c49b1SMark Rutland
try_emulate_armv8_deprecated(struct pt_regs * regs,u32 insn)576124c49b1SMark Rutland bool try_emulate_armv8_deprecated(struct pt_regs *regs, u32 insn)
577124c49b1SMark Rutland {
578124c49b1SMark Rutland for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
579124c49b1SMark Rutland struct insn_emulation *ie = insn_emulations[i];
580124c49b1SMark Rutland
581124c49b1SMark Rutland if (ie->status == INSN_UNAVAILABLE)
582124c49b1SMark Rutland continue;
583124c49b1SMark Rutland
584124c49b1SMark Rutland /*
585124c49b1SMark Rutland * A trap may race with the mode being changed
586124c49b1SMark Rutland * INSN_EMULATE<->INSN_HW. Try to emulate the instruction to
587124c49b1SMark Rutland * avoid a spurious UNDEF.
588124c49b1SMark Rutland */
589124c49b1SMark Rutland if (READ_ONCE(ie->current_mode) == INSN_UNDEF)
590124c49b1SMark Rutland continue;
591124c49b1SMark Rutland
592124c49b1SMark Rutland if (ie->try_emulate(regs, insn))
593124c49b1SMark Rutland return true;
594124c49b1SMark Rutland }
595124c49b1SMark Rutland
596124c49b1SMark Rutland return false;
59725eeac0cSMark Rutland }
59825eeac0cSMark Rutland
599bd35a4adSPunit Agrawal /*
60026415330SHanjun Guo * Invoked as core_initcall, which guarantees that the instruction
60126415330SHanjun Guo * emulation is ready for userspace.
602587064b6SPunit Agrawal */
armv8_deprecated_init(void)603587064b6SPunit Agrawal static int __init armv8_deprecated_init(void)
604587064b6SPunit Agrawal {
605124c49b1SMark Rutland #ifdef CONFIG_SETEND_EMULATION
606124c49b1SMark Rutland if (!system_supports_mixed_endian_el0()) {
607124c49b1SMark Rutland insn_setend.status = INSN_UNAVAILABLE;
608117f5727SMark Rutland pr_info("setend instruction emulation is not supported on this system\n");
6092d888f48SSuzuki K. Poulose }
6102d888f48SSuzuki K. Poulose
611124c49b1SMark Rutland #endif
612124c49b1SMark Rutland for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
613124c49b1SMark Rutland struct insn_emulation *ie = insn_emulations[i];
614124c49b1SMark Rutland
615124c49b1SMark Rutland if (ie->status == INSN_UNAVAILABLE)
616124c49b1SMark Rutland continue;
617124c49b1SMark Rutland
618124c49b1SMark Rutland register_insn_emulation(ie);
619124c49b1SMark Rutland }
620124c49b1SMark Rutland
62127c01a8cSSebastian Andrzej Siewior cpuhp_setup_state_nocalls(CPUHP_AP_ARM64_ISNDEP_STARTING,
62273c1b41eSThomas Gleixner "arm64/isndep:starting",
62327c01a8cSSebastian Andrzej Siewior run_all_insn_set_hw_mode, NULL);
624587064b6SPunit Agrawal return 0;
625587064b6SPunit Agrawal }
626587064b6SPunit Agrawal
627c0d8832eSSuzuki K Poulose core_initcall(armv8_deprecated_init);
628