1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
264d2dc38SLeif Lindholm /*
364d2dc38SLeif Lindholm * linux/arch/arm/kernel/swp_emulate.c
464d2dc38SLeif Lindholm *
564d2dc38SLeif Lindholm * Copyright (C) 2009 ARM Limited
664d2dc38SLeif Lindholm * __user_* functions adapted from include/asm/uaccess.h
764d2dc38SLeif Lindholm *
864d2dc38SLeif Lindholm * Implements emulation of the SWP/SWPB instructions using load-exclusive and
964d2dc38SLeif Lindholm * store-exclusive for processors that have them disabled (or future ones that
1064d2dc38SLeif Lindholm * might not implement them).
1164d2dc38SLeif Lindholm *
1264d2dc38SLeif Lindholm * Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>]
1364d2dc38SLeif Lindholm * Where: Rt = destination
1464d2dc38SLeif Lindholm * Rt2 = source
1564d2dc38SLeif Lindholm * Rn = address
1664d2dc38SLeif Lindholm */
1764d2dc38SLeif Lindholm
1864d2dc38SLeif Lindholm #include <linux/init.h>
1964d2dc38SLeif Lindholm #include <linux/kernel.h>
2064d2dc38SLeif Lindholm #include <linux/proc_fs.h>
21526c5978SDavid Howells #include <linux/seq_file.h>
2264d2dc38SLeif Lindholm #include <linux/sched.h>
23589ee628SIngo Molnar #include <linux/sched/mm.h>
2464d2dc38SLeif Lindholm #include <linux/syscalls.h>
2564d2dc38SLeif Lindholm #include <linux/perf_event.h>
2664d2dc38SLeif Lindholm
27c245dcd3SLeif Lindholm #include <asm/opcodes.h>
287397aa48SRussell King #include <asm/system_info.h>
2964d2dc38SLeif Lindholm #include <asm/traps.h>
307c0f6ba6SLinus Torvalds #include <linux/uaccess.h>
3164d2dc38SLeif Lindholm
3264d2dc38SLeif Lindholm /*
3364d2dc38SLeif Lindholm * Error-checking SWP macros implemented using ldrex{b}/strex{b}
3464d2dc38SLeif Lindholm */
3564d2dc38SLeif Lindholm #define __user_swpX_asm(data, addr, res, temp, B) \
3664d2dc38SLeif Lindholm __asm__ __volatile__( \
37*a2faac39SNick Desaulniers ".arch armv7-a\n" \
3834bfbae3SShengjiu Wang "0: ldrex"B" %2, [%3]\n" \
3934bfbae3SShengjiu Wang "1: strex"B" %0, %1, [%3]\n" \
4064d2dc38SLeif Lindholm " cmp %0, #0\n" \
4134bfbae3SShengjiu Wang " moveq %1, %2\n" \
4264d2dc38SLeif Lindholm " movne %0, %4\n" \
4364d2dc38SLeif Lindholm "2:\n" \
44c4a84ae3SArd Biesheuvel " .section .text.fixup,\"ax\"\n" \
4564d2dc38SLeif Lindholm " .align 2\n" \
4664d2dc38SLeif Lindholm "3: mov %0, %5\n" \
4764d2dc38SLeif Lindholm " b 2b\n" \
4864d2dc38SLeif Lindholm " .previous\n" \
4964d2dc38SLeif Lindholm " .section __ex_table,\"a\"\n" \
5064d2dc38SLeif Lindholm " .align 3\n" \
5164d2dc38SLeif Lindholm " .long 0b, 3b\n" \
5264d2dc38SLeif Lindholm " .long 1b, 3b\n" \
5364d2dc38SLeif Lindholm " .previous" \
5464d2dc38SLeif Lindholm : "=&r" (res), "+r" (data), "=&r" (temp) \
5564d2dc38SLeif Lindholm : "r" (addr), "i" (-EAGAIN), "i" (-EFAULT) \
5664d2dc38SLeif Lindholm : "cc", "memory")
5764d2dc38SLeif Lindholm
5864d2dc38SLeif Lindholm #define __user_swp_asm(data, addr, res, temp) \
5964d2dc38SLeif Lindholm __user_swpX_asm(data, addr, res, temp, "")
6064d2dc38SLeif Lindholm #define __user_swpb_asm(data, addr, res, temp) \
6164d2dc38SLeif Lindholm __user_swpX_asm(data, addr, res, temp, "b")
6264d2dc38SLeif Lindholm
6364d2dc38SLeif Lindholm /*
6464d2dc38SLeif Lindholm * Macros/defines for extracting register numbers from instruction.
6564d2dc38SLeif Lindholm */
6664d2dc38SLeif Lindholm #define EXTRACT_REG_NUM(instruction, offset) \
6764d2dc38SLeif Lindholm (((instruction) & (0xf << (offset))) >> (offset))
6864d2dc38SLeif Lindholm #define RN_OFFSET 16
6964d2dc38SLeif Lindholm #define RT_OFFSET 12
7064d2dc38SLeif Lindholm #define RT2_OFFSET 0
7164d2dc38SLeif Lindholm /*
7264d2dc38SLeif Lindholm * Bit 22 of the instruction encoding distinguishes between
7364d2dc38SLeif Lindholm * the SWP and SWPB variants (bit set means SWPB).
7464d2dc38SLeif Lindholm */
7564d2dc38SLeif Lindholm #define TYPE_SWPB (1 << 22)
7664d2dc38SLeif Lindholm
7764d2dc38SLeif Lindholm static unsigned long swpcounter;
7864d2dc38SLeif Lindholm static unsigned long swpbcounter;
7964d2dc38SLeif Lindholm static unsigned long abtcounter;
8064d2dc38SLeif Lindholm static pid_t previous_pid;
8164d2dc38SLeif Lindholm
8264d2dc38SLeif Lindholm #ifdef CONFIG_PROC_FS
proc_status_show(struct seq_file * m,void * v)83526c5978SDavid Howells static int proc_status_show(struct seq_file *m, void *v)
8464d2dc38SLeif Lindholm {
85526c5978SDavid Howells seq_printf(m, "Emulated SWP:\t\t%lu\n", swpcounter);
86526c5978SDavid Howells seq_printf(m, "Emulated SWPB:\t\t%lu\n", swpbcounter);
87526c5978SDavid Howells seq_printf(m, "Aborted SWP{B}:\t\t%lu\n", abtcounter);
8864d2dc38SLeif Lindholm if (previous_pid != 0)
89526c5978SDavid Howells seq_printf(m, "Last process:\t\t%d\n", previous_pid);
90526c5978SDavid Howells return 0;
9164d2dc38SLeif Lindholm }
9264d2dc38SLeif Lindholm #endif
9364d2dc38SLeif Lindholm
9464d2dc38SLeif Lindholm /*
9564d2dc38SLeif Lindholm * Set up process info to signal segmentation fault - called on access error.
9664d2dc38SLeif Lindholm */
set_segfault(struct pt_regs * regs,unsigned long addr)9764d2dc38SLeif Lindholm static void set_segfault(struct pt_regs *regs, unsigned long addr)
9864d2dc38SLeif Lindholm {
9905e792e3SEric W. Biederman int si_code;
10064d2dc38SLeif Lindholm
101d8ed45c5SMichel Lespinasse mmap_read_lock(current->mm);
10264d2dc38SLeif Lindholm if (find_vma(current->mm, addr) == NULL)
10305e792e3SEric W. Biederman si_code = SEGV_MAPERR;
10464d2dc38SLeif Lindholm else
10505e792e3SEric W. Biederman si_code = SEGV_ACCERR;
106d8ed45c5SMichel Lespinasse mmap_read_unlock(current->mm);
10764d2dc38SLeif Lindholm
10864d2dc38SLeif Lindholm pr_debug("SWP{B} emulation: access caused memory abort!\n");
10905e792e3SEric W. Biederman arm_notify_die("Illegal memory access", regs,
11005e792e3SEric W. Biederman SIGSEGV, si_code,
11105e792e3SEric W. Biederman (void __user *)instruction_pointer(regs),
11205e792e3SEric W. Biederman 0, 0);
11364d2dc38SLeif Lindholm
11464d2dc38SLeif Lindholm abtcounter++;
11564d2dc38SLeif Lindholm }
11664d2dc38SLeif Lindholm
emulate_swpX(unsigned int address,unsigned int * data,unsigned int type)11764d2dc38SLeif Lindholm static int emulate_swpX(unsigned int address, unsigned int *data,
11864d2dc38SLeif Lindholm unsigned int type)
11964d2dc38SLeif Lindholm {
12064d2dc38SLeif Lindholm unsigned int res = 0;
12164d2dc38SLeif Lindholm
12264d2dc38SLeif Lindholm if ((type != TYPE_SWPB) && (address & 0x3)) {
12364d2dc38SLeif Lindholm /* SWP to unaligned address not permitted */
12464d2dc38SLeif Lindholm pr_debug("SWP instruction on unaligned pointer!\n");
12564d2dc38SLeif Lindholm return -EFAULT;
12664d2dc38SLeif Lindholm }
12764d2dc38SLeif Lindholm
12864d2dc38SLeif Lindholm while (1) {
12964d2dc38SLeif Lindholm unsigned long temp;
130a5e090acSRussell King unsigned int __ua_flags;
13164d2dc38SLeif Lindholm
132a5e090acSRussell King __ua_flags = uaccess_save_and_enable();
13364d2dc38SLeif Lindholm if (type == TYPE_SWPB)
13464d2dc38SLeif Lindholm __user_swpb_asm(*data, address, res, temp);
13564d2dc38SLeif Lindholm else
13664d2dc38SLeif Lindholm __user_swp_asm(*data, address, res, temp);
137a5e090acSRussell King uaccess_restore(__ua_flags);
13864d2dc38SLeif Lindholm
13964d2dc38SLeif Lindholm if (likely(res != -EAGAIN) || signal_pending(current))
14064d2dc38SLeif Lindholm break;
14164d2dc38SLeif Lindholm
14264d2dc38SLeif Lindholm cond_resched();
14364d2dc38SLeif Lindholm }
14464d2dc38SLeif Lindholm
14564d2dc38SLeif Lindholm if (res == 0) {
14664d2dc38SLeif Lindholm if (type == TYPE_SWPB)
14764d2dc38SLeif Lindholm swpbcounter++;
14864d2dc38SLeif Lindholm else
14964d2dc38SLeif Lindholm swpcounter++;
15064d2dc38SLeif Lindholm }
15164d2dc38SLeif Lindholm
15264d2dc38SLeif Lindholm return res;
15364d2dc38SLeif Lindholm }
15464d2dc38SLeif Lindholm
15564d2dc38SLeif Lindholm /*
15664d2dc38SLeif Lindholm * swp_handler logs the id of calling process, dissects the instruction, sanity
15764d2dc38SLeif Lindholm * checks the memory location, calls emulate_swpX for the actual operation and
15864d2dc38SLeif Lindholm * deals with fixup/error handling before returning
15964d2dc38SLeif Lindholm */
swp_handler(struct pt_regs * regs,unsigned int instr)16064d2dc38SLeif Lindholm static int swp_handler(struct pt_regs *regs, unsigned int instr)
16164d2dc38SLeif Lindholm {
16264d2dc38SLeif Lindholm unsigned int address, destreg, data, type;
16364d2dc38SLeif Lindholm unsigned int res = 0;
16464d2dc38SLeif Lindholm
165a8b0ca17SPeter Zijlstra perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->ARM_pc);
16664d2dc38SLeif Lindholm
167c245dcd3SLeif Lindholm res = arm_check_condition(instr, regs->ARM_cpsr);
168c245dcd3SLeif Lindholm switch (res) {
169c245dcd3SLeif Lindholm case ARM_OPCODE_CONDTEST_PASS:
170c245dcd3SLeif Lindholm break;
171c245dcd3SLeif Lindholm case ARM_OPCODE_CONDTEST_FAIL:
172c245dcd3SLeif Lindholm /* Condition failed - return to next instruction */
173c245dcd3SLeif Lindholm regs->ARM_pc += 4;
174c245dcd3SLeif Lindholm return 0;
175c245dcd3SLeif Lindholm case ARM_OPCODE_CONDTEST_UNCOND:
176c245dcd3SLeif Lindholm /* If unconditional encoding - not a SWP, undef */
177c245dcd3SLeif Lindholm return -EFAULT;
178c245dcd3SLeif Lindholm default:
179c245dcd3SLeif Lindholm return -EINVAL;
180c245dcd3SLeif Lindholm }
181c245dcd3SLeif Lindholm
18264d2dc38SLeif Lindholm if (current->pid != previous_pid) {
18364d2dc38SLeif Lindholm pr_debug("\"%s\" (%ld) uses deprecated SWP{B} instruction\n",
18464d2dc38SLeif Lindholm current->comm, (unsigned long)current->pid);
18564d2dc38SLeif Lindholm previous_pid = current->pid;
18664d2dc38SLeif Lindholm }
18764d2dc38SLeif Lindholm
18864d2dc38SLeif Lindholm address = regs->uregs[EXTRACT_REG_NUM(instr, RN_OFFSET)];
18964d2dc38SLeif Lindholm data = regs->uregs[EXTRACT_REG_NUM(instr, RT2_OFFSET)];
19064d2dc38SLeif Lindholm destreg = EXTRACT_REG_NUM(instr, RT_OFFSET);
19164d2dc38SLeif Lindholm
19264d2dc38SLeif Lindholm type = instr & TYPE_SWPB;
19364d2dc38SLeif Lindholm
19464d2dc38SLeif Lindholm pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n",
19564d2dc38SLeif Lindholm EXTRACT_REG_NUM(instr, RN_OFFSET), address,
19664d2dc38SLeif Lindholm destreg, EXTRACT_REG_NUM(instr, RT2_OFFSET), data);
19764d2dc38SLeif Lindholm
19864d2dc38SLeif Lindholm /* Check access in reasonable access range for both SWP and SWPB */
19923fc539eSArnd Bergmann if (!access_ok((void __user *)(address & ~3), 4)) {
20064d2dc38SLeif Lindholm pr_debug("SWP{B} emulation: access to %p not allowed!\n",
20164d2dc38SLeif Lindholm (void *)address);
20264d2dc38SLeif Lindholm res = -EFAULT;
20364d2dc38SLeif Lindholm } else {
20464d2dc38SLeif Lindholm res = emulate_swpX(address, &data, type);
20564d2dc38SLeif Lindholm }
20664d2dc38SLeif Lindholm
20764d2dc38SLeif Lindholm if (res == 0) {
20864d2dc38SLeif Lindholm /*
20964d2dc38SLeif Lindholm * On successful emulation, revert the adjustment to the PC
21064d2dc38SLeif Lindholm * made in kernel/traps.c in order to resume execution at the
21164d2dc38SLeif Lindholm * instruction following the SWP{B}.
21264d2dc38SLeif Lindholm */
21364d2dc38SLeif Lindholm regs->ARM_pc += 4;
21464d2dc38SLeif Lindholm regs->uregs[destreg] = data;
21564d2dc38SLeif Lindholm } else if (res == -EFAULT) {
21664d2dc38SLeif Lindholm /*
21764d2dc38SLeif Lindholm * Memory errors do not mean emulation failed.
21864d2dc38SLeif Lindholm * Set up signal info to return SEGV, then return OK
21964d2dc38SLeif Lindholm */
22064d2dc38SLeif Lindholm set_segfault(regs, address);
22164d2dc38SLeif Lindholm }
22264d2dc38SLeif Lindholm
22364d2dc38SLeif Lindholm return 0;
22464d2dc38SLeif Lindholm }
22564d2dc38SLeif Lindholm
22664d2dc38SLeif Lindholm /*
22764d2dc38SLeif Lindholm * Only emulate SWP/SWPB executed in ARM state/User mode.
22864d2dc38SLeif Lindholm * The kernel must be SWP free and SWP{B} does not exist in Thumb/ThumbEE.
22964d2dc38SLeif Lindholm */
23064d2dc38SLeif Lindholm static struct undef_hook swp_hook = {
23164d2dc38SLeif Lindholm .instr_mask = 0x0fb00ff0,
23264d2dc38SLeif Lindholm .instr_val = 0x01000090,
23364d2dc38SLeif Lindholm .cpsr_mask = MODE_MASK | PSR_T_BIT | PSR_J_BIT,
23464d2dc38SLeif Lindholm .cpsr_val = USR_MODE,
23564d2dc38SLeif Lindholm .fn = swp_handler
23664d2dc38SLeif Lindholm };
23764d2dc38SLeif Lindholm
23864d2dc38SLeif Lindholm /*
23964d2dc38SLeif Lindholm * Register handler and create status file in /proc/cpu
24064d2dc38SLeif Lindholm * Invoked as late_initcall, since not needed before init spawned.
24164d2dc38SLeif Lindholm */
swp_emulation_init(void)24264d2dc38SLeif Lindholm static int __init swp_emulation_init(void)
24364d2dc38SLeif Lindholm {
2447397aa48SRussell King if (cpu_architecture() < CPU_ARCH_ARMv7)
2457397aa48SRussell King return 0;
2467397aa48SRussell King
24764d2dc38SLeif Lindholm #ifdef CONFIG_PROC_FS
2483f3942acSChristoph Hellwig if (!proc_create_single("cpu/swp_emulation", S_IRUGO, NULL,
2493f3942acSChristoph Hellwig proc_status_show))
25064d2dc38SLeif Lindholm return -ENOMEM;
25164d2dc38SLeif Lindholm #endif /* CONFIG_PROC_FS */
25264d2dc38SLeif Lindholm
2534ed89f22SRussell King pr_notice("Registering SWP/SWPB emulation handler\n");
25464d2dc38SLeif Lindholm register_undef_hook(&swp_hook);
25564d2dc38SLeif Lindholm
25664d2dc38SLeif Lindholm return 0;
25764d2dc38SLeif Lindholm }
25864d2dc38SLeif Lindholm
25964d2dc38SLeif Lindholm late_initcall(swp_emulation_init);
260