1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
2d670bd4fSSam Ravnborg /*
3d670bd4fSSam Ravnborg * unaligned.c: Unaligned load/store trap handling with special
4d670bd4fSSam Ravnborg * cases for the kernel to do them more quickly.
5d670bd4fSSam Ravnborg *
6d670bd4fSSam Ravnborg * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu)
7d670bd4fSSam Ravnborg * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
8d670bd4fSSam Ravnborg */
9d670bd4fSSam Ravnborg
10d670bd4fSSam Ravnborg
11d670bd4fSSam Ravnborg #include <linux/kernel.h>
123f07c014SIngo Molnar #include <linux/sched/signal.h>
13d670bd4fSSam Ravnborg #include <linux/mm.h>
14d670bd4fSSam Ravnborg #include <asm/ptrace.h>
15d670bd4fSSam Ravnborg #include <asm/processor.h>
167c0f6ba6SLinus Torvalds #include <linux/uaccess.h>
17d670bd4fSSam Ravnborg #include <linux/smp.h>
18121dd5f2SDavid S. Miller #include <linux/perf_event.h>
19b4edf06cSAl Viro #include <linux/extable.h>
20d670bd4fSSam Ravnborg
219edfae3fSSam Ravnborg #include <asm/setup.h>
229edfae3fSSam Ravnborg
239edfae3fSSam Ravnborg #include "kernel.h"
249edfae3fSSam Ravnborg
25d670bd4fSSam Ravnborg enum direction {
26d670bd4fSSam Ravnborg load, /* ld, ldd, ldh, ldsh */
27d670bd4fSSam Ravnborg store, /* st, std, sth, stsh */
28d670bd4fSSam Ravnborg both, /* Swap, ldstub, etc. */
29d670bd4fSSam Ravnborg fpload,
30d670bd4fSSam Ravnborg fpstore,
31d670bd4fSSam Ravnborg invalid,
32d670bd4fSSam Ravnborg };
33d670bd4fSSam Ravnborg
decode_direction(unsigned int insn)34d670bd4fSSam Ravnborg static inline enum direction decode_direction(unsigned int insn)
35d670bd4fSSam Ravnborg {
36d670bd4fSSam Ravnborg unsigned long tmp = (insn >> 21) & 1;
37d670bd4fSSam Ravnborg
38d670bd4fSSam Ravnborg if(!tmp)
39d670bd4fSSam Ravnborg return load;
40d670bd4fSSam Ravnborg else {
41d670bd4fSSam Ravnborg if(((insn>>19)&0x3f) == 15)
42d670bd4fSSam Ravnborg return both;
43d670bd4fSSam Ravnborg else
44d670bd4fSSam Ravnborg return store;
45d670bd4fSSam Ravnborg }
46d670bd4fSSam Ravnborg }
47d670bd4fSSam Ravnborg
48d670bd4fSSam Ravnborg /* 8 = double-word, 4 = word, 2 = half-word */
decode_access_size(unsigned int insn)49d670bd4fSSam Ravnborg static inline int decode_access_size(unsigned int insn)
50d670bd4fSSam Ravnborg {
51d670bd4fSSam Ravnborg insn = (insn >> 19) & 3;
52d670bd4fSSam Ravnborg
53d670bd4fSSam Ravnborg if(!insn)
54d670bd4fSSam Ravnborg return 4;
55d670bd4fSSam Ravnborg else if(insn == 3)
56d670bd4fSSam Ravnborg return 8;
57d670bd4fSSam Ravnborg else if(insn == 2)
58d670bd4fSSam Ravnborg return 2;
59d670bd4fSSam Ravnborg else {
60d670bd4fSSam Ravnborg printk("Impossible unaligned trap. insn=%08x\n", insn);
61d670bd4fSSam Ravnborg die_if_kernel("Byte sized unaligned access?!?!", current->thread.kregs);
62d670bd4fSSam Ravnborg return 4; /* just to keep gcc happy. */
63d670bd4fSSam Ravnborg }
64d670bd4fSSam Ravnborg }
65d670bd4fSSam Ravnborg
66d670bd4fSSam Ravnborg /* 0x400000 = signed, 0 = unsigned */
decode_signedness(unsigned int insn)67d670bd4fSSam Ravnborg static inline int decode_signedness(unsigned int insn)
68d670bd4fSSam Ravnborg {
69d670bd4fSSam Ravnborg return (insn & 0x400000);
70d670bd4fSSam Ravnborg }
71d670bd4fSSam Ravnborg
maybe_flush_windows(unsigned int rs1,unsigned int rs2,unsigned int rd)72d670bd4fSSam Ravnborg static inline void maybe_flush_windows(unsigned int rs1, unsigned int rs2,
73d670bd4fSSam Ravnborg unsigned int rd)
74d670bd4fSSam Ravnborg {
75d670bd4fSSam Ravnborg if(rs2 >= 16 || rs1 >= 16 || rd >= 16) {
76d670bd4fSSam Ravnborg /* Wheee... */
77d670bd4fSSam Ravnborg __asm__ __volatile__("save %sp, -0x40, %sp\n\t"
78d670bd4fSSam Ravnborg "save %sp, -0x40, %sp\n\t"
79d670bd4fSSam Ravnborg "save %sp, -0x40, %sp\n\t"
80d670bd4fSSam Ravnborg "save %sp, -0x40, %sp\n\t"
81d670bd4fSSam Ravnborg "save %sp, -0x40, %sp\n\t"
82d670bd4fSSam Ravnborg "save %sp, -0x40, %sp\n\t"
83d670bd4fSSam Ravnborg "save %sp, -0x40, %sp\n\t"
84d670bd4fSSam Ravnborg "restore; restore; restore; restore;\n\t"
85d670bd4fSSam Ravnborg "restore; restore; restore;\n\t");
86d670bd4fSSam Ravnborg }
87d670bd4fSSam Ravnborg }
88d670bd4fSSam Ravnborg
sign_extend_imm13(int imm)89d670bd4fSSam Ravnborg static inline int sign_extend_imm13(int imm)
90d670bd4fSSam Ravnborg {
91d670bd4fSSam Ravnborg return imm << 19 >> 19;
92d670bd4fSSam Ravnborg }
93d670bd4fSSam Ravnborg
fetch_reg(unsigned int reg,struct pt_regs * regs)94d670bd4fSSam Ravnborg static inline unsigned long fetch_reg(unsigned int reg, struct pt_regs *regs)
95d670bd4fSSam Ravnborg {
964d7b92adSSam Ravnborg struct reg_window32 *win;
97d670bd4fSSam Ravnborg
98d670bd4fSSam Ravnborg if(reg < 16)
99d670bd4fSSam Ravnborg return (!reg ? 0 : regs->u_regs[reg]);
100d670bd4fSSam Ravnborg
101d670bd4fSSam Ravnborg /* Ho hum, the slightly complicated case. */
1024d7b92adSSam Ravnborg win = (struct reg_window32 *) regs->u_regs[UREG_FP];
103d670bd4fSSam Ravnborg return win->locals[reg - 16]; /* yes, I know what this does... */
104d670bd4fSSam Ravnborg }
105d670bd4fSSam Ravnborg
safe_fetch_reg(unsigned int reg,struct pt_regs * regs)106d670bd4fSSam Ravnborg static inline unsigned long safe_fetch_reg(unsigned int reg, struct pt_regs *regs)
107d670bd4fSSam Ravnborg {
1084d7b92adSSam Ravnborg struct reg_window32 __user *win;
109d670bd4fSSam Ravnborg unsigned long ret;
110d670bd4fSSam Ravnborg
111d670bd4fSSam Ravnborg if (reg < 16)
112d670bd4fSSam Ravnborg return (!reg ? 0 : regs->u_regs[reg]);
113d670bd4fSSam Ravnborg
114d670bd4fSSam Ravnborg /* Ho hum, the slightly complicated case. */
1154d7b92adSSam Ravnborg win = (struct reg_window32 __user *) regs->u_regs[UREG_FP];
116d670bd4fSSam Ravnborg
117d670bd4fSSam Ravnborg if ((unsigned long)win & 3)
118d670bd4fSSam Ravnborg return -1;
119d670bd4fSSam Ravnborg
120d670bd4fSSam Ravnborg if (get_user(ret, &win->locals[reg - 16]))
121d670bd4fSSam Ravnborg return -1;
122d670bd4fSSam Ravnborg
123d670bd4fSSam Ravnborg return ret;
124d670bd4fSSam Ravnborg }
125d670bd4fSSam Ravnborg
fetch_reg_addr(unsigned int reg,struct pt_regs * regs)126d670bd4fSSam Ravnborg static inline unsigned long *fetch_reg_addr(unsigned int reg, struct pt_regs *regs)
127d670bd4fSSam Ravnborg {
1284d7b92adSSam Ravnborg struct reg_window32 *win;
129d670bd4fSSam Ravnborg
130d670bd4fSSam Ravnborg if(reg < 16)
131d670bd4fSSam Ravnborg return ®s->u_regs[reg];
1324d7b92adSSam Ravnborg win = (struct reg_window32 *) regs->u_regs[UREG_FP];
133d670bd4fSSam Ravnborg return &win->locals[reg - 16];
134d670bd4fSSam Ravnborg }
135d670bd4fSSam Ravnborg
compute_effective_address(struct pt_regs * regs,unsigned int insn)136d670bd4fSSam Ravnborg static unsigned long compute_effective_address(struct pt_regs *regs,
137d670bd4fSSam Ravnborg unsigned int insn)
138d670bd4fSSam Ravnborg {
139d670bd4fSSam Ravnborg unsigned int rs1 = (insn >> 14) & 0x1f;
140d670bd4fSSam Ravnborg unsigned int rs2 = insn & 0x1f;
141d670bd4fSSam Ravnborg unsigned int rd = (insn >> 25) & 0x1f;
142d670bd4fSSam Ravnborg
143d670bd4fSSam Ravnborg if(insn & 0x2000) {
144d670bd4fSSam Ravnborg maybe_flush_windows(rs1, 0, rd);
145d670bd4fSSam Ravnborg return (fetch_reg(rs1, regs) + sign_extend_imm13(insn));
146d670bd4fSSam Ravnborg } else {
147d670bd4fSSam Ravnborg maybe_flush_windows(rs1, rs2, rd);
148d670bd4fSSam Ravnborg return (fetch_reg(rs1, regs) + fetch_reg(rs2, regs));
149d670bd4fSSam Ravnborg }
150d670bd4fSSam Ravnborg }
151d670bd4fSSam Ravnborg
safe_compute_effective_address(struct pt_regs * regs,unsigned int insn)152d670bd4fSSam Ravnborg unsigned long safe_compute_effective_address(struct pt_regs *regs,
153d670bd4fSSam Ravnborg unsigned int insn)
154d670bd4fSSam Ravnborg {
155d670bd4fSSam Ravnborg unsigned int rs1 = (insn >> 14) & 0x1f;
156d670bd4fSSam Ravnborg unsigned int rs2 = insn & 0x1f;
157d670bd4fSSam Ravnborg unsigned int rd = (insn >> 25) & 0x1f;
158d670bd4fSSam Ravnborg
159d670bd4fSSam Ravnborg if(insn & 0x2000) {
160d670bd4fSSam Ravnborg maybe_flush_windows(rs1, 0, rd);
161d670bd4fSSam Ravnborg return (safe_fetch_reg(rs1, regs) + sign_extend_imm13(insn));
162d670bd4fSSam Ravnborg } else {
163d670bd4fSSam Ravnborg maybe_flush_windows(rs1, rs2, rd);
164d670bd4fSSam Ravnborg return (safe_fetch_reg(rs1, regs) + safe_fetch_reg(rs2, regs));
165d670bd4fSSam Ravnborg }
166d670bd4fSSam Ravnborg }
167d670bd4fSSam Ravnborg
168d670bd4fSSam Ravnborg /* This is just to make gcc think panic does return... */
unaligned_panic(char * str)169d670bd4fSSam Ravnborg static void unaligned_panic(char *str)
170d670bd4fSSam Ravnborg {
171e7fbaf01SKees Cook panic("%s", str);
172d670bd4fSSam Ravnborg }
173d670bd4fSSam Ravnborg
174d670bd4fSSam Ravnborg /* una_asm.S */
175d670bd4fSSam Ravnborg extern int do_int_load(unsigned long *dest_reg, int size,
176d670bd4fSSam Ravnborg unsigned long *saddr, int is_signed);
177d670bd4fSSam Ravnborg extern int __do_int_store(unsigned long *dst_addr, int size,
178d670bd4fSSam Ravnborg unsigned long *src_val);
179d670bd4fSSam Ravnborg
do_int_store(int reg_num,int size,unsigned long * dst_addr,struct pt_regs * regs)180d670bd4fSSam Ravnborg static int do_int_store(int reg_num, int size, unsigned long *dst_addr,
181d670bd4fSSam Ravnborg struct pt_regs *regs)
182d670bd4fSSam Ravnborg {
183d670bd4fSSam Ravnborg unsigned long zero[2] = { 0, 0 };
184d670bd4fSSam Ravnborg unsigned long *src_val;
185d670bd4fSSam Ravnborg
186d670bd4fSSam Ravnborg if (reg_num)
187d670bd4fSSam Ravnborg src_val = fetch_reg_addr(reg_num, regs);
188d670bd4fSSam Ravnborg else {
189d670bd4fSSam Ravnborg src_val = &zero[0];
190d670bd4fSSam Ravnborg if (size == 8)
191d670bd4fSSam Ravnborg zero[1] = fetch_reg(1, regs);
192d670bd4fSSam Ravnborg }
193d670bd4fSSam Ravnborg return __do_int_store(dst_addr, size, src_val);
194d670bd4fSSam Ravnborg }
195d670bd4fSSam Ravnborg
196d670bd4fSSam Ravnborg extern void smp_capture(void);
197d670bd4fSSam Ravnborg extern void smp_release(void);
198d670bd4fSSam Ravnborg
advance(struct pt_regs * regs)199d670bd4fSSam Ravnborg static inline void advance(struct pt_regs *regs)
200d670bd4fSSam Ravnborg {
201d670bd4fSSam Ravnborg regs->pc = regs->npc;
202d670bd4fSSam Ravnborg regs->npc += 4;
203d670bd4fSSam Ravnborg }
204d670bd4fSSam Ravnborg
floating_point_load_or_store_p(unsigned int insn)205d670bd4fSSam Ravnborg static inline int floating_point_load_or_store_p(unsigned int insn)
206d670bd4fSSam Ravnborg {
207d670bd4fSSam Ravnborg return (insn >> 24) & 1;
208d670bd4fSSam Ravnborg }
209d670bd4fSSam Ravnborg
ok_for_kernel(unsigned int insn)210d670bd4fSSam Ravnborg static inline int ok_for_kernel(unsigned int insn)
211d670bd4fSSam Ravnborg {
212d670bd4fSSam Ravnborg return !floating_point_load_or_store_p(insn);
213d670bd4fSSam Ravnborg }
214d670bd4fSSam Ravnborg
kernel_mna_trap_fault(struct pt_regs * regs,unsigned int insn)215d670bd4fSSam Ravnborg static void kernel_mna_trap_fault(struct pt_regs *regs, unsigned int insn)
216d670bd4fSSam Ravnborg {
217b4edf06cSAl Viro const struct exception_table_entry *entry;
218d670bd4fSSam Ravnborg
219b4edf06cSAl Viro entry = search_exception_tables(regs->pc);
220b4edf06cSAl Viro if (!entry) {
221d670bd4fSSam Ravnborg unsigned long address = compute_effective_address(regs, insn);
222d670bd4fSSam Ravnborg if(address < PAGE_SIZE) {
223d670bd4fSSam Ravnborg printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference in mna handler");
224d670bd4fSSam Ravnborg } else
225d670bd4fSSam Ravnborg printk(KERN_ALERT "Unable to handle kernel paging request in mna handler");
226d670bd4fSSam Ravnborg printk(KERN_ALERT " at virtual address %08lx\n",address);
227d670bd4fSSam Ravnborg printk(KERN_ALERT "current->{mm,active_mm}->context = %08lx\n",
228d670bd4fSSam Ravnborg (current->mm ? current->mm->context :
229d670bd4fSSam Ravnborg current->active_mm->context));
230d670bd4fSSam Ravnborg printk(KERN_ALERT "current->{mm,active_mm}->pgd = %08lx\n",
231d670bd4fSSam Ravnborg (current->mm ? (unsigned long) current->mm->pgd :
232d670bd4fSSam Ravnborg (unsigned long) current->active_mm->pgd));
233d670bd4fSSam Ravnborg die_if_kernel("Oops", regs);
234d670bd4fSSam Ravnborg /* Not reached */
235d670bd4fSSam Ravnborg }
236b4edf06cSAl Viro regs->pc = entry->fixup;
237d670bd4fSSam Ravnborg regs->npc = regs->pc + 4;
238d670bd4fSSam Ravnborg }
239d670bd4fSSam Ravnborg
kernel_unaligned_trap(struct pt_regs * regs,unsigned int insn)240d670bd4fSSam Ravnborg asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn)
241d670bd4fSSam Ravnborg {
242d670bd4fSSam Ravnborg enum direction dir = decode_direction(insn);
243d670bd4fSSam Ravnborg int size = decode_access_size(insn);
244d670bd4fSSam Ravnborg
245d670bd4fSSam Ravnborg if(!ok_for_kernel(insn) || dir == both) {
246d670bd4fSSam Ravnborg printk("Unsupported unaligned load/store trap for kernel at <%08lx>.\n",
247d670bd4fSSam Ravnborg regs->pc);
248d670bd4fSSam Ravnborg unaligned_panic("Wheee. Kernel does fpu/atomic unaligned load/store.");
249d670bd4fSSam Ravnborg } else {
250d670bd4fSSam Ravnborg unsigned long addr = compute_effective_address(regs, insn);
251d670bd4fSSam Ravnborg int err;
252d670bd4fSSam Ravnborg
253a8b0ca17SPeter Zijlstra perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr);
254d670bd4fSSam Ravnborg switch (dir) {
255d670bd4fSSam Ravnborg case load:
256d670bd4fSSam Ravnborg err = do_int_load(fetch_reg_addr(((insn>>25)&0x1f),
257d670bd4fSSam Ravnborg regs),
258d670bd4fSSam Ravnborg size, (unsigned long *) addr,
259d670bd4fSSam Ravnborg decode_signedness(insn));
260d670bd4fSSam Ravnborg break;
261d670bd4fSSam Ravnborg
262d670bd4fSSam Ravnborg case store:
263d670bd4fSSam Ravnborg err = do_int_store(((insn>>25)&0x1f), size,
264d670bd4fSSam Ravnborg (unsigned long *) addr, regs);
265d670bd4fSSam Ravnborg break;
266d670bd4fSSam Ravnborg default:
267d670bd4fSSam Ravnborg panic("Impossible kernel unaligned trap.");
268d670bd4fSSam Ravnborg /* Not reached... */
269d670bd4fSSam Ravnborg }
270d670bd4fSSam Ravnborg if (err)
271d670bd4fSSam Ravnborg kernel_mna_trap_fault(regs, insn);
272d670bd4fSSam Ravnborg else
273d670bd4fSSam Ravnborg advance(regs);
274d670bd4fSSam Ravnborg }
275d670bd4fSSam Ravnborg }
276d670bd4fSSam Ravnborg
user_unaligned_trap(struct pt_regs * regs,unsigned int insn)277415ddc3bSAl Viro asmlinkage void user_unaligned_trap(struct pt_regs *regs, unsigned int insn)
278d670bd4fSSam Ravnborg {
2790e3d9f1eSEric W. Biederman send_sig_fault(SIGBUS, BUS_ADRALN,
2800e3d9f1eSEric W. Biederman (void __user *)safe_compute_effective_address(regs, insn),
281*2c9f7eafSEric W. Biederman current);
282d670bd4fSSam Ravnborg }
283