1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2020 Western Digital Corporation or its affiliates.
4 */
5 #include <linux/kernel.h>
6 #include <linux/init.h>
7 #include <linux/mm.h>
8 #include <linux/module.h>
9 #include <linux/irq.h>
10 #include <linux/stringify.h>
11
12 #include <asm/processor.h>
13 #include <asm/ptrace.h>
14 #include <asm/csr.h>
15
16 #define INSN_MATCH_LB 0x3
17 #define INSN_MASK_LB 0x707f
18 #define INSN_MATCH_LH 0x1003
19 #define INSN_MASK_LH 0x707f
20 #define INSN_MATCH_LW 0x2003
21 #define INSN_MASK_LW 0x707f
22 #define INSN_MATCH_LD 0x3003
23 #define INSN_MASK_LD 0x707f
24 #define INSN_MATCH_LBU 0x4003
25 #define INSN_MASK_LBU 0x707f
26 #define INSN_MATCH_LHU 0x5003
27 #define INSN_MASK_LHU 0x707f
28 #define INSN_MATCH_LWU 0x6003
29 #define INSN_MASK_LWU 0x707f
30 #define INSN_MATCH_SB 0x23
31 #define INSN_MASK_SB 0x707f
32 #define INSN_MATCH_SH 0x1023
33 #define INSN_MASK_SH 0x707f
34 #define INSN_MATCH_SW 0x2023
35 #define INSN_MASK_SW 0x707f
36 #define INSN_MATCH_SD 0x3023
37 #define INSN_MASK_SD 0x707f
38
39 #define INSN_MATCH_FLW 0x2007
40 #define INSN_MASK_FLW 0x707f
41 #define INSN_MATCH_FLD 0x3007
42 #define INSN_MASK_FLD 0x707f
43 #define INSN_MATCH_FLQ 0x4007
44 #define INSN_MASK_FLQ 0x707f
45 #define INSN_MATCH_FSW 0x2027
46 #define INSN_MASK_FSW 0x707f
47 #define INSN_MATCH_FSD 0x3027
48 #define INSN_MASK_FSD 0x707f
49 #define INSN_MATCH_FSQ 0x4027
50 #define INSN_MASK_FSQ 0x707f
51
52 #define INSN_MATCH_C_LD 0x6000
53 #define INSN_MASK_C_LD 0xe003
54 #define INSN_MATCH_C_SD 0xe000
55 #define INSN_MASK_C_SD 0xe003
56 #define INSN_MATCH_C_LW 0x4000
57 #define INSN_MASK_C_LW 0xe003
58 #define INSN_MATCH_C_SW 0xc000
59 #define INSN_MASK_C_SW 0xe003
60 #define INSN_MATCH_C_LDSP 0x6002
61 #define INSN_MASK_C_LDSP 0xe003
62 #define INSN_MATCH_C_SDSP 0xe002
63 #define INSN_MASK_C_SDSP 0xe003
64 #define INSN_MATCH_C_LWSP 0x4002
65 #define INSN_MASK_C_LWSP 0xe003
66 #define INSN_MATCH_C_SWSP 0xc002
67 #define INSN_MASK_C_SWSP 0xe003
68
69 #define INSN_MATCH_C_FLD 0x2000
70 #define INSN_MASK_C_FLD 0xe003
71 #define INSN_MATCH_C_FLW 0x6000
72 #define INSN_MASK_C_FLW 0xe003
73 #define INSN_MATCH_C_FSD 0xa000
74 #define INSN_MASK_C_FSD 0xe003
75 #define INSN_MATCH_C_FSW 0xe000
76 #define INSN_MASK_C_FSW 0xe003
77 #define INSN_MATCH_C_FLDSP 0x2002
78 #define INSN_MASK_C_FLDSP 0xe003
79 #define INSN_MATCH_C_FSDSP 0xa002
80 #define INSN_MASK_C_FSDSP 0xe003
81 #define INSN_MATCH_C_FLWSP 0x6002
82 #define INSN_MASK_C_FLWSP 0xe003
83 #define INSN_MATCH_C_FSWSP 0xe002
84 #define INSN_MASK_C_FSWSP 0xe003
85
86 #define INSN_LEN(insn) ((((insn) & 0x3) < 0x3) ? 2 : 4)
87
88 #if defined(CONFIG_64BIT)
89 #define LOG_REGBYTES 3
90 #define XLEN 64
91 #else
92 #define LOG_REGBYTES 2
93 #define XLEN 32
94 #endif
95 #define REGBYTES (1 << LOG_REGBYTES)
96 #define XLEN_MINUS_16 ((XLEN) - 16)
97
98 #define SH_RD 7
99 #define SH_RS1 15
100 #define SH_RS2 20
101 #define SH_RS2C 2
102
103 #define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1))
104 #define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \
105 (RV_X(x, 10, 3) << 3) | \
106 (RV_X(x, 5, 1) << 6))
107 #define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \
108 (RV_X(x, 5, 2) << 6))
109 #define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \
110 (RV_X(x, 12, 1) << 5) | \
111 (RV_X(x, 2, 2) << 6))
112 #define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \
113 (RV_X(x, 12, 1) << 5) | \
114 (RV_X(x, 2, 3) << 6))
115 #define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \
116 (RV_X(x, 7, 2) << 6))
117 #define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \
118 (RV_X(x, 7, 3) << 6))
119 #define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3))
120 #define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3))
121 #define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5)
122
123 #define SHIFT_RIGHT(x, y) \
124 ((y) < 0 ? ((x) << -(y)) : ((x) >> (y)))
125
126 #define REG_MASK \
127 ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES))
128
129 #define REG_OFFSET(insn, pos) \
130 (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK)
131
132 #define REG_PTR(insn, pos, regs) \
133 (ulong *)((ulong)(regs) + REG_OFFSET(insn, pos))
134
135 #define GET_RM(insn) (((insn) >> 12) & 7)
136
137 #define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs))
138 #define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs))
139 #define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs))
140 #define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs))
141 #define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs))
142 #define GET_SP(regs) (*REG_PTR(2, 0, regs))
143 #define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val))
144 #define IMM_I(insn) ((s32)(insn) >> 20)
145 #define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \
146 (s32)(((insn) >> 7) & 0x1f))
147 #define MASK_FUNCT3 0x7000
148
149 #define GET_PRECISION(insn) (((insn) >> 25) & 3)
150 #define GET_RM(insn) (((insn) >> 12) & 7)
151 #define PRECISION_S 0
152 #define PRECISION_D 1
153
load_u8(const u8 * addr)154 static inline u8 load_u8(const u8 *addr)
155 {
156 u8 val;
157
158 asm volatile("lbu %0, %1" : "=&r" (val) : "m" (*addr));
159
160 return val;
161 }
162
store_u8(u8 * addr,u8 val)163 static inline void store_u8(u8 *addr, u8 val)
164 {
165 asm volatile ("sb %0, %1\n" : : "r" (val), "m" (*addr));
166 }
167
get_insn(ulong mepc)168 static inline ulong get_insn(ulong mepc)
169 {
170 register ulong __mepc asm ("a2") = mepc;
171 ulong val, rvc_mask = 3, tmp;
172
173 asm ("and %[tmp], %[addr], 2\n"
174 "bnez %[tmp], 1f\n"
175 #if defined(CONFIG_64BIT)
176 __stringify(LWU) " %[insn], (%[addr])\n"
177 #else
178 __stringify(LW) " %[insn], (%[addr])\n"
179 #endif
180 "and %[tmp], %[insn], %[rvc_mask]\n"
181 "beq %[tmp], %[rvc_mask], 2f\n"
182 "sll %[insn], %[insn], %[xlen_minus_16]\n"
183 "srl %[insn], %[insn], %[xlen_minus_16]\n"
184 "j 2f\n"
185 "1:\n"
186 "lhu %[insn], (%[addr])\n"
187 "and %[tmp], %[insn], %[rvc_mask]\n"
188 "bne %[tmp], %[rvc_mask], 2f\n"
189 "lhu %[tmp], 2(%[addr])\n"
190 "sll %[tmp], %[tmp], 16\n"
191 "add %[insn], %[insn], %[tmp]\n"
192 "2:"
193 : [insn] "=&r" (val), [tmp] "=&r" (tmp)
194 : [addr] "r" (__mepc), [rvc_mask] "r" (rvc_mask),
195 [xlen_minus_16] "i" (XLEN_MINUS_16));
196
197 return val;
198 }
199
200 union reg_data {
201 u8 data_bytes[8];
202 ulong data_ulong;
203 u64 data_u64;
204 };
205
handle_misaligned_load(struct pt_regs * regs)206 int handle_misaligned_load(struct pt_regs *regs)
207 {
208 union reg_data val;
209 unsigned long epc = regs->epc;
210 unsigned long insn = get_insn(epc);
211 unsigned long addr = csr_read(mtval);
212 int i, fp = 0, shift = 0, len = 0;
213
214 regs->epc = 0;
215
216 if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) {
217 len = 4;
218 shift = 8 * (sizeof(unsigned long) - len);
219 #if defined(CONFIG_64BIT)
220 } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) {
221 len = 8;
222 shift = 8 * (sizeof(unsigned long) - len);
223 } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) {
224 len = 4;
225 #endif
226 } else if ((insn & INSN_MASK_FLD) == INSN_MATCH_FLD) {
227 fp = 1;
228 len = 8;
229 } else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) {
230 fp = 1;
231 len = 4;
232 } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) {
233 len = 2;
234 shift = 8 * (sizeof(unsigned long) - len);
235 } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) {
236 len = 2;
237 #if defined(CONFIG_64BIT)
238 } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) {
239 len = 8;
240 shift = 8 * (sizeof(unsigned long) - len);
241 insn = RVC_RS2S(insn) << SH_RD;
242 } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP &&
243 ((insn >> SH_RD) & 0x1f)) {
244 len = 8;
245 shift = 8 * (sizeof(unsigned long) - len);
246 #endif
247 } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) {
248 len = 4;
249 shift = 8 * (sizeof(unsigned long) - len);
250 insn = RVC_RS2S(insn) << SH_RD;
251 } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP &&
252 ((insn >> SH_RD) & 0x1f)) {
253 len = 4;
254 shift = 8 * (sizeof(unsigned long) - len);
255 } else if ((insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD) {
256 fp = 1;
257 len = 8;
258 insn = RVC_RS2S(insn) << SH_RD;
259 } else if ((insn & INSN_MASK_C_FLDSP) == INSN_MATCH_C_FLDSP) {
260 fp = 1;
261 len = 8;
262 #if defined(CONFIG_32BIT)
263 } else if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) {
264 fp = 1;
265 len = 4;
266 insn = RVC_RS2S(insn) << SH_RD;
267 } else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) {
268 fp = 1;
269 len = 4;
270 #endif
271 } else {
272 regs->epc = epc;
273 return -1;
274 }
275
276 val.data_u64 = 0;
277 for (i = 0; i < len; i++)
278 val.data_bytes[i] = load_u8((void *)(addr + i));
279
280 if (fp)
281 return -1;
282 SET_RD(insn, regs, val.data_ulong << shift >> shift);
283
284 regs->epc = epc + INSN_LEN(insn);
285
286 return 0;
287 }
288
handle_misaligned_store(struct pt_regs * regs)289 int handle_misaligned_store(struct pt_regs *regs)
290 {
291 union reg_data val;
292 unsigned long epc = regs->epc;
293 unsigned long insn = get_insn(epc);
294 unsigned long addr = csr_read(mtval);
295 int i, len = 0;
296
297 regs->epc = 0;
298
299 val.data_ulong = GET_RS2(insn, regs);
300
301 if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) {
302 len = 4;
303 #if defined(CONFIG_64BIT)
304 } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) {
305 len = 8;
306 #endif
307 } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) {
308 len = 2;
309 #if defined(CONFIG_64BIT)
310 } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) {
311 len = 8;
312 val.data_ulong = GET_RS2S(insn, regs);
313 } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP) {
314 len = 8;
315 val.data_ulong = GET_RS2C(insn, regs);
316 #endif
317 } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) {
318 len = 4;
319 val.data_ulong = GET_RS2S(insn, regs);
320 } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP) {
321 len = 4;
322 val.data_ulong = GET_RS2C(insn, regs);
323 } else {
324 regs->epc = epc;
325 return -1;
326 }
327
328 for (i = 0; i < len; i++)
329 store_u8((void *)(addr + i), val.data_bytes[i]);
330
331 regs->epc = epc + INSN_LEN(insn);
332
333 return 0;
334 }
335