1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 Western Digital Corporation or its affiliates. 4 */ 5 #include <linux/kernel.h> 6 #include <linux/init.h> 7 #include <linux/mm.h> 8 #include <linux/module.h> 9 #include <linux/irq.h> 10 #include <linux/stringify.h> 11 12 #include <asm/processor.h> 13 #include <asm/ptrace.h> 14 #include <asm/csr.h> 15 16 #define INSN_MATCH_LB 0x3 17 #define INSN_MASK_LB 0x707f 18 #define INSN_MATCH_LH 0x1003 19 #define INSN_MASK_LH 0x707f 20 #define INSN_MATCH_LW 0x2003 21 #define INSN_MASK_LW 0x707f 22 #define INSN_MATCH_LD 0x3003 23 #define INSN_MASK_LD 0x707f 24 #define INSN_MATCH_LBU 0x4003 25 #define INSN_MASK_LBU 0x707f 26 #define INSN_MATCH_LHU 0x5003 27 #define INSN_MASK_LHU 0x707f 28 #define INSN_MATCH_LWU 0x6003 29 #define INSN_MASK_LWU 0x707f 30 #define INSN_MATCH_SB 0x23 31 #define INSN_MASK_SB 0x707f 32 #define INSN_MATCH_SH 0x1023 33 #define INSN_MASK_SH 0x707f 34 #define INSN_MATCH_SW 0x2023 35 #define INSN_MASK_SW 0x707f 36 #define INSN_MATCH_SD 0x3023 37 #define INSN_MASK_SD 0x707f 38 39 #define INSN_MATCH_FLW 0x2007 40 #define INSN_MASK_FLW 0x707f 41 #define INSN_MATCH_FLD 0x3007 42 #define INSN_MASK_FLD 0x707f 43 #define INSN_MATCH_FLQ 0x4007 44 #define INSN_MASK_FLQ 0x707f 45 #define INSN_MATCH_FSW 0x2027 46 #define INSN_MASK_FSW 0x707f 47 #define INSN_MATCH_FSD 0x3027 48 #define INSN_MASK_FSD 0x707f 49 #define INSN_MATCH_FSQ 0x4027 50 #define INSN_MASK_FSQ 0x707f 51 52 #define INSN_MATCH_C_LD 0x6000 53 #define INSN_MASK_C_LD 0xe003 54 #define INSN_MATCH_C_SD 0xe000 55 #define INSN_MASK_C_SD 0xe003 56 #define INSN_MATCH_C_LW 0x4000 57 #define INSN_MASK_C_LW 0xe003 58 #define INSN_MATCH_C_SW 0xc000 59 #define INSN_MASK_C_SW 0xe003 60 #define INSN_MATCH_C_LDSP 0x6002 61 #define INSN_MASK_C_LDSP 0xe003 62 #define INSN_MATCH_C_SDSP 0xe002 63 #define INSN_MASK_C_SDSP 0xe003 64 #define INSN_MATCH_C_LWSP 0x4002 65 #define INSN_MASK_C_LWSP 0xe003 66 #define INSN_MATCH_C_SWSP 0xc002 67 #define INSN_MASK_C_SWSP 0xe003 68 69 #define INSN_MATCH_C_FLD 0x2000 70 #define INSN_MASK_C_FLD 0xe003 71 #define INSN_MATCH_C_FLW 0x6000 72 #define INSN_MASK_C_FLW 0xe003 73 #define INSN_MATCH_C_FSD 0xa000 74 #define INSN_MASK_C_FSD 0xe003 75 #define INSN_MATCH_C_FSW 0xe000 76 #define INSN_MASK_C_FSW 0xe003 77 #define INSN_MATCH_C_FLDSP 0x2002 78 #define INSN_MASK_C_FLDSP 0xe003 79 #define INSN_MATCH_C_FSDSP 0xa002 80 #define INSN_MASK_C_FSDSP 0xe003 81 #define INSN_MATCH_C_FLWSP 0x6002 82 #define INSN_MASK_C_FLWSP 0xe003 83 #define INSN_MATCH_C_FSWSP 0xe002 84 #define INSN_MASK_C_FSWSP 0xe003 85 86 #define INSN_LEN(insn) ((((insn) & 0x3) < 0x3) ? 2 : 4) 87 88 #if defined(CONFIG_64BIT) 89 #define LOG_REGBYTES 3 90 #define XLEN 64 91 #else 92 #define LOG_REGBYTES 2 93 #define XLEN 32 94 #endif 95 #define REGBYTES (1 << LOG_REGBYTES) 96 #define XLEN_MINUS_16 ((XLEN) - 16) 97 98 #define SH_RD 7 99 #define SH_RS1 15 100 #define SH_RS2 20 101 #define SH_RS2C 2 102 103 #define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1)) 104 #define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \ 105 (RV_X(x, 10, 3) << 3) | \ 106 (RV_X(x, 5, 1) << 6)) 107 #define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \ 108 (RV_X(x, 5, 2) << 6)) 109 #define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \ 110 (RV_X(x, 12, 1) << 5) | \ 111 (RV_X(x, 2, 2) << 6)) 112 #define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \ 113 (RV_X(x, 12, 1) << 5) | \ 114 (RV_X(x, 2, 3) << 6)) 115 #define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \ 116 (RV_X(x, 7, 2) << 6)) 117 #define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \ 118 (RV_X(x, 7, 3) << 6)) 119 #define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3)) 120 #define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3)) 121 #define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5) 122 123 #define SHIFT_RIGHT(x, y) \ 124 ((y) < 0 ? ((x) << -(y)) : ((x) >> (y))) 125 126 #define REG_MASK \ 127 ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES)) 128 129 #define REG_OFFSET(insn, pos) \ 130 (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK) 131 132 #define REG_PTR(insn, pos, regs) \ 133 (ulong *)((ulong)(regs) + REG_OFFSET(insn, pos)) 134 135 #define GET_RM(insn) (((insn) >> 12) & 7) 136 137 #define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs)) 138 #define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs)) 139 #define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs)) 140 #define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs)) 141 #define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs)) 142 #define GET_SP(regs) (*REG_PTR(2, 0, regs)) 143 #define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val)) 144 #define IMM_I(insn) ((s32)(insn) >> 20) 145 #define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \ 146 (s32)(((insn) >> 7) & 0x1f)) 147 #define MASK_FUNCT3 0x7000 148 149 #define GET_PRECISION(insn) (((insn) >> 25) & 3) 150 #define GET_RM(insn) (((insn) >> 12) & 7) 151 #define PRECISION_S 0 152 #define PRECISION_D 1 153 154 static inline u8 load_u8(const u8 *addr) 155 { 156 u8 val; 157 158 asm volatile("lbu %0, %1" : "=&r" (val) : "m" (*addr)); 159 160 return val; 161 } 162 163 static inline void store_u8(u8 *addr, u8 val) 164 { 165 asm volatile ("sb %0, %1\n" : : "r" (val), "m" (*addr)); 166 } 167 168 static inline ulong get_insn(ulong mepc) 169 { 170 register ulong __mepc asm ("a2") = mepc; 171 ulong val, rvc_mask = 3, tmp; 172 173 asm ("and %[tmp], %[addr], 2\n" 174 "bnez %[tmp], 1f\n" 175 #if defined(CONFIG_64BIT) 176 __stringify(LWU) " %[insn], (%[addr])\n" 177 #else 178 __stringify(LW) " %[insn], (%[addr])\n" 179 #endif 180 "and %[tmp], %[insn], %[rvc_mask]\n" 181 "beq %[tmp], %[rvc_mask], 2f\n" 182 "sll %[insn], %[insn], %[xlen_minus_16]\n" 183 "srl %[insn], %[insn], %[xlen_minus_16]\n" 184 "j 2f\n" 185 "1:\n" 186 "lhu %[insn], (%[addr])\n" 187 "and %[tmp], %[insn], %[rvc_mask]\n" 188 "bne %[tmp], %[rvc_mask], 2f\n" 189 "lhu %[tmp], 2(%[addr])\n" 190 "sll %[tmp], %[tmp], 16\n" 191 "add %[insn], %[insn], %[tmp]\n" 192 "2:" 193 : [insn] "=&r" (val), [tmp] "=&r" (tmp) 194 : [addr] "r" (__mepc), [rvc_mask] "r" (rvc_mask), 195 [xlen_minus_16] "i" (XLEN_MINUS_16)); 196 197 return val; 198 } 199 200 union reg_data { 201 u8 data_bytes[8]; 202 ulong data_ulong; 203 u64 data_u64; 204 }; 205 206 int handle_misaligned_load(struct pt_regs *regs) 207 { 208 union reg_data val; 209 unsigned long epc = regs->epc; 210 unsigned long insn = get_insn(epc); 211 unsigned long addr = csr_read(mtval); 212 int i, fp = 0, shift = 0, len = 0; 213 214 regs->epc = 0; 215 216 if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) { 217 len = 4; 218 shift = 8 * (sizeof(unsigned long) - len); 219 #if defined(CONFIG_64BIT) 220 } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) { 221 len = 8; 222 shift = 8 * (sizeof(unsigned long) - len); 223 } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) { 224 len = 4; 225 #endif 226 } else if ((insn & INSN_MASK_FLD) == INSN_MATCH_FLD) { 227 fp = 1; 228 len = 8; 229 } else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) { 230 fp = 1; 231 len = 4; 232 } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) { 233 len = 2; 234 shift = 8 * (sizeof(unsigned long) - len); 235 } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) { 236 len = 2; 237 #if defined(CONFIG_64BIT) 238 } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) { 239 len = 8; 240 shift = 8 * (sizeof(unsigned long) - len); 241 insn = RVC_RS2S(insn) << SH_RD; 242 } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP && 243 ((insn >> SH_RD) & 0x1f)) { 244 len = 8; 245 shift = 8 * (sizeof(unsigned long) - len); 246 #endif 247 } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) { 248 len = 4; 249 shift = 8 * (sizeof(unsigned long) - len); 250 insn = RVC_RS2S(insn) << SH_RD; 251 } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP && 252 ((insn >> SH_RD) & 0x1f)) { 253 len = 4; 254 shift = 8 * (sizeof(unsigned long) - len); 255 } else if ((insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD) { 256 fp = 1; 257 len = 8; 258 insn = RVC_RS2S(insn) << SH_RD; 259 } else if ((insn & INSN_MASK_C_FLDSP) == INSN_MATCH_C_FLDSP) { 260 fp = 1; 261 len = 8; 262 #if defined(CONFIG_32BIT) 263 } else if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) { 264 fp = 1; 265 len = 4; 266 insn = RVC_RS2S(insn) << SH_RD; 267 } else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) { 268 fp = 1; 269 len = 4; 270 #endif 271 } else { 272 regs->epc = epc; 273 return -1; 274 } 275 276 val.data_u64 = 0; 277 for (i = 0; i < len; i++) 278 val.data_bytes[i] = load_u8((void *)(addr + i)); 279 280 if (fp) 281 return -1; 282 SET_RD(insn, regs, val.data_ulong << shift >> shift); 283 284 regs->epc = epc + INSN_LEN(insn); 285 286 return 0; 287 } 288 289 int handle_misaligned_store(struct pt_regs *regs) 290 { 291 union reg_data val; 292 unsigned long epc = regs->epc; 293 unsigned long insn = get_insn(epc); 294 unsigned long addr = csr_read(mtval); 295 int i, len = 0; 296 297 regs->epc = 0; 298 299 val.data_ulong = GET_RS2(insn, regs); 300 301 if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) { 302 len = 4; 303 #if defined(CONFIG_64BIT) 304 } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) { 305 len = 8; 306 #endif 307 } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) { 308 len = 2; 309 #if defined(CONFIG_64BIT) 310 } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) { 311 len = 8; 312 val.data_ulong = GET_RS2S(insn, regs); 313 } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP) { 314 len = 8; 315 val.data_ulong = GET_RS2C(insn, regs); 316 #endif 317 } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) { 318 len = 4; 319 val.data_ulong = GET_RS2S(insn, regs); 320 } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP) { 321 len = 4; 322 val.data_ulong = GET_RS2C(insn, regs); 323 } else { 324 regs->epc = epc; 325 return -1; 326 } 327 328 for (i = 0; i < len; i++) 329 store_u8((void *)(addr + i), val.data_bytes[i]); 330 331 regs->epc = epc + INSN_LEN(insn); 332 333 return 0; 334 } 335