1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* align.c - handle alignment exceptions for the Power PC. 3 * 4 * Copyright (c) 1996 Paul Mackerras <paulus@cs.anu.edu.au> 5 * Copyright (c) 1998-1999 TiVo, Inc. 6 * PowerPC 403GCX modifications. 7 * Copyright (c) 1999 Grant Erickson <grant@lcse.umn.edu> 8 * PowerPC 403GCX/405GP modifications. 9 * Copyright (c) 2001-2002 PPC64 team, IBM Corp 10 * 64-bit and Power4 support 11 * Copyright (c) 2005 Benjamin Herrenschmidt, IBM Corp 12 * <benh@kernel.crashing.org> 13 * Merge ppc32 and ppc64 implementations 14 */ 15 16 #include <linux/kernel.h> 17 #include <linux/mm.h> 18 #include <asm/processor.h> 19 #include <linux/uaccess.h> 20 #include <asm/cache.h> 21 #include <asm/cputable.h> 22 #include <asm/emulated_ops.h> 23 #include <asm/switch_to.h> 24 #include <asm/disassemble.h> 25 #include <asm/cpu_has_feature.h> 26 #include <asm/sstep.h> 27 #include <asm/inst.h> 28 29 struct aligninfo { 30 unsigned char len; 31 unsigned char flags; 32 }; 33 34 35 #define INVALID { 0, 0 } 36 37 /* Bits in the flags field */ 38 #define LD 0 /* load */ 39 #define ST 1 /* store */ 40 #define SE 2 /* sign-extend value, or FP ld/st as word */ 41 #define SW 0x20 /* byte swap */ 42 #define E4 0x40 /* SPE endianness is word */ 43 #define E8 0x80 /* SPE endianness is double word */ 44 45 #ifdef CONFIG_SPE 46 47 static struct aligninfo spe_aligninfo[32] = { 48 { 8, LD+E8 }, /* 0 00 00: evldd[x] */ 49 { 8, LD+E4 }, /* 0 00 01: evldw[x] */ 50 { 8, LD }, /* 0 00 10: evldh[x] */ 51 INVALID, /* 0 00 11 */ 52 { 2, LD }, /* 0 01 00: evlhhesplat[x] */ 53 INVALID, /* 0 01 01 */ 54 { 2, LD }, /* 0 01 10: evlhhousplat[x] */ 55 { 2, LD+SE }, /* 0 01 11: evlhhossplat[x] */ 56 { 4, LD }, /* 0 10 00: evlwhe[x] */ 57 INVALID, /* 0 10 01 */ 58 { 4, LD }, /* 0 10 10: evlwhou[x] */ 59 { 4, LD+SE }, /* 0 10 11: evlwhos[x] */ 60 { 4, LD+E4 }, /* 0 11 00: evlwwsplat[x] */ 61 INVALID, /* 0 11 01 */ 62 { 4, LD }, /* 0 11 10: evlwhsplat[x] */ 63 INVALID, /* 0 11 11 */ 64 65 { 8, ST+E8 }, /* 1 00 00: evstdd[x] */ 66 { 8, ST+E4 }, /* 1 00 01: evstdw[x] */ 67 { 8, ST }, /* 1 00 10: evstdh[x] */ 68 INVALID, /* 1 00 11 */ 69 INVALID, /* 1 01 00 */ 70 INVALID, /* 1 01 01 */ 71 INVALID, /* 1 01 10 */ 72 INVALID, /* 1 01 11 */ 73 { 4, ST }, /* 1 10 00: evstwhe[x] */ 74 INVALID, /* 1 10 01 */ 75 { 4, ST }, /* 1 10 10: evstwho[x] */ 76 INVALID, /* 1 10 11 */ 77 { 4, ST+E4 }, /* 1 11 00: evstwwe[x] */ 78 INVALID, /* 1 11 01 */ 79 { 4, ST+E4 }, /* 1 11 10: evstwwo[x] */ 80 INVALID, /* 1 11 11 */ 81 }; 82 83 #define EVLDD 0x00 84 #define EVLDW 0x01 85 #define EVLDH 0x02 86 #define EVLHHESPLAT 0x04 87 #define EVLHHOUSPLAT 0x06 88 #define EVLHHOSSPLAT 0x07 89 #define EVLWHE 0x08 90 #define EVLWHOU 0x0A 91 #define EVLWHOS 0x0B 92 #define EVLWWSPLAT 0x0C 93 #define EVLWHSPLAT 0x0E 94 #define EVSTDD 0x10 95 #define EVSTDW 0x11 96 #define EVSTDH 0x12 97 #define EVSTWHE 0x18 98 #define EVSTWHO 0x1A 99 #define EVSTWWE 0x1C 100 #define EVSTWWO 0x1E 101 102 /* 103 * Emulate SPE loads and stores. 104 * Only Book-E has these instructions, and it does true little-endian, 105 * so we don't need the address swizzling. 106 */ 107 static int emulate_spe(struct pt_regs *regs, unsigned int reg, 108 ppc_inst_t ppc_instr) 109 { 110 union { 111 u64 ll; 112 u32 w[2]; 113 u16 h[4]; 114 u8 v[8]; 115 } data, temp; 116 unsigned char __user *p, *addr; 117 unsigned long *evr = ¤t->thread.evr[reg]; 118 unsigned int nb, flags, instr; 119 120 instr = ppc_inst_val(ppc_instr); 121 instr = (instr >> 1) & 0x1f; 122 123 /* DAR has the operand effective address */ 124 addr = (unsigned char __user *)regs->dar; 125 126 nb = spe_aligninfo[instr].len; 127 flags = spe_aligninfo[instr].flags; 128 129 /* userland only */ 130 if (unlikely(!user_mode(regs))) 131 return 0; 132 133 flush_spe_to_thread(current); 134 135 /* If we are loading, get the data from user space, else 136 * get it from register values 137 */ 138 if (flags & ST) { 139 data.ll = 0; 140 switch (instr) { 141 case EVSTDD: 142 case EVSTDW: 143 case EVSTDH: 144 data.w[0] = *evr; 145 data.w[1] = regs->gpr[reg]; 146 break; 147 case EVSTWHE: 148 data.h[2] = *evr >> 16; 149 data.h[3] = regs->gpr[reg] >> 16; 150 break; 151 case EVSTWHO: 152 data.h[2] = *evr & 0xffff; 153 data.h[3] = regs->gpr[reg] & 0xffff; 154 break; 155 case EVSTWWE: 156 data.w[1] = *evr; 157 break; 158 case EVSTWWO: 159 data.w[1] = regs->gpr[reg]; 160 break; 161 default: 162 return -EINVAL; 163 } 164 } else { 165 temp.ll = data.ll = 0; 166 p = addr; 167 168 if (!user_read_access_begin(addr, nb)) 169 return -EFAULT; 170 171 switch (nb) { 172 case 8: 173 unsafe_get_user(temp.v[0], p++, Efault_read); 174 unsafe_get_user(temp.v[1], p++, Efault_read); 175 unsafe_get_user(temp.v[2], p++, Efault_read); 176 unsafe_get_user(temp.v[3], p++, Efault_read); 177 fallthrough; 178 case 4: 179 unsafe_get_user(temp.v[4], p++, Efault_read); 180 unsafe_get_user(temp.v[5], p++, Efault_read); 181 fallthrough; 182 case 2: 183 unsafe_get_user(temp.v[6], p++, Efault_read); 184 unsafe_get_user(temp.v[7], p++, Efault_read); 185 } 186 user_read_access_end(); 187 188 switch (instr) { 189 case EVLDD: 190 case EVLDW: 191 case EVLDH: 192 data.ll = temp.ll; 193 break; 194 case EVLHHESPLAT: 195 data.h[0] = temp.h[3]; 196 data.h[2] = temp.h[3]; 197 break; 198 case EVLHHOUSPLAT: 199 case EVLHHOSSPLAT: 200 data.h[1] = temp.h[3]; 201 data.h[3] = temp.h[3]; 202 break; 203 case EVLWHE: 204 data.h[0] = temp.h[2]; 205 data.h[2] = temp.h[3]; 206 break; 207 case EVLWHOU: 208 case EVLWHOS: 209 data.h[1] = temp.h[2]; 210 data.h[3] = temp.h[3]; 211 break; 212 case EVLWWSPLAT: 213 data.w[0] = temp.w[1]; 214 data.w[1] = temp.w[1]; 215 break; 216 case EVLWHSPLAT: 217 data.h[0] = temp.h[2]; 218 data.h[1] = temp.h[2]; 219 data.h[2] = temp.h[3]; 220 data.h[3] = temp.h[3]; 221 break; 222 default: 223 return -EINVAL; 224 } 225 } 226 227 if (flags & SW) { 228 switch (flags & 0xf0) { 229 case E8: 230 data.ll = swab64(data.ll); 231 break; 232 case E4: 233 data.w[0] = swab32(data.w[0]); 234 data.w[1] = swab32(data.w[1]); 235 break; 236 /* Its half word endian */ 237 default: 238 data.h[0] = swab16(data.h[0]); 239 data.h[1] = swab16(data.h[1]); 240 data.h[2] = swab16(data.h[2]); 241 data.h[3] = swab16(data.h[3]); 242 break; 243 } 244 } 245 246 if (flags & SE) { 247 data.w[0] = (s16)data.h[1]; 248 data.w[1] = (s16)data.h[3]; 249 } 250 251 /* Store result to memory or update registers */ 252 if (flags & ST) { 253 p = addr; 254 255 if (!user_write_access_begin(addr, nb)) 256 return -EFAULT; 257 258 switch (nb) { 259 case 8: 260 unsafe_put_user(data.v[0], p++, Efault_write); 261 unsafe_put_user(data.v[1], p++, Efault_write); 262 unsafe_put_user(data.v[2], p++, Efault_write); 263 unsafe_put_user(data.v[3], p++, Efault_write); 264 fallthrough; 265 case 4: 266 unsafe_put_user(data.v[4], p++, Efault_write); 267 unsafe_put_user(data.v[5], p++, Efault_write); 268 fallthrough; 269 case 2: 270 unsafe_put_user(data.v[6], p++, Efault_write); 271 unsafe_put_user(data.v[7], p++, Efault_write); 272 } 273 user_write_access_end(); 274 } else { 275 *evr = data.w[0]; 276 regs->gpr[reg] = data.w[1]; 277 } 278 279 return 1; 280 281 Efault_read: 282 user_read_access_end(); 283 return -EFAULT; 284 285 Efault_write: 286 user_write_access_end(); 287 return -EFAULT; 288 } 289 #endif /* CONFIG_SPE */ 290 291 /* 292 * Called on alignment exception. Attempts to fixup 293 * 294 * Return 1 on success 295 * Return 0 if unable to handle the interrupt 296 * Return -EFAULT if data address is bad 297 * Other negative return values indicate that the instruction can't 298 * be emulated, and the process should be given a SIGBUS. 299 */ 300 301 int fix_alignment(struct pt_regs *regs) 302 { 303 ppc_inst_t instr; 304 struct instruction_op op; 305 int r, type; 306 307 if (is_kernel_addr(regs->nip)) 308 r = copy_inst_from_kernel_nofault(&instr, (void *)regs->nip); 309 else 310 r = __get_user_instr(instr, (void __user *)regs->nip); 311 312 if (unlikely(r)) 313 return -EFAULT; 314 if ((regs->msr & MSR_LE) != (MSR_KERNEL & MSR_LE)) { 315 /* We don't handle PPC little-endian any more... */ 316 if (cpu_has_feature(CPU_FTR_PPC_LE)) 317 return -EIO; 318 instr = ppc_inst_swab(instr); 319 } 320 321 #ifdef CONFIG_SPE 322 if (ppc_inst_primary_opcode(instr) == 0x4) { 323 int reg = (ppc_inst_val(instr) >> 21) & 0x1f; 324 PPC_WARN_ALIGNMENT(spe, regs); 325 return emulate_spe(regs, reg, instr); 326 } 327 #endif 328 329 330 /* 331 * ISA 3.0 (such as P9) copy, copy_first, paste and paste_last alignment 332 * check. 333 * 334 * Send a SIGBUS to the process that caused the fault. 335 * 336 * We do not emulate these because paste may contain additional metadata 337 * when pasting to a co-processor. Furthermore, paste_last is the 338 * synchronisation point for preceding copy/paste sequences. 339 */ 340 if ((ppc_inst_val(instr) & 0xfc0006fe) == (PPC_INST_COPY & 0xfc0006fe)) 341 return -EIO; 342 343 r = analyse_instr(&op, regs, instr); 344 if (r < 0) 345 return -EINVAL; 346 347 type = GETTYPE(op.type); 348 if (!OP_IS_LOAD_STORE(type)) { 349 if (op.type != CACHEOP + DCBZ) 350 return -EINVAL; 351 PPC_WARN_ALIGNMENT(dcbz, regs); 352 WARN_ON_ONCE(!user_mode(regs)); 353 r = emulate_dcbz(op.ea, regs); 354 } else { 355 if (type == LARX || type == STCX) 356 return -EIO; 357 PPC_WARN_ALIGNMENT(unaligned, regs); 358 r = emulate_loadstore(regs, &op); 359 } 360 361 if (!r) 362 return 1; 363 return r; 364 } 365