1 /* align.c - handle alignment exceptions for the Power PC. 2 * 3 * Copyright (c) 1996 Paul Mackerras <paulus@cs.anu.edu.au> 4 * Copyright (c) 1998-1999 TiVo, Inc. 5 * PowerPC 403GCX modifications. 6 * Copyright (c) 1999 Grant Erickson <grant@lcse.umn.edu> 7 * PowerPC 403GCX/405GP modifications. 8 * Copyright (c) 2001-2002 PPC64 team, IBM Corp 9 * 64-bit and Power4 support 10 * Copyright (c) 2005 Benjamin Herrenschmidt, IBM Corp 11 * <benh@kernel.crashing.org> 12 * Merge ppc32 and ppc64 implementations 13 * 14 * This program is free software; you can redistribute it and/or 15 * modify it under the terms of the GNU General Public License 16 * as published by the Free Software Foundation; either version 17 * 2 of the License, or (at your option) any later version. 18 */ 19 20 #include <linux/kernel.h> 21 #include <linux/mm.h> 22 #include <asm/processor.h> 23 #include <asm/uaccess.h> 24 #include <asm/system.h> 25 #include <asm/cache.h> 26 #include <asm/cputable.h> 27 28 struct aligninfo { 29 unsigned char len; 30 unsigned char flags; 31 }; 32 33 #define IS_XFORM(inst) (((inst) >> 26) == 31) 34 #define IS_DSFORM(inst) (((inst) >> 26) >= 56) 35 36 #define INVALID { 0, 0 } 37 38 #define LD 1 /* load */ 39 #define ST 2 /* store */ 40 #define SE 4 /* sign-extend value */ 41 #define F 8 /* to/from fp regs */ 42 #define U 0x10 /* update index register */ 43 #define M 0x20 /* multiple load/store */ 44 #define SW 0x40 /* byte swap int or ... */ 45 #define S 0x40 /* ... single-precision fp */ 46 #define SX 0x40 /* byte count in XER */ 47 #define HARD 0x80 /* string, stwcx. */ 48 49 #define DCBZ 0x5f /* 8xx/82xx dcbz faults when cache not enabled */ 50 51 #define SWAP(a, b) (t = (a), (a) = (b), (b) = t) 52 53 /* 54 * The PowerPC stores certain bits of the instruction that caused the 55 * alignment exception in the DSISR register. This array maps those 56 * bits to information about the operand length and what the 57 * instruction would do. 58 */ 59 static struct aligninfo aligninfo[128] = { 60 { 4, LD }, /* 00 0 0000: lwz / lwarx */ 61 INVALID, /* 00 0 0001 */ 62 { 4, ST }, /* 00 0 0010: stw */ 63 INVALID, /* 00 0 0011 */ 64 { 2, LD }, /* 00 0 0100: lhz */ 65 { 2, LD+SE }, /* 00 0 0101: lha */ 66 { 2, ST }, /* 00 0 0110: sth */ 67 { 4, LD+M }, /* 00 0 0111: lmw */ 68 { 4, LD+F+S }, /* 00 0 1000: lfs */ 69 { 8, LD+F }, /* 00 0 1001: lfd */ 70 { 4, ST+F+S }, /* 00 0 1010: stfs */ 71 { 8, ST+F }, /* 00 0 1011: stfd */ 72 INVALID, /* 00 0 1100 */ 73 { 8, LD }, /* 00 0 1101: ld/ldu/lwa */ 74 INVALID, /* 00 0 1110 */ 75 { 8, ST }, /* 00 0 1111: std/stdu */ 76 { 4, LD+U }, /* 00 1 0000: lwzu */ 77 INVALID, /* 00 1 0001 */ 78 { 4, ST+U }, /* 00 1 0010: stwu */ 79 INVALID, /* 00 1 0011 */ 80 { 2, LD+U }, /* 00 1 0100: lhzu */ 81 { 2, LD+SE+U }, /* 00 1 0101: lhau */ 82 { 2, ST+U }, /* 00 1 0110: sthu */ 83 { 4, ST+M }, /* 00 1 0111: stmw */ 84 { 4, LD+F+S+U }, /* 00 1 1000: lfsu */ 85 { 8, LD+F+U }, /* 00 1 1001: lfdu */ 86 { 4, ST+F+S+U }, /* 00 1 1010: stfsu */ 87 { 8, ST+F+U }, /* 00 1 1011: stfdu */ 88 INVALID, /* 00 1 1100 */ 89 INVALID, /* 00 1 1101 */ 90 INVALID, /* 00 1 1110 */ 91 INVALID, /* 00 1 1111 */ 92 { 8, LD }, /* 01 0 0000: ldx */ 93 INVALID, /* 01 0 0001 */ 94 { 8, ST }, /* 01 0 0010: stdx */ 95 INVALID, /* 01 0 0011 */ 96 INVALID, /* 01 0 0100 */ 97 { 4, LD+SE }, /* 01 0 0101: lwax */ 98 INVALID, /* 01 0 0110 */ 99 INVALID, /* 01 0 0111 */ 100 { 4, LD+M+HARD+SX }, /* 01 0 1000: lswx */ 101 { 4, LD+M+HARD }, /* 01 0 1001: lswi */ 102 { 4, ST+M+HARD+SX }, /* 01 0 1010: stswx */ 103 { 4, ST+M+HARD }, /* 01 0 1011: stswi */ 104 INVALID, /* 01 0 1100 */ 105 { 8, LD+U }, /* 01 0 1101: ldu */ 106 INVALID, /* 01 0 1110 */ 107 { 8, ST+U }, /* 01 0 1111: stdu */ 108 { 8, LD+U }, /* 01 1 0000: ldux */ 109 INVALID, /* 01 1 0001 */ 110 { 8, ST+U }, /* 01 1 0010: stdux */ 111 INVALID, /* 01 1 0011 */ 112 INVALID, /* 01 1 0100 */ 113 { 4, LD+SE+U }, /* 01 1 0101: lwaux */ 114 INVALID, /* 01 1 0110 */ 115 INVALID, /* 01 1 0111 */ 116 INVALID, /* 01 1 1000 */ 117 INVALID, /* 01 1 1001 */ 118 INVALID, /* 01 1 1010 */ 119 INVALID, /* 01 1 1011 */ 120 INVALID, /* 01 1 1100 */ 121 INVALID, /* 01 1 1101 */ 122 INVALID, /* 01 1 1110 */ 123 INVALID, /* 01 1 1111 */ 124 INVALID, /* 10 0 0000 */ 125 INVALID, /* 10 0 0001 */ 126 INVALID, /* 10 0 0010: stwcx. */ 127 INVALID, /* 10 0 0011 */ 128 INVALID, /* 10 0 0100 */ 129 INVALID, /* 10 0 0101 */ 130 INVALID, /* 10 0 0110 */ 131 INVALID, /* 10 0 0111 */ 132 { 4, LD+SW }, /* 10 0 1000: lwbrx */ 133 INVALID, /* 10 0 1001 */ 134 { 4, ST+SW }, /* 10 0 1010: stwbrx */ 135 INVALID, /* 10 0 1011 */ 136 { 2, LD+SW }, /* 10 0 1100: lhbrx */ 137 { 4, LD+SE }, /* 10 0 1101 lwa */ 138 { 2, ST+SW }, /* 10 0 1110: sthbrx */ 139 INVALID, /* 10 0 1111 */ 140 INVALID, /* 10 1 0000 */ 141 INVALID, /* 10 1 0001 */ 142 INVALID, /* 10 1 0010 */ 143 INVALID, /* 10 1 0011 */ 144 INVALID, /* 10 1 0100 */ 145 INVALID, /* 10 1 0101 */ 146 INVALID, /* 10 1 0110 */ 147 INVALID, /* 10 1 0111 */ 148 INVALID, /* 10 1 1000 */ 149 INVALID, /* 10 1 1001 */ 150 INVALID, /* 10 1 1010 */ 151 INVALID, /* 10 1 1011 */ 152 INVALID, /* 10 1 1100 */ 153 INVALID, /* 10 1 1101 */ 154 INVALID, /* 10 1 1110 */ 155 { 0, ST+HARD }, /* 10 1 1111: dcbz */ 156 { 4, LD }, /* 11 0 0000: lwzx */ 157 INVALID, /* 11 0 0001 */ 158 { 4, ST }, /* 11 0 0010: stwx */ 159 INVALID, /* 11 0 0011 */ 160 { 2, LD }, /* 11 0 0100: lhzx */ 161 { 2, LD+SE }, /* 11 0 0101: lhax */ 162 { 2, ST }, /* 11 0 0110: sthx */ 163 INVALID, /* 11 0 0111 */ 164 { 4, LD+F+S }, /* 11 0 1000: lfsx */ 165 { 8, LD+F }, /* 11 0 1001: lfdx */ 166 { 4, ST+F+S }, /* 11 0 1010: stfsx */ 167 { 8, ST+F }, /* 11 0 1011: stfdx */ 168 INVALID, /* 11 0 1100 */ 169 { 8, LD+M }, /* 11 0 1101: lmd */ 170 INVALID, /* 11 0 1110 */ 171 { 8, ST+M }, /* 11 0 1111: stmd */ 172 { 4, LD+U }, /* 11 1 0000: lwzux */ 173 INVALID, /* 11 1 0001 */ 174 { 4, ST+U }, /* 11 1 0010: stwux */ 175 INVALID, /* 11 1 0011 */ 176 { 2, LD+U }, /* 11 1 0100: lhzux */ 177 { 2, LD+SE+U }, /* 11 1 0101: lhaux */ 178 { 2, ST+U }, /* 11 1 0110: sthux */ 179 INVALID, /* 11 1 0111 */ 180 { 4, LD+F+S+U }, /* 11 1 1000: lfsux */ 181 { 8, LD+F+U }, /* 11 1 1001: lfdux */ 182 { 4, ST+F+S+U }, /* 11 1 1010: stfsux */ 183 { 8, ST+F+U }, /* 11 1 1011: stfdux */ 184 INVALID, /* 11 1 1100 */ 185 INVALID, /* 11 1 1101 */ 186 INVALID, /* 11 1 1110 */ 187 INVALID, /* 11 1 1111 */ 188 }; 189 190 /* 191 * Create a DSISR value from the instruction 192 */ 193 static inline unsigned make_dsisr(unsigned instr) 194 { 195 unsigned dsisr; 196 197 198 /* bits 6:15 --> 22:31 */ 199 dsisr = (instr & 0x03ff0000) >> 16; 200 201 if (IS_XFORM(instr)) { 202 /* bits 29:30 --> 15:16 */ 203 dsisr |= (instr & 0x00000006) << 14; 204 /* bit 25 --> 17 */ 205 dsisr |= (instr & 0x00000040) << 8; 206 /* bits 21:24 --> 18:21 */ 207 dsisr |= (instr & 0x00000780) << 3; 208 } else { 209 /* bit 5 --> 17 */ 210 dsisr |= (instr & 0x04000000) >> 12; 211 /* bits 1: 4 --> 18:21 */ 212 dsisr |= (instr & 0x78000000) >> 17; 213 /* bits 30:31 --> 12:13 */ 214 if (IS_DSFORM(instr)) 215 dsisr |= (instr & 0x00000003) << 18; 216 } 217 218 return dsisr; 219 } 220 221 /* 222 * The dcbz (data cache block zero) instruction 223 * gives an alignment fault if used on non-cacheable 224 * memory. We handle the fault mainly for the 225 * case when we are running with the cache disabled 226 * for debugging. 227 */ 228 static int emulate_dcbz(struct pt_regs *regs, unsigned char __user *addr) 229 { 230 long __user *p; 231 int i, size; 232 233 #ifdef __powerpc64__ 234 size = ppc64_caches.dline_size; 235 #else 236 size = L1_CACHE_BYTES; 237 #endif 238 p = (long __user *) (regs->dar & -size); 239 if (user_mode(regs) && !access_ok(VERIFY_WRITE, p, size)) 240 return -EFAULT; 241 for (i = 0; i < size / sizeof(long); ++i) 242 if (__put_user(0, p+i)) 243 return -EFAULT; 244 return 1; 245 } 246 247 /* 248 * Emulate load & store multiple instructions 249 * On 64-bit machines, these instructions only affect/use the 250 * bottom 4 bytes of each register, and the loads clear the 251 * top 4 bytes of the affected register. 252 */ 253 #ifdef CONFIG_PPC64 254 #define REG_BYTE(rp, i) *((u8 *)((rp) + ((i) >> 2)) + ((i) & 3) + 4) 255 #else 256 #define REG_BYTE(rp, i) *((u8 *)(rp) + (i)) 257 #endif 258 259 static int emulate_multiple(struct pt_regs *regs, unsigned char __user *addr, 260 unsigned int reg, unsigned int nb, 261 unsigned int flags, unsigned int instr) 262 { 263 unsigned long *rptr; 264 unsigned int nb0, i; 265 266 /* 267 * We do not try to emulate 8 bytes multiple as they aren't really 268 * available in our operating environments and we don't try to 269 * emulate multiples operations in kernel land as they should never 270 * be used/generated there at least not on unaligned boundaries 271 */ 272 if (unlikely((nb > 4) || !user_mode(regs))) 273 return 0; 274 275 /* lmw, stmw, lswi/x, stswi/x */ 276 nb0 = 0; 277 if (flags & HARD) { 278 if (flags & SX) { 279 nb = regs->xer & 127; 280 if (nb == 0) 281 return 1; 282 } else { 283 if (__get_user(instr, 284 (unsigned int __user *)regs->nip)) 285 return -EFAULT; 286 nb = (instr >> 11) & 0x1f; 287 if (nb == 0) 288 nb = 32; 289 } 290 if (nb + reg * 4 > 128) { 291 nb0 = nb + reg * 4 - 128; 292 nb = 128 - reg * 4; 293 } 294 } else { 295 /* lwm, stmw */ 296 nb = (32 - reg) * 4; 297 } 298 299 if (!access_ok((flags & ST ? VERIFY_WRITE: VERIFY_READ), addr, nb+nb0)) 300 return -EFAULT; /* bad address */ 301 302 rptr = ®s->gpr[reg]; 303 if (flags & LD) { 304 /* 305 * This zeroes the top 4 bytes of the affected registers 306 * in 64-bit mode, and also zeroes out any remaining 307 * bytes of the last register for lsw*. 308 */ 309 memset(rptr, 0, ((nb + 3) / 4) * sizeof(unsigned long)); 310 if (nb0 > 0) 311 memset(®s->gpr[0], 0, 312 ((nb0 + 3) / 4) * sizeof(unsigned long)); 313 314 for (i = 0; i < nb; ++i) 315 if (__get_user(REG_BYTE(rptr, i), addr + i)) 316 return -EFAULT; 317 if (nb0 > 0) { 318 rptr = ®s->gpr[0]; 319 addr += nb; 320 for (i = 0; i < nb0; ++i) 321 if (__get_user(REG_BYTE(rptr, i), addr + i)) 322 return -EFAULT; 323 } 324 325 } else { 326 for (i = 0; i < nb; ++i) 327 if (__put_user(REG_BYTE(rptr, i), addr + i)) 328 return -EFAULT; 329 if (nb0 > 0) { 330 rptr = ®s->gpr[0]; 331 addr += nb; 332 for (i = 0; i < nb0; ++i) 333 if (__put_user(REG_BYTE(rptr, i), addr + i)) 334 return -EFAULT; 335 } 336 } 337 return 1; 338 } 339 340 341 /* 342 * Called on alignment exception. Attempts to fixup 343 * 344 * Return 1 on success 345 * Return 0 if unable to handle the interrupt 346 * Return -EFAULT if data address is bad 347 */ 348 349 int fix_alignment(struct pt_regs *regs) 350 { 351 unsigned int instr, nb, flags; 352 unsigned int reg, areg; 353 unsigned int dsisr; 354 unsigned char __user *addr; 355 unsigned char __user *p; 356 int ret, t; 357 union { 358 u64 ll; 359 double dd; 360 unsigned char v[8]; 361 struct { 362 unsigned hi32; 363 int low32; 364 } x32; 365 struct { 366 unsigned char hi48[6]; 367 short low16; 368 } x16; 369 } data; 370 371 /* 372 * We require a complete register set, if not, then our assembly 373 * is broken 374 */ 375 CHECK_FULL_REGS(regs); 376 377 dsisr = regs->dsisr; 378 379 /* Some processors don't provide us with a DSISR we can use here, 380 * let's make one up from the instruction 381 */ 382 if (cpu_has_feature(CPU_FTR_NODSISRALIGN)) { 383 unsigned int real_instr; 384 if (unlikely(__get_user(real_instr, 385 (unsigned int __user *)regs->nip))) 386 return -EFAULT; 387 dsisr = make_dsisr(real_instr); 388 } 389 390 /* extract the operation and registers from the dsisr */ 391 reg = (dsisr >> 5) & 0x1f; /* source/dest register */ 392 areg = dsisr & 0x1f; /* register to update */ 393 instr = (dsisr >> 10) & 0x7f; 394 instr |= (dsisr >> 13) & 0x60; 395 396 /* Lookup the operation in our table */ 397 nb = aligninfo[instr].len; 398 flags = aligninfo[instr].flags; 399 400 /* DAR has the operand effective address */ 401 addr = (unsigned char __user *)regs->dar; 402 403 /* A size of 0 indicates an instruction we don't support, with 404 * the exception of DCBZ which is handled as a special case here 405 */ 406 if (instr == DCBZ) 407 return emulate_dcbz(regs, addr); 408 if (unlikely(nb == 0)) 409 return 0; 410 411 /* Load/Store Multiple instructions are handled in their own 412 * function 413 */ 414 if (flags & M) 415 return emulate_multiple(regs, addr, reg, nb, flags, instr); 416 417 /* Verify the address of the operand */ 418 if (unlikely(user_mode(regs) && 419 !access_ok((flags & ST ? VERIFY_WRITE : VERIFY_READ), 420 addr, nb))) 421 return -EFAULT; 422 423 /* Force the fprs into the save area so we can reference them */ 424 if (flags & F) { 425 /* userland only */ 426 if (unlikely(!user_mode(regs))) 427 return 0; 428 flush_fp_to_thread(current); 429 } 430 431 /* If we are loading, get the data from user space, else 432 * get it from register values 433 */ 434 if (flags & LD) { 435 data.ll = 0; 436 ret = 0; 437 p = addr; 438 switch (nb) { 439 case 8: 440 ret |= __get_user(data.v[0], p++); 441 ret |= __get_user(data.v[1], p++); 442 ret |= __get_user(data.v[2], p++); 443 ret |= __get_user(data.v[3], p++); 444 case 4: 445 ret |= __get_user(data.v[4], p++); 446 ret |= __get_user(data.v[5], p++); 447 case 2: 448 ret |= __get_user(data.v[6], p++); 449 ret |= __get_user(data.v[7], p++); 450 if (unlikely(ret)) 451 return -EFAULT; 452 } 453 } else if (flags & F) 454 data.dd = current->thread.fpr[reg]; 455 else 456 data.ll = regs->gpr[reg]; 457 458 /* Perform other misc operations like sign extension, byteswap, 459 * or floating point single precision conversion 460 */ 461 switch (flags & ~U) { 462 case LD+SE: /* sign extend */ 463 if ( nb == 2 ) 464 data.ll = data.x16.low16; 465 else /* nb must be 4 */ 466 data.ll = data.x32.low32; 467 break; 468 case LD+S: /* byte-swap */ 469 case ST+S: 470 if (nb == 2) { 471 SWAP(data.v[6], data.v[7]); 472 } else { 473 SWAP(data.v[4], data.v[7]); 474 SWAP(data.v[5], data.v[6]); 475 } 476 break; 477 478 /* Single-precision FP load and store require conversions... */ 479 case LD+F+S: 480 #ifdef CONFIG_PPC_FPU 481 preempt_disable(); 482 enable_kernel_fp(); 483 cvt_fd((float *)&data.v[4], &data.dd, ¤t->thread); 484 preempt_enable(); 485 #else 486 return 0; 487 #endif 488 break; 489 case ST+F+S: 490 #ifdef CONFIG_PPC_FPU 491 preempt_disable(); 492 enable_kernel_fp(); 493 cvt_df(&data.dd, (float *)&data.v[4], ¤t->thread); 494 preempt_enable(); 495 #else 496 return 0; 497 #endif 498 break; 499 } 500 501 /* Store result to memory or update registers */ 502 if (flags & ST) { 503 ret = 0; 504 p = addr; 505 switch (nb) { 506 case 8: 507 ret |= __put_user(data.v[0], p++); 508 ret |= __put_user(data.v[1], p++); 509 ret |= __put_user(data.v[2], p++); 510 ret |= __put_user(data.v[3], p++); 511 case 4: 512 ret |= __put_user(data.v[4], p++); 513 ret |= __put_user(data.v[5], p++); 514 case 2: 515 ret |= __put_user(data.v[6], p++); 516 ret |= __put_user(data.v[7], p++); 517 } 518 if (unlikely(ret)) 519 return -EFAULT; 520 } else if (flags & F) 521 current->thread.fpr[reg] = data.dd; 522 else 523 regs->gpr[reg] = data.ll; 524 525 /* Update RA as needed */ 526 if (flags & U) 527 regs->gpr[areg] = regs->dar; 528 529 return 1; 530 } 531