1/* 2 * arch/xtensa/kernel/align.S 3 * 4 * Handle unalignment and load/store exceptions. 5 * 6 * This file is subject to the terms and conditions of the GNU General 7 * Public License. See the file "COPYING" in the main directory of 8 * this archive for more details. 9 * 10 * Copyright (C) 2001 - 2005 Tensilica, Inc. 11 * Copyright (C) 2014 Cadence Design Systems Inc. 12 * 13 * Rewritten by Chris Zankel <chris@zankel.net> 14 * 15 * Based on work from Joe Taylor <joe@tensilica.com, joetylr@yahoo.com> 16 * and Marc Gauthier <marc@tensilica.com, marc@alimni.uwaterloo.ca> 17 */ 18 19#include <linux/linkage.h> 20#include <asm/current.h> 21#include <asm/asm-offsets.h> 22#include <asm/asmmacro.h> 23#include <asm/processor.h> 24 25#if XCHAL_UNALIGNED_LOAD_EXCEPTION || defined CONFIG_XTENSA_LOAD_STORE 26#define LOAD_EXCEPTION_HANDLER 27#endif 28 29#if XCHAL_UNALIGNED_STORE_EXCEPTION || defined CONFIG_XTENSA_LOAD_STORE 30#define STORE_EXCEPTION_HANDLER 31#endif 32 33#if defined LOAD_EXCEPTION_HANDLER || defined STORE_EXCEPTION_HANDLER 34#define ANY_EXCEPTION_HANDLER 35#endif 36 37#if XCHAL_HAVE_WINDOWED && defined CONFIG_MMU 38#define UNALIGNED_USER_EXCEPTION 39#endif 40 41/* Big and little endian 16-bit values are located in 42 * different halves of a register. HWORD_START helps to 43 * abstract the notion of extracting a 16-bit value from a 44 * register. 45 * We also have to define new shifting instructions because 46 * lsb and msb are on 'opposite' ends in a register for 47 * different endian machines. 48 * 49 * Assume a memory region in ascending address: 50 * 0 1 2 3|4 5 6 7 51 * 52 * When loading one word into a register, the content of that register is: 53 * LE 3 2 1 0, 7 6 5 4 54 * BE 0 1 2 3, 4 5 6 7 55 * 56 * Masking the bits of the higher/lower address means: 57 * LE X X 0 0, 0 0 X X 58 * BE 0 0 X X, X X 0 0 59 * 60 * Shifting to higher/lower addresses, means: 61 * LE shift left / shift right 62 * BE shift right / shift left 63 * 64 * Extracting 16 bits from a 32 bit reg. value to higher/lower address means: 65 * LE mask 0 0 X X / shift left 66 * BE shift left / mask 0 0 X X 67 */ 68 69#if XCHAL_HAVE_BE 70 71#define HWORD_START 16 72#define INSN_OP0 28 73#define INSN_T 24 74#define INSN_OP1 16 75 76.macro __ssa8r r; ssa8l \r; .endm 77.macro __sh r, s; srl \r, \s; .endm 78.macro __sl r, s; sll \r, \s; .endm 79.macro __exth r, s; extui \r, \s, 0, 16; .endm 80.macro __extl r, s; slli \r, \s, 16; .endm 81 82#else 83 84#define HWORD_START 0 85#define INSN_OP0 0 86#define INSN_T 4 87#define INSN_OP1 12 88 89.macro __ssa8r r; ssa8b \r; .endm 90.macro __sh r, s; sll \r, \s; .endm 91.macro __sl r, s; srl \r, \s; .endm 92.macro __exth r, s; slli \r, \s, 16; .endm 93.macro __extl r, s; extui \r, \s, 0, 16; .endm 94 95#endif 96 97/* 98 * xxxx xxxx = imm8 field 99 * yyyy = imm4 field 100 * ssss = s field 101 * tttt = t field 102 * 103 * 16 0 104 * ------------------- 105 * L32I.N yyyy ssss tttt 1000 106 * S32I.N yyyy ssss tttt 1001 107 * 108 * 23 0 109 * ----------------------------- 110 * L8UI xxxx xxxx 0000 ssss tttt 0010 111 * L16UI xxxx xxxx 0001 ssss tttt 0010 112 * L32I xxxx xxxx 0010 ssss tttt 0010 113 * XXX 0011 ssss tttt 0010 114 * XXX 0100 ssss tttt 0010 115 * S16I xxxx xxxx 0101 ssss tttt 0010 116 * S32I xxxx xxxx 0110 ssss tttt 0010 117 * XXX 0111 ssss tttt 0010 118 * XXX 1000 ssss tttt 0010 119 * L16SI xxxx xxxx 1001 ssss tttt 0010 120 * XXX 1010 0010 121 * **L32AI xxxx xxxx 1011 ssss tttt 0010 unsupported 122 * XXX 1100 0010 123 * XXX 1101 0010 124 * XXX 1110 0010 125 * **S32RI xxxx xxxx 1111 ssss tttt 0010 unsupported 126 * ----------------------------- 127 * ^ ^ ^ 128 * sub-opcode (NIBBLE_R) -+ | | 129 * t field (NIBBLE_T) -----------+ | 130 * major opcode (NIBBLE_OP0) --------------+ 131 */ 132 133#define OP0_L32I_N 0x8 /* load immediate narrow */ 134#define OP0_S32I_N 0x9 /* store immediate narrow */ 135#define OP0_LSAI 0x2 /* load/store */ 136#define OP1_SI_MASK 0x4 /* OP1 bit set for stores */ 137#define OP1_SI_BIT 2 /* OP1 bit number for stores */ 138 139#define OP1_L8UI 0x0 140#define OP1_L32I 0x2 141#define OP1_L16UI 0x1 142#define OP1_L16SI 0x9 143#define OP1_L32AI 0xb 144 145#define OP1_S32I 0x6 146#define OP1_S16I 0x5 147#define OP1_S32RI 0xf 148 149/* 150 * Entry condition: 151 * 152 * a0: trashed, original value saved on stack (PT_AREG0) 153 * a1: a1 154 * a2: new stack pointer, original in DEPC 155 * a3: a3 156 * depc: a2, original value saved on stack (PT_DEPC) 157 * excsave_1: dispatch table 158 * 159 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC 160 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception 161 */ 162 163 .literal_position 164#ifdef CONFIG_XTENSA_LOAD_STORE 165ENTRY(fast_load_store) 166 167 call0 .Lsave_and_load_instruction 168 169 /* Analyze the instruction (load or store?). */ 170 171 extui a0, a4, INSN_OP0, 4 # get insn.op0 nibble 172 173#if XCHAL_HAVE_DENSITY 174 _beqi a0, OP0_L32I_N, 1f # L32I.N, jump 175#endif 176 bnei a0, OP0_LSAI, .Linvalid_instruction 177 /* 'store indicator bit' set, jump */ 178 bbsi.l a4, OP1_SI_BIT + INSN_OP1, .Linvalid_instruction 179 1801: 181 movi a3, ~3 182 and a3, a3, a8 # align memory address 183 184 __ssa8 a8 185 186#ifdef CONFIG_MMU 187 /* l32e can't be used here even when it's available. */ 188 /* TODO access_ok(a3) could be used here */ 189 j .Linvalid_instruction 190#endif 191 l32i a5, a3, 0 192 l32i a6, a3, 4 193 __src_b a3, a5, a6 # a3 has the data word 194 195#if XCHAL_HAVE_DENSITY 196 addi a7, a7, 2 # increment PC (assume 16-bit insn) 197 _beqi a0, OP0_L32I_N, .Lload_w# l32i.n: jump 198 addi a7, a7, 1 199#else 200 addi a7, a7, 3 201#endif 202 203 extui a5, a4, INSN_OP1, 4 204 _beqi a5, OP1_L32I, .Lload_w 205 bnei a5, OP1_L8UI, .Lload16 206 extui a3, a3, 0, 8 207 j .Lload_w 208 209ENDPROC(fast_load_store) 210#endif 211 212/* 213 * Entry condition: 214 * 215 * a0: trashed, original value saved on stack (PT_AREG0) 216 * a1: a1 217 * a2: new stack pointer, original in DEPC 218 * a3: a3 219 * depc: a2, original value saved on stack (PT_DEPC) 220 * excsave_1: dispatch table 221 * 222 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC 223 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception 224 */ 225 226#ifdef ANY_EXCEPTION_HANDLER 227ENTRY(fast_unaligned) 228 229 call0 .Lsave_and_load_instruction 230 231 /* Analyze the instruction (load or store?). */ 232 233 extui a5, a4, INSN_OP0, 4 # get insn.op0 nibble 234 235#if XCHAL_HAVE_DENSITY 236 _beqi a5, OP0_L32I_N, .Lload # L32I.N, jump 237 addi a6, a5, -OP0_S32I_N 238 _beqz a6, .Lstore # S32I.N, do a store 239#endif 240 /* 'store indicator bit' not set, jump */ 241 _bbci.l a4, OP1_SI_BIT + INSN_OP1, .Lload 242 243#ifdef STORE_EXCEPTION_HANDLER 244 245 /* Store: Jump to table entry to get the value in the source register.*/ 246 247.Lstore:movi a5, .Lstore_table # table 248 extui a6, a4, INSN_T, 4 # get source register 249 addx8 a5, a6, a5 250 jx a5 # jump into table 251#endif 252#ifdef LOAD_EXCEPTION_HANDLER 253 254 /* Load: Load memory address. */ 255 256.Lload: movi a3, ~3 257 and a3, a3, a8 # align memory address 258 259 __ssa8 a8 260#ifdef UNALIGNED_USER_EXCEPTION 261 addi a3, a3, 8 262 l32e a5, a3, -8 263 l32e a6, a3, -4 264#else 265 l32i a5, a3, 0 266 l32i a6, a3, 4 267#endif 268 __src_b a3, a5, a6 # a3 has the data word 269 270#if XCHAL_HAVE_DENSITY 271 addi a7, a7, 2 # increment PC (assume 16-bit insn) 272 273 extui a5, a4, INSN_OP0, 4 274 _beqi a5, OP0_L32I_N, .Lload_w# l32i.n: jump 275 276 addi a7, a7, 1 277#else 278 addi a7, a7, 3 279#endif 280 281 extui a5, a4, INSN_OP1, 4 282 _beqi a5, OP1_L32I, .Lload_w # l32i: jump 283#endif 284#ifdef LOAD_EXCEPTION_HANDLER 285.Lload16: 286 extui a3, a3, 0, 16 # extract lower 16 bits 287 _beqi a5, OP1_L16UI, .Lload_w 288 addi a5, a5, -OP1_L16SI 289 _bnez a5, .Linvalid_instruction 290 291 /* sign extend value */ 292#if XCHAL_HAVE_SEXT 293 sext a3, a3, 15 294#else 295 slli a3, a3, 16 296 srai a3, a3, 16 297#endif 298 299 /* Set target register. */ 300 301.Lload_w: 302 extui a4, a4, INSN_T, 4 # extract target register 303 movi a5, .Lload_table 304 addx8 a4, a4, a5 305 jx a4 # jump to entry for target register 306 307 .align 8 308.Lload_table: 309 s32i a3, a2, PT_AREG0; _j .Lexit; .align 8 310 mov a1, a3; _j .Lexit; .align 8 # fishy?? 311 s32i a3, a2, PT_AREG2; _j .Lexit; .align 8 312 s32i a3, a2, PT_AREG3; _j .Lexit; .align 8 313 s32i a3, a2, PT_AREG4; _j .Lexit; .align 8 314 s32i a3, a2, PT_AREG5; _j .Lexit; .align 8 315 s32i a3, a2, PT_AREG6; _j .Lexit; .align 8 316 s32i a3, a2, PT_AREG7; _j .Lexit; .align 8 317 s32i a3, a2, PT_AREG8; _j .Lexit; .align 8 318 mov a9, a3 ; _j .Lexit; .align 8 319 mov a10, a3 ; _j .Lexit; .align 8 320 mov a11, a3 ; _j .Lexit; .align 8 321 mov a12, a3 ; _j .Lexit; .align 8 322 mov a13, a3 ; _j .Lexit; .align 8 323 mov a14, a3 ; _j .Lexit; .align 8 324 mov a15, a3 ; _j .Lexit; .align 8 325#endif 326#ifdef STORE_EXCEPTION_HANDLER 327.Lstore_table: 328 l32i a3, a2, PT_AREG0; _j .Lstore_w; .align 8 329 mov a3, a1; _j .Lstore_w; .align 8 # fishy?? 330 l32i a3, a2, PT_AREG2; _j .Lstore_w; .align 8 331 l32i a3, a2, PT_AREG3; _j .Lstore_w; .align 8 332 l32i a3, a2, PT_AREG4; _j .Lstore_w; .align 8 333 l32i a3, a2, PT_AREG5; _j .Lstore_w; .align 8 334 l32i a3, a2, PT_AREG6; _j .Lstore_w; .align 8 335 l32i a3, a2, PT_AREG7; _j .Lstore_w; .align 8 336 l32i a3, a2, PT_AREG8; _j .Lstore_w; .align 8 337 mov a3, a9 ; _j .Lstore_w; .align 8 338 mov a3, a10 ; _j .Lstore_w; .align 8 339 mov a3, a11 ; _j .Lstore_w; .align 8 340 mov a3, a12 ; _j .Lstore_w; .align 8 341 mov a3, a13 ; _j .Lstore_w; .align 8 342 mov a3, a14 ; _j .Lstore_w; .align 8 343 mov a3, a15 ; _j .Lstore_w; .align 8 344#endif 345 346 /* We cannot handle this exception. */ 347 348 .extern _kernel_exception 349.Linvalid_instruction: 350 351 movi a4, 0 352 rsr a3, excsave1 353 s32i a4, a3, EXC_TABLE_FIXUP 354 355 /* Restore a4...a8 and SAR, set SP, and jump to default exception. */ 356 357 l32i a0, a2, PT_SAR 358 l32i a8, a2, PT_AREG8 359 l32i a7, a2, PT_AREG7 360 l32i a6, a2, PT_AREG6 361 l32i a5, a2, PT_AREG5 362 l32i a4, a2, PT_AREG4 363 wsr a0, sar 364 mov a1, a2 365 366 rsr a0, ps 367 bbsi.l a0, PS_UM_BIT, 2f # jump if user mode 368 369 movi a0, _kernel_exception 370 jx a0 371 3722: movi a0, _user_exception 373 jx a0 374 375#ifdef STORE_EXCEPTION_HANDLER 376 377 # a7: instruction pointer, a4: instruction, a3: value 378.Lstore_w: 379 movi a6, 0 # mask: ffffffff:00000000 380 381#if XCHAL_HAVE_DENSITY 382 addi a7, a7, 2 # incr. PC,assume 16-bit instruction 383 384 extui a5, a4, INSN_OP0, 4 # extract OP0 385 addi a5, a5, -OP0_S32I_N 386 _beqz a5, 1f # s32i.n: jump 387 388 addi a7, a7, 1 # increment PC, 32-bit instruction 389#else 390 addi a7, a7, 3 # increment PC, 32-bit instruction 391#endif 392 393 extui a5, a4, INSN_OP1, 4 # extract OP1 394 _beqi a5, OP1_S32I, 1f # jump if 32 bit store 395 _bnei a5, OP1_S16I, .Linvalid_instruction 396 397 movi a5, -1 398 __extl a3, a3 # get 16-bit value 399 __exth a6, a5 # get 16-bit mask ffffffff:ffff0000 400 401 /* Get memory address */ 402 4031: 404 movi a4, ~3 405 and a4, a4, a8 # align memory address 406 407 /* Insert value into memory */ 408 409 movi a5, -1 # mask: ffffffff:XXXX0000 410#ifdef UNALIGNED_USER_EXCEPTION 411 addi a4, a4, 8 412#endif 413 414 __ssa8r a8 415 __src_b a8, a5, a6 # lo-mask F..F0..0 (BE) 0..0F..F (LE) 416 __src_b a6, a6, a5 # hi-mask 0..0F..F (BE) F..F0..0 (LE) 417#ifdef UNALIGNED_USER_EXCEPTION 418 l32e a5, a4, -8 419#else 420 l32i a5, a4, 0 # load lower address word 421#endif 422 and a5, a5, a8 # mask 423 __sh a8, a3 # shift value 424 or a5, a5, a8 # or with original value 425#ifdef UNALIGNED_USER_EXCEPTION 426 s32e a5, a4, -8 427 l32e a8, a4, -4 428#else 429 s32i a5, a4, 0 # store 430 l32i a8, a4, 4 # same for upper address word 431#endif 432 __sl a5, a3 433 and a6, a8, a6 434 or a6, a6, a5 435#ifdef UNALIGNED_USER_EXCEPTION 436 s32e a6, a4, -4 437#else 438 s32i a6, a4, 4 439#endif 440#endif 441 442.Lexit: 443#if XCHAL_HAVE_LOOPS 444 rsr a4, lend # check if we reached LEND 445 bne a7, a4, 1f 446 rsr a4, lcount # and LCOUNT != 0 447 beqz a4, 1f 448 addi a4, a4, -1 # decrement LCOUNT and set 449 rsr a7, lbeg # set PC to LBEGIN 450 wsr a4, lcount 451#endif 452 4531: wsr a7, epc1 # skip emulated instruction 454 455 /* Update icount if we're single-stepping in userspace. */ 456 rsr a4, icountlevel 457 beqz a4, 1f 458 bgeui a4, LOCKLEVEL + 1, 1f 459 rsr a4, icount 460 addi a4, a4, 1 461 wsr a4, icount 4621: 463 movi a4, 0 464 rsr a3, excsave1 465 s32i a4, a3, EXC_TABLE_FIXUP 466 467 /* Restore working register */ 468 469 l32i a0, a2, PT_SAR 470 l32i a8, a2, PT_AREG8 471 l32i a7, a2, PT_AREG7 472 l32i a6, a2, PT_AREG6 473 l32i a5, a2, PT_AREG5 474 l32i a4, a2, PT_AREG4 475 l32i a3, a2, PT_AREG3 476 477 /* restore SAR and return */ 478 479 wsr a0, sar 480 l32i a0, a2, PT_AREG0 481 l32i a2, a2, PT_AREG2 482 rfe 483 484 .align 4 485.Lsave_and_load_instruction: 486 487 /* Save some working register */ 488 489 s32i a3, a2, PT_AREG3 490 s32i a4, a2, PT_AREG4 491 s32i a5, a2, PT_AREG5 492 s32i a6, a2, PT_AREG6 493 s32i a7, a2, PT_AREG7 494 s32i a8, a2, PT_AREG8 495 496 rsr a4, depc 497 s32i a4, a2, PT_AREG2 498 499 rsr a5, sar 500 s32i a5, a2, PT_SAR 501 502 rsr a3, excsave1 503 movi a4, fast_unaligned_fixup 504 s32i a4, a3, EXC_TABLE_FIXUP 505 506 rsr a8, excvaddr # load unaligned memory address 507 508 /* Now, identify one of the following load/store instructions. 509 * 510 * The only possible danger of a double exception on the 511 * following l32i instructions is kernel code in vmalloc 512 * memory. The processor was just executing at the EPC_1 513 * address, and indeed, already fetched the instruction. That 514 * guarantees a TLB mapping, which hasn't been replaced by 515 * this unaligned exception handler that uses only static TLB 516 * mappings. However, high-level interrupt handlers might 517 * modify TLB entries, so for the generic case, we register a 518 * TABLE_FIXUP handler here, too. 519 */ 520 521 /* a3...a6 saved on stack, a2 = SP */ 522 523 /* Extract the instruction that caused the unaligned access. */ 524 525 rsr a7, epc1 # load exception address 526 movi a3, ~3 527 and a3, a3, a7 # mask lower bits 528 529 l32i a4, a3, 0 # load 2 words 530 l32i a5, a3, 4 531 532 __ssa8 a7 533 __src_b a4, a4, a5 # a4 has the instruction 534 535 ret 536 537ENDPROC(fast_unaligned) 538 539ENTRY(fast_unaligned_fixup) 540 541 l32i a2, a3, EXC_TABLE_DOUBLE_SAVE 542 wsr a3, excsave1 543 544 l32i a8, a2, PT_AREG8 545 l32i a7, a2, PT_AREG7 546 l32i a6, a2, PT_AREG6 547 l32i a5, a2, PT_AREG5 548 l32i a4, a2, PT_SAR 549 l32i a0, a2, PT_AREG2 550 wsr a4, sar 551 wsr a0, depc # restore depc and a0 552 l32i a4, a2, PT_AREG4 553 554 rsr a0, exccause 555 s32i a0, a2, PT_DEPC # mark as a regular exception 556 557 rsr a0, ps 558 bbsi.l a0, PS_UM_BIT, 1f # jump if user mode 559 560 rsr a0, exccause 561 addx4 a0, a0, a3 # find entry in table 562 l32i a0, a0, EXC_TABLE_FAST_KERNEL # load handler 563 l32i a3, a2, PT_AREG3 564 jx a0 5651: 566 rsr a0, exccause 567 addx4 a0, a0, a3 # find entry in table 568 l32i a0, a0, EXC_TABLE_FAST_USER # load handler 569 l32i a3, a2, PT_AREG3 570 jx a0 571 572ENDPROC(fast_unaligned_fixup) 573#endif 574