1/* 2 * arch/xtensa/kernel/align.S 3 * 4 * Handle unalignment exceptions in kernel space. 5 * 6 * This file is subject to the terms and conditions of the GNU General 7 * Public License. See the file "COPYING" in the main directory of 8 * this archive for more details. 9 * 10 * Copyright (C) 2001 - 2005 Tensilica, Inc. 11 * Copyright (C) 2014 Cadence Design Systems Inc. 12 * 13 * Rewritten by Chris Zankel <chris@zankel.net> 14 * 15 * Based on work from Joe Taylor <joe@tensilica.com, joetylr@yahoo.com> 16 * and Marc Gauthier <marc@tensilica.com, marc@alimni.uwaterloo.ca> 17 */ 18 19#include <linux/linkage.h> 20#include <asm/current.h> 21#include <asm/asm-offsets.h> 22#include <asm/asmmacro.h> 23#include <asm/processor.h> 24 25#if XCHAL_UNALIGNED_LOAD_EXCEPTION || defined CONFIG_XTENSA_LOAD_STORE 26#define LOAD_EXCEPTION_HANDLER 27#endif 28 29#if XCHAL_UNALIGNED_STORE_EXCEPTION || defined LOAD_EXCEPTION_HANDLER 30#define ANY_EXCEPTION_HANDLER 31#endif 32 33#if XCHAL_HAVE_WINDOWED 34#define UNALIGNED_USER_EXCEPTION 35#endif 36 37/* First-level exception handler for unaligned exceptions. 38 * 39 * Note: This handler works only for kernel exceptions. Unaligned user 40 * access should get a seg fault. 41 */ 42 43/* Big and little endian 16-bit values are located in 44 * different halves of a register. HWORD_START helps to 45 * abstract the notion of extracting a 16-bit value from a 46 * register. 47 * We also have to define new shifting instructions because 48 * lsb and msb are on 'opposite' ends in a register for 49 * different endian machines. 50 * 51 * Assume a memory region in ascending address: 52 * 0 1 2 3|4 5 6 7 53 * 54 * When loading one word into a register, the content of that register is: 55 * LE 3 2 1 0, 7 6 5 4 56 * BE 0 1 2 3, 4 5 6 7 57 * 58 * Masking the bits of the higher/lower address means: 59 * LE X X 0 0, 0 0 X X 60 * BE 0 0 X X, X X 0 0 61 * 62 * Shifting to higher/lower addresses, means: 63 * LE shift left / shift right 64 * BE shift right / shift left 65 * 66 * Extracting 16 bits from a 32 bit reg. value to higher/lower address means: 67 * LE mask 0 0 X X / shift left 68 * BE shift left / mask 0 0 X X 69 */ 70 71#if XCHAL_HAVE_BE 72 73#define HWORD_START 16 74#define INSN_OP0 28 75#define INSN_T 24 76#define INSN_OP1 16 77 78.macro __ssa8r r; ssa8l \r; .endm 79.macro __sh r, s; srl \r, \s; .endm 80.macro __sl r, s; sll \r, \s; .endm 81.macro __exth r, s; extui \r, \s, 0, 16; .endm 82.macro __extl r, s; slli \r, \s, 16; .endm 83 84#else 85 86#define HWORD_START 0 87#define INSN_OP0 0 88#define INSN_T 4 89#define INSN_OP1 12 90 91.macro __ssa8r r; ssa8b \r; .endm 92.macro __sh r, s; sll \r, \s; .endm 93.macro __sl r, s; srl \r, \s; .endm 94.macro __exth r, s; slli \r, \s, 16; .endm 95.macro __extl r, s; extui \r, \s, 0, 16; .endm 96 97#endif 98 99/* 100 * xxxx xxxx = imm8 field 101 * yyyy = imm4 field 102 * ssss = s field 103 * tttt = t field 104 * 105 * 16 0 106 * ------------------- 107 * L32I.N yyyy ssss tttt 1000 108 * S32I.N yyyy ssss tttt 1001 109 * 110 * 23 0 111 * ----------------------------- 112 * L8UI xxxx xxxx 0000 ssss tttt 0010 113 * L16UI xxxx xxxx 0001 ssss tttt 0010 114 * L32I xxxx xxxx 0010 ssss tttt 0010 115 * XXX 0011 ssss tttt 0010 116 * XXX 0100 ssss tttt 0010 117 * S16I xxxx xxxx 0101 ssss tttt 0010 118 * S32I xxxx xxxx 0110 ssss tttt 0010 119 * XXX 0111 ssss tttt 0010 120 * XXX 1000 ssss tttt 0010 121 * L16SI xxxx xxxx 1001 ssss tttt 0010 122 * XXX 1010 0010 123 * **L32AI xxxx xxxx 1011 ssss tttt 0010 unsupported 124 * XXX 1100 0010 125 * XXX 1101 0010 126 * XXX 1110 0010 127 * **S32RI xxxx xxxx 1111 ssss tttt 0010 unsupported 128 * ----------------------------- 129 * ^ ^ ^ 130 * sub-opcode (NIBBLE_R) -+ | | 131 * t field (NIBBLE_T) -----------+ | 132 * major opcode (NIBBLE_OP0) --------------+ 133 */ 134 135#define OP0_L32I_N 0x8 /* load immediate narrow */ 136#define OP0_S32I_N 0x9 /* store immediate narrow */ 137#define OP0_LSAI 0x2 /* load/store */ 138#define OP1_SI_MASK 0x4 /* OP1 bit set for stores */ 139#define OP1_SI_BIT 2 /* OP1 bit number for stores */ 140 141#define OP1_L8UI 0x0 142#define OP1_L32I 0x2 143#define OP1_L16UI 0x1 144#define OP1_L16SI 0x9 145#define OP1_L32AI 0xb 146 147#define OP1_S32I 0x6 148#define OP1_S16I 0x5 149#define OP1_S32RI 0xf 150 151/* 152 * Entry condition: 153 * 154 * a0: trashed, original value saved on stack (PT_AREG0) 155 * a1: a1 156 * a2: new stack pointer, original in DEPC 157 * a3: a3 158 * depc: a2, original value saved on stack (PT_DEPC) 159 * excsave_1: dispatch table 160 * 161 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC 162 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception 163 */ 164 165 .literal_position 166#ifdef CONFIG_XTENSA_LOAD_STORE 167ENTRY(fast_load_store) 168 169 call0 .Lsave_and_load_instruction 170 171 /* Analyze the instruction (load or store?). */ 172 173 extui a0, a4, INSN_OP0, 4 # get insn.op0 nibble 174 175#if XCHAL_HAVE_DENSITY 176 _beqi a0, OP0_L32I_N, 1f # L32I.N, jump 177#endif 178 bnei a0, OP0_LSAI, .Linvalid_instruction 179 /* 'store indicator bit' set, jump */ 180 bbsi.l a4, OP1_SI_BIT + INSN_OP1, .Linvalid_instruction 181 1821: 183 movi a3, ~3 184 and a3, a3, a8 # align memory address 185 186 __ssa8 a8 187 188#ifdef CONFIG_MMU 189 /* l32e can't be used here even when it's available. */ 190 /* TODO access_ok(a3) could be used here */ 191 j .Linvalid_instruction 192#endif 193 l32i a5, a3, 0 194 l32i a6, a3, 4 195 __src_b a3, a5, a6 # a3 has the data word 196 197#if XCHAL_HAVE_DENSITY 198 addi a7, a7, 2 # increment PC (assume 16-bit insn) 199 _beqi a0, OP0_L32I_N, .Lload_w# l32i.n: jump 200 addi a7, a7, 1 201#else 202 addi a7, a7, 3 203#endif 204 205 extui a5, a4, INSN_OP1, 4 206 _beqi a5, OP1_L32I, .Lload_w 207 bnei a5, OP1_L8UI, .Lload16 208 extui a3, a3, 0, 8 209 j .Lload_w 210 211ENDPROC(fast_load_store) 212#endif 213 214/* 215 * Entry condition: 216 * 217 * a0: trashed, original value saved on stack (PT_AREG0) 218 * a1: a1 219 * a2: new stack pointer, original in DEPC 220 * a3: a3 221 * depc: a2, original value saved on stack (PT_DEPC) 222 * excsave_1: dispatch table 223 * 224 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC 225 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception 226 */ 227 228#ifdef ANY_EXCEPTION_HANDLER 229ENTRY(fast_unaligned) 230 231#if XCHAL_UNALIGNED_LOAD_EXCEPTION || XCHAL_UNALIGNED_STORE_EXCEPTION 232 233 call0 .Lsave_and_load_instruction 234 235 /* Analyze the instruction (load or store?). */ 236 237 extui a5, a4, INSN_OP0, 4 # get insn.op0 nibble 238 239#if XCHAL_HAVE_DENSITY 240 _beqi a5, OP0_L32I_N, .Lload # L32I.N, jump 241 addi a6, a5, -OP0_S32I_N 242 _beqz a6, .Lstore # S32I.N, do a store 243#endif 244 /* 'store indicator bit' not set, jump */ 245 _bbci.l a4, OP1_SI_BIT + INSN_OP1, .Lload 246 247#endif 248#if XCHAL_UNALIGNED_STORE_EXCEPTION 249 250 /* Store: Jump to table entry to get the value in the source register.*/ 251 252.Lstore:movi a5, .Lstore_table # table 253 extui a6, a4, INSN_T, 4 # get source register 254 addx8 a5, a6, a5 255 jx a5 # jump into table 256#endif 257#if XCHAL_UNALIGNED_LOAD_EXCEPTION 258 259 /* Load: Load memory address. */ 260 261.Lload: movi a3, ~3 262 and a3, a3, a8 # align memory address 263 264 __ssa8 a8 265#ifdef UNALIGNED_USER_EXCEPTION 266 addi a3, a3, 8 267 l32e a5, a3, -8 268 l32e a6, a3, -4 269#else 270 l32i a5, a3, 0 271 l32i a6, a3, 4 272#endif 273 __src_b a3, a5, a6 # a3 has the data word 274 275#if XCHAL_HAVE_DENSITY 276 addi a7, a7, 2 # increment PC (assume 16-bit insn) 277 278 extui a5, a4, INSN_OP0, 4 279 _beqi a5, OP0_L32I_N, .Lload_w# l32i.n: jump 280 281 addi a7, a7, 1 282#else 283 addi a7, a7, 3 284#endif 285 286 extui a5, a4, INSN_OP1, 4 287 _beqi a5, OP1_L32I, .Lload_w # l32i: jump 288#endif 289#ifdef LOAD_EXCEPTION_HANDLER 290.Lload16: 291 extui a3, a3, 0, 16 # extract lower 16 bits 292 _beqi a5, OP1_L16UI, .Lload_w 293 addi a5, a5, -OP1_L16SI 294 _bnez a5, .Linvalid_instruction 295 296 /* sign extend value */ 297#if XCHAL_HAVE_SEXT 298 sext a3, a3, 15 299#else 300 slli a3, a3, 16 301 srai a3, a3, 16 302#endif 303 304 /* Set target register. */ 305 306.Lload_w: 307 extui a4, a4, INSN_T, 4 # extract target register 308 movi a5, .Lload_table 309 addx8 a4, a4, a5 310 jx a4 # jump to entry for target register 311 312 .align 8 313.Lload_table: 314 s32i a3, a2, PT_AREG0; _j .Lexit; .align 8 315 mov a1, a3; _j .Lexit; .align 8 # fishy?? 316 s32i a3, a2, PT_AREG2; _j .Lexit; .align 8 317 s32i a3, a2, PT_AREG3; _j .Lexit; .align 8 318 s32i a3, a2, PT_AREG4; _j .Lexit; .align 8 319 s32i a3, a2, PT_AREG5; _j .Lexit; .align 8 320 s32i a3, a2, PT_AREG6; _j .Lexit; .align 8 321 s32i a3, a2, PT_AREG7; _j .Lexit; .align 8 322 s32i a3, a2, PT_AREG8; _j .Lexit; .align 8 323 mov a9, a3 ; _j .Lexit; .align 8 324 mov a10, a3 ; _j .Lexit; .align 8 325 mov a11, a3 ; _j .Lexit; .align 8 326 mov a12, a3 ; _j .Lexit; .align 8 327 mov a13, a3 ; _j .Lexit; .align 8 328 mov a14, a3 ; _j .Lexit; .align 8 329 mov a15, a3 ; _j .Lexit; .align 8 330#endif 331#if XCHAL_UNALIGNED_STORE_EXCEPTION 332.Lstore_table: 333 l32i a3, a2, PT_AREG0; _j .Lstore_w; .align 8 334 mov a3, a1; _j .Lstore_w; .align 8 # fishy?? 335 l32i a3, a2, PT_AREG2; _j .Lstore_w; .align 8 336 l32i a3, a2, PT_AREG3; _j .Lstore_w; .align 8 337 l32i a3, a2, PT_AREG4; _j .Lstore_w; .align 8 338 l32i a3, a2, PT_AREG5; _j .Lstore_w; .align 8 339 l32i a3, a2, PT_AREG6; _j .Lstore_w; .align 8 340 l32i a3, a2, PT_AREG7; _j .Lstore_w; .align 8 341 l32i a3, a2, PT_AREG8; _j .Lstore_w; .align 8 342 mov a3, a9 ; _j .Lstore_w; .align 8 343 mov a3, a10 ; _j .Lstore_w; .align 8 344 mov a3, a11 ; _j .Lstore_w; .align 8 345 mov a3, a12 ; _j .Lstore_w; .align 8 346 mov a3, a13 ; _j .Lstore_w; .align 8 347 mov a3, a14 ; _j .Lstore_w; .align 8 348 mov a3, a15 ; _j .Lstore_w; .align 8 349#endif 350 351#ifdef ANY_EXCEPTION_HANDLER 352 /* We cannot handle this exception. */ 353 354 .extern _kernel_exception 355.Linvalid_instruction: 356 357 movi a4, 0 358 rsr a3, excsave1 359 s32i a4, a3, EXC_TABLE_FIXUP 360 361 /* Restore a4...a8 and SAR, set SP, and jump to default exception. */ 362 363 l32i a0, a2, PT_SAR 364 l32i a8, a2, PT_AREG8 365 l32i a7, a2, PT_AREG7 366 l32i a6, a2, PT_AREG6 367 l32i a5, a2, PT_AREG5 368 l32i a4, a2, PT_AREG4 369 wsr a0, sar 370 mov a1, a2 371 372 rsr a0, ps 373 bbsi.l a0, PS_UM_BIT, 2f # jump if user mode 374 375 movi a0, _kernel_exception 376 jx a0 377 3782: movi a0, _user_exception 379 jx a0 380#endif 381#if XCHAL_UNALIGNED_STORE_EXCEPTION 382 383 # a7: instruction pointer, a4: instruction, a3: value 384.Lstore_w: 385 movi a6, 0 # mask: ffffffff:00000000 386 387#if XCHAL_HAVE_DENSITY 388 addi a7, a7, 2 # incr. PC,assume 16-bit instruction 389 390 extui a5, a4, INSN_OP0, 4 # extract OP0 391 addi a5, a5, -OP0_S32I_N 392 _beqz a5, 1f # s32i.n: jump 393 394 addi a7, a7, 1 # increment PC, 32-bit instruction 395#else 396 addi a7, a7, 3 # increment PC, 32-bit instruction 397#endif 398 399 extui a5, a4, INSN_OP1, 4 # extract OP1 400 _beqi a5, OP1_S32I, 1f # jump if 32 bit store 401 _bnei a5, OP1_S16I, .Linvalid_instruction 402 403 movi a5, -1 404 __extl a3, a3 # get 16-bit value 405 __exth a6, a5 # get 16-bit mask ffffffff:ffff0000 406 407 /* Get memory address */ 408 4091: 410 movi a4, ~3 411 and a4, a4, a8 # align memory address 412 413 /* Insert value into memory */ 414 415 movi a5, -1 # mask: ffffffff:XXXX0000 416#ifdef UNALIGNED_USER_EXCEPTION 417 addi a4, a4, 8 418#endif 419 420 __ssa8r a8 421 __src_b a8, a5, a6 # lo-mask F..F0..0 (BE) 0..0F..F (LE) 422 __src_b a6, a6, a5 # hi-mask 0..0F..F (BE) F..F0..0 (LE) 423#ifdef UNALIGNED_USER_EXCEPTION 424 l32e a5, a4, -8 425#else 426 l32i a5, a4, 0 # load lower address word 427#endif 428 and a5, a5, a8 # mask 429 __sh a8, a3 # shift value 430 or a5, a5, a8 # or with original value 431#ifdef UNALIGNED_USER_EXCEPTION 432 s32e a5, a4, -8 433 l32e a8, a4, -4 434#else 435 s32i a5, a4, 0 # store 436 l32i a8, a4, 4 # same for upper address word 437#endif 438 __sl a5, a3 439 and a6, a8, a6 440 or a6, a6, a5 441#ifdef UNALIGNED_USER_EXCEPTION 442 s32e a6, a4, -4 443#else 444 s32i a6, a4, 4 445#endif 446#endif 447#ifdef ANY_EXCEPTION_HANDLER 448.Lexit: 449#if XCHAL_HAVE_LOOPS 450 rsr a4, lend # check if we reached LEND 451 bne a7, a4, 1f 452 rsr a4, lcount # and LCOUNT != 0 453 beqz a4, 1f 454 addi a4, a4, -1 # decrement LCOUNT and set 455 rsr a7, lbeg # set PC to LBEGIN 456 wsr a4, lcount 457#endif 458 4591: wsr a7, epc1 # skip emulated instruction 460 461 /* Update icount if we're single-stepping in userspace. */ 462 rsr a4, icountlevel 463 beqz a4, 1f 464 bgeui a4, LOCKLEVEL + 1, 1f 465 rsr a4, icount 466 addi a4, a4, 1 467 wsr a4, icount 4681: 469 movi a4, 0 470 rsr a3, excsave1 471 s32i a4, a3, EXC_TABLE_FIXUP 472 473 /* Restore working register */ 474 475 l32i a0, a2, PT_SAR 476 l32i a8, a2, PT_AREG8 477 l32i a7, a2, PT_AREG7 478 l32i a6, a2, PT_AREG6 479 l32i a5, a2, PT_AREG5 480 l32i a4, a2, PT_AREG4 481 l32i a3, a2, PT_AREG3 482 483 /* restore SAR and return */ 484 485 wsr a0, sar 486 l32i a0, a2, PT_AREG0 487 l32i a2, a2, PT_AREG2 488 rfe 489 490 .align 4 491.Lsave_and_load_instruction: 492 493 /* Save some working register */ 494 495 s32i a3, a2, PT_AREG3 496 s32i a4, a2, PT_AREG4 497 s32i a5, a2, PT_AREG5 498 s32i a6, a2, PT_AREG6 499 s32i a7, a2, PT_AREG7 500 s32i a8, a2, PT_AREG8 501 502 rsr a4, depc 503 s32i a4, a2, PT_AREG2 504 505 rsr a5, sar 506 s32i a5, a2, PT_SAR 507 508 rsr a3, excsave1 509 movi a4, fast_unaligned_fixup 510 s32i a4, a3, EXC_TABLE_FIXUP 511 512 rsr a8, excvaddr # load unaligned memory address 513 514 /* Now, identify one of the following load/store instructions. 515 * 516 * The only possible danger of a double exception on the 517 * following l32i instructions is kernel code in vmalloc 518 * memory. The processor was just executing at the EPC_1 519 * address, and indeed, already fetched the instruction. That 520 * guarantees a TLB mapping, which hasn't been replaced by 521 * this unaligned exception handler that uses only static TLB 522 * mappings. However, high-level interrupt handlers might 523 * modify TLB entries, so for the generic case, we register a 524 * TABLE_FIXUP handler here, too. 525 */ 526 527 /* a3...a6 saved on stack, a2 = SP */ 528 529 /* Extract the instruction that caused the unaligned access. */ 530 531 rsr a7, epc1 # load exception address 532 movi a3, ~3 533 and a3, a3, a7 # mask lower bits 534 535 l32i a4, a3, 0 # load 2 words 536 l32i a5, a3, 4 537 538 __ssa8 a7 539 __src_b a4, a4, a5 # a4 has the instruction 540 541 ret 542#endif 543ENDPROC(fast_unaligned) 544 545ENTRY(fast_unaligned_fixup) 546 547 l32i a2, a3, EXC_TABLE_DOUBLE_SAVE 548 wsr a3, excsave1 549 550 l32i a8, a2, PT_AREG8 551 l32i a7, a2, PT_AREG7 552 l32i a6, a2, PT_AREG6 553 l32i a5, a2, PT_AREG5 554 l32i a4, a2, PT_SAR 555 l32i a0, a2, PT_AREG2 556 wsr a4, sar 557 wsr a0, depc # restore depc and a0 558 l32i a4, a2, PT_AREG4 559 560 rsr a0, exccause 561 s32i a0, a2, PT_DEPC # mark as a regular exception 562 563 rsr a0, ps 564 bbsi.l a0, PS_UM_BIT, 1f # jump if user mode 565 566 rsr a0, exccause 567 addx4 a0, a0, a3 # find entry in table 568 l32i a0, a0, EXC_TABLE_FAST_KERNEL # load handler 569 l32i a3, a2, PT_AREG3 570 jx a0 5711: 572 rsr a0, exccause 573 addx4 a0, a0, a3 # find entry in table 574 l32i a0, a0, EXC_TABLE_FAST_USER # load handler 575 l32i a3, a2, PT_AREG3 576 jx a0 577 578ENDPROC(fast_unaligned_fixup) 579#endif 580