1/* 2 * arch/xtensa/kernel/align.S 3 * 4 * Handle unalignment exceptions in kernel space. 5 * 6 * This file is subject to the terms and conditions of the GNU General 7 * Public License. See the file "COPYING" in the main directory of 8 * this archive for more details. 9 * 10 * Copyright (C) 2001 - 2005 Tensilica, Inc. 11 * Copyright (C) 2014 Cadence Design Systems Inc. 12 * 13 * Rewritten by Chris Zankel <chris@zankel.net> 14 * 15 * Based on work from Joe Taylor <joe@tensilica.com, joetylr@yahoo.com> 16 * and Marc Gauthier <marc@tensilica.com, marc@alimni.uwaterloo.ca> 17 */ 18 19#include <linux/linkage.h> 20#include <asm/current.h> 21#include <asm/asm-offsets.h> 22#include <asm/processor.h> 23 24#if XCHAL_UNALIGNED_LOAD_EXCEPTION || XCHAL_UNALIGNED_STORE_EXCEPTION 25 26/* First-level exception handler for unaligned exceptions. 27 * 28 * Note: This handler works only for kernel exceptions. Unaligned user 29 * access should get a seg fault. 30 */ 31 32/* Big and little endian 16-bit values are located in 33 * different halves of a register. HWORD_START helps to 34 * abstract the notion of extracting a 16-bit value from a 35 * register. 36 * We also have to define new shifting instructions because 37 * lsb and msb are on 'opposite' ends in a register for 38 * different endian machines. 39 * 40 * Assume a memory region in ascending address: 41 * 0 1 2 3|4 5 6 7 42 * 43 * When loading one word into a register, the content of that register is: 44 * LE 3 2 1 0, 7 6 5 4 45 * BE 0 1 2 3, 4 5 6 7 46 * 47 * Masking the bits of the higher/lower address means: 48 * LE X X 0 0, 0 0 X X 49 * BE 0 0 X X, X X 0 0 50 * 51 * Shifting to higher/lower addresses, means: 52 * LE shift left / shift right 53 * BE shift right / shift left 54 * 55 * Extracting 16 bits from a 32 bit reg. value to higher/lower address means: 56 * LE mask 0 0 X X / shift left 57 * BE shift left / mask 0 0 X X 58 */ 59 60#define UNALIGNED_USER_EXCEPTION 61 62#if XCHAL_HAVE_BE 63 64#define HWORD_START 16 65#define INSN_OP0 28 66#define INSN_T 24 67#define INSN_OP1 16 68 69.macro __src_b r, w0, w1; src \r, \w0, \w1; .endm 70.macro __ssa8 r; ssa8b \r; .endm 71.macro __ssa8r r; ssa8l \r; .endm 72.macro __sh r, s; srl \r, \s; .endm 73.macro __sl r, s; sll \r, \s; .endm 74.macro __exth r, s; extui \r, \s, 0, 16; .endm 75.macro __extl r, s; slli \r, \s, 16; .endm 76 77#else 78 79#define HWORD_START 0 80#define INSN_OP0 0 81#define INSN_T 4 82#define INSN_OP1 12 83 84.macro __src_b r, w0, w1; src \r, \w1, \w0; .endm 85.macro __ssa8 r; ssa8l \r; .endm 86.macro __ssa8r r; ssa8b \r; .endm 87.macro __sh r, s; sll \r, \s; .endm 88.macro __sl r, s; srl \r, \s; .endm 89.macro __exth r, s; slli \r, \s, 16; .endm 90.macro __extl r, s; extui \r, \s, 0, 16; .endm 91 92#endif 93 94/* 95 * xxxx xxxx = imm8 field 96 * yyyy = imm4 field 97 * ssss = s field 98 * tttt = t field 99 * 100 * 16 0 101 * ------------------- 102 * L32I.N yyyy ssss tttt 1000 103 * S32I.N yyyy ssss tttt 1001 104 * 105 * 23 0 106 * ----------------------------- 107 * res 0000 0010 108 * L16UI xxxx xxxx 0001 ssss tttt 0010 109 * L32I xxxx xxxx 0010 ssss tttt 0010 110 * XXX 0011 ssss tttt 0010 111 * XXX 0100 ssss tttt 0010 112 * S16I xxxx xxxx 0101 ssss tttt 0010 113 * S32I xxxx xxxx 0110 ssss tttt 0010 114 * XXX 0111 ssss tttt 0010 115 * XXX 1000 ssss tttt 0010 116 * L16SI xxxx xxxx 1001 ssss tttt 0010 117 * XXX 1010 0010 118 * **L32AI xxxx xxxx 1011 ssss tttt 0010 unsupported 119 * XXX 1100 0010 120 * XXX 1101 0010 121 * XXX 1110 0010 122 * **S32RI xxxx xxxx 1111 ssss tttt 0010 unsupported 123 * ----------------------------- 124 * ^ ^ ^ 125 * sub-opcode (NIBBLE_R) -+ | | 126 * t field (NIBBLE_T) -----------+ | 127 * major opcode (NIBBLE_OP0) --------------+ 128 */ 129 130#define OP0_L32I_N 0x8 /* load immediate narrow */ 131#define OP0_S32I_N 0x9 /* store immediate narrow */ 132#define OP1_SI_MASK 0x4 /* OP1 bit set for stores */ 133#define OP1_SI_BIT 2 /* OP1 bit number for stores */ 134 135#define OP1_L32I 0x2 136#define OP1_L16UI 0x1 137#define OP1_L16SI 0x9 138#define OP1_L32AI 0xb 139 140#define OP1_S32I 0x6 141#define OP1_S16I 0x5 142#define OP1_S32RI 0xf 143 144/* 145 * Entry condition: 146 * 147 * a0: trashed, original value saved on stack (PT_AREG0) 148 * a1: a1 149 * a2: new stack pointer, original in DEPC 150 * a3: a3 151 * depc: a2, original value saved on stack (PT_DEPC) 152 * excsave_1: dispatch table 153 * 154 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC 155 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception 156 */ 157 158 159ENTRY(fast_unaligned) 160 161 /* Note: We don't expect the address to be aligned on a word 162 * boundary. After all, the processor generated that exception 163 * and it would be a hardware fault. 164 */ 165 166 /* Save some working register */ 167 168 s32i a4, a2, PT_AREG4 169 s32i a5, a2, PT_AREG5 170 s32i a6, a2, PT_AREG6 171 s32i a7, a2, PT_AREG7 172 s32i a8, a2, PT_AREG8 173 174 rsr a0, depc 175 s32i a0, a2, PT_AREG2 176 s32i a3, a2, PT_AREG3 177 178 rsr a3, excsave1 179 movi a4, fast_unaligned_fixup 180 s32i a4, a3, EXC_TABLE_FIXUP 181 182 /* Keep value of SAR in a0 */ 183 184 rsr a0, sar 185 rsr a8, excvaddr # load unaligned memory address 186 187 /* Now, identify one of the following load/store instructions. 188 * 189 * The only possible danger of a double exception on the 190 * following l32i instructions is kernel code in vmalloc 191 * memory. The processor was just executing at the EPC_1 192 * address, and indeed, already fetched the instruction. That 193 * guarantees a TLB mapping, which hasn't been replaced by 194 * this unaligned exception handler that uses only static TLB 195 * mappings. However, high-level interrupt handlers might 196 * modify TLB entries, so for the generic case, we register a 197 * TABLE_FIXUP handler here, too. 198 */ 199 200 /* a3...a6 saved on stack, a2 = SP */ 201 202 /* Extract the instruction that caused the unaligned access. */ 203 204 rsr a7, epc1 # load exception address 205 movi a3, ~3 206 and a3, a3, a7 # mask lower bits 207 208 l32i a4, a3, 0 # load 2 words 209 l32i a5, a3, 4 210 211 __ssa8 a7 212 __src_b a4, a4, a5 # a4 has the instruction 213 214 /* Analyze the instruction (load or store?). */ 215 216 extui a5, a4, INSN_OP0, 4 # get insn.op0 nibble 217 218#if XCHAL_HAVE_DENSITY 219 _beqi a5, OP0_L32I_N, .Lload # L32I.N, jump 220 addi a6, a5, -OP0_S32I_N 221 _beqz a6, .Lstore # S32I.N, do a store 222#endif 223 /* 'store indicator bit' not set, jump */ 224 _bbci.l a4, OP1_SI_BIT + INSN_OP1, .Lload 225 226 /* Store: Jump to table entry to get the value in the source register.*/ 227 228.Lstore:movi a5, .Lstore_table # table 229 extui a6, a4, INSN_T, 4 # get source register 230 addx8 a5, a6, a5 231 jx a5 # jump into table 232 233 /* Invalid instruction, CRITICAL! */ 234.Linvalid_instruction_load: 235 j .Linvalid_instruction 236 237 /* Load: Load memory address. */ 238 239.Lload: movi a3, ~3 240 and a3, a3, a8 # align memory address 241 242 __ssa8 a8 243#ifdef UNALIGNED_USER_EXCEPTION 244 addi a3, a3, 8 245 l32e a5, a3, -8 246 l32e a6, a3, -4 247#else 248 l32i a5, a3, 0 249 l32i a6, a3, 4 250#endif 251 __src_b a3, a5, a6 # a3 has the data word 252 253#if XCHAL_HAVE_DENSITY 254 addi a7, a7, 2 # increment PC (assume 16-bit insn) 255 256 extui a5, a4, INSN_OP0, 4 257 _beqi a5, OP0_L32I_N, 1f # l32i.n: jump 258 259 addi a7, a7, 1 260#else 261 addi a7, a7, 3 262#endif 263 264 extui a5, a4, INSN_OP1, 4 265 _beqi a5, OP1_L32I, 1f # l32i: jump 266 267 extui a3, a3, 0, 16 # extract lower 16 bits 268 _beqi a5, OP1_L16UI, 1f 269 addi a5, a5, -OP1_L16SI 270 _bnez a5, .Linvalid_instruction_load 271 272 /* sign extend value */ 273 274 slli a3, a3, 16 275 srai a3, a3, 16 276 277 /* Set target register. */ 278 2791: 280 extui a4, a4, INSN_T, 4 # extract target register 281 movi a5, .Lload_table 282 addx8 a4, a4, a5 283 jx a4 # jump to entry for target register 284 285 .align 8 286.Lload_table: 287 s32i a3, a2, PT_AREG0; _j .Lexit; .align 8 288 mov a1, a3; _j .Lexit; .align 8 # fishy?? 289 s32i a3, a2, PT_AREG2; _j .Lexit; .align 8 290 s32i a3, a2, PT_AREG3; _j .Lexit; .align 8 291 s32i a3, a2, PT_AREG4; _j .Lexit; .align 8 292 s32i a3, a2, PT_AREG5; _j .Lexit; .align 8 293 s32i a3, a2, PT_AREG6; _j .Lexit; .align 8 294 s32i a3, a2, PT_AREG7; _j .Lexit; .align 8 295 s32i a3, a2, PT_AREG8; _j .Lexit; .align 8 296 mov a9, a3 ; _j .Lexit; .align 8 297 mov a10, a3 ; _j .Lexit; .align 8 298 mov a11, a3 ; _j .Lexit; .align 8 299 mov a12, a3 ; _j .Lexit; .align 8 300 mov a13, a3 ; _j .Lexit; .align 8 301 mov a14, a3 ; _j .Lexit; .align 8 302 mov a15, a3 ; _j .Lexit; .align 8 303 304.Lstore_table: 305 l32i a3, a2, PT_AREG0; _j 1f; .align 8 306 mov a3, a1; _j 1f; .align 8 # fishy?? 307 l32i a3, a2, PT_AREG2; _j 1f; .align 8 308 l32i a3, a2, PT_AREG3; _j 1f; .align 8 309 l32i a3, a2, PT_AREG4; _j 1f; .align 8 310 l32i a3, a2, PT_AREG5; _j 1f; .align 8 311 l32i a3, a2, PT_AREG6; _j 1f; .align 8 312 l32i a3, a2, PT_AREG7; _j 1f; .align 8 313 l32i a3, a2, PT_AREG8; _j 1f; .align 8 314 mov a3, a9 ; _j 1f; .align 8 315 mov a3, a10 ; _j 1f; .align 8 316 mov a3, a11 ; _j 1f; .align 8 317 mov a3, a12 ; _j 1f; .align 8 318 mov a3, a13 ; _j 1f; .align 8 319 mov a3, a14 ; _j 1f; .align 8 320 mov a3, a15 ; _j 1f; .align 8 321 3221: # a7: instruction pointer, a4: instruction, a3: value 323 324 movi a6, 0 # mask: ffffffff:00000000 325 326#if XCHAL_HAVE_DENSITY 327 addi a7, a7, 2 # incr. PC,assume 16-bit instruction 328 329 extui a5, a4, INSN_OP0, 4 # extract OP0 330 addi a5, a5, -OP0_S32I_N 331 _beqz a5, 1f # s32i.n: jump 332 333 addi a7, a7, 1 # increment PC, 32-bit instruction 334#else 335 addi a7, a7, 3 # increment PC, 32-bit instruction 336#endif 337 338 extui a5, a4, INSN_OP1, 4 # extract OP1 339 _beqi a5, OP1_S32I, 1f # jump if 32 bit store 340 _bnei a5, OP1_S16I, .Linvalid_instruction_store 341 342 movi a5, -1 343 __extl a3, a3 # get 16-bit value 344 __exth a6, a5 # get 16-bit mask ffffffff:ffff0000 345 346 /* Get memory address */ 347 3481: 349 movi a4, ~3 350 and a4, a4, a8 # align memory address 351 352 /* Insert value into memory */ 353 354 movi a5, -1 # mask: ffffffff:XXXX0000 355#ifdef UNALIGNED_USER_EXCEPTION 356 addi a4, a4, 8 357#endif 358 359 __ssa8r a8 360 __src_b a8, a5, a6 # lo-mask F..F0..0 (BE) 0..0F..F (LE) 361 __src_b a6, a6, a5 # hi-mask 0..0F..F (BE) F..F0..0 (LE) 362#ifdef UNALIGNED_USER_EXCEPTION 363 l32e a5, a4, -8 364#else 365 l32i a5, a4, 0 # load lower address word 366#endif 367 and a5, a5, a8 # mask 368 __sh a8, a3 # shift value 369 or a5, a5, a8 # or with original value 370#ifdef UNALIGNED_USER_EXCEPTION 371 s32e a5, a4, -8 372 l32e a8, a4, -4 373#else 374 s32i a5, a4, 0 # store 375 l32i a8, a4, 4 # same for upper address word 376#endif 377 __sl a5, a3 378 and a6, a8, a6 379 or a6, a6, a5 380#ifdef UNALIGNED_USER_EXCEPTION 381 s32e a6, a4, -4 382#else 383 s32i a6, a4, 4 384#endif 385 386.Lexit: 387#if XCHAL_HAVE_LOOPS 388 rsr a4, lend # check if we reached LEND 389 bne a7, a4, 1f 390 rsr a4, lcount # and LCOUNT != 0 391 beqz a4, 1f 392 addi a4, a4, -1 # decrement LCOUNT and set 393 rsr a7, lbeg # set PC to LBEGIN 394 wsr a4, lcount 395#endif 396 3971: wsr a7, epc1 # skip emulated instruction 398 399 movi a4, 0 400 rsr a3, excsave1 401 s32i a4, a3, EXC_TABLE_FIXUP 402 403 /* Restore working register */ 404 405 l32i a8, a2, PT_AREG8 406 l32i a7, a2, PT_AREG7 407 l32i a6, a2, PT_AREG6 408 l32i a5, a2, PT_AREG5 409 l32i a4, a2, PT_AREG4 410 l32i a3, a2, PT_AREG3 411 412 /* restore SAR and return */ 413 414 wsr a0, sar 415 l32i a0, a2, PT_AREG0 416 l32i a2, a2, PT_AREG2 417 rfe 418 419 /* We cannot handle this exception. */ 420 421 .extern _kernel_exception 422.Linvalid_instruction_store: 423.Linvalid_instruction: 424 425 movi a4, 0 426 rsr a3, excsave1 427 s32i a4, a3, EXC_TABLE_FIXUP 428 429 /* Restore a4...a8 and SAR, set SP, and jump to default exception. */ 430 431 l32i a8, a2, PT_AREG8 432 l32i a7, a2, PT_AREG7 433 l32i a6, a2, PT_AREG6 434 l32i a5, a2, PT_AREG5 435 l32i a4, a2, PT_AREG4 436 wsr a0, sar 437 mov a1, a2 438 439 rsr a0, ps 440 bbsi.l a0, PS_UM_BIT, 1f # jump if user mode 441 442 movi a0, _kernel_exception 443 jx a0 444 4451: movi a0, _user_exception 446 jx a0 447 448ENDPROC(fast_unaligned) 449 450ENTRY(fast_unaligned_fixup) 451 452 l32i a2, a3, EXC_TABLE_DOUBLE_SAVE 453 wsr a3, excsave1 454 455 l32i a8, a2, PT_AREG8 456 l32i a7, a2, PT_AREG7 457 l32i a6, a2, PT_AREG6 458 l32i a5, a2, PT_AREG5 459 l32i a4, a2, PT_AREG4 460 l32i a0, a2, PT_AREG2 461 xsr a0, depc # restore depc and a0 462 wsr a0, sar 463 464 rsr a0, exccause 465 s32i a0, a2, PT_DEPC # mark as a regular exception 466 467 rsr a0, ps 468 bbsi.l a0, PS_UM_BIT, 1f # jump if user mode 469 470 rsr a0, exccause 471 addx4 a0, a0, a3 # find entry in table 472 l32i a0, a0, EXC_TABLE_FAST_KERNEL # load handler 473 l32i a3, a2, PT_AREG3 474 jx a0 4751: 476 rsr a0, exccause 477 addx4 a0, a0, a3 # find entry in table 478 l32i a0, a0, EXC_TABLE_FAST_USER # load handler 479 l32i a3, a2, PT_AREG3 480 jx a0 481 482ENDPROC(fast_unaligned_fixup) 483 484#endif /* XCHAL_UNALIGNED_LOAD_EXCEPTION || XCHAL_UNALIGNED_STORE_EXCEPTION */ 485