1/* 2 * linux/arch/arm/kernel/entry-armv.S 3 * 4 * Copyright (C) 1996,1997,1998 Russell King. 5 * ARM700 fix by Matthew Godbolt (linux-user@willothewisp.demon.co.uk) 6 * nommu support by Hyok S. Choi (hyok.choi@samsung.com) 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 * 12 * Low-level vector interface routines 13 * 14 * Note: there is a StrongARM bug in the STMIA rn, {regs}^ instruction 15 * that causes it to save wrong values... Be aware! 16 */ 17 18#include <asm/memory.h> 19#include <asm/glue.h> 20#include <asm/vfpmacros.h> 21#include <mach/entry-macro.S> 22#include <asm/thread_notify.h> 23#include <asm/unwind.h> 24#include <asm/unistd.h> 25 26#include "entry-header.S" 27 28/* 29 * Interrupt handling. Preserves r7, r8, r9 30 */ 31 .macro irq_handler 32 get_irqnr_preamble r5, lr 331: get_irqnr_and_base r0, r6, r5, lr 34 movne r1, sp 35 @ 36 @ routine called with r0 = irq number, r1 = struct pt_regs * 37 @ 38 adrne lr, BSYM(1b) 39 bne asm_do_IRQ 40 41#ifdef CONFIG_SMP 42 /* 43 * XXX 44 * 45 * this macro assumes that irqstat (r6) and base (r5) are 46 * preserved from get_irqnr_and_base above 47 */ 48 test_for_ipi r0, r6, r5, lr 49 movne r0, sp 50 adrne lr, BSYM(1b) 51 bne do_IPI 52 53#ifdef CONFIG_LOCAL_TIMERS 54 test_for_ltirq r0, r6, r5, lr 55 movne r0, sp 56 adrne lr, BSYM(1b) 57 bne do_local_timer 58#endif 59#endif 60 61 .endm 62 63#ifdef CONFIG_KPROBES 64 .section .kprobes.text,"ax",%progbits 65#else 66 .text 67#endif 68 69/* 70 * Invalid mode handlers 71 */ 72 .macro inv_entry, reason 73 sub sp, sp, #S_FRAME_SIZE 74 ARM( stmib sp, {r1 - lr} ) 75 THUMB( stmia sp, {r0 - r12} ) 76 THUMB( str sp, [sp, #S_SP] ) 77 THUMB( str lr, [sp, #S_LR] ) 78 mov r1, #\reason 79 .endm 80 81__pabt_invalid: 82 inv_entry BAD_PREFETCH 83 b common_invalid 84ENDPROC(__pabt_invalid) 85 86__dabt_invalid: 87 inv_entry BAD_DATA 88 b common_invalid 89ENDPROC(__dabt_invalid) 90 91__irq_invalid: 92 inv_entry BAD_IRQ 93 b common_invalid 94ENDPROC(__irq_invalid) 95 96__und_invalid: 97 inv_entry BAD_UNDEFINSTR 98 99 @ 100 @ XXX fall through to common_invalid 101 @ 102 103@ 104@ common_invalid - generic code for failed exception (re-entrant version of handlers) 105@ 106common_invalid: 107 zero_fp 108 109 ldmia r0, {r4 - r6} 110 add r0, sp, #S_PC @ here for interlock avoidance 111 mov r7, #-1 @ "" "" "" "" 112 str r4, [sp] @ save preserved r0 113 stmia r0, {r5 - r7} @ lr_<exception>, 114 @ cpsr_<exception>, "old_r0" 115 116 mov r0, sp 117 b bad_mode 118ENDPROC(__und_invalid) 119 120/* 121 * SVC mode handlers 122 */ 123 124#if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5) 125#define SPFIX(code...) code 126#else 127#define SPFIX(code...) 128#endif 129 130 .macro svc_entry, stack_hole=0 131 UNWIND(.fnstart ) 132 UNWIND(.save {r0 - pc} ) 133 sub sp, sp, #(S_FRAME_SIZE + \stack_hole - 4) 134#ifdef CONFIG_THUMB2_KERNEL 135 SPFIX( str r0, [sp] ) @ temporarily saved 136 SPFIX( mov r0, sp ) 137 SPFIX( tst r0, #4 ) @ test original stack alignment 138 SPFIX( ldr r0, [sp] ) @ restored 139#else 140 SPFIX( tst sp, #4 ) 141#endif 142 SPFIX( subeq sp, sp, #4 ) 143 stmia sp, {r1 - r12} 144 145 ldmia r0, {r1 - r3} 146 add r5, sp, #S_SP - 4 @ here for interlock avoidance 147 mov r4, #-1 @ "" "" "" "" 148 add r0, sp, #(S_FRAME_SIZE + \stack_hole - 4) 149 SPFIX( addeq r0, r0, #4 ) 150 str r1, [sp, #-4]! @ save the "real" r0 copied 151 @ from the exception stack 152 153 mov r1, lr 154 155 @ 156 @ We are now ready to fill in the remaining blanks on the stack: 157 @ 158 @ r0 - sp_svc 159 @ r1 - lr_svc 160 @ r2 - lr_<exception>, already fixed up for correct return/restart 161 @ r3 - spsr_<exception> 162 @ r4 - orig_r0 (see pt_regs definition in ptrace.h) 163 @ 164 stmia r5, {r0 - r4} 165 166 asm_trace_hardirqs_off 167 .endm 168 169 .align 5 170__dabt_svc: 171 svc_entry 172 173 @ 174 @ get ready to re-enable interrupts if appropriate 175 @ 176 mrs r9, cpsr 177 tst r3, #PSR_I_BIT 178 biceq r9, r9, #PSR_I_BIT 179 180 @ 181 @ Call the processor-specific abort handler: 182 @ 183 @ r2 - aborted context pc 184 @ r3 - aborted context cpsr 185 @ 186 @ The abort handler must return the aborted address in r0, and 187 @ the fault status register in r1. r9 must be preserved. 188 @ 189#ifdef MULTI_DABORT 190 ldr r4, .LCprocfns 191 mov lr, pc 192 ldr pc, [r4, #PROCESSOR_DABT_FUNC] 193#else 194 bl CPU_DABORT_HANDLER 195#endif 196 197 @ 198 @ set desired IRQ state, then call main handler 199 @ 200 msr cpsr_c, r9 201 mov r2, sp 202 bl do_DataAbort 203 204 @ 205 @ IRQs off again before pulling preserved data off the stack 206 @ 207 disable_irq 208 209 @ 210 @ restore SPSR and restart the instruction 211 @ 212 ldr r2, [sp, #S_PSR] 213 svc_exit r2 @ return from exception 214 UNWIND(.fnend ) 215ENDPROC(__dabt_svc) 216 217 .align 5 218__irq_svc: 219 svc_entry 220 221#ifdef CONFIG_PREEMPT 222 get_thread_info tsk 223 ldr r8, [tsk, #TI_PREEMPT] @ get preempt count 224 add r7, r8, #1 @ increment it 225 str r7, [tsk, #TI_PREEMPT] 226#endif 227 228 irq_handler 229#ifdef CONFIG_PREEMPT 230 str r8, [tsk, #TI_PREEMPT] @ restore preempt count 231 ldr r0, [tsk, #TI_FLAGS] @ get flags 232 teq r8, #0 @ if preempt count != 0 233 movne r0, #0 @ force flags to 0 234 tst r0, #_TIF_NEED_RESCHED 235 blne svc_preempt 236#endif 237 ldr r4, [sp, #S_PSR] @ irqs are already disabled 238#ifdef CONFIG_TRACE_IRQFLAGS 239 tst r4, #PSR_I_BIT 240 bleq trace_hardirqs_on 241#endif 242 svc_exit r4 @ return from exception 243 UNWIND(.fnend ) 244ENDPROC(__irq_svc) 245 246 .ltorg 247 248#ifdef CONFIG_PREEMPT 249svc_preempt: 250 mov r8, lr 2511: bl preempt_schedule_irq @ irq en/disable is done inside 252 ldr r0, [tsk, #TI_FLAGS] @ get new tasks TI_FLAGS 253 tst r0, #_TIF_NEED_RESCHED 254 moveq pc, r8 @ go again 255 b 1b 256#endif 257 258 .align 5 259__und_svc: 260#ifdef CONFIG_KPROBES 261 @ If a kprobe is about to simulate a "stmdb sp..." instruction, 262 @ it obviously needs free stack space which then will belong to 263 @ the saved context. 264 svc_entry 64 265#else 266 svc_entry 267#endif 268 269 @ 270 @ call emulation code, which returns using r9 if it has emulated 271 @ the instruction, or the more conventional lr if we are to treat 272 @ this as a real undefined instruction 273 @ 274 @ r0 - instruction 275 @ 276#ifndef CONFIG_THUMB2_KERNEL 277 ldr r0, [r2, #-4] 278#else 279 ldrh r0, [r2, #-2] @ Thumb instruction at LR - 2 280 and r9, r0, #0xf800 281 cmp r9, #0xe800 @ 32-bit instruction if xx >= 0 282 ldrhhs r9, [r2] @ bottom 16 bits 283 orrhs r0, r9, r0, lsl #16 284#endif 285 adr r9, BSYM(1f) 286 bl call_fpe 287 288 mov r0, sp @ struct pt_regs *regs 289 bl do_undefinstr 290 291 @ 292 @ IRQs off again before pulling preserved data off the stack 293 @ 2941: disable_irq 295 296 @ 297 @ restore SPSR and restart the instruction 298 @ 299 ldr r2, [sp, #S_PSR] @ Get SVC cpsr 300 svc_exit r2 @ return from exception 301 UNWIND(.fnend ) 302ENDPROC(__und_svc) 303 304 .align 5 305__pabt_svc: 306 svc_entry 307 308 @ 309 @ re-enable interrupts if appropriate 310 @ 311 mrs r9, cpsr 312 tst r3, #PSR_I_BIT 313 biceq r9, r9, #PSR_I_BIT 314 315 mov r0, r2 @ pass address of aborted instruction. 316#ifdef MULTI_PABORT 317 ldr r4, .LCprocfns 318 mov lr, pc 319 ldr pc, [r4, #PROCESSOR_PABT_FUNC] 320#else 321 bl CPU_PABORT_HANDLER 322#endif 323 msr cpsr_c, r9 @ Maybe enable interrupts 324 mov r2, sp @ regs 325 bl do_PrefetchAbort @ call abort handler 326 327 @ 328 @ IRQs off again before pulling preserved data off the stack 329 @ 330 disable_irq 331 332 @ 333 @ restore SPSR and restart the instruction 334 @ 335 ldr r2, [sp, #S_PSR] 336 svc_exit r2 @ return from exception 337 UNWIND(.fnend ) 338ENDPROC(__pabt_svc) 339 340 .align 5 341.LCcralign: 342 .word cr_alignment 343#ifdef MULTI_DABORT 344.LCprocfns: 345 .word processor 346#endif 347.LCfp: 348 .word fp_enter 349 350/* 351 * User mode handlers 352 * 353 * EABI note: sp_svc is always 64-bit aligned here, so should S_FRAME_SIZE 354 */ 355 356#if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5) && (S_FRAME_SIZE & 7) 357#error "sizeof(struct pt_regs) must be a multiple of 8" 358#endif 359 360 .macro usr_entry 361 UNWIND(.fnstart ) 362 UNWIND(.cantunwind ) @ don't unwind the user space 363 sub sp, sp, #S_FRAME_SIZE 364 ARM( stmib sp, {r1 - r12} ) 365 THUMB( stmia sp, {r0 - r12} ) 366 367 ldmia r0, {r1 - r3} 368 add r0, sp, #S_PC @ here for interlock avoidance 369 mov r4, #-1 @ "" "" "" "" 370 371 str r1, [sp] @ save the "real" r0 copied 372 @ from the exception stack 373 374 @ 375 @ We are now ready to fill in the remaining blanks on the stack: 376 @ 377 @ r2 - lr_<exception>, already fixed up for correct return/restart 378 @ r3 - spsr_<exception> 379 @ r4 - orig_r0 (see pt_regs definition in ptrace.h) 380 @ 381 @ Also, separately save sp_usr and lr_usr 382 @ 383 stmia r0, {r2 - r4} 384 ARM( stmdb r0, {sp, lr}^ ) 385 THUMB( store_user_sp_lr r0, r1, S_SP - S_PC ) 386 387 @ 388 @ Enable the alignment trap while in kernel mode 389 @ 390 alignment_trap r0 391 392 @ 393 @ Clear FP to mark the first stack frame 394 @ 395 zero_fp 396 397 asm_trace_hardirqs_off 398 .endm 399 400 .macro kuser_cmpxchg_check 401#if __LINUX_ARM_ARCH__ < 6 && !defined(CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG) 402#ifndef CONFIG_MMU 403#warning "NPTL on non MMU needs fixing" 404#else 405 @ Make sure our user space atomic helper is restarted 406 @ if it was interrupted in a critical region. Here we 407 @ perform a quick test inline since it should be false 408 @ 99.9999% of the time. The rest is done out of line. 409 cmp r2, #TASK_SIZE 410 blhs kuser_cmpxchg_fixup 411#endif 412#endif 413 .endm 414 415 .align 5 416__dabt_usr: 417 usr_entry 418 kuser_cmpxchg_check 419 420 @ 421 @ Call the processor-specific abort handler: 422 @ 423 @ r2 - aborted context pc 424 @ r3 - aborted context cpsr 425 @ 426 @ The abort handler must return the aborted address in r0, and 427 @ the fault status register in r1. 428 @ 429#ifdef MULTI_DABORT 430 ldr r4, .LCprocfns 431 mov lr, pc 432 ldr pc, [r4, #PROCESSOR_DABT_FUNC] 433#else 434 bl CPU_DABORT_HANDLER 435#endif 436 437 @ 438 @ IRQs on, then call the main handler 439 @ 440 enable_irq 441 mov r2, sp 442 adr lr, BSYM(ret_from_exception) 443 b do_DataAbort 444 UNWIND(.fnend ) 445ENDPROC(__dabt_usr) 446 447 .align 5 448__irq_usr: 449 usr_entry 450 kuser_cmpxchg_check 451 452 get_thread_info tsk 453#ifdef CONFIG_PREEMPT 454 ldr r8, [tsk, #TI_PREEMPT] @ get preempt count 455 add r7, r8, #1 @ increment it 456 str r7, [tsk, #TI_PREEMPT] 457#endif 458 459 irq_handler 460#ifdef CONFIG_PREEMPT 461 ldr r0, [tsk, #TI_PREEMPT] 462 str r8, [tsk, #TI_PREEMPT] 463 teq r0, r7 464 ARM( strne r0, [r0, -r0] ) 465 THUMB( movne r0, #0 ) 466 THUMB( strne r0, [r0] ) 467#endif 468#ifdef CONFIG_TRACE_IRQFLAGS 469 bl trace_hardirqs_on 470#endif 471 472 mov why, #0 473 b ret_to_user 474 UNWIND(.fnend ) 475ENDPROC(__irq_usr) 476 477 .ltorg 478 479 .align 5 480__und_usr: 481 usr_entry 482 483 @ 484 @ fall through to the emulation code, which returns using r9 if 485 @ it has emulated the instruction, or the more conventional lr 486 @ if we are to treat this as a real undefined instruction 487 @ 488 @ r0 - instruction 489 @ 490 adr r9, BSYM(ret_from_exception) 491 adr lr, BSYM(__und_usr_unknown) 492 tst r3, #PSR_T_BIT @ Thumb mode? 493 itet eq @ explicit IT needed for the 1f label 494 subeq r4, r2, #4 @ ARM instr at LR - 4 495 subne r4, r2, #2 @ Thumb instr at LR - 2 4961: ldreqt r0, [r4] 497#ifdef CONFIG_CPU_ENDIAN_BE8 498 reveq r0, r0 @ little endian instruction 499#endif 500 beq call_fpe 501 @ Thumb instruction 502#if __LINUX_ARM_ARCH__ >= 7 5032: 504 ARM( ldrht r5, [r4], #2 ) 505 THUMB( ldrht r5, [r4] ) 506 THUMB( add r4, r4, #2 ) 507 and r0, r5, #0xf800 @ mask bits 111x x... .... .... 508 cmp r0, #0xe800 @ 32bit instruction if xx != 0 509 blo __und_usr_unknown 5103: ldrht r0, [r4] 511 add r2, r2, #2 @ r2 is PC + 2, make it PC + 4 512 orr r0, r0, r5, lsl #16 513#else 514 b __und_usr_unknown 515#endif 516 UNWIND(.fnend ) 517ENDPROC(__und_usr) 518 519 @ 520 @ fallthrough to call_fpe 521 @ 522 523/* 524 * The out of line fixup for the ldrt above. 525 */ 526 .section .fixup, "ax" 5274: mov pc, r9 528 .previous 529 .section __ex_table,"a" 530 .long 1b, 4b 531#if __LINUX_ARM_ARCH__ >= 7 532 .long 2b, 4b 533 .long 3b, 4b 534#endif 535 .previous 536 537/* 538 * Check whether the instruction is a co-processor instruction. 539 * If yes, we need to call the relevant co-processor handler. 540 * 541 * Note that we don't do a full check here for the co-processor 542 * instructions; all instructions with bit 27 set are well 543 * defined. The only instructions that should fault are the 544 * co-processor instructions. However, we have to watch out 545 * for the ARM6/ARM7 SWI bug. 546 * 547 * NEON is a special case that has to be handled here. Not all 548 * NEON instructions are co-processor instructions, so we have 549 * to make a special case of checking for them. Plus, there's 550 * five groups of them, so we have a table of mask/opcode pairs 551 * to check against, and if any match then we branch off into the 552 * NEON handler code. 553 * 554 * Emulators may wish to make use of the following registers: 555 * r0 = instruction opcode. 556 * r2 = PC+4 557 * r9 = normal "successful" return address 558 * r10 = this threads thread_info structure. 559 * lr = unrecognised instruction return address 560 */ 561 @ 562 @ Fall-through from Thumb-2 __und_usr 563 @ 564#ifdef CONFIG_NEON 565 adr r6, .LCneon_thumb_opcodes 566 b 2f 567#endif 568call_fpe: 569#ifdef CONFIG_NEON 570 adr r6, .LCneon_arm_opcodes 5712: 572 ldr r7, [r6], #4 @ mask value 573 cmp r7, #0 @ end mask? 574 beq 1f 575 and r8, r0, r7 576 ldr r7, [r6], #4 @ opcode bits matching in mask 577 cmp r8, r7 @ NEON instruction? 578 bne 2b 579 get_thread_info r10 580 mov r7, #1 581 strb r7, [r10, #TI_USED_CP + 10] @ mark CP#10 as used 582 strb r7, [r10, #TI_USED_CP + 11] @ mark CP#11 as used 583 b do_vfp @ let VFP handler handle this 5841: 585#endif 586 tst r0, #0x08000000 @ only CDP/CPRT/LDC/STC have bit 27 587 tstne r0, #0x04000000 @ bit 26 set on both ARM and Thumb-2 588#if defined(CONFIG_CPU_ARM610) || defined(CONFIG_CPU_ARM710) 589 and r8, r0, #0x0f000000 @ mask out op-code bits 590 teqne r8, #0x0f000000 @ SWI (ARM6/7 bug)? 591#endif 592 moveq pc, lr 593 get_thread_info r10 @ get current thread 594 and r8, r0, #0x00000f00 @ mask out CP number 595 THUMB( lsr r8, r8, #8 ) 596 mov r7, #1 597 add r6, r10, #TI_USED_CP 598 ARM( strb r7, [r6, r8, lsr #8] ) @ set appropriate used_cp[] 599 THUMB( strb r7, [r6, r8] ) @ set appropriate used_cp[] 600#ifdef CONFIG_IWMMXT 601 @ Test if we need to give access to iWMMXt coprocessors 602 ldr r5, [r10, #TI_FLAGS] 603 rsbs r7, r8, #(1 << 8) @ CP 0 or 1 only 604 movcss r7, r5, lsr #(TIF_USING_IWMMXT + 1) 605 bcs iwmmxt_task_enable 606#endif 607 ARM( add pc, pc, r8, lsr #6 ) 608 THUMB( lsl r8, r8, #2 ) 609 THUMB( add pc, r8 ) 610 nop 611 612 movw_pc lr @ CP#0 613 W(b) do_fpe @ CP#1 (FPE) 614 W(b) do_fpe @ CP#2 (FPE) 615 movw_pc lr @ CP#3 616#ifdef CONFIG_CRUNCH 617 b crunch_task_enable @ CP#4 (MaverickCrunch) 618 b crunch_task_enable @ CP#5 (MaverickCrunch) 619 b crunch_task_enable @ CP#6 (MaverickCrunch) 620#else 621 movw_pc lr @ CP#4 622 movw_pc lr @ CP#5 623 movw_pc lr @ CP#6 624#endif 625 movw_pc lr @ CP#7 626 movw_pc lr @ CP#8 627 movw_pc lr @ CP#9 628#ifdef CONFIG_VFP 629 W(b) do_vfp @ CP#10 (VFP) 630 W(b) do_vfp @ CP#11 (VFP) 631#else 632 movw_pc lr @ CP#10 (VFP) 633 movw_pc lr @ CP#11 (VFP) 634#endif 635 movw_pc lr @ CP#12 636 movw_pc lr @ CP#13 637 movw_pc lr @ CP#14 (Debug) 638 movw_pc lr @ CP#15 (Control) 639 640#ifdef CONFIG_NEON 641 .align 6 642 643.LCneon_arm_opcodes: 644 .word 0xfe000000 @ mask 645 .word 0xf2000000 @ opcode 646 647 .word 0xff100000 @ mask 648 .word 0xf4000000 @ opcode 649 650 .word 0x00000000 @ mask 651 .word 0x00000000 @ opcode 652 653.LCneon_thumb_opcodes: 654 .word 0xef000000 @ mask 655 .word 0xef000000 @ opcode 656 657 .word 0xff100000 @ mask 658 .word 0xf9000000 @ opcode 659 660 .word 0x00000000 @ mask 661 .word 0x00000000 @ opcode 662#endif 663 664do_fpe: 665 enable_irq 666 ldr r4, .LCfp 667 add r10, r10, #TI_FPSTATE @ r10 = workspace 668 ldr pc, [r4] @ Call FP module USR entry point 669 670/* 671 * The FP module is called with these registers set: 672 * r0 = instruction 673 * r2 = PC+4 674 * r9 = normal "successful" return address 675 * r10 = FP workspace 676 * lr = unrecognised FP instruction return address 677 */ 678 679 .data 680ENTRY(fp_enter) 681 .word no_fp 682 .previous 683 684ENTRY(no_fp) 685 mov pc, lr 686ENDPROC(no_fp) 687 688__und_usr_unknown: 689 enable_irq 690 mov r0, sp 691 adr lr, BSYM(ret_from_exception) 692 b do_undefinstr 693ENDPROC(__und_usr_unknown) 694 695 .align 5 696__pabt_usr: 697 usr_entry 698 699 mov r0, r2 @ pass address of aborted instruction. 700#ifdef MULTI_PABORT 701 ldr r4, .LCprocfns 702 mov lr, pc 703 ldr pc, [r4, #PROCESSOR_PABT_FUNC] 704#else 705 bl CPU_PABORT_HANDLER 706#endif 707 enable_irq @ Enable interrupts 708 mov r2, sp @ regs 709 bl do_PrefetchAbort @ call abort handler 710 UNWIND(.fnend ) 711 /* fall through */ 712/* 713 * This is the return code to user mode for abort handlers 714 */ 715ENTRY(ret_from_exception) 716 UNWIND(.fnstart ) 717 UNWIND(.cantunwind ) 718 get_thread_info tsk 719 mov why, #0 720 b ret_to_user 721 UNWIND(.fnend ) 722ENDPROC(__pabt_usr) 723ENDPROC(ret_from_exception) 724 725/* 726 * Register switch for ARMv3 and ARMv4 processors 727 * r0 = previous task_struct, r1 = previous thread_info, r2 = next thread_info 728 * previous and next are guaranteed not to be the same. 729 */ 730ENTRY(__switch_to) 731 UNWIND(.fnstart ) 732 UNWIND(.cantunwind ) 733 add ip, r1, #TI_CPU_SAVE 734 ldr r3, [r2, #TI_TP_VALUE] 735 ARM( stmia ip!, {r4 - sl, fp, sp, lr} ) @ Store most regs on stack 736 THUMB( stmia ip!, {r4 - sl, fp} ) @ Store most regs on stack 737 THUMB( str sp, [ip], #4 ) 738 THUMB( str lr, [ip], #4 ) 739#ifdef CONFIG_MMU 740 ldr r6, [r2, #TI_CPU_DOMAIN] 741#endif 742#if defined(CONFIG_HAS_TLS_REG) 743 mcr p15, 0, r3, c13, c0, 3 @ set TLS register 744#elif !defined(CONFIG_TLS_REG_EMUL) 745 mov r4, #0xffff0fff 746 str r3, [r4, #-15] @ TLS val at 0xffff0ff0 747#endif 748#ifdef CONFIG_MMU 749 mcr p15, 0, r6, c3, c0, 0 @ Set domain register 750#endif 751 mov r5, r0 752 add r4, r2, #TI_CPU_SAVE 753 ldr r0, =thread_notify_head 754 mov r1, #THREAD_NOTIFY_SWITCH 755 bl atomic_notifier_call_chain 756 THUMB( mov ip, r4 ) 757 mov r0, r5 758 ARM( ldmia r4, {r4 - sl, fp, sp, pc} ) @ Load all regs saved previously 759 THUMB( ldmia ip!, {r4 - sl, fp} ) @ Load all regs saved previously 760 THUMB( ldr sp, [ip], #4 ) 761 THUMB( ldr pc, [ip] ) 762 UNWIND(.fnend ) 763ENDPROC(__switch_to) 764 765 __INIT 766 767/* 768 * User helpers. 769 * 770 * These are segment of kernel provided user code reachable from user space 771 * at a fixed address in kernel memory. This is used to provide user space 772 * with some operations which require kernel help because of unimplemented 773 * native feature and/or instructions in many ARM CPUs. The idea is for 774 * this code to be executed directly in user mode for best efficiency but 775 * which is too intimate with the kernel counter part to be left to user 776 * libraries. In fact this code might even differ from one CPU to another 777 * depending on the available instruction set and restrictions like on 778 * SMP systems. In other words, the kernel reserves the right to change 779 * this code as needed without warning. Only the entry points and their 780 * results are guaranteed to be stable. 781 * 782 * Each segment is 32-byte aligned and will be moved to the top of the high 783 * vector page. New segments (if ever needed) must be added in front of 784 * existing ones. This mechanism should be used only for things that are 785 * really small and justified, and not be abused freely. 786 * 787 * User space is expected to implement those things inline when optimizing 788 * for a processor that has the necessary native support, but only if such 789 * resulting binaries are already to be incompatible with earlier ARM 790 * processors due to the use of unsupported instructions other than what 791 * is provided here. In other words don't make binaries unable to run on 792 * earlier processors just for the sake of not using these kernel helpers 793 * if your compiled code is not going to use the new instructions for other 794 * purpose. 795 */ 796 THUMB( .arm ) 797 798 .macro usr_ret, reg 799#ifdef CONFIG_ARM_THUMB 800 bx \reg 801#else 802 mov pc, \reg 803#endif 804 .endm 805 806 .align 5 807 .globl __kuser_helper_start 808__kuser_helper_start: 809 810/* 811 * Reference prototype: 812 * 813 * void __kernel_memory_barrier(void) 814 * 815 * Input: 816 * 817 * lr = return address 818 * 819 * Output: 820 * 821 * none 822 * 823 * Clobbered: 824 * 825 * none 826 * 827 * Definition and user space usage example: 828 * 829 * typedef void (__kernel_dmb_t)(void); 830 * #define __kernel_dmb (*(__kernel_dmb_t *)0xffff0fa0) 831 * 832 * Apply any needed memory barrier to preserve consistency with data modified 833 * manually and __kuser_cmpxchg usage. 834 * 835 * This could be used as follows: 836 * 837 * #define __kernel_dmb() \ 838 * asm volatile ( "mov r0, #0xffff0fff; mov lr, pc; sub pc, r0, #95" \ 839 * : : : "r0", "lr","cc" ) 840 */ 841 842__kuser_memory_barrier: @ 0xffff0fa0 843 smp_dmb 844 usr_ret lr 845 846 .align 5 847 848/* 849 * Reference prototype: 850 * 851 * int __kernel_cmpxchg(int oldval, int newval, int *ptr) 852 * 853 * Input: 854 * 855 * r0 = oldval 856 * r1 = newval 857 * r2 = ptr 858 * lr = return address 859 * 860 * Output: 861 * 862 * r0 = returned value (zero or non-zero) 863 * C flag = set if r0 == 0, clear if r0 != 0 864 * 865 * Clobbered: 866 * 867 * r3, ip, flags 868 * 869 * Definition and user space usage example: 870 * 871 * typedef int (__kernel_cmpxchg_t)(int oldval, int newval, int *ptr); 872 * #define __kernel_cmpxchg (*(__kernel_cmpxchg_t *)0xffff0fc0) 873 * 874 * Atomically store newval in *ptr if *ptr is equal to oldval for user space. 875 * Return zero if *ptr was changed or non-zero if no exchange happened. 876 * The C flag is also set if *ptr was changed to allow for assembly 877 * optimization in the calling code. 878 * 879 * Notes: 880 * 881 * - This routine already includes memory barriers as needed. 882 * 883 * For example, a user space atomic_add implementation could look like this: 884 * 885 * #define atomic_add(ptr, val) \ 886 * ({ register unsigned int *__ptr asm("r2") = (ptr); \ 887 * register unsigned int __result asm("r1"); \ 888 * asm volatile ( \ 889 * "1: @ atomic_add\n\t" \ 890 * "ldr r0, [r2]\n\t" \ 891 * "mov r3, #0xffff0fff\n\t" \ 892 * "add lr, pc, #4\n\t" \ 893 * "add r1, r0, %2\n\t" \ 894 * "add pc, r3, #(0xffff0fc0 - 0xffff0fff)\n\t" \ 895 * "bcc 1b" \ 896 * : "=&r" (__result) \ 897 * : "r" (__ptr), "rIL" (val) \ 898 * : "r0","r3","ip","lr","cc","memory" ); \ 899 * __result; }) 900 */ 901 902__kuser_cmpxchg: @ 0xffff0fc0 903 904#if defined(CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG) 905 906 /* 907 * Poor you. No fast solution possible... 908 * The kernel itself must perform the operation. 909 * A special ghost syscall is used for that (see traps.c). 910 */ 911 stmfd sp!, {r7, lr} 912 ldr r7, =1f @ it's 20 bits 913 swi __ARM_NR_cmpxchg 914 ldmfd sp!, {r7, pc} 9151: .word __ARM_NR_cmpxchg 916 917#elif __LINUX_ARM_ARCH__ < 6 918 919#ifdef CONFIG_MMU 920 921 /* 922 * The only thing that can break atomicity in this cmpxchg 923 * implementation is either an IRQ or a data abort exception 924 * causing another process/thread to be scheduled in the middle 925 * of the critical sequence. To prevent this, code is added to 926 * the IRQ and data abort exception handlers to set the pc back 927 * to the beginning of the critical section if it is found to be 928 * within that critical section (see kuser_cmpxchg_fixup). 929 */ 9301: ldr r3, [r2] @ load current val 931 subs r3, r3, r0 @ compare with oldval 9322: streq r1, [r2] @ store newval if eq 933 rsbs r0, r3, #0 @ set return val and C flag 934 usr_ret lr 935 936 .text 937kuser_cmpxchg_fixup: 938 @ Called from kuser_cmpxchg_check macro. 939 @ r2 = address of interrupted insn (must be preserved). 940 @ sp = saved regs. r7 and r8 are clobbered. 941 @ 1b = first critical insn, 2b = last critical insn. 942 @ If r2 >= 1b and r2 <= 2b then saved pc_usr is set to 1b. 943 mov r7, #0xffff0fff 944 sub r7, r7, #(0xffff0fff - (0xffff0fc0 + (1b - __kuser_cmpxchg))) 945 subs r8, r2, r7 946 rsbcss r8, r8, #(2b - 1b) 947 strcs r7, [sp, #S_PC] 948 mov pc, lr 949 .previous 950 951#else 952#warning "NPTL on non MMU needs fixing" 953 mov r0, #-1 954 adds r0, r0, #0 955 usr_ret lr 956#endif 957 958#else 959 960#ifdef CONFIG_SMP 961 mcr p15, 0, r0, c7, c10, 5 @ dmb 962#endif 9631: ldrex r3, [r2] 964 subs r3, r3, r0 965 strexeq r3, r1, [r2] 966 teqeq r3, #1 967 beq 1b 968 rsbs r0, r3, #0 969 /* beware -- each __kuser slot must be 8 instructions max */ 970#ifdef CONFIG_SMP 971 b __kuser_memory_barrier 972#else 973 usr_ret lr 974#endif 975 976#endif 977 978 .align 5 979 980/* 981 * Reference prototype: 982 * 983 * int __kernel_get_tls(void) 984 * 985 * Input: 986 * 987 * lr = return address 988 * 989 * Output: 990 * 991 * r0 = TLS value 992 * 993 * Clobbered: 994 * 995 * none 996 * 997 * Definition and user space usage example: 998 * 999 * typedef int (__kernel_get_tls_t)(void); 1000 * #define __kernel_get_tls (*(__kernel_get_tls_t *)0xffff0fe0) 1001 * 1002 * Get the TLS value as previously set via the __ARM_NR_set_tls syscall. 1003 * 1004 * This could be used as follows: 1005 * 1006 * #define __kernel_get_tls() \ 1007 * ({ register unsigned int __val asm("r0"); \ 1008 * asm( "mov r0, #0xffff0fff; mov lr, pc; sub pc, r0, #31" \ 1009 * : "=r" (__val) : : "lr","cc" ); \ 1010 * __val; }) 1011 */ 1012 1013__kuser_get_tls: @ 0xffff0fe0 1014 1015#if !defined(CONFIG_HAS_TLS_REG) && !defined(CONFIG_TLS_REG_EMUL) 1016 ldr r0, [pc, #(16 - 8)] @ TLS stored at 0xffff0ff0 1017#else 1018 mrc p15, 0, r0, c13, c0, 3 @ read TLS register 1019#endif 1020 usr_ret lr 1021 1022 .rep 5 1023 .word 0 @ pad up to __kuser_helper_version 1024 .endr 1025 1026/* 1027 * Reference declaration: 1028 * 1029 * extern unsigned int __kernel_helper_version; 1030 * 1031 * Definition and user space usage example: 1032 * 1033 * #define __kernel_helper_version (*(unsigned int *)0xffff0ffc) 1034 * 1035 * User space may read this to determine the curent number of helpers 1036 * available. 1037 */ 1038 1039__kuser_helper_version: @ 0xffff0ffc 1040 .word ((__kuser_helper_end - __kuser_helper_start) >> 5) 1041 1042 .globl __kuser_helper_end 1043__kuser_helper_end: 1044 1045 THUMB( .thumb ) 1046 1047/* 1048 * Vector stubs. 1049 * 1050 * This code is copied to 0xffff0200 so we can use branches in the 1051 * vectors, rather than ldr's. Note that this code must not 1052 * exceed 0x300 bytes. 1053 * 1054 * Common stub entry macro: 1055 * Enter in IRQ mode, spsr = SVC/USR CPSR, lr = SVC/USR PC 1056 * 1057 * SP points to a minimal amount of processor-private memory, the address 1058 * of which is copied into r0 for the mode specific abort handler. 1059 */ 1060 .macro vector_stub, name, mode, correction=0 1061 .align 5 1062 1063vector_\name: 1064 .if \correction 1065 sub lr, lr, #\correction 1066 .endif 1067 1068 @ 1069 @ Save r0, lr_<exception> (parent PC) and spsr_<exception> 1070 @ (parent CPSR) 1071 @ 1072 stmia sp, {r0, lr} @ save r0, lr 1073 mrs lr, spsr 1074 str lr, [sp, #8] @ save spsr 1075 1076 @ 1077 @ Prepare for SVC32 mode. IRQs remain disabled. 1078 @ 1079 mrs r0, cpsr 1080 eor r0, r0, #(\mode ^ SVC_MODE | PSR_ISETSTATE) 1081 msr spsr_cxsf, r0 1082 1083 @ 1084 @ the branch table must immediately follow this code 1085 @ 1086 and lr, lr, #0x0f 1087 THUMB( adr r0, 1f ) 1088 THUMB( ldr lr, [r0, lr, lsl #2] ) 1089 mov r0, sp 1090 ARM( ldr lr, [pc, lr, lsl #2] ) 1091 movs pc, lr @ branch to handler in SVC mode 1092ENDPROC(vector_\name) 1093 1094 .align 2 1095 @ handler addresses follow this label 10961: 1097 .endm 1098 1099 .globl __stubs_start 1100__stubs_start: 1101/* 1102 * Interrupt dispatcher 1103 */ 1104 vector_stub irq, IRQ_MODE, 4 1105 1106 .long __irq_usr @ 0 (USR_26 / USR_32) 1107 .long __irq_invalid @ 1 (FIQ_26 / FIQ_32) 1108 .long __irq_invalid @ 2 (IRQ_26 / IRQ_32) 1109 .long __irq_svc @ 3 (SVC_26 / SVC_32) 1110 .long __irq_invalid @ 4 1111 .long __irq_invalid @ 5 1112 .long __irq_invalid @ 6 1113 .long __irq_invalid @ 7 1114 .long __irq_invalid @ 8 1115 .long __irq_invalid @ 9 1116 .long __irq_invalid @ a 1117 .long __irq_invalid @ b 1118 .long __irq_invalid @ c 1119 .long __irq_invalid @ d 1120 .long __irq_invalid @ e 1121 .long __irq_invalid @ f 1122 1123/* 1124 * Data abort dispatcher 1125 * Enter in ABT mode, spsr = USR CPSR, lr = USR PC 1126 */ 1127 vector_stub dabt, ABT_MODE, 8 1128 1129 .long __dabt_usr @ 0 (USR_26 / USR_32) 1130 .long __dabt_invalid @ 1 (FIQ_26 / FIQ_32) 1131 .long __dabt_invalid @ 2 (IRQ_26 / IRQ_32) 1132 .long __dabt_svc @ 3 (SVC_26 / SVC_32) 1133 .long __dabt_invalid @ 4 1134 .long __dabt_invalid @ 5 1135 .long __dabt_invalid @ 6 1136 .long __dabt_invalid @ 7 1137 .long __dabt_invalid @ 8 1138 .long __dabt_invalid @ 9 1139 .long __dabt_invalid @ a 1140 .long __dabt_invalid @ b 1141 .long __dabt_invalid @ c 1142 .long __dabt_invalid @ d 1143 .long __dabt_invalid @ e 1144 .long __dabt_invalid @ f 1145 1146/* 1147 * Prefetch abort dispatcher 1148 * Enter in ABT mode, spsr = USR CPSR, lr = USR PC 1149 */ 1150 vector_stub pabt, ABT_MODE, 4 1151 1152 .long __pabt_usr @ 0 (USR_26 / USR_32) 1153 .long __pabt_invalid @ 1 (FIQ_26 / FIQ_32) 1154 .long __pabt_invalid @ 2 (IRQ_26 / IRQ_32) 1155 .long __pabt_svc @ 3 (SVC_26 / SVC_32) 1156 .long __pabt_invalid @ 4 1157 .long __pabt_invalid @ 5 1158 .long __pabt_invalid @ 6 1159 .long __pabt_invalid @ 7 1160 .long __pabt_invalid @ 8 1161 .long __pabt_invalid @ 9 1162 .long __pabt_invalid @ a 1163 .long __pabt_invalid @ b 1164 .long __pabt_invalid @ c 1165 .long __pabt_invalid @ d 1166 .long __pabt_invalid @ e 1167 .long __pabt_invalid @ f 1168 1169/* 1170 * Undef instr entry dispatcher 1171 * Enter in UND mode, spsr = SVC/USR CPSR, lr = SVC/USR PC 1172 */ 1173 vector_stub und, UND_MODE 1174 1175 .long __und_usr @ 0 (USR_26 / USR_32) 1176 .long __und_invalid @ 1 (FIQ_26 / FIQ_32) 1177 .long __und_invalid @ 2 (IRQ_26 / IRQ_32) 1178 .long __und_svc @ 3 (SVC_26 / SVC_32) 1179 .long __und_invalid @ 4 1180 .long __und_invalid @ 5 1181 .long __und_invalid @ 6 1182 .long __und_invalid @ 7 1183 .long __und_invalid @ 8 1184 .long __und_invalid @ 9 1185 .long __und_invalid @ a 1186 .long __und_invalid @ b 1187 .long __und_invalid @ c 1188 .long __und_invalid @ d 1189 .long __und_invalid @ e 1190 .long __und_invalid @ f 1191 1192 .align 5 1193 1194/*============================================================================= 1195 * Undefined FIQs 1196 *----------------------------------------------------------------------------- 1197 * Enter in FIQ mode, spsr = ANY CPSR, lr = ANY PC 1198 * MUST PRESERVE SVC SPSR, but need to switch to SVC mode to show our msg. 1199 * Basically to switch modes, we *HAVE* to clobber one register... brain 1200 * damage alert! I don't think that we can execute any code in here in any 1201 * other mode than FIQ... Ok you can switch to another mode, but you can't 1202 * get out of that mode without clobbering one register. 1203 */ 1204vector_fiq: 1205 disable_fiq 1206 subs pc, lr, #4 1207 1208/*============================================================================= 1209 * Address exception handler 1210 *----------------------------------------------------------------------------- 1211 * These aren't too critical. 1212 * (they're not supposed to happen, and won't happen in 32-bit data mode). 1213 */ 1214 1215vector_addrexcptn: 1216 b vector_addrexcptn 1217 1218/* 1219 * We group all the following data together to optimise 1220 * for CPUs with separate I & D caches. 1221 */ 1222 .align 5 1223 1224.LCvswi: 1225 .word vector_swi 1226 1227 .globl __stubs_end 1228__stubs_end: 1229 1230 .equ stubs_offset, __vectors_start + 0x200 - __stubs_start 1231 1232 .globl __vectors_start 1233__vectors_start: 1234 ARM( swi SYS_ERROR0 ) 1235 THUMB( svc #0 ) 1236 THUMB( nop ) 1237 W(b) vector_und + stubs_offset 1238 W(ldr) pc, .LCvswi + stubs_offset 1239 W(b) vector_pabt + stubs_offset 1240 W(b) vector_dabt + stubs_offset 1241 W(b) vector_addrexcptn + stubs_offset 1242 W(b) vector_irq + stubs_offset 1243 W(b) vector_fiq + stubs_offset 1244 1245 .globl __vectors_end 1246__vectors_end: 1247 1248 .data 1249 1250 .globl cr_alignment 1251 .globl cr_no_alignment 1252cr_alignment: 1253 .space 4 1254cr_no_alignment: 1255 .space 4 1256