1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S 4 * 5 * Copyright (C) 1996-2000 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 */ 8 #ifndef __ASSEMBLY__ 9 #error "Only include this from assembly code" 10 #endif 11 12 #ifndef __ASM_ASSEMBLER_H 13 #define __ASM_ASSEMBLER_H 14 15 #include <asm-generic/export.h> 16 17 #include <asm/asm-offsets.h> 18 #include <asm/cpufeature.h> 19 #include <asm/cputype.h> 20 #include <asm/debug-monitors.h> 21 #include <asm/page.h> 22 #include <asm/pgtable-hwdef.h> 23 #include <asm/ptrace.h> 24 #include <asm/thread_info.h> 25 26 .macro save_and_disable_daif, flags 27 mrs \flags, daif 28 msr daifset, #0xf 29 .endm 30 31 .macro disable_daif 32 msr daifset, #0xf 33 .endm 34 35 .macro enable_daif 36 msr daifclr, #0xf 37 .endm 38 39 .macro restore_daif, flags:req 40 msr daif, \flags 41 .endm 42 43 /* Only on aarch64 pstate, PSR_D_BIT is different for aarch32 */ 44 .macro inherit_daif, pstate:req, tmp:req 45 and \tmp, \pstate, #(PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT) 46 msr daif, \tmp 47 .endm 48 49 /* IRQ is the lowest priority flag, unconditionally unmask the rest. */ 50 .macro enable_da_f 51 msr daifclr, #(8 | 4 | 1) 52 .endm 53 54 /* 55 * Save/restore interrupts. 56 */ 57 .macro save_and_disable_irq, flags 58 mrs \flags, daif 59 msr daifset, #2 60 .endm 61 62 .macro restore_irq, flags 63 msr daif, \flags 64 .endm 65 66 .macro enable_dbg 67 msr daifclr, #8 68 .endm 69 70 .macro disable_step_tsk, flgs, tmp 71 tbz \flgs, #TIF_SINGLESTEP, 9990f 72 mrs \tmp, mdscr_el1 73 bic \tmp, \tmp, #DBG_MDSCR_SS 74 msr mdscr_el1, \tmp 75 isb // Synchronise with enable_dbg 76 9990: 77 .endm 78 79 /* call with daif masked */ 80 .macro enable_step_tsk, flgs, tmp 81 tbz \flgs, #TIF_SINGLESTEP, 9990f 82 mrs \tmp, mdscr_el1 83 orr \tmp, \tmp, #DBG_MDSCR_SS 84 msr mdscr_el1, \tmp 85 9990: 86 .endm 87 88 /* 89 * SMP data memory barrier 90 */ 91 .macro smp_dmb, opt 92 dmb \opt 93 .endm 94 95 /* 96 * RAS Error Synchronization barrier 97 */ 98 .macro esb 99 #ifdef CONFIG_ARM64_RAS_EXTN 100 hint #16 101 #else 102 nop 103 #endif 104 .endm 105 106 /* 107 * Value prediction barrier 108 */ 109 .macro csdb 110 hint #20 111 .endm 112 113 /* 114 * Speculation barrier 115 */ 116 .macro sb 117 alternative_if_not ARM64_HAS_SB 118 dsb nsh 119 isb 120 alternative_else 121 SB_BARRIER_INSN 122 nop 123 alternative_endif 124 .endm 125 126 /* 127 * Sanitise a 64-bit bounded index wrt speculation, returning zero if out 128 * of bounds. 129 */ 130 .macro mask_nospec64, idx, limit, tmp 131 sub \tmp, \idx, \limit 132 bic \tmp, \tmp, \idx 133 and \idx, \idx, \tmp, asr #63 134 csdb 135 .endm 136 137 /* 138 * NOP sequence 139 */ 140 .macro nops, num 141 .rept \num 142 nop 143 .endr 144 .endm 145 146 /* 147 * Emit an entry into the exception table 148 */ 149 .macro _asm_extable, from, to 150 .pushsection __ex_table, "a" 151 .align 3 152 .long (\from - .), (\to - .) 153 .popsection 154 .endm 155 156 #define USER(l, x...) \ 157 9999: x; \ 158 _asm_extable 9999b, l 159 160 /* 161 * Register aliases. 162 */ 163 lr .req x30 // link register 164 165 /* 166 * Vector entry 167 */ 168 .macro ventry label 169 .align 7 170 b \label 171 .endm 172 173 /* 174 * Select code when configured for BE. 175 */ 176 #ifdef CONFIG_CPU_BIG_ENDIAN 177 #define CPU_BE(code...) code 178 #else 179 #define CPU_BE(code...) 180 #endif 181 182 /* 183 * Select code when configured for LE. 184 */ 185 #ifdef CONFIG_CPU_BIG_ENDIAN 186 #define CPU_LE(code...) 187 #else 188 #define CPU_LE(code...) code 189 #endif 190 191 /* 192 * Define a macro that constructs a 64-bit value by concatenating two 193 * 32-bit registers. Note that on big endian systems the order of the 194 * registers is swapped. 195 */ 196 #ifndef CONFIG_CPU_BIG_ENDIAN 197 .macro regs_to_64, rd, lbits, hbits 198 #else 199 .macro regs_to_64, rd, hbits, lbits 200 #endif 201 orr \rd, \lbits, \hbits, lsl #32 202 .endm 203 204 /* 205 * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where 206 * <symbol> is within the range +/- 4 GB of the PC. 207 */ 208 /* 209 * @dst: destination register (64 bit wide) 210 * @sym: name of the symbol 211 */ 212 .macro adr_l, dst, sym 213 adrp \dst, \sym 214 add \dst, \dst, :lo12:\sym 215 .endm 216 217 /* 218 * @dst: destination register (32 or 64 bit wide) 219 * @sym: name of the symbol 220 * @tmp: optional 64-bit scratch register to be used if <dst> is a 221 * 32-bit wide register, in which case it cannot be used to hold 222 * the address 223 */ 224 .macro ldr_l, dst, sym, tmp= 225 .ifb \tmp 226 adrp \dst, \sym 227 ldr \dst, [\dst, :lo12:\sym] 228 .else 229 adrp \tmp, \sym 230 ldr \dst, [\tmp, :lo12:\sym] 231 .endif 232 .endm 233 234 /* 235 * @src: source register (32 or 64 bit wide) 236 * @sym: name of the symbol 237 * @tmp: mandatory 64-bit scratch register to calculate the address 238 * while <src> needs to be preserved. 239 */ 240 .macro str_l, src, sym, tmp 241 adrp \tmp, \sym 242 str \src, [\tmp, :lo12:\sym] 243 .endm 244 245 /* 246 * @dst: Result of per_cpu(sym, smp_processor_id()) (can be SP) 247 * @sym: The name of the per-cpu variable 248 * @tmp: scratch register 249 */ 250 .macro adr_this_cpu, dst, sym, tmp 251 adrp \tmp, \sym 252 add \dst, \tmp, #:lo12:\sym 253 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN 254 mrs \tmp, tpidr_el1 255 alternative_else 256 mrs \tmp, tpidr_el2 257 alternative_endif 258 add \dst, \dst, \tmp 259 .endm 260 261 /* 262 * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id())) 263 * @sym: The name of the per-cpu variable 264 * @tmp: scratch register 265 */ 266 .macro ldr_this_cpu dst, sym, tmp 267 adr_l \dst, \sym 268 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN 269 mrs \tmp, tpidr_el1 270 alternative_else 271 mrs \tmp, tpidr_el2 272 alternative_endif 273 ldr \dst, [\dst, \tmp] 274 .endm 275 276 /* 277 * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm) 278 */ 279 .macro vma_vm_mm, rd, rn 280 ldr \rd, [\rn, #VMA_VM_MM] 281 .endm 282 283 /* 284 * mmid - get context id from mm pointer (mm->context.id) 285 */ 286 .macro mmid, rd, rn 287 ldr \rd, [\rn, #MM_CONTEXT_ID] 288 .endm 289 /* 290 * read_ctr - read CTR_EL0. If the system has mismatched register fields, 291 * provide the system wide safe value from arm64_ftr_reg_ctrel0.sys_val 292 */ 293 .macro read_ctr, reg 294 alternative_if_not ARM64_MISMATCHED_CACHE_TYPE 295 mrs \reg, ctr_el0 // read CTR 296 nop 297 alternative_else 298 ldr_l \reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL 299 alternative_endif 300 .endm 301 302 303 /* 304 * raw_dcache_line_size - get the minimum D-cache line size on this CPU 305 * from the CTR register. 306 */ 307 .macro raw_dcache_line_size, reg, tmp 308 mrs \tmp, ctr_el0 // read CTR 309 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 310 mov \reg, #4 // bytes per word 311 lsl \reg, \reg, \tmp // actual cache line size 312 .endm 313 314 /* 315 * dcache_line_size - get the safe D-cache line size across all CPUs 316 */ 317 .macro dcache_line_size, reg, tmp 318 read_ctr \tmp 319 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 320 mov \reg, #4 // bytes per word 321 lsl \reg, \reg, \tmp // actual cache line size 322 .endm 323 324 /* 325 * raw_icache_line_size - get the minimum I-cache line size on this CPU 326 * from the CTR register. 327 */ 328 .macro raw_icache_line_size, reg, tmp 329 mrs \tmp, ctr_el0 // read CTR 330 and \tmp, \tmp, #0xf // cache line size encoding 331 mov \reg, #4 // bytes per word 332 lsl \reg, \reg, \tmp // actual cache line size 333 .endm 334 335 /* 336 * icache_line_size - get the safe I-cache line size across all CPUs 337 */ 338 .macro icache_line_size, reg, tmp 339 read_ctr \tmp 340 and \tmp, \tmp, #0xf // cache line size encoding 341 mov \reg, #4 // bytes per word 342 lsl \reg, \reg, \tmp // actual cache line size 343 .endm 344 345 /* 346 * tcr_set_t0sz - update TCR.T0SZ so that we can load the ID map 347 */ 348 .macro tcr_set_t0sz, valreg, t0sz 349 bfi \valreg, \t0sz, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH 350 .endm 351 352 /* 353 * tcr_compute_pa_size - set TCR.(I)PS to the highest supported 354 * ID_AA64MMFR0_EL1.PARange value 355 * 356 * tcr: register with the TCR_ELx value to be updated 357 * pos: IPS or PS bitfield position 358 * tmp{0,1}: temporary registers 359 */ 360 .macro tcr_compute_pa_size, tcr, pos, tmp0, tmp1 361 mrs \tmp0, ID_AA64MMFR0_EL1 362 // Narrow PARange to fit the PS field in TCR_ELx 363 ubfx \tmp0, \tmp0, #ID_AA64MMFR0_PARANGE_SHIFT, #3 364 mov \tmp1, #ID_AA64MMFR0_PARANGE_MAX 365 cmp \tmp0, \tmp1 366 csel \tmp0, \tmp1, \tmp0, hi 367 bfi \tcr, \tmp0, \pos, #3 368 .endm 369 370 /* 371 * Macro to perform a data cache maintenance for the interval 372 * [kaddr, kaddr + size) 373 * 374 * op: operation passed to dc instruction 375 * domain: domain used in dsb instruciton 376 * kaddr: starting virtual address of the region 377 * size: size of the region 378 * Corrupts: kaddr, size, tmp1, tmp2 379 */ 380 .macro __dcache_op_workaround_clean_cache, op, kaddr 381 alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE 382 dc \op, \kaddr 383 alternative_else 384 dc civac, \kaddr 385 alternative_endif 386 .endm 387 388 .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2 389 dcache_line_size \tmp1, \tmp2 390 add \size, \kaddr, \size 391 sub \tmp2, \tmp1, #1 392 bic \kaddr, \kaddr, \tmp2 393 9998: 394 .ifc \op, cvau 395 __dcache_op_workaround_clean_cache \op, \kaddr 396 .else 397 .ifc \op, cvac 398 __dcache_op_workaround_clean_cache \op, \kaddr 399 .else 400 .ifc \op, cvap 401 sys 3, c7, c12, 1, \kaddr // dc cvap 402 .else 403 .ifc \op, cvadp 404 sys 3, c7, c13, 1, \kaddr // dc cvadp 405 .else 406 dc \op, \kaddr 407 .endif 408 .endif 409 .endif 410 .endif 411 add \kaddr, \kaddr, \tmp1 412 cmp \kaddr, \size 413 b.lo 9998b 414 dsb \domain 415 .endm 416 417 /* 418 * Macro to perform an instruction cache maintenance for the interval 419 * [start, end) 420 * 421 * start, end: virtual addresses describing the region 422 * label: A label to branch to on user fault. 423 * Corrupts: tmp1, tmp2 424 */ 425 .macro invalidate_icache_by_line start, end, tmp1, tmp2, label 426 icache_line_size \tmp1, \tmp2 427 sub \tmp2, \tmp1, #1 428 bic \tmp2, \start, \tmp2 429 9997: 430 USER(\label, ic ivau, \tmp2) // invalidate I line PoU 431 add \tmp2, \tmp2, \tmp1 432 cmp \tmp2, \end 433 b.lo 9997b 434 dsb ish 435 isb 436 .endm 437 438 /* 439 * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present 440 */ 441 .macro reset_pmuserenr_el0, tmpreg 442 mrs \tmpreg, id_aa64dfr0_el1 443 sbfx \tmpreg, \tmpreg, #ID_AA64DFR0_PMUVER_SHIFT, #4 444 cmp \tmpreg, #1 // Skip if no PMU present 445 b.lt 9000f 446 msr pmuserenr_el0, xzr // Disable PMU access from EL0 447 9000: 448 .endm 449 450 /* 451 * copy_page - copy src to dest using temp registers t1-t8 452 */ 453 .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req 454 9998: ldp \t1, \t2, [\src] 455 ldp \t3, \t4, [\src, #16] 456 ldp \t5, \t6, [\src, #32] 457 ldp \t7, \t8, [\src, #48] 458 add \src, \src, #64 459 stnp \t1, \t2, [\dest] 460 stnp \t3, \t4, [\dest, #16] 461 stnp \t5, \t6, [\dest, #32] 462 stnp \t7, \t8, [\dest, #48] 463 add \dest, \dest, #64 464 tst \src, #(PAGE_SIZE - 1) 465 b.ne 9998b 466 .endm 467 468 /* 469 * Annotate a function as position independent, i.e., safe to be called before 470 * the kernel virtual mapping is activated. 471 */ 472 #define ENDPIPROC(x) \ 473 .globl __pi_##x; \ 474 .type __pi_##x, %function; \ 475 .set __pi_##x, x; \ 476 .size __pi_##x, . - x; \ 477 ENDPROC(x) 478 479 /* 480 * Annotate a function as being unsuitable for kprobes. 481 */ 482 #ifdef CONFIG_KPROBES 483 #define NOKPROBE(x) \ 484 .pushsection "_kprobe_blacklist", "aw"; \ 485 .quad x; \ 486 .popsection; 487 #else 488 #define NOKPROBE(x) 489 #endif 490 491 #ifdef CONFIG_KASAN 492 #define EXPORT_SYMBOL_NOKASAN(name) 493 #else 494 #define EXPORT_SYMBOL_NOKASAN(name) EXPORT_SYMBOL(name) 495 #endif 496 497 /* 498 * Emit a 64-bit absolute little endian symbol reference in a way that 499 * ensures that it will be resolved at build time, even when building a 500 * PIE binary. This requires cooperation from the linker script, which 501 * must emit the lo32/hi32 halves individually. 502 */ 503 .macro le64sym, sym 504 .long \sym\()_lo32 505 .long \sym\()_hi32 506 .endm 507 508 /* 509 * mov_q - move an immediate constant into a 64-bit register using 510 * between 2 and 4 movz/movk instructions (depending on the 511 * magnitude and sign of the operand) 512 */ 513 .macro mov_q, reg, val 514 .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff) 515 movz \reg, :abs_g1_s:\val 516 .else 517 .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff) 518 movz \reg, :abs_g2_s:\val 519 .else 520 movz \reg, :abs_g3:\val 521 movk \reg, :abs_g2_nc:\val 522 .endif 523 movk \reg, :abs_g1_nc:\val 524 .endif 525 movk \reg, :abs_g0_nc:\val 526 .endm 527 528 /* 529 * Return the current task_struct. 530 */ 531 .macro get_current_task, rd 532 mrs \rd, sp_el0 533 .endm 534 535 /* 536 * Offset ttbr1 to allow for 48-bit kernel VAs set with 52-bit PTRS_PER_PGD. 537 * orr is used as it can cover the immediate value (and is idempotent). 538 * In future this may be nop'ed out when dealing with 52-bit kernel VAs. 539 * ttbr: Value of ttbr to set, modified. 540 */ 541 .macro offset_ttbr1, ttbr 542 #ifdef CONFIG_ARM64_USER_VA_BITS_52 543 orr \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET 544 #endif 545 .endm 546 547 /* 548 * Perform the reverse of offset_ttbr1. 549 * bic is used as it can cover the immediate value and, in future, won't need 550 * to be nop'ed out when dealing with 52-bit kernel VAs. 551 */ 552 .macro restore_ttbr1, ttbr 553 #ifdef CONFIG_ARM64_USER_VA_BITS_52 554 bic \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET 555 #endif 556 .endm 557 558 /* 559 * Arrange a physical address in a TTBR register, taking care of 52-bit 560 * addresses. 561 * 562 * phys: physical address, preserved 563 * ttbr: returns the TTBR value 564 */ 565 .macro phys_to_ttbr, ttbr, phys 566 #ifdef CONFIG_ARM64_PA_BITS_52 567 orr \ttbr, \phys, \phys, lsr #46 568 and \ttbr, \ttbr, #TTBR_BADDR_MASK_52 569 #else 570 mov \ttbr, \phys 571 #endif 572 .endm 573 574 .macro phys_to_pte, pte, phys 575 #ifdef CONFIG_ARM64_PA_BITS_52 576 /* 577 * We assume \phys is 64K aligned and this is guaranteed by only 578 * supporting this configuration with 64K pages. 579 */ 580 orr \pte, \phys, \phys, lsr #36 581 and \pte, \pte, #PTE_ADDR_MASK 582 #else 583 mov \pte, \phys 584 #endif 585 .endm 586 587 .macro pte_to_phys, phys, pte 588 #ifdef CONFIG_ARM64_PA_BITS_52 589 ubfiz \phys, \pte, #(48 - 16 - 12), #16 590 bfxil \phys, \pte, #16, #32 591 lsl \phys, \phys, #16 592 #else 593 and \phys, \pte, #PTE_ADDR_MASK 594 #endif 595 .endm 596 597 /* 598 * tcr_clear_errata_bits - Clear TCR bits that trigger an errata on this CPU. 599 */ 600 .macro tcr_clear_errata_bits, tcr, tmp1, tmp2 601 #ifdef CONFIG_FUJITSU_ERRATUM_010001 602 mrs \tmp1, midr_el1 603 604 mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001_MASK 605 and \tmp1, \tmp1, \tmp2 606 mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001 607 cmp \tmp1, \tmp2 608 b.ne 10f 609 610 mov_q \tmp2, TCR_CLEAR_FUJITSU_ERRATUM_010001 611 bic \tcr, \tcr, \tmp2 612 10: 613 #endif /* CONFIG_FUJITSU_ERRATUM_010001 */ 614 .endm 615 616 /** 617 * Errata workaround prior to disable MMU. Insert an ISB immediately prior 618 * to executing the MSR that will change SCTLR_ELn[M] from a value of 1 to 0. 619 */ 620 .macro pre_disable_mmu_workaround 621 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_E1041 622 isb 623 #endif 624 .endm 625 626 /* 627 * frame_push - Push @regcount callee saved registers to the stack, 628 * starting at x19, as well as x29/x30, and set x29 to 629 * the new value of sp. Add @extra bytes of stack space 630 * for locals. 631 */ 632 .macro frame_push, regcount:req, extra 633 __frame st, \regcount, \extra 634 .endm 635 636 /* 637 * frame_pop - Pop the callee saved registers from the stack that were 638 * pushed in the most recent call to frame_push, as well 639 * as x29/x30 and any extra stack space that may have been 640 * allocated. 641 */ 642 .macro frame_pop 643 __frame ld 644 .endm 645 646 .macro __frame_regs, reg1, reg2, op, num 647 .if .Lframe_regcount == \num 648 \op\()r \reg1, [sp, #(\num + 1) * 8] 649 .elseif .Lframe_regcount > \num 650 \op\()p \reg1, \reg2, [sp, #(\num + 1) * 8] 651 .endif 652 .endm 653 654 .macro __frame, op, regcount, extra=0 655 .ifc \op, st 656 .if (\regcount) < 0 || (\regcount) > 10 657 .error "regcount should be in the range [0 ... 10]" 658 .endif 659 .if ((\extra) % 16) != 0 660 .error "extra should be a multiple of 16 bytes" 661 .endif 662 .ifdef .Lframe_regcount 663 .if .Lframe_regcount != -1 664 .error "frame_push/frame_pop may not be nested" 665 .endif 666 .endif 667 .set .Lframe_regcount, \regcount 668 .set .Lframe_extra, \extra 669 .set .Lframe_local_offset, ((\regcount + 3) / 2) * 16 670 stp x29, x30, [sp, #-.Lframe_local_offset - .Lframe_extra]! 671 mov x29, sp 672 .endif 673 674 __frame_regs x19, x20, \op, 1 675 __frame_regs x21, x22, \op, 3 676 __frame_regs x23, x24, \op, 5 677 __frame_regs x25, x26, \op, 7 678 __frame_regs x27, x28, \op, 9 679 680 .ifc \op, ld 681 .if .Lframe_regcount == -1 682 .error "frame_push/frame_pop may not be nested" 683 .endif 684 ldp x29, x30, [sp], #.Lframe_local_offset + .Lframe_extra 685 .set .Lframe_regcount, -1 686 .endif 687 .endm 688 689 /* 690 * Check whether to yield to another runnable task from kernel mode NEON code 691 * (which runs with preemption disabled). 692 * 693 * if_will_cond_yield_neon 694 * // pre-yield patchup code 695 * do_cond_yield_neon 696 * // post-yield patchup code 697 * endif_yield_neon <label> 698 * 699 * where <label> is optional, and marks the point where execution will resume 700 * after a yield has been performed. If omitted, execution resumes right after 701 * the endif_yield_neon invocation. Note that the entire sequence, including 702 * the provided patchup code, will be omitted from the image if CONFIG_PREEMPT 703 * is not defined. 704 * 705 * As a convenience, in the case where no patchup code is required, the above 706 * sequence may be abbreviated to 707 * 708 * cond_yield_neon <label> 709 * 710 * Note that the patchup code does not support assembler directives that change 711 * the output section, any use of such directives is undefined. 712 * 713 * The yield itself consists of the following: 714 * - Check whether the preempt count is exactly 1 and a reschedule is also 715 * needed. If so, calling of preempt_enable() in kernel_neon_end() will 716 * trigger a reschedule. If it is not the case, yielding is pointless. 717 * - Disable and re-enable kernel mode NEON, and branch to the yield fixup 718 * code. 719 * 720 * This macro sequence may clobber all CPU state that is not guaranteed by the 721 * AAPCS to be preserved across an ordinary function call. 722 */ 723 724 .macro cond_yield_neon, lbl 725 if_will_cond_yield_neon 726 do_cond_yield_neon 727 endif_yield_neon \lbl 728 .endm 729 730 .macro if_will_cond_yield_neon 731 #ifdef CONFIG_PREEMPT 732 get_current_task x0 733 ldr x0, [x0, #TSK_TI_PREEMPT] 734 sub x0, x0, #PREEMPT_DISABLE_OFFSET 735 cbz x0, .Lyield_\@ 736 /* fall through to endif_yield_neon */ 737 .subsection 1 738 .Lyield_\@ : 739 #else 740 .section ".discard.cond_yield_neon", "ax" 741 #endif 742 .endm 743 744 .macro do_cond_yield_neon 745 bl kernel_neon_end 746 bl kernel_neon_begin 747 .endm 748 749 .macro endif_yield_neon, lbl 750 .ifnb \lbl 751 b \lbl 752 .else 753 b .Lyield_out_\@ 754 .endif 755 .previous 756 .Lyield_out_\@ : 757 .endm 758 759 #endif /* __ASM_ASSEMBLER_H */ 760