1 /* 2 * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S 3 * 4 * Copyright (C) 1996-2000 Russell King 5 * Copyright (C) 2012 ARM Ltd. 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program. If not, see <http://www.gnu.org/licenses/>. 18 */ 19 #ifndef __ASSEMBLY__ 20 #error "Only include this from assembly code" 21 #endif 22 23 #ifndef __ASM_ASSEMBLER_H 24 #define __ASM_ASSEMBLER_H 25 26 #include <asm-generic/export.h> 27 28 #include <asm/asm-offsets.h> 29 #include <asm/cpufeature.h> 30 #include <asm/debug-monitors.h> 31 #include <asm/page.h> 32 #include <asm/pgtable-hwdef.h> 33 #include <asm/ptrace.h> 34 #include <asm/thread_info.h> 35 36 .macro save_and_disable_daif, flags 37 mrs \flags, daif 38 msr daifset, #0xf 39 .endm 40 41 .macro disable_daif 42 msr daifset, #0xf 43 .endm 44 45 .macro enable_daif 46 msr daifclr, #0xf 47 .endm 48 49 .macro restore_daif, flags:req 50 msr daif, \flags 51 .endm 52 53 /* Only on aarch64 pstate, PSR_D_BIT is different for aarch32 */ 54 .macro inherit_daif, pstate:req, tmp:req 55 and \tmp, \pstate, #(PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT) 56 msr daif, \tmp 57 .endm 58 59 /* IRQ is the lowest priority flag, unconditionally unmask the rest. */ 60 .macro enable_da_f 61 msr daifclr, #(8 | 4 | 1) 62 .endm 63 64 /* 65 * Enable and disable interrupts. 66 */ 67 .macro disable_irq 68 msr daifset, #2 69 .endm 70 71 .macro enable_irq 72 msr daifclr, #2 73 .endm 74 75 .macro save_and_disable_irq, flags 76 mrs \flags, daif 77 msr daifset, #2 78 .endm 79 80 .macro restore_irq, flags 81 msr daif, \flags 82 .endm 83 84 .macro enable_dbg 85 msr daifclr, #8 86 .endm 87 88 .macro disable_step_tsk, flgs, tmp 89 tbz \flgs, #TIF_SINGLESTEP, 9990f 90 mrs \tmp, mdscr_el1 91 bic \tmp, \tmp, #DBG_MDSCR_SS 92 msr mdscr_el1, \tmp 93 isb // Synchronise with enable_dbg 94 9990: 95 .endm 96 97 /* call with daif masked */ 98 .macro enable_step_tsk, flgs, tmp 99 tbz \flgs, #TIF_SINGLESTEP, 9990f 100 mrs \tmp, mdscr_el1 101 orr \tmp, \tmp, #DBG_MDSCR_SS 102 msr mdscr_el1, \tmp 103 9990: 104 .endm 105 106 /* 107 * SMP data memory barrier 108 */ 109 .macro smp_dmb, opt 110 dmb \opt 111 .endm 112 113 /* 114 * RAS Error Synchronization barrier 115 */ 116 .macro esb 117 hint #16 118 .endm 119 120 /* 121 * Value prediction barrier 122 */ 123 .macro csdb 124 hint #20 125 .endm 126 127 /* 128 * Speculation barrier 129 */ 130 .macro sb 131 alternative_if_not ARM64_HAS_SB 132 dsb nsh 133 isb 134 alternative_else 135 SB_BARRIER_INSN 136 nop 137 alternative_endif 138 .endm 139 140 /* 141 * Sanitise a 64-bit bounded index wrt speculation, returning zero if out 142 * of bounds. 143 */ 144 .macro mask_nospec64, idx, limit, tmp 145 sub \tmp, \idx, \limit 146 bic \tmp, \tmp, \idx 147 and \idx, \idx, \tmp, asr #63 148 csdb 149 .endm 150 151 /* 152 * NOP sequence 153 */ 154 .macro nops, num 155 .rept \num 156 nop 157 .endr 158 .endm 159 160 /* 161 * Emit an entry into the exception table 162 */ 163 .macro _asm_extable, from, to 164 .pushsection __ex_table, "a" 165 .align 3 166 .long (\from - .), (\to - .) 167 .popsection 168 .endm 169 170 #define USER(l, x...) \ 171 9999: x; \ 172 _asm_extable 9999b, l 173 174 /* 175 * Register aliases. 176 */ 177 lr .req x30 // link register 178 179 /* 180 * Vector entry 181 */ 182 .macro ventry label 183 .align 7 184 b \label 185 .endm 186 187 /* 188 * Select code when configured for BE. 189 */ 190 #ifdef CONFIG_CPU_BIG_ENDIAN 191 #define CPU_BE(code...) code 192 #else 193 #define CPU_BE(code...) 194 #endif 195 196 /* 197 * Select code when configured for LE. 198 */ 199 #ifdef CONFIG_CPU_BIG_ENDIAN 200 #define CPU_LE(code...) 201 #else 202 #define CPU_LE(code...) code 203 #endif 204 205 /* 206 * Define a macro that constructs a 64-bit value by concatenating two 207 * 32-bit registers. Note that on big endian systems the order of the 208 * registers is swapped. 209 */ 210 #ifndef CONFIG_CPU_BIG_ENDIAN 211 .macro regs_to_64, rd, lbits, hbits 212 #else 213 .macro regs_to_64, rd, hbits, lbits 214 #endif 215 orr \rd, \lbits, \hbits, lsl #32 216 .endm 217 218 /* 219 * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where 220 * <symbol> is within the range +/- 4 GB of the PC. 221 */ 222 /* 223 * @dst: destination register (64 bit wide) 224 * @sym: name of the symbol 225 */ 226 .macro adr_l, dst, sym 227 adrp \dst, \sym 228 add \dst, \dst, :lo12:\sym 229 .endm 230 231 /* 232 * @dst: destination register (32 or 64 bit wide) 233 * @sym: name of the symbol 234 * @tmp: optional 64-bit scratch register to be used if <dst> is a 235 * 32-bit wide register, in which case it cannot be used to hold 236 * the address 237 */ 238 .macro ldr_l, dst, sym, tmp= 239 .ifb \tmp 240 adrp \dst, \sym 241 ldr \dst, [\dst, :lo12:\sym] 242 .else 243 adrp \tmp, \sym 244 ldr \dst, [\tmp, :lo12:\sym] 245 .endif 246 .endm 247 248 /* 249 * @src: source register (32 or 64 bit wide) 250 * @sym: name of the symbol 251 * @tmp: mandatory 64-bit scratch register to calculate the address 252 * while <src> needs to be preserved. 253 */ 254 .macro str_l, src, sym, tmp 255 adrp \tmp, \sym 256 str \src, [\tmp, :lo12:\sym] 257 .endm 258 259 /* 260 * @dst: Result of per_cpu(sym, smp_processor_id()) (can be SP) 261 * @sym: The name of the per-cpu variable 262 * @tmp: scratch register 263 */ 264 .macro adr_this_cpu, dst, sym, tmp 265 adrp \tmp, \sym 266 add \dst, \tmp, #:lo12:\sym 267 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN 268 mrs \tmp, tpidr_el1 269 alternative_else 270 mrs \tmp, tpidr_el2 271 alternative_endif 272 add \dst, \dst, \tmp 273 .endm 274 275 /* 276 * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id())) 277 * @sym: The name of the per-cpu variable 278 * @tmp: scratch register 279 */ 280 .macro ldr_this_cpu dst, sym, tmp 281 adr_l \dst, \sym 282 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN 283 mrs \tmp, tpidr_el1 284 alternative_else 285 mrs \tmp, tpidr_el2 286 alternative_endif 287 ldr \dst, [\dst, \tmp] 288 .endm 289 290 /* 291 * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm) 292 */ 293 .macro vma_vm_mm, rd, rn 294 ldr \rd, [\rn, #VMA_VM_MM] 295 .endm 296 297 /* 298 * mmid - get context id from mm pointer (mm->context.id) 299 */ 300 .macro mmid, rd, rn 301 ldr \rd, [\rn, #MM_CONTEXT_ID] 302 .endm 303 /* 304 * read_ctr - read CTR_EL0. If the system has mismatched register fields, 305 * provide the system wide safe value from arm64_ftr_reg_ctrel0.sys_val 306 */ 307 .macro read_ctr, reg 308 alternative_if_not ARM64_MISMATCHED_CACHE_TYPE 309 mrs \reg, ctr_el0 // read CTR 310 nop 311 alternative_else 312 ldr_l \reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL 313 alternative_endif 314 .endm 315 316 317 /* 318 * raw_dcache_line_size - get the minimum D-cache line size on this CPU 319 * from the CTR register. 320 */ 321 .macro raw_dcache_line_size, reg, tmp 322 mrs \tmp, ctr_el0 // read CTR 323 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 324 mov \reg, #4 // bytes per word 325 lsl \reg, \reg, \tmp // actual cache line size 326 .endm 327 328 /* 329 * dcache_line_size - get the safe D-cache line size across all CPUs 330 */ 331 .macro dcache_line_size, reg, tmp 332 read_ctr \tmp 333 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 334 mov \reg, #4 // bytes per word 335 lsl \reg, \reg, \tmp // actual cache line size 336 .endm 337 338 /* 339 * raw_icache_line_size - get the minimum I-cache line size on this CPU 340 * from the CTR register. 341 */ 342 .macro raw_icache_line_size, reg, tmp 343 mrs \tmp, ctr_el0 // read CTR 344 and \tmp, \tmp, #0xf // cache line size encoding 345 mov \reg, #4 // bytes per word 346 lsl \reg, \reg, \tmp // actual cache line size 347 .endm 348 349 /* 350 * icache_line_size - get the safe I-cache line size across all CPUs 351 */ 352 .macro icache_line_size, reg, tmp 353 read_ctr \tmp 354 and \tmp, \tmp, #0xf // cache line size encoding 355 mov \reg, #4 // bytes per word 356 lsl \reg, \reg, \tmp // actual cache line size 357 .endm 358 359 /* 360 * tcr_set_t0sz - update TCR.T0SZ so that we can load the ID map 361 */ 362 .macro tcr_set_t0sz, valreg, t0sz 363 bfi \valreg, \t0sz, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH 364 .endm 365 366 /* 367 * tcr_compute_pa_size - set TCR.(I)PS to the highest supported 368 * ID_AA64MMFR0_EL1.PARange value 369 * 370 * tcr: register with the TCR_ELx value to be updated 371 * pos: IPS or PS bitfield position 372 * tmp{0,1}: temporary registers 373 */ 374 .macro tcr_compute_pa_size, tcr, pos, tmp0, tmp1 375 mrs \tmp0, ID_AA64MMFR0_EL1 376 // Narrow PARange to fit the PS field in TCR_ELx 377 ubfx \tmp0, \tmp0, #ID_AA64MMFR0_PARANGE_SHIFT, #3 378 mov \tmp1, #ID_AA64MMFR0_PARANGE_MAX 379 cmp \tmp0, \tmp1 380 csel \tmp0, \tmp1, \tmp0, hi 381 bfi \tcr, \tmp0, \pos, #3 382 .endm 383 384 /* 385 * Macro to perform a data cache maintenance for the interval 386 * [kaddr, kaddr + size) 387 * 388 * op: operation passed to dc instruction 389 * domain: domain used in dsb instruciton 390 * kaddr: starting virtual address of the region 391 * size: size of the region 392 * Corrupts: kaddr, size, tmp1, tmp2 393 */ 394 .macro __dcache_op_workaround_clean_cache, op, kaddr 395 alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE 396 dc \op, \kaddr 397 alternative_else 398 dc civac, \kaddr 399 alternative_endif 400 .endm 401 402 .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2 403 dcache_line_size \tmp1, \tmp2 404 add \size, \kaddr, \size 405 sub \tmp2, \tmp1, #1 406 bic \kaddr, \kaddr, \tmp2 407 9998: 408 .ifc \op, cvau 409 __dcache_op_workaround_clean_cache \op, \kaddr 410 .else 411 .ifc \op, cvac 412 __dcache_op_workaround_clean_cache \op, \kaddr 413 .else 414 .ifc \op, cvap 415 sys 3, c7, c12, 1, \kaddr // dc cvap 416 .else 417 dc \op, \kaddr 418 .endif 419 .endif 420 .endif 421 add \kaddr, \kaddr, \tmp1 422 cmp \kaddr, \size 423 b.lo 9998b 424 dsb \domain 425 .endm 426 427 /* 428 * Macro to perform an instruction cache maintenance for the interval 429 * [start, end) 430 * 431 * start, end: virtual addresses describing the region 432 * label: A label to branch to on user fault. 433 * Corrupts: tmp1, tmp2 434 */ 435 .macro invalidate_icache_by_line start, end, tmp1, tmp2, label 436 icache_line_size \tmp1, \tmp2 437 sub \tmp2, \tmp1, #1 438 bic \tmp2, \start, \tmp2 439 9997: 440 USER(\label, ic ivau, \tmp2) // invalidate I line PoU 441 add \tmp2, \tmp2, \tmp1 442 cmp \tmp2, \end 443 b.lo 9997b 444 dsb ish 445 isb 446 .endm 447 448 /* 449 * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present 450 */ 451 .macro reset_pmuserenr_el0, tmpreg 452 mrs \tmpreg, id_aa64dfr0_el1 // Check ID_AA64DFR0_EL1 PMUVer 453 sbfx \tmpreg, \tmpreg, #8, #4 454 cmp \tmpreg, #1 // Skip if no PMU present 455 b.lt 9000f 456 msr pmuserenr_el0, xzr // Disable PMU access from EL0 457 9000: 458 .endm 459 460 /* 461 * copy_page - copy src to dest using temp registers t1-t8 462 */ 463 .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req 464 9998: ldp \t1, \t2, [\src] 465 ldp \t3, \t4, [\src, #16] 466 ldp \t5, \t6, [\src, #32] 467 ldp \t7, \t8, [\src, #48] 468 add \src, \src, #64 469 stnp \t1, \t2, [\dest] 470 stnp \t3, \t4, [\dest, #16] 471 stnp \t5, \t6, [\dest, #32] 472 stnp \t7, \t8, [\dest, #48] 473 add \dest, \dest, #64 474 tst \src, #(PAGE_SIZE - 1) 475 b.ne 9998b 476 .endm 477 478 /* 479 * Annotate a function as position independent, i.e., safe to be called before 480 * the kernel virtual mapping is activated. 481 */ 482 #define ENDPIPROC(x) \ 483 .globl __pi_##x; \ 484 .type __pi_##x, %function; \ 485 .set __pi_##x, x; \ 486 .size __pi_##x, . - x; \ 487 ENDPROC(x) 488 489 /* 490 * Annotate a function as being unsuitable for kprobes. 491 */ 492 #ifdef CONFIG_KPROBES 493 #define NOKPROBE(x) \ 494 .pushsection "_kprobe_blacklist", "aw"; \ 495 .quad x; \ 496 .popsection; 497 #else 498 #define NOKPROBE(x) 499 #endif 500 501 #ifdef CONFIG_KASAN 502 #define EXPORT_SYMBOL_NOKASAN(name) 503 #else 504 #define EXPORT_SYMBOL_NOKASAN(name) EXPORT_SYMBOL(name) 505 #endif 506 507 /* 508 * Emit a 64-bit absolute little endian symbol reference in a way that 509 * ensures that it will be resolved at build time, even when building a 510 * PIE binary. This requires cooperation from the linker script, which 511 * must emit the lo32/hi32 halves individually. 512 */ 513 .macro le64sym, sym 514 .long \sym\()_lo32 515 .long \sym\()_hi32 516 .endm 517 518 /* 519 * mov_q - move an immediate constant into a 64-bit register using 520 * between 2 and 4 movz/movk instructions (depending on the 521 * magnitude and sign of the operand) 522 */ 523 .macro mov_q, reg, val 524 .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff) 525 movz \reg, :abs_g1_s:\val 526 .else 527 .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff) 528 movz \reg, :abs_g2_s:\val 529 .else 530 movz \reg, :abs_g3:\val 531 movk \reg, :abs_g2_nc:\val 532 .endif 533 movk \reg, :abs_g1_nc:\val 534 .endif 535 movk \reg, :abs_g0_nc:\val 536 .endm 537 538 /* 539 * Return the current thread_info. 540 */ 541 .macro get_thread_info, rd 542 mrs \rd, sp_el0 543 .endm 544 545 /* 546 * Offset ttbr1 to allow for 48-bit kernel VAs set with 52-bit PTRS_PER_PGD. 547 * orr is used as it can cover the immediate value (and is idempotent). 548 * In future this may be nop'ed out when dealing with 52-bit kernel VAs. 549 * ttbr: Value of ttbr to set, modified. 550 */ 551 .macro offset_ttbr1, ttbr 552 #ifdef CONFIG_ARM64_USER_VA_BITS_52 553 orr \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET 554 #endif 555 .endm 556 557 /* 558 * Perform the reverse of offset_ttbr1. 559 * bic is used as it can cover the immediate value and, in future, won't need 560 * to be nop'ed out when dealing with 52-bit kernel VAs. 561 */ 562 .macro restore_ttbr1, ttbr 563 #ifdef CONFIG_ARM64_USER_VA_BITS_52 564 bic \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET 565 #endif 566 .endm 567 568 /* 569 * Arrange a physical address in a TTBR register, taking care of 52-bit 570 * addresses. 571 * 572 * phys: physical address, preserved 573 * ttbr: returns the TTBR value 574 */ 575 .macro phys_to_ttbr, ttbr, phys 576 #ifdef CONFIG_ARM64_PA_BITS_52 577 orr \ttbr, \phys, \phys, lsr #46 578 and \ttbr, \ttbr, #TTBR_BADDR_MASK_52 579 #else 580 mov \ttbr, \phys 581 #endif 582 .endm 583 584 .macro phys_to_pte, pte, phys 585 #ifdef CONFIG_ARM64_PA_BITS_52 586 /* 587 * We assume \phys is 64K aligned and this is guaranteed by only 588 * supporting this configuration with 64K pages. 589 */ 590 orr \pte, \phys, \phys, lsr #36 591 and \pte, \pte, #PTE_ADDR_MASK 592 #else 593 mov \pte, \phys 594 #endif 595 .endm 596 597 .macro pte_to_phys, phys, pte 598 #ifdef CONFIG_ARM64_PA_BITS_52 599 ubfiz \phys, \pte, #(48 - 16 - 12), #16 600 bfxil \phys, \pte, #16, #32 601 lsl \phys, \phys, #16 602 #else 603 and \phys, \pte, #PTE_ADDR_MASK 604 #endif 605 .endm 606 607 /** 608 * Errata workaround prior to disable MMU. Insert an ISB immediately prior 609 * to executing the MSR that will change SCTLR_ELn[M] from a value of 1 to 0. 610 */ 611 .macro pre_disable_mmu_workaround 612 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_E1041 613 isb 614 #endif 615 .endm 616 617 /* 618 * frame_push - Push @regcount callee saved registers to the stack, 619 * starting at x19, as well as x29/x30, and set x29 to 620 * the new value of sp. Add @extra bytes of stack space 621 * for locals. 622 */ 623 .macro frame_push, regcount:req, extra 624 __frame st, \regcount, \extra 625 .endm 626 627 /* 628 * frame_pop - Pop the callee saved registers from the stack that were 629 * pushed in the most recent call to frame_push, as well 630 * as x29/x30 and any extra stack space that may have been 631 * allocated. 632 */ 633 .macro frame_pop 634 __frame ld 635 .endm 636 637 .macro __frame_regs, reg1, reg2, op, num 638 .if .Lframe_regcount == \num 639 \op\()r \reg1, [sp, #(\num + 1) * 8] 640 .elseif .Lframe_regcount > \num 641 \op\()p \reg1, \reg2, [sp, #(\num + 1) * 8] 642 .endif 643 .endm 644 645 .macro __frame, op, regcount, extra=0 646 .ifc \op, st 647 .if (\regcount) < 0 || (\regcount) > 10 648 .error "regcount should be in the range [0 ... 10]" 649 .endif 650 .if ((\extra) % 16) != 0 651 .error "extra should be a multiple of 16 bytes" 652 .endif 653 .ifdef .Lframe_regcount 654 .if .Lframe_regcount != -1 655 .error "frame_push/frame_pop may not be nested" 656 .endif 657 .endif 658 .set .Lframe_regcount, \regcount 659 .set .Lframe_extra, \extra 660 .set .Lframe_local_offset, ((\regcount + 3) / 2) * 16 661 stp x29, x30, [sp, #-.Lframe_local_offset - .Lframe_extra]! 662 mov x29, sp 663 .endif 664 665 __frame_regs x19, x20, \op, 1 666 __frame_regs x21, x22, \op, 3 667 __frame_regs x23, x24, \op, 5 668 __frame_regs x25, x26, \op, 7 669 __frame_regs x27, x28, \op, 9 670 671 .ifc \op, ld 672 .if .Lframe_regcount == -1 673 .error "frame_push/frame_pop may not be nested" 674 .endif 675 ldp x29, x30, [sp], #.Lframe_local_offset + .Lframe_extra 676 .set .Lframe_regcount, -1 677 .endif 678 .endm 679 680 /* 681 * Check whether to yield to another runnable task from kernel mode NEON code 682 * (which runs with preemption disabled). 683 * 684 * if_will_cond_yield_neon 685 * // pre-yield patchup code 686 * do_cond_yield_neon 687 * // post-yield patchup code 688 * endif_yield_neon <label> 689 * 690 * where <label> is optional, and marks the point where execution will resume 691 * after a yield has been performed. If omitted, execution resumes right after 692 * the endif_yield_neon invocation. Note that the entire sequence, including 693 * the provided patchup code, will be omitted from the image if CONFIG_PREEMPT 694 * is not defined. 695 * 696 * As a convenience, in the case where no patchup code is required, the above 697 * sequence may be abbreviated to 698 * 699 * cond_yield_neon <label> 700 * 701 * Note that the patchup code does not support assembler directives that change 702 * the output section, any use of such directives is undefined. 703 * 704 * The yield itself consists of the following: 705 * - Check whether the preempt count is exactly 1, in which case disabling 706 * preemption once will make the task preemptible. If this is not the case, 707 * yielding is pointless. 708 * - Check whether TIF_NEED_RESCHED is set, and if so, disable and re-enable 709 * kernel mode NEON (which will trigger a reschedule), and branch to the 710 * yield fixup code. 711 * 712 * This macro sequence may clobber all CPU state that is not guaranteed by the 713 * AAPCS to be preserved across an ordinary function call. 714 */ 715 716 .macro cond_yield_neon, lbl 717 if_will_cond_yield_neon 718 do_cond_yield_neon 719 endif_yield_neon \lbl 720 .endm 721 722 .macro if_will_cond_yield_neon 723 #ifdef CONFIG_PREEMPT 724 get_thread_info x0 725 ldr x0, [x0, #TSK_TI_PREEMPT] 726 sub x0, x0, #PREEMPT_DISABLE_OFFSET 727 cbz x0, .Lyield_\@ 728 /* fall through to endif_yield_neon */ 729 .subsection 1 730 .Lyield_\@ : 731 #else 732 .section ".discard.cond_yield_neon", "ax" 733 #endif 734 .endm 735 736 .macro do_cond_yield_neon 737 bl kernel_neon_end 738 bl kernel_neon_begin 739 .endm 740 741 .macro endif_yield_neon, lbl 742 .ifnb \lbl 743 b \lbl 744 .else 745 b .Lyield_out_\@ 746 .endif 747 .previous 748 .Lyield_out_\@ : 749 .endm 750 751 #endif /* __ASM_ASSEMBLER_H */ 752