1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S 4 * 5 * Copyright (C) 1996-2000 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 */ 8 #ifndef __ASSEMBLY__ 9 #error "Only include this from assembly code" 10 #endif 11 12 #ifndef __ASM_ASSEMBLER_H 13 #define __ASM_ASSEMBLER_H 14 15 #include <asm-generic/export.h> 16 17 #include <asm/alternative.h> 18 #include <asm/asm-bug.h> 19 #include <asm/asm-extable.h> 20 #include <asm/asm-offsets.h> 21 #include <asm/cpufeature.h> 22 #include <asm/cputype.h> 23 #include <asm/debug-monitors.h> 24 #include <asm/page.h> 25 #include <asm/pgtable-hwdef.h> 26 #include <asm/ptrace.h> 27 #include <asm/thread_info.h> 28 29 /* 30 * Provide a wxN alias for each wN register so what we can paste a xN 31 * reference after a 'w' to obtain the 32-bit version. 32 */ 33 .irp n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 34 wx\n .req w\n 35 .endr 36 37 .macro disable_daif 38 msr daifset, #0xf 39 .endm 40 41 .macro enable_daif 42 msr daifclr, #0xf 43 .endm 44 45 /* 46 * Save/restore interrupts. 47 */ 48 .macro save_and_disable_irq, flags 49 mrs \flags, daif 50 msr daifset, #3 51 .endm 52 53 .macro restore_irq, flags 54 msr daif, \flags 55 .endm 56 57 .macro enable_dbg 58 msr daifclr, #8 59 .endm 60 61 .macro disable_step_tsk, flgs, tmp 62 tbz \flgs, #TIF_SINGLESTEP, 9990f 63 mrs \tmp, mdscr_el1 64 bic \tmp, \tmp, #DBG_MDSCR_SS 65 msr mdscr_el1, \tmp 66 isb // Synchronise with enable_dbg 67 9990: 68 .endm 69 70 /* call with daif masked */ 71 .macro enable_step_tsk, flgs, tmp 72 tbz \flgs, #TIF_SINGLESTEP, 9990f 73 mrs \tmp, mdscr_el1 74 orr \tmp, \tmp, #DBG_MDSCR_SS 75 msr mdscr_el1, \tmp 76 9990: 77 .endm 78 79 /* 80 * RAS Error Synchronization barrier 81 */ 82 .macro esb 83 #ifdef CONFIG_ARM64_RAS_EXTN 84 hint #16 85 #else 86 nop 87 #endif 88 .endm 89 90 /* 91 * Value prediction barrier 92 */ 93 .macro csdb 94 hint #20 95 .endm 96 97 /* 98 * Clear Branch History instruction 99 */ 100 .macro clearbhb 101 hint #22 102 .endm 103 104 /* 105 * Speculation barrier 106 */ 107 .macro sb 108 alternative_if_not ARM64_HAS_SB 109 dsb nsh 110 isb 111 alternative_else 112 SB_BARRIER_INSN 113 nop 114 alternative_endif 115 .endm 116 117 /* 118 * NOP sequence 119 */ 120 .macro nops, num 121 .rept \num 122 nop 123 .endr 124 .endm 125 126 /* 127 * Register aliases. 128 */ 129 lr .req x30 // link register 130 131 /* 132 * Vector entry 133 */ 134 .macro ventry label 135 .align 7 136 b \label 137 .endm 138 139 /* 140 * Select code when configured for BE. 141 */ 142 #ifdef CONFIG_CPU_BIG_ENDIAN 143 #define CPU_BE(code...) code 144 #else 145 #define CPU_BE(code...) 146 #endif 147 148 /* 149 * Select code when configured for LE. 150 */ 151 #ifdef CONFIG_CPU_BIG_ENDIAN 152 #define CPU_LE(code...) 153 #else 154 #define CPU_LE(code...) code 155 #endif 156 157 /* 158 * Define a macro that constructs a 64-bit value by concatenating two 159 * 32-bit registers. Note that on big endian systems the order of the 160 * registers is swapped. 161 */ 162 #ifndef CONFIG_CPU_BIG_ENDIAN 163 .macro regs_to_64, rd, lbits, hbits 164 #else 165 .macro regs_to_64, rd, hbits, lbits 166 #endif 167 orr \rd, \lbits, \hbits, lsl #32 168 .endm 169 170 /* 171 * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where 172 * <symbol> is within the range +/- 4 GB of the PC. 173 */ 174 /* 175 * @dst: destination register (64 bit wide) 176 * @sym: name of the symbol 177 */ 178 .macro adr_l, dst, sym 179 adrp \dst, \sym 180 add \dst, \dst, :lo12:\sym 181 .endm 182 183 /* 184 * @dst: destination register (32 or 64 bit wide) 185 * @sym: name of the symbol 186 * @tmp: optional 64-bit scratch register to be used if <dst> is a 187 * 32-bit wide register, in which case it cannot be used to hold 188 * the address 189 */ 190 .macro ldr_l, dst, sym, tmp= 191 .ifb \tmp 192 adrp \dst, \sym 193 ldr \dst, [\dst, :lo12:\sym] 194 .else 195 adrp \tmp, \sym 196 ldr \dst, [\tmp, :lo12:\sym] 197 .endif 198 .endm 199 200 /* 201 * @src: source register (32 or 64 bit wide) 202 * @sym: name of the symbol 203 * @tmp: mandatory 64-bit scratch register to calculate the address 204 * while <src> needs to be preserved. 205 */ 206 .macro str_l, src, sym, tmp 207 adrp \tmp, \sym 208 str \src, [\tmp, :lo12:\sym] 209 .endm 210 211 /* 212 * @dst: destination register 213 */ 214 #if defined(__KVM_NVHE_HYPERVISOR__) || defined(__KVM_VHE_HYPERVISOR__) 215 .macro get_this_cpu_offset, dst 216 mrs \dst, tpidr_el2 217 .endm 218 #else 219 .macro get_this_cpu_offset, dst 220 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN 221 mrs \dst, tpidr_el1 222 alternative_else 223 mrs \dst, tpidr_el2 224 alternative_endif 225 .endm 226 227 .macro set_this_cpu_offset, src 228 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN 229 msr tpidr_el1, \src 230 alternative_else 231 msr tpidr_el2, \src 232 alternative_endif 233 .endm 234 #endif 235 236 /* 237 * @dst: Result of per_cpu(sym, smp_processor_id()) (can be SP) 238 * @sym: The name of the per-cpu variable 239 * @tmp: scratch register 240 */ 241 .macro adr_this_cpu, dst, sym, tmp 242 adrp \tmp, \sym 243 add \dst, \tmp, #:lo12:\sym 244 get_this_cpu_offset \tmp 245 add \dst, \dst, \tmp 246 .endm 247 248 /* 249 * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id())) 250 * @sym: The name of the per-cpu variable 251 * @tmp: scratch register 252 */ 253 .macro ldr_this_cpu dst, sym, tmp 254 adr_l \dst, \sym 255 get_this_cpu_offset \tmp 256 ldr \dst, [\dst, \tmp] 257 .endm 258 259 /* 260 * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm) 261 */ 262 .macro vma_vm_mm, rd, rn 263 ldr \rd, [\rn, #VMA_VM_MM] 264 .endm 265 266 /* 267 * read_ctr - read CTR_EL0. If the system has mismatched register fields, 268 * provide the system wide safe value from arm64_ftr_reg_ctrel0.sys_val 269 */ 270 .macro read_ctr, reg 271 #ifndef __KVM_NVHE_HYPERVISOR__ 272 alternative_if_not ARM64_MISMATCHED_CACHE_TYPE 273 mrs \reg, ctr_el0 // read CTR 274 nop 275 alternative_else 276 ldr_l \reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL 277 alternative_endif 278 #else 279 alternative_if_not ARM64_KVM_PROTECTED_MODE 280 ASM_BUG() 281 alternative_else_nop_endif 282 alternative_cb ARM64_ALWAYS_SYSTEM, kvm_compute_final_ctr_el0 283 movz \reg, #0 284 movk \reg, #0, lsl #16 285 movk \reg, #0, lsl #32 286 movk \reg, #0, lsl #48 287 alternative_cb_end 288 #endif 289 .endm 290 291 292 /* 293 * raw_dcache_line_size - get the minimum D-cache line size on this CPU 294 * from the CTR register. 295 */ 296 .macro raw_dcache_line_size, reg, tmp 297 mrs \tmp, ctr_el0 // read CTR 298 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 299 mov \reg, #4 // bytes per word 300 lsl \reg, \reg, \tmp // actual cache line size 301 .endm 302 303 /* 304 * dcache_line_size - get the safe D-cache line size across all CPUs 305 */ 306 .macro dcache_line_size, reg, tmp 307 read_ctr \tmp 308 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 309 mov \reg, #4 // bytes per word 310 lsl \reg, \reg, \tmp // actual cache line size 311 .endm 312 313 /* 314 * raw_icache_line_size - get the minimum I-cache line size on this CPU 315 * from the CTR register. 316 */ 317 .macro raw_icache_line_size, reg, tmp 318 mrs \tmp, ctr_el0 // read CTR 319 and \tmp, \tmp, #0xf // cache line size encoding 320 mov \reg, #4 // bytes per word 321 lsl \reg, \reg, \tmp // actual cache line size 322 .endm 323 324 /* 325 * icache_line_size - get the safe I-cache line size across all CPUs 326 */ 327 .macro icache_line_size, reg, tmp 328 read_ctr \tmp 329 and \tmp, \tmp, #0xf // cache line size encoding 330 mov \reg, #4 // bytes per word 331 lsl \reg, \reg, \tmp // actual cache line size 332 .endm 333 334 /* 335 * tcr_set_t0sz - update TCR.T0SZ so that we can load the ID map 336 */ 337 .macro tcr_set_t0sz, valreg, t0sz 338 bfi \valreg, \t0sz, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH 339 .endm 340 341 /* 342 * tcr_set_t1sz - update TCR.T1SZ 343 */ 344 .macro tcr_set_t1sz, valreg, t1sz 345 bfi \valreg, \t1sz, #TCR_T1SZ_OFFSET, #TCR_TxSZ_WIDTH 346 .endm 347 348 /* 349 * idmap_get_t0sz - get the T0SZ value needed to cover the ID map 350 * 351 * Calculate the maximum allowed value for TCR_EL1.T0SZ so that the 352 * entire ID map region can be mapped. As T0SZ == (64 - #bits used), 353 * this number conveniently equals the number of leading zeroes in 354 * the physical address of _end. 355 */ 356 .macro idmap_get_t0sz, reg 357 adrp \reg, _end 358 orr \reg, \reg, #(1 << VA_BITS_MIN) - 1 359 clz \reg, \reg 360 .endm 361 362 /* 363 * tcr_compute_pa_size - set TCR.(I)PS to the highest supported 364 * ID_AA64MMFR0_EL1.PARange value 365 * 366 * tcr: register with the TCR_ELx value to be updated 367 * pos: IPS or PS bitfield position 368 * tmp{0,1}: temporary registers 369 */ 370 .macro tcr_compute_pa_size, tcr, pos, tmp0, tmp1 371 mrs \tmp0, ID_AA64MMFR0_EL1 372 // Narrow PARange to fit the PS field in TCR_ELx 373 ubfx \tmp0, \tmp0, #ID_AA64MMFR0_EL1_PARANGE_SHIFT, #3 374 mov \tmp1, #ID_AA64MMFR0_EL1_PARANGE_MAX 375 cmp \tmp0, \tmp1 376 csel \tmp0, \tmp1, \tmp0, hi 377 bfi \tcr, \tmp0, \pos, #3 378 .endm 379 380 .macro __dcache_op_workaround_clean_cache, op, addr 381 alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE 382 dc \op, \addr 383 alternative_else 384 dc civac, \addr 385 alternative_endif 386 .endm 387 388 /* 389 * Macro to perform a data cache maintenance for the interval 390 * [start, end) with dcache line size explicitly provided. 391 * 392 * op: operation passed to dc instruction 393 * domain: domain used in dsb instruciton 394 * start: starting virtual address of the region 395 * end: end virtual address of the region 396 * linesz: dcache line size 397 * fixup: optional label to branch to on user fault 398 * Corrupts: start, end, tmp 399 */ 400 .macro dcache_by_myline_op op, domain, start, end, linesz, tmp, fixup 401 sub \tmp, \linesz, #1 402 bic \start, \start, \tmp 403 .Ldcache_op\@: 404 .ifc \op, cvau 405 __dcache_op_workaround_clean_cache \op, \start 406 .else 407 .ifc \op, cvac 408 __dcache_op_workaround_clean_cache \op, \start 409 .else 410 .ifc \op, cvap 411 sys 3, c7, c12, 1, \start // dc cvap 412 .else 413 .ifc \op, cvadp 414 sys 3, c7, c13, 1, \start // dc cvadp 415 .else 416 dc \op, \start 417 .endif 418 .endif 419 .endif 420 .endif 421 add \start, \start, \linesz 422 cmp \start, \end 423 b.lo .Ldcache_op\@ 424 dsb \domain 425 426 _cond_uaccess_extable .Ldcache_op\@, \fixup 427 .endm 428 429 /* 430 * Macro to perform a data cache maintenance for the interval 431 * [start, end) 432 * 433 * op: operation passed to dc instruction 434 * domain: domain used in dsb instruciton 435 * start: starting virtual address of the region 436 * end: end virtual address of the region 437 * fixup: optional label to branch to on user fault 438 * Corrupts: start, end, tmp1, tmp2 439 */ 440 .macro dcache_by_line_op op, domain, start, end, tmp1, tmp2, fixup 441 dcache_line_size \tmp1, \tmp2 442 dcache_by_myline_op \op, \domain, \start, \end, \tmp1, \tmp2, \fixup 443 .endm 444 445 /* 446 * Macro to perform an instruction cache maintenance for the interval 447 * [start, end) 448 * 449 * start, end: virtual addresses describing the region 450 * fixup: optional label to branch to on user fault 451 * Corrupts: tmp1, tmp2 452 */ 453 .macro invalidate_icache_by_line start, end, tmp1, tmp2, fixup 454 icache_line_size \tmp1, \tmp2 455 sub \tmp2, \tmp1, #1 456 bic \tmp2, \start, \tmp2 457 .Licache_op\@: 458 ic ivau, \tmp2 // invalidate I line PoU 459 add \tmp2, \tmp2, \tmp1 460 cmp \tmp2, \end 461 b.lo .Licache_op\@ 462 dsb ish 463 isb 464 465 _cond_uaccess_extable .Licache_op\@, \fixup 466 .endm 467 468 /* 469 * load_ttbr1 - install @pgtbl as a TTBR1 page table 470 * pgtbl preserved 471 * tmp1/tmp2 clobbered, either may overlap with pgtbl 472 */ 473 .macro load_ttbr1, pgtbl, tmp1, tmp2 474 phys_to_ttbr \tmp1, \pgtbl 475 offset_ttbr1 \tmp1, \tmp2 476 msr ttbr1_el1, \tmp1 477 isb 478 .endm 479 480 /* 481 * To prevent the possibility of old and new partial table walks being visible 482 * in the tlb, switch the ttbr to a zero page when we invalidate the old 483 * records. D4.7.1 'General TLB maintenance requirements' in ARM DDI 0487A.i 484 * Even switching to our copied tables will cause a changed output address at 485 * each stage of the walk. 486 */ 487 .macro break_before_make_ttbr_switch zero_page, page_table, tmp, tmp2 488 phys_to_ttbr \tmp, \zero_page 489 msr ttbr1_el1, \tmp 490 isb 491 tlbi vmalle1 492 dsb nsh 493 load_ttbr1 \page_table, \tmp, \tmp2 494 .endm 495 496 /* 497 * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present 498 */ 499 .macro reset_pmuserenr_el0, tmpreg 500 mrs \tmpreg, id_aa64dfr0_el1 501 sbfx \tmpreg, \tmpreg, #ID_AA64DFR0_EL1_PMUVer_SHIFT, #4 502 cmp \tmpreg, #1 // Skip if no PMU present 503 b.lt 9000f 504 msr pmuserenr_el0, xzr // Disable PMU access from EL0 505 9000: 506 .endm 507 508 /* 509 * reset_amuserenr_el0 - reset AMUSERENR_EL0 if AMUv1 present 510 */ 511 .macro reset_amuserenr_el0, tmpreg 512 mrs \tmpreg, id_aa64pfr0_el1 // Check ID_AA64PFR0_EL1 513 ubfx \tmpreg, \tmpreg, #ID_AA64PFR0_EL1_AMU_SHIFT, #4 514 cbz \tmpreg, .Lskip_\@ // Skip if no AMU present 515 msr_s SYS_AMUSERENR_EL0, xzr // Disable AMU access from EL0 516 .Lskip_\@: 517 .endm 518 /* 519 * copy_page - copy src to dest using temp registers t1-t8 520 */ 521 .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req 522 9998: ldp \t1, \t2, [\src] 523 ldp \t3, \t4, [\src, #16] 524 ldp \t5, \t6, [\src, #32] 525 ldp \t7, \t8, [\src, #48] 526 add \src, \src, #64 527 stnp \t1, \t2, [\dest] 528 stnp \t3, \t4, [\dest, #16] 529 stnp \t5, \t6, [\dest, #32] 530 stnp \t7, \t8, [\dest, #48] 531 add \dest, \dest, #64 532 tst \src, #(PAGE_SIZE - 1) 533 b.ne 9998b 534 .endm 535 536 /* 537 * Annotate a function as being unsuitable for kprobes. 538 */ 539 #ifdef CONFIG_KPROBES 540 #define NOKPROBE(x) \ 541 .pushsection "_kprobe_blacklist", "aw"; \ 542 .quad x; \ 543 .popsection; 544 #else 545 #define NOKPROBE(x) 546 #endif 547 548 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) 549 #define EXPORT_SYMBOL_NOKASAN(name) 550 #else 551 #define EXPORT_SYMBOL_NOKASAN(name) EXPORT_SYMBOL(name) 552 #endif 553 554 /* 555 * Emit a 64-bit absolute little endian symbol reference in a way that 556 * ensures that it will be resolved at build time, even when building a 557 * PIE binary. This requires cooperation from the linker script, which 558 * must emit the lo32/hi32 halves individually. 559 */ 560 .macro le64sym, sym 561 .long \sym\()_lo32 562 .long \sym\()_hi32 563 .endm 564 565 /* 566 * mov_q - move an immediate constant into a 64-bit register using 567 * between 2 and 4 movz/movk instructions (depending on the 568 * magnitude and sign of the operand) 569 */ 570 .macro mov_q, reg, val 571 .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff) 572 movz \reg, :abs_g1_s:\val 573 .else 574 .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff) 575 movz \reg, :abs_g2_s:\val 576 .else 577 movz \reg, :abs_g3:\val 578 movk \reg, :abs_g2_nc:\val 579 .endif 580 movk \reg, :abs_g1_nc:\val 581 .endif 582 movk \reg, :abs_g0_nc:\val 583 .endm 584 585 /* 586 * Return the current task_struct. 587 */ 588 .macro get_current_task, rd 589 mrs \rd, sp_el0 590 .endm 591 592 /* 593 * Offset ttbr1 to allow for 48-bit kernel VAs set with 52-bit PTRS_PER_PGD. 594 * orr is used as it can cover the immediate value (and is idempotent). 595 * In future this may be nop'ed out when dealing with 52-bit kernel VAs. 596 * ttbr: Value of ttbr to set, modified. 597 */ 598 .macro offset_ttbr1, ttbr, tmp 599 #ifdef CONFIG_ARM64_VA_BITS_52 600 mrs_s \tmp, SYS_ID_AA64MMFR2_EL1 601 and \tmp, \tmp, #(0xf << ID_AA64MMFR2_EL1_VARange_SHIFT) 602 cbnz \tmp, .Lskipoffs_\@ 603 orr \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET 604 .Lskipoffs_\@ : 605 #endif 606 .endm 607 608 /* 609 * Arrange a physical address in a TTBR register, taking care of 52-bit 610 * addresses. 611 * 612 * phys: physical address, preserved 613 * ttbr: returns the TTBR value 614 */ 615 .macro phys_to_ttbr, ttbr, phys 616 #ifdef CONFIG_ARM64_PA_BITS_52 617 orr \ttbr, \phys, \phys, lsr #46 618 and \ttbr, \ttbr, #TTBR_BADDR_MASK_52 619 #else 620 mov \ttbr, \phys 621 #endif 622 .endm 623 624 .macro phys_to_pte, pte, phys 625 #ifdef CONFIG_ARM64_PA_BITS_52 626 /* 627 * We assume \phys is 64K aligned and this is guaranteed by only 628 * supporting this configuration with 64K pages. 629 */ 630 orr \pte, \phys, \phys, lsr #36 631 and \pte, \pte, #PTE_ADDR_MASK 632 #else 633 mov \pte, \phys 634 #endif 635 .endm 636 637 .macro pte_to_phys, phys, pte 638 and \phys, \pte, #PTE_ADDR_MASK 639 #ifdef CONFIG_ARM64_PA_BITS_52 640 orr \phys, \phys, \phys, lsl #PTE_ADDR_HIGH_SHIFT 641 and \phys, \phys, GENMASK_ULL(PHYS_MASK_SHIFT - 1, PAGE_SHIFT) 642 #endif 643 .endm 644 645 /* 646 * tcr_clear_errata_bits - Clear TCR bits that trigger an errata on this CPU. 647 */ 648 .macro tcr_clear_errata_bits, tcr, tmp1, tmp2 649 #ifdef CONFIG_FUJITSU_ERRATUM_010001 650 mrs \tmp1, midr_el1 651 652 mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001_MASK 653 and \tmp1, \tmp1, \tmp2 654 mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001 655 cmp \tmp1, \tmp2 656 b.ne 10f 657 658 mov_q \tmp2, TCR_CLEAR_FUJITSU_ERRATUM_010001 659 bic \tcr, \tcr, \tmp2 660 10: 661 #endif /* CONFIG_FUJITSU_ERRATUM_010001 */ 662 .endm 663 664 /** 665 * Errata workaround prior to disable MMU. Insert an ISB immediately prior 666 * to executing the MSR that will change SCTLR_ELn[M] from a value of 1 to 0. 667 */ 668 .macro pre_disable_mmu_workaround 669 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_E1041 670 isb 671 #endif 672 .endm 673 674 /* 675 * frame_push - Push @regcount callee saved registers to the stack, 676 * starting at x19, as well as x29/x30, and set x29 to 677 * the new value of sp. Add @extra bytes of stack space 678 * for locals. 679 */ 680 .macro frame_push, regcount:req, extra 681 __frame st, \regcount, \extra 682 .endm 683 684 /* 685 * frame_pop - Pop the callee saved registers from the stack that were 686 * pushed in the most recent call to frame_push, as well 687 * as x29/x30 and any extra stack space that may have been 688 * allocated. 689 */ 690 .macro frame_pop 691 __frame ld 692 .endm 693 694 .macro __frame_regs, reg1, reg2, op, num 695 .if .Lframe_regcount == \num 696 \op\()r \reg1, [sp, #(\num + 1) * 8] 697 .elseif .Lframe_regcount > \num 698 \op\()p \reg1, \reg2, [sp, #(\num + 1) * 8] 699 .endif 700 .endm 701 702 .macro __frame, op, regcount, extra=0 703 .ifc \op, st 704 .if (\regcount) < 0 || (\regcount) > 10 705 .error "regcount should be in the range [0 ... 10]" 706 .endif 707 .if ((\extra) % 16) != 0 708 .error "extra should be a multiple of 16 bytes" 709 .endif 710 .ifdef .Lframe_regcount 711 .if .Lframe_regcount != -1 712 .error "frame_push/frame_pop may not be nested" 713 .endif 714 .endif 715 .set .Lframe_regcount, \regcount 716 .set .Lframe_extra, \extra 717 .set .Lframe_local_offset, ((\regcount + 3) / 2) * 16 718 stp x29, x30, [sp, #-.Lframe_local_offset - .Lframe_extra]! 719 mov x29, sp 720 .endif 721 722 __frame_regs x19, x20, \op, 1 723 __frame_regs x21, x22, \op, 3 724 __frame_regs x23, x24, \op, 5 725 __frame_regs x25, x26, \op, 7 726 __frame_regs x27, x28, \op, 9 727 728 .ifc \op, ld 729 .if .Lframe_regcount == -1 730 .error "frame_push/frame_pop may not be nested" 731 .endif 732 ldp x29, x30, [sp], #.Lframe_local_offset + .Lframe_extra 733 .set .Lframe_regcount, -1 734 .endif 735 .endm 736 737 /* 738 * Set SCTLR_ELx to the @reg value, and invalidate the local icache 739 * in the process. This is called when setting the MMU on. 740 */ 741 .macro set_sctlr, sreg, reg 742 msr \sreg, \reg 743 isb 744 /* 745 * Invalidate the local I-cache so that any instructions fetched 746 * speculatively from the PoC are discarded, since they may have 747 * been dynamically patched at the PoU. 748 */ 749 ic iallu 750 dsb nsh 751 isb 752 .endm 753 754 .macro set_sctlr_el1, reg 755 set_sctlr sctlr_el1, \reg 756 .endm 757 758 .macro set_sctlr_el2, reg 759 set_sctlr sctlr_el2, \reg 760 .endm 761 762 /* 763 * Check whether preempt/bh-disabled asm code should yield as soon as 764 * it is able. This is the case if we are currently running in task 765 * context, and either a softirq is pending, or the TIF_NEED_RESCHED 766 * flag is set and re-enabling preemption a single time would result in 767 * a preempt count of zero. (Note that the TIF_NEED_RESCHED flag is 768 * stored negated in the top word of the thread_info::preempt_count 769 * field) 770 */ 771 .macro cond_yield, lbl:req, tmp:req, tmp2:req 772 get_current_task \tmp 773 ldr \tmp, [\tmp, #TSK_TI_PREEMPT] 774 /* 775 * If we are serving a softirq, there is no point in yielding: the 776 * softirq will not be preempted no matter what we do, so we should 777 * run to completion as quickly as we can. 778 */ 779 tbnz \tmp, #SOFTIRQ_SHIFT, .Lnoyield_\@ 780 #ifdef CONFIG_PREEMPTION 781 sub \tmp, \tmp, #PREEMPT_DISABLE_OFFSET 782 cbz \tmp, \lbl 783 #endif 784 adr_l \tmp, irq_stat + IRQ_CPUSTAT_SOFTIRQ_PENDING 785 get_this_cpu_offset \tmp2 786 ldr w\tmp, [\tmp, \tmp2] 787 cbnz w\tmp, \lbl // yield on pending softirq in task context 788 .Lnoyield_\@: 789 .endm 790 791 /* 792 * Branch Target Identifier (BTI) 793 */ 794 .macro bti, targets 795 .equ .L__bti_targets_c, 34 796 .equ .L__bti_targets_j, 36 797 .equ .L__bti_targets_jc,38 798 hint #.L__bti_targets_\targets 799 .endm 800 801 /* 802 * This macro emits a program property note section identifying 803 * architecture features which require special handling, mainly for 804 * use in assembly files included in the VDSO. 805 */ 806 807 #define NT_GNU_PROPERTY_TYPE_0 5 808 #define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000 809 810 #define GNU_PROPERTY_AARCH64_FEATURE_1_BTI (1U << 0) 811 #define GNU_PROPERTY_AARCH64_FEATURE_1_PAC (1U << 1) 812 813 #ifdef CONFIG_ARM64_BTI_KERNEL 814 #define GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT \ 815 ((GNU_PROPERTY_AARCH64_FEATURE_1_BTI | \ 816 GNU_PROPERTY_AARCH64_FEATURE_1_PAC)) 817 #endif 818 819 #ifdef GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT 820 .macro emit_aarch64_feature_1_and, feat=GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT 821 .pushsection .note.gnu.property, "a" 822 .align 3 823 .long 2f - 1f 824 .long 6f - 3f 825 .long NT_GNU_PROPERTY_TYPE_0 826 1: .string "GNU" 827 2: 828 .align 3 829 3: .long GNU_PROPERTY_AARCH64_FEATURE_1_AND 830 .long 5f - 4f 831 4: 832 /* 833 * This is described with an array of char in the Linux API 834 * spec but the text and all other usage (including binutils, 835 * clang and GCC) treat this as a 32 bit value so no swizzling 836 * is required for big endian. 837 */ 838 .long \feat 839 5: 840 .align 3 841 6: 842 .popsection 843 .endm 844 845 #else 846 .macro emit_aarch64_feature_1_and, feat=0 847 .endm 848 849 #endif /* GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT */ 850 851 .macro __mitigate_spectre_bhb_loop tmp 852 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 853 alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_loop_iter 854 mov \tmp, #32 // Patched to correct the immediate 855 alternative_cb_end 856 .Lspectre_bhb_loop\@: 857 b . + 4 858 subs \tmp, \tmp, #1 859 b.ne .Lspectre_bhb_loop\@ 860 sb 861 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 862 .endm 863 864 .macro mitigate_spectre_bhb_loop tmp 865 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 866 alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_loop_mitigation_enable 867 b .L_spectre_bhb_loop_done\@ // Patched to NOP 868 alternative_cb_end 869 __mitigate_spectre_bhb_loop \tmp 870 .L_spectre_bhb_loop_done\@: 871 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 872 .endm 873 874 /* Save/restores x0-x3 to the stack */ 875 .macro __mitigate_spectre_bhb_fw 876 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 877 stp x0, x1, [sp, #-16]! 878 stp x2, x3, [sp, #-16]! 879 mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3 880 alternative_cb ARM64_ALWAYS_SYSTEM, smccc_patch_fw_mitigation_conduit 881 nop // Patched to SMC/HVC #0 882 alternative_cb_end 883 ldp x2, x3, [sp], #16 884 ldp x0, x1, [sp], #16 885 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 886 .endm 887 888 .macro mitigate_spectre_bhb_clear_insn 889 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 890 alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_clearbhb 891 /* Patched to NOP when not supported */ 892 clearbhb 893 isb 894 alternative_cb_end 895 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 896 .endm 897 #endif /* __ASM_ASSEMBLER_H */ 898