1 /* 2 * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S 3 * 4 * Copyright (C) 1996-2000 Russell King 5 * Copyright (C) 2012 ARM Ltd. 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program. If not, see <http://www.gnu.org/licenses/>. 18 */ 19 #ifndef __ASSEMBLY__ 20 #error "Only include this from assembly code" 21 #endif 22 23 #ifndef __ASM_ASSEMBLER_H 24 #define __ASM_ASSEMBLER_H 25 26 #include <asm/asm-offsets.h> 27 #include <asm/cpufeature.h> 28 #include <asm/debug-monitors.h> 29 #include <asm/mmu_context.h> 30 #include <asm/page.h> 31 #include <asm/pgtable-hwdef.h> 32 #include <asm/ptrace.h> 33 #include <asm/thread_info.h> 34 35 .macro save_and_disable_daif, flags 36 mrs \flags, daif 37 msr daifset, #0xf 38 .endm 39 40 .macro disable_daif 41 msr daifset, #0xf 42 .endm 43 44 .macro enable_daif 45 msr daifclr, #0xf 46 .endm 47 48 .macro restore_daif, flags:req 49 msr daif, \flags 50 .endm 51 52 /* Only on aarch64 pstate, PSR_D_BIT is different for aarch32 */ 53 .macro inherit_daif, pstate:req, tmp:req 54 and \tmp, \pstate, #(PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT) 55 msr daif, \tmp 56 .endm 57 58 /* IRQ is the lowest priority flag, unconditionally unmask the rest. */ 59 .macro enable_da_f 60 msr daifclr, #(8 | 4 | 1) 61 .endm 62 63 /* 64 * Enable and disable interrupts. 65 */ 66 .macro disable_irq 67 msr daifset, #2 68 .endm 69 70 .macro enable_irq 71 msr daifclr, #2 72 .endm 73 74 .macro save_and_disable_irq, flags 75 mrs \flags, daif 76 msr daifset, #2 77 .endm 78 79 .macro restore_irq, flags 80 msr daif, \flags 81 .endm 82 83 .macro enable_dbg 84 msr daifclr, #8 85 .endm 86 87 .macro disable_step_tsk, flgs, tmp 88 tbz \flgs, #TIF_SINGLESTEP, 9990f 89 mrs \tmp, mdscr_el1 90 bic \tmp, \tmp, #DBG_MDSCR_SS 91 msr mdscr_el1, \tmp 92 isb // Synchronise with enable_dbg 93 9990: 94 .endm 95 96 /* call with daif masked */ 97 .macro enable_step_tsk, flgs, tmp 98 tbz \flgs, #TIF_SINGLESTEP, 9990f 99 mrs \tmp, mdscr_el1 100 orr \tmp, \tmp, #DBG_MDSCR_SS 101 msr mdscr_el1, \tmp 102 9990: 103 .endm 104 105 /* 106 * SMP data memory barrier 107 */ 108 .macro smp_dmb, opt 109 dmb \opt 110 .endm 111 112 /* 113 * NOP sequence 114 */ 115 .macro nops, num 116 .rept \num 117 nop 118 .endr 119 .endm 120 121 /* 122 * Emit an entry into the exception table 123 */ 124 .macro _asm_extable, from, to 125 .pushsection __ex_table, "a" 126 .align 3 127 .long (\from - .), (\to - .) 128 .popsection 129 .endm 130 131 #define USER(l, x...) \ 132 9999: x; \ 133 _asm_extable 9999b, l 134 135 /* 136 * Register aliases. 137 */ 138 lr .req x30 // link register 139 140 /* 141 * Vector entry 142 */ 143 .macro ventry label 144 .align 7 145 b \label 146 .endm 147 148 /* 149 * Select code when configured for BE. 150 */ 151 #ifdef CONFIG_CPU_BIG_ENDIAN 152 #define CPU_BE(code...) code 153 #else 154 #define CPU_BE(code...) 155 #endif 156 157 /* 158 * Select code when configured for LE. 159 */ 160 #ifdef CONFIG_CPU_BIG_ENDIAN 161 #define CPU_LE(code...) 162 #else 163 #define CPU_LE(code...) code 164 #endif 165 166 /* 167 * Define a macro that constructs a 64-bit value by concatenating two 168 * 32-bit registers. Note that on big endian systems the order of the 169 * registers is swapped. 170 */ 171 #ifndef CONFIG_CPU_BIG_ENDIAN 172 .macro regs_to_64, rd, lbits, hbits 173 #else 174 .macro regs_to_64, rd, hbits, lbits 175 #endif 176 orr \rd, \lbits, \hbits, lsl #32 177 .endm 178 179 /* 180 * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where 181 * <symbol> is within the range +/- 4 GB of the PC when running 182 * in core kernel context. In module context, a movz/movk sequence 183 * is used, since modules may be loaded far away from the kernel 184 * when KASLR is in effect. 185 */ 186 /* 187 * @dst: destination register (64 bit wide) 188 * @sym: name of the symbol 189 */ 190 .macro adr_l, dst, sym 191 #ifndef MODULE 192 adrp \dst, \sym 193 add \dst, \dst, :lo12:\sym 194 #else 195 movz \dst, #:abs_g3:\sym 196 movk \dst, #:abs_g2_nc:\sym 197 movk \dst, #:abs_g1_nc:\sym 198 movk \dst, #:abs_g0_nc:\sym 199 #endif 200 .endm 201 202 /* 203 * @dst: destination register (32 or 64 bit wide) 204 * @sym: name of the symbol 205 * @tmp: optional 64-bit scratch register to be used if <dst> is a 206 * 32-bit wide register, in which case it cannot be used to hold 207 * the address 208 */ 209 .macro ldr_l, dst, sym, tmp= 210 #ifndef MODULE 211 .ifb \tmp 212 adrp \dst, \sym 213 ldr \dst, [\dst, :lo12:\sym] 214 .else 215 adrp \tmp, \sym 216 ldr \dst, [\tmp, :lo12:\sym] 217 .endif 218 #else 219 .ifb \tmp 220 adr_l \dst, \sym 221 ldr \dst, [\dst] 222 .else 223 adr_l \tmp, \sym 224 ldr \dst, [\tmp] 225 .endif 226 #endif 227 .endm 228 229 /* 230 * @src: source register (32 or 64 bit wide) 231 * @sym: name of the symbol 232 * @tmp: mandatory 64-bit scratch register to calculate the address 233 * while <src> needs to be preserved. 234 */ 235 .macro str_l, src, sym, tmp 236 #ifndef MODULE 237 adrp \tmp, \sym 238 str \src, [\tmp, :lo12:\sym] 239 #else 240 adr_l \tmp, \sym 241 str \src, [\tmp] 242 #endif 243 .endm 244 245 /* 246 * @dst: Result of per_cpu(sym, smp_processor_id()), can be SP for 247 * non-module code 248 * @sym: The name of the per-cpu variable 249 * @tmp: scratch register 250 */ 251 .macro adr_this_cpu, dst, sym, tmp 252 #ifndef MODULE 253 adrp \tmp, \sym 254 add \dst, \tmp, #:lo12:\sym 255 #else 256 adr_l \dst, \sym 257 #endif 258 mrs \tmp, tpidr_el1 259 add \dst, \dst, \tmp 260 .endm 261 262 /* 263 * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id())) 264 * @sym: The name of the per-cpu variable 265 * @tmp: scratch register 266 */ 267 .macro ldr_this_cpu dst, sym, tmp 268 adr_l \dst, \sym 269 mrs \tmp, tpidr_el1 270 ldr \dst, [\dst, \tmp] 271 .endm 272 273 /* 274 * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm) 275 */ 276 .macro vma_vm_mm, rd, rn 277 ldr \rd, [\rn, #VMA_VM_MM] 278 .endm 279 280 /* 281 * mmid - get context id from mm pointer (mm->context.id) 282 */ 283 .macro mmid, rd, rn 284 ldr \rd, [\rn, #MM_CONTEXT_ID] 285 .endm 286 /* 287 * read_ctr - read CTR_EL0. If the system has mismatched 288 * cache line sizes, provide the system wide safe value 289 * from arm64_ftr_reg_ctrel0.sys_val 290 */ 291 .macro read_ctr, reg 292 alternative_if_not ARM64_MISMATCHED_CACHE_LINE_SIZE 293 mrs \reg, ctr_el0 // read CTR 294 nop 295 alternative_else 296 ldr_l \reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL 297 alternative_endif 298 .endm 299 300 301 /* 302 * raw_dcache_line_size - get the minimum D-cache line size on this CPU 303 * from the CTR register. 304 */ 305 .macro raw_dcache_line_size, reg, tmp 306 mrs \tmp, ctr_el0 // read CTR 307 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 308 mov \reg, #4 // bytes per word 309 lsl \reg, \reg, \tmp // actual cache line size 310 .endm 311 312 /* 313 * dcache_line_size - get the safe D-cache line size across all CPUs 314 */ 315 .macro dcache_line_size, reg, tmp 316 read_ctr \tmp 317 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 318 mov \reg, #4 // bytes per word 319 lsl \reg, \reg, \tmp // actual cache line size 320 .endm 321 322 /* 323 * raw_icache_line_size - get the minimum I-cache line size on this CPU 324 * from the CTR register. 325 */ 326 .macro raw_icache_line_size, reg, tmp 327 mrs \tmp, ctr_el0 // read CTR 328 and \tmp, \tmp, #0xf // cache line size encoding 329 mov \reg, #4 // bytes per word 330 lsl \reg, \reg, \tmp // actual cache line size 331 .endm 332 333 /* 334 * icache_line_size - get the safe I-cache line size across all CPUs 335 */ 336 .macro icache_line_size, reg, tmp 337 read_ctr \tmp 338 and \tmp, \tmp, #0xf // cache line size encoding 339 mov \reg, #4 // bytes per word 340 lsl \reg, \reg, \tmp // actual cache line size 341 .endm 342 343 /* 344 * tcr_set_idmap_t0sz - update TCR.T0SZ so that we can load the ID map 345 */ 346 .macro tcr_set_idmap_t0sz, valreg, tmpreg 347 #ifndef CONFIG_ARM64_VA_BITS_48 348 ldr_l \tmpreg, idmap_t0sz 349 bfi \valreg, \tmpreg, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH 350 #endif 351 .endm 352 353 /* 354 * Macro to perform a data cache maintenance for the interval 355 * [kaddr, kaddr + size) 356 * 357 * op: operation passed to dc instruction 358 * domain: domain used in dsb instruciton 359 * kaddr: starting virtual address of the region 360 * size: size of the region 361 * Corrupts: kaddr, size, tmp1, tmp2 362 */ 363 .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2 364 dcache_line_size \tmp1, \tmp2 365 add \size, \kaddr, \size 366 sub \tmp2, \tmp1, #1 367 bic \kaddr, \kaddr, \tmp2 368 9998: 369 .if (\op == cvau || \op == cvac) 370 alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE 371 dc \op, \kaddr 372 alternative_else 373 dc civac, \kaddr 374 alternative_endif 375 .elseif (\op == cvap) 376 alternative_if ARM64_HAS_DCPOP 377 sys 3, c7, c12, 1, \kaddr // dc cvap 378 alternative_else 379 dc cvac, \kaddr 380 alternative_endif 381 .else 382 dc \op, \kaddr 383 .endif 384 add \kaddr, \kaddr, \tmp1 385 cmp \kaddr, \size 386 b.lo 9998b 387 dsb \domain 388 .endm 389 390 /* 391 * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present 392 */ 393 .macro reset_pmuserenr_el0, tmpreg 394 mrs \tmpreg, id_aa64dfr0_el1 // Check ID_AA64DFR0_EL1 PMUVer 395 sbfx \tmpreg, \tmpreg, #8, #4 396 cmp \tmpreg, #1 // Skip if no PMU present 397 b.lt 9000f 398 msr pmuserenr_el0, xzr // Disable PMU access from EL0 399 9000: 400 .endm 401 402 /* 403 * copy_page - copy src to dest using temp registers t1-t8 404 */ 405 .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req 406 9998: ldp \t1, \t2, [\src] 407 ldp \t3, \t4, [\src, #16] 408 ldp \t5, \t6, [\src, #32] 409 ldp \t7, \t8, [\src, #48] 410 add \src, \src, #64 411 stnp \t1, \t2, [\dest] 412 stnp \t3, \t4, [\dest, #16] 413 stnp \t5, \t6, [\dest, #32] 414 stnp \t7, \t8, [\dest, #48] 415 add \dest, \dest, #64 416 tst \src, #(PAGE_SIZE - 1) 417 b.ne 9998b 418 .endm 419 420 /* 421 * Annotate a function as position independent, i.e., safe to be called before 422 * the kernel virtual mapping is activated. 423 */ 424 #define ENDPIPROC(x) \ 425 .globl __pi_##x; \ 426 .type __pi_##x, %function; \ 427 .set __pi_##x, x; \ 428 .size __pi_##x, . - x; \ 429 ENDPROC(x) 430 431 /* 432 * Annotate a function as being unsuitable for kprobes. 433 */ 434 #ifdef CONFIG_KPROBES 435 #define NOKPROBE(x) \ 436 .pushsection "_kprobe_blacklist", "aw"; \ 437 .quad x; \ 438 .popsection; 439 #else 440 #define NOKPROBE(x) 441 #endif 442 /* 443 * Emit a 64-bit absolute little endian symbol reference in a way that 444 * ensures that it will be resolved at build time, even when building a 445 * PIE binary. This requires cooperation from the linker script, which 446 * must emit the lo32/hi32 halves individually. 447 */ 448 .macro le64sym, sym 449 .long \sym\()_lo32 450 .long \sym\()_hi32 451 .endm 452 453 /* 454 * mov_q - move an immediate constant into a 64-bit register using 455 * between 2 and 4 movz/movk instructions (depending on the 456 * magnitude and sign of the operand) 457 */ 458 .macro mov_q, reg, val 459 .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff) 460 movz \reg, :abs_g1_s:\val 461 .else 462 .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff) 463 movz \reg, :abs_g2_s:\val 464 .else 465 movz \reg, :abs_g3:\val 466 movk \reg, :abs_g2_nc:\val 467 .endif 468 movk \reg, :abs_g1_nc:\val 469 .endif 470 movk \reg, :abs_g0_nc:\val 471 .endm 472 473 /* 474 * Return the current thread_info. 475 */ 476 .macro get_thread_info, rd 477 mrs \rd, sp_el0 478 .endm 479 480 /* 481 * Errata workaround prior to TTBR0_EL1 update 482 * 483 * val: TTBR value with new BADDR, preserved 484 * tmp0: temporary register, clobbered 485 * tmp1: other temporary register, clobbered 486 */ 487 .macro pre_ttbr0_update_workaround, val, tmp0, tmp1 488 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003 489 alternative_if ARM64_WORKAROUND_QCOM_FALKOR_E1003 490 mrs \tmp0, ttbr0_el1 491 mov \tmp1, #FALKOR_RESERVED_ASID 492 bfi \tmp0, \tmp1, #48, #16 // reserved ASID + old BADDR 493 msr ttbr0_el1, \tmp0 494 isb 495 bfi \tmp0, \val, #0, #48 // reserved ASID + new BADDR 496 msr ttbr0_el1, \tmp0 497 isb 498 alternative_else_nop_endif 499 #endif 500 .endm 501 502 /* 503 * Errata workaround post TTBR0_EL1 update. 504 */ 505 .macro post_ttbr0_update_workaround 506 #ifdef CONFIG_CAVIUM_ERRATUM_27456 507 alternative_if ARM64_WORKAROUND_CAVIUM_27456 508 ic iallu 509 dsb nsh 510 isb 511 alternative_else_nop_endif 512 #endif 513 .endm 514 515 #endif /* __ASM_ASSEMBLER_H */ 516