1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */ 24f3db074SArnaldo Carvalho de Melo #ifndef _TOOLS_LINUX_ASM_AARCH64_BARRIER_H 34f3db074SArnaldo Carvalho de Melo #define _TOOLS_LINUX_ASM_AARCH64_BARRIER_H 44f3db074SArnaldo Carvalho de Melo 54f3db074SArnaldo Carvalho de Melo /* 64f3db074SArnaldo Carvalho de Melo * From tools/perf/perf-sys.h, last modified in: 74f3db074SArnaldo Carvalho de Melo * f428ebd184c82a7914b2aa7e9f868918aaf7ea78 perf tools: Fix AAAAARGH64 memory barriers 84f3db074SArnaldo Carvalho de Melo * 94f3db074SArnaldo Carvalho de Melo * XXX: arch/arm64/include/asm/barrier.h in the kernel sources use dsb, is this 104f3db074SArnaldo Carvalho de Melo * a case like for arm32 where we do things differently in userspace? 114f3db074SArnaldo Carvalho de Melo */ 124f3db074SArnaldo Carvalho de Melo 134f3db074SArnaldo Carvalho de Melo #define mb() asm volatile("dmb ish" ::: "memory") 144f3db074SArnaldo Carvalho de Melo #define wmb() asm volatile("dmb ishst" ::: "memory") 154f3db074SArnaldo Carvalho de Melo #define rmb() asm volatile("dmb ishld" ::: "memory") 164f3db074SArnaldo Carvalho de Melo 17*6b7a2114SDaniel Borkmann /* 18*6b7a2114SDaniel Borkmann * Kernel uses dmb variants on arm64 for smp_*() barriers. Pretty much the same 19*6b7a2114SDaniel Borkmann * implementation as above mb()/wmb()/rmb(), though for the latter kernel uses 20*6b7a2114SDaniel Borkmann * dsb. In any case, should above mb()/wmb()/rmb() change, make sure the below 21*6b7a2114SDaniel Borkmann * smp_*() don't. 22*6b7a2114SDaniel Borkmann */ 23*6b7a2114SDaniel Borkmann #define smp_mb() asm volatile("dmb ish" ::: "memory") 24*6b7a2114SDaniel Borkmann #define smp_wmb() asm volatile("dmb ishst" ::: "memory") 25*6b7a2114SDaniel Borkmann #define smp_rmb() asm volatile("dmb ishld" ::: "memory") 26*6b7a2114SDaniel Borkmann 2709d62154SDaniel Borkmann #define smp_store_release(p, v) \ 2809d62154SDaniel Borkmann do { \ 2909d62154SDaniel Borkmann union { typeof(*p) __val; char __c[1]; } __u = \ 3051f5fd2eSWill Deacon { .__val = (v) }; \ 3109d62154SDaniel Borkmann \ 3209d62154SDaniel Borkmann switch (sizeof(*p)) { \ 3309d62154SDaniel Borkmann case 1: \ 3409d62154SDaniel Borkmann asm volatile ("stlrb %w1, %0" \ 3509d62154SDaniel Borkmann : "=Q" (*p) \ 3651f5fd2eSWill Deacon : "r" (*(__u8_alias_t *)__u.__c) \ 3709d62154SDaniel Borkmann : "memory"); \ 3809d62154SDaniel Borkmann break; \ 3909d62154SDaniel Borkmann case 2: \ 4009d62154SDaniel Borkmann asm volatile ("stlrh %w1, %0" \ 4109d62154SDaniel Borkmann : "=Q" (*p) \ 4251f5fd2eSWill Deacon : "r" (*(__u16_alias_t *)__u.__c) \ 4309d62154SDaniel Borkmann : "memory"); \ 4409d62154SDaniel Borkmann break; \ 4509d62154SDaniel Borkmann case 4: \ 4609d62154SDaniel Borkmann asm volatile ("stlr %w1, %0" \ 4709d62154SDaniel Borkmann : "=Q" (*p) \ 4851f5fd2eSWill Deacon : "r" (*(__u32_alias_t *)__u.__c) \ 4909d62154SDaniel Borkmann : "memory"); \ 5009d62154SDaniel Borkmann break; \ 5109d62154SDaniel Borkmann case 8: \ 5209d62154SDaniel Borkmann asm volatile ("stlr %1, %0" \ 5309d62154SDaniel Borkmann : "=Q" (*p) \ 5451f5fd2eSWill Deacon : "r" (*(__u64_alias_t *)__u.__c) \ 5509d62154SDaniel Borkmann : "memory"); \ 5609d62154SDaniel Borkmann break; \ 5709d62154SDaniel Borkmann default: \ 5809d62154SDaniel Borkmann /* Only to shut up gcc ... */ \ 5909d62154SDaniel Borkmann mb(); \ 6009d62154SDaniel Borkmann break; \ 6109d62154SDaniel Borkmann } \ 6209d62154SDaniel Borkmann } while (0) 6309d62154SDaniel Borkmann 6409d62154SDaniel Borkmann #define smp_load_acquire(p) \ 6509d62154SDaniel Borkmann ({ \ 6651f5fd2eSWill Deacon union { typeof(*p) __val; char __c[1]; } __u = \ 6751f5fd2eSWill Deacon { .__c = { 0 } }; \ 6809d62154SDaniel Borkmann \ 6909d62154SDaniel Borkmann switch (sizeof(*p)) { \ 7009d62154SDaniel Borkmann case 1: \ 7109d62154SDaniel Borkmann asm volatile ("ldarb %w0, %1" \ 7251f5fd2eSWill Deacon : "=r" (*(__u8_alias_t *)__u.__c) \ 7309d62154SDaniel Borkmann : "Q" (*p) : "memory"); \ 7409d62154SDaniel Borkmann break; \ 7509d62154SDaniel Borkmann case 2: \ 7609d62154SDaniel Borkmann asm volatile ("ldarh %w0, %1" \ 7751f5fd2eSWill Deacon : "=r" (*(__u16_alias_t *)__u.__c) \ 7809d62154SDaniel Borkmann : "Q" (*p) : "memory"); \ 7909d62154SDaniel Borkmann break; \ 8009d62154SDaniel Borkmann case 4: \ 8109d62154SDaniel Borkmann asm volatile ("ldar %w0, %1" \ 8251f5fd2eSWill Deacon : "=r" (*(__u32_alias_t *)__u.__c) \ 8309d62154SDaniel Borkmann : "Q" (*p) : "memory"); \ 8409d62154SDaniel Borkmann break; \ 8509d62154SDaniel Borkmann case 8: \ 8609d62154SDaniel Borkmann asm volatile ("ldar %0, %1" \ 8751f5fd2eSWill Deacon : "=r" (*(__u64_alias_t *)__u.__c) \ 8809d62154SDaniel Borkmann : "Q" (*p) : "memory"); \ 8909d62154SDaniel Borkmann break; \ 9009d62154SDaniel Borkmann default: \ 9109d62154SDaniel Borkmann /* Only to shut up gcc ... */ \ 9209d62154SDaniel Borkmann mb(); \ 9309d62154SDaniel Borkmann break; \ 9409d62154SDaniel Borkmann } \ 9509d62154SDaniel Borkmann __u.__val; \ 9609d62154SDaniel Borkmann }) 9709d62154SDaniel Borkmann 984f3db074SArnaldo Carvalho de Melo #endif /* _TOOLS_LINUX_ASM_AARCH64_BARRIER_H */ 99