1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/include/asm/barrier.h 4 * 5 * Copyright (C) 2012 ARM Ltd. 6 */ 7 #ifndef __ASM_BARRIER_H 8 #define __ASM_BARRIER_H 9 10 #ifndef __ASSEMBLY__ 11 12 #include <linux/kasan-checks.h> 13 14 #define __nops(n) ".rept " #n "\nnop\n.endr\n" 15 #define nops(n) asm volatile(__nops(n)) 16 17 #define sev() asm volatile("sev" : : : "memory") 18 #define wfe() asm volatile("wfe" : : : "memory") 19 #define wfet(val) asm volatile("msr s0_3_c1_c0_0, %0" \ 20 : : "r" (val) : "memory") 21 #define wfi() asm volatile("wfi" : : : "memory") 22 #define wfit(val) asm volatile("msr s0_3_c1_c0_1, %0" \ 23 : : "r" (val) : "memory") 24 25 #define isb() asm volatile("isb" : : : "memory") 26 #define dmb(opt) asm volatile("dmb " #opt : : : "memory") 27 #define dsb(opt) asm volatile("dsb " #opt : : : "memory") 28 29 #define psb_csync() asm volatile("hint #17" : : : "memory") 30 #define __tsb_csync() asm volatile("hint #18" : : : "memory") 31 #define csdb() asm volatile("hint #20" : : : "memory") 32 33 /* 34 * Data Gathering Hint: 35 * This instruction prevents merging memory accesses with Normal-NC or 36 * Device-GRE attributes before the hint instruction with any memory accesses 37 * appearing after the hint instruction. 38 */ 39 #define dgh() asm volatile("hint #6" : : : "memory") 40 41 #ifdef CONFIG_ARM64_PSEUDO_NMI 42 #define pmr_sync() \ 43 do { \ 44 extern struct static_key_false gic_pmr_sync; \ 45 \ 46 if (static_branch_unlikely(&gic_pmr_sync)) \ 47 dsb(sy); \ 48 } while(0) 49 #else 50 #define pmr_sync() do {} while (0) 51 #endif 52 53 #define __mb() dsb(sy) 54 #define __rmb() dsb(ld) 55 #define __wmb() dsb(st) 56 57 #define __dma_mb() dmb(osh) 58 #define __dma_rmb() dmb(oshld) 59 #define __dma_wmb() dmb(oshst) 60 61 #define io_stop_wc() dgh() 62 63 #define tsb_csync() \ 64 do { \ 65 /* \ 66 * CPUs affected by Arm Erratum 2054223 or 2067961 needs \ 67 * another TSB to ensure the trace is flushed. The barriers \ 68 * don't have to be strictly back to back, as long as the \ 69 * CPU is in trace prohibited state. \ 70 */ \ 71 if (cpus_have_final_cap(ARM64_WORKAROUND_TSB_FLUSH_FAILURE)) \ 72 __tsb_csync(); \ 73 __tsb_csync(); \ 74 } while (0) 75 76 /* 77 * Generate a mask for array_index__nospec() that is ~0UL when 0 <= idx < sz 78 * and 0 otherwise. 79 */ 80 #define array_index_mask_nospec array_index_mask_nospec 81 static inline unsigned long array_index_mask_nospec(unsigned long idx, 82 unsigned long sz) 83 { 84 unsigned long mask; 85 86 asm volatile( 87 " cmp %1, %2\n" 88 " sbc %0, xzr, xzr\n" 89 : "=r" (mask) 90 : "r" (idx), "Ir" (sz) 91 : "cc"); 92 93 csdb(); 94 return mask; 95 } 96 97 /* 98 * Ensure that reads of the counter are treated the same as memory reads 99 * for the purposes of ordering by subsequent memory barriers. 100 * 101 * This insanity brought to you by speculative system register reads, 102 * out-of-order memory accesses, sequence locks and Thomas Gleixner. 103 * 104 * https://lore.kernel.org/r/alpine.DEB.2.21.1902081950260.1662@nanos.tec.linutronix.de/ 105 */ 106 #define arch_counter_enforce_ordering(val) do { \ 107 u64 tmp, _val = (val); \ 108 \ 109 asm volatile( \ 110 " eor %0, %1, %1\n" \ 111 " add %0, sp, %0\n" \ 112 " ldr xzr, [%0]" \ 113 : "=r" (tmp) : "r" (_val)); \ 114 } while (0) 115 116 #define __smp_mb() dmb(ish) 117 #define __smp_rmb() dmb(ishld) 118 #define __smp_wmb() dmb(ishst) 119 120 #define __smp_store_release(p, v) \ 121 do { \ 122 typeof(p) __p = (p); \ 123 union { __unqual_scalar_typeof(*p) __val; char __c[1]; } __u = \ 124 { .__val = (__force __unqual_scalar_typeof(*p)) (v) }; \ 125 compiletime_assert_atomic_type(*p); \ 126 kasan_check_write(__p, sizeof(*p)); \ 127 switch (sizeof(*p)) { \ 128 case 1: \ 129 asm volatile ("stlrb %w1, %0" \ 130 : "=Q" (*__p) \ 131 : "r" (*(__u8 *)__u.__c) \ 132 : "memory"); \ 133 break; \ 134 case 2: \ 135 asm volatile ("stlrh %w1, %0" \ 136 : "=Q" (*__p) \ 137 : "r" (*(__u16 *)__u.__c) \ 138 : "memory"); \ 139 break; \ 140 case 4: \ 141 asm volatile ("stlr %w1, %0" \ 142 : "=Q" (*__p) \ 143 : "r" (*(__u32 *)__u.__c) \ 144 : "memory"); \ 145 break; \ 146 case 8: \ 147 asm volatile ("stlr %1, %0" \ 148 : "=Q" (*__p) \ 149 : "r" (*(__u64 *)__u.__c) \ 150 : "memory"); \ 151 break; \ 152 } \ 153 } while (0) 154 155 #define __smp_load_acquire(p) \ 156 ({ \ 157 union { __unqual_scalar_typeof(*p) __val; char __c[1]; } __u; \ 158 typeof(p) __p = (p); \ 159 compiletime_assert_atomic_type(*p); \ 160 kasan_check_read(__p, sizeof(*p)); \ 161 switch (sizeof(*p)) { \ 162 case 1: \ 163 asm volatile ("ldarb %w0, %1" \ 164 : "=r" (*(__u8 *)__u.__c) \ 165 : "Q" (*__p) : "memory"); \ 166 break; \ 167 case 2: \ 168 asm volatile ("ldarh %w0, %1" \ 169 : "=r" (*(__u16 *)__u.__c) \ 170 : "Q" (*__p) : "memory"); \ 171 break; \ 172 case 4: \ 173 asm volatile ("ldar %w0, %1" \ 174 : "=r" (*(__u32 *)__u.__c) \ 175 : "Q" (*__p) : "memory"); \ 176 break; \ 177 case 8: \ 178 asm volatile ("ldar %0, %1" \ 179 : "=r" (*(__u64 *)__u.__c) \ 180 : "Q" (*__p) : "memory"); \ 181 break; \ 182 } \ 183 (typeof(*p))__u.__val; \ 184 }) 185 186 #define smp_cond_load_relaxed(ptr, cond_expr) \ 187 ({ \ 188 typeof(ptr) __PTR = (ptr); \ 189 __unqual_scalar_typeof(*ptr) VAL; \ 190 for (;;) { \ 191 VAL = READ_ONCE(*__PTR); \ 192 if (cond_expr) \ 193 break; \ 194 __cmpwait_relaxed(__PTR, VAL); \ 195 } \ 196 (typeof(*ptr))VAL; \ 197 }) 198 199 #define smp_cond_load_acquire(ptr, cond_expr) \ 200 ({ \ 201 typeof(ptr) __PTR = (ptr); \ 202 __unqual_scalar_typeof(*ptr) VAL; \ 203 for (;;) { \ 204 VAL = smp_load_acquire(__PTR); \ 205 if (cond_expr) \ 206 break; \ 207 __cmpwait_relaxed(__PTR, VAL); \ 208 } \ 209 (typeof(*ptr))VAL; \ 210 }) 211 212 #include <asm-generic/barrier.h> 213 214 #endif /* __ASSEMBLY__ */ 215 216 #endif /* __ASM_BARRIER_H */ 217