1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/include/asm/barrier.h 4 * 5 * Copyright (C) 2012 ARM Ltd. 6 */ 7 #ifndef __ASM_BARRIER_H 8 #define __ASM_BARRIER_H 9 10 #ifndef __ASSEMBLY__ 11 12 #include <linux/kasan-checks.h> 13 14 #define __nops(n) ".rept " #n "\nnop\n.endr\n" 15 #define nops(n) asm volatile(__nops(n)) 16 17 #define sev() asm volatile("sev" : : : "memory") 18 #define wfe() asm volatile("wfe" : : : "memory") 19 #define wfi() asm volatile("wfi" : : : "memory") 20 21 #define isb() asm volatile("isb" : : : "memory") 22 #define dmb(opt) asm volatile("dmb " #opt : : : "memory") 23 #define dsb(opt) asm volatile("dsb " #opt : : : "memory") 24 25 #define psb_csync() asm volatile("hint #17" : : : "memory") 26 #define csdb() asm volatile("hint #20" : : : "memory") 27 28 #define spec_bar() asm volatile(ALTERNATIVE("dsb nsh\nisb\n", \ 29 SB_BARRIER_INSN"nop\n", \ 30 ARM64_HAS_SB)) 31 32 #define mb() dsb(sy) 33 #define rmb() dsb(ld) 34 #define wmb() dsb(st) 35 36 #define dma_rmb() dmb(oshld) 37 #define dma_wmb() dmb(oshst) 38 39 /* 40 * Generate a mask for array_index__nospec() that is ~0UL when 0 <= idx < sz 41 * and 0 otherwise. 42 */ 43 #define array_index_mask_nospec array_index_mask_nospec 44 static inline unsigned long array_index_mask_nospec(unsigned long idx, 45 unsigned long sz) 46 { 47 unsigned long mask; 48 49 asm volatile( 50 " cmp %1, %2\n" 51 " sbc %0, xzr, xzr\n" 52 : "=r" (mask) 53 : "r" (idx), "Ir" (sz) 54 : "cc"); 55 56 csdb(); 57 return mask; 58 } 59 60 #define __smp_mb() dmb(ish) 61 #define __smp_rmb() dmb(ishld) 62 #define __smp_wmb() dmb(ishst) 63 64 #define __smp_store_release(p, v) \ 65 do { \ 66 typeof(p) __p = (p); \ 67 union { typeof(*p) __val; char __c[1]; } __u = \ 68 { .__val = (__force typeof(*p)) (v) }; \ 69 compiletime_assert_atomic_type(*p); \ 70 kasan_check_write(__p, sizeof(*p)); \ 71 switch (sizeof(*p)) { \ 72 case 1: \ 73 asm volatile ("stlrb %w1, %0" \ 74 : "=Q" (*__p) \ 75 : "r" (*(__u8 *)__u.__c) \ 76 : "memory"); \ 77 break; \ 78 case 2: \ 79 asm volatile ("stlrh %w1, %0" \ 80 : "=Q" (*__p) \ 81 : "r" (*(__u16 *)__u.__c) \ 82 : "memory"); \ 83 break; \ 84 case 4: \ 85 asm volatile ("stlr %w1, %0" \ 86 : "=Q" (*__p) \ 87 : "r" (*(__u32 *)__u.__c) \ 88 : "memory"); \ 89 break; \ 90 case 8: \ 91 asm volatile ("stlr %1, %0" \ 92 : "=Q" (*__p) \ 93 : "r" (*(__u64 *)__u.__c) \ 94 : "memory"); \ 95 break; \ 96 } \ 97 } while (0) 98 99 #define __smp_load_acquire(p) \ 100 ({ \ 101 union { typeof(*p) __val; char __c[1]; } __u; \ 102 typeof(p) __p = (p); \ 103 compiletime_assert_atomic_type(*p); \ 104 kasan_check_read(__p, sizeof(*p)); \ 105 switch (sizeof(*p)) { \ 106 case 1: \ 107 asm volatile ("ldarb %w0, %1" \ 108 : "=r" (*(__u8 *)__u.__c) \ 109 : "Q" (*__p) : "memory"); \ 110 break; \ 111 case 2: \ 112 asm volatile ("ldarh %w0, %1" \ 113 : "=r" (*(__u16 *)__u.__c) \ 114 : "Q" (*__p) : "memory"); \ 115 break; \ 116 case 4: \ 117 asm volatile ("ldar %w0, %1" \ 118 : "=r" (*(__u32 *)__u.__c) \ 119 : "Q" (*__p) : "memory"); \ 120 break; \ 121 case 8: \ 122 asm volatile ("ldar %0, %1" \ 123 : "=r" (*(__u64 *)__u.__c) \ 124 : "Q" (*__p) : "memory"); \ 125 break; \ 126 } \ 127 __u.__val; \ 128 }) 129 130 #define smp_cond_load_relaxed(ptr, cond_expr) \ 131 ({ \ 132 typeof(ptr) __PTR = (ptr); \ 133 typeof(*ptr) VAL; \ 134 for (;;) { \ 135 VAL = READ_ONCE(*__PTR); \ 136 if (cond_expr) \ 137 break; \ 138 __cmpwait_relaxed(__PTR, VAL); \ 139 } \ 140 VAL; \ 141 }) 142 143 #define smp_cond_load_acquire(ptr, cond_expr) \ 144 ({ \ 145 typeof(ptr) __PTR = (ptr); \ 146 typeof(*ptr) VAL; \ 147 for (;;) { \ 148 VAL = smp_load_acquire(__PTR); \ 149 if (cond_expr) \ 150 break; \ 151 __cmpwait_relaxed(__PTR, VAL); \ 152 } \ 153 VAL; \ 154 }) 155 156 #include <asm-generic/barrier.h> 157 158 #endif /* __ASSEMBLY__ */ 159 160 #endif /* __ASM_BARRIER_H */ 161