1 /* 2 * Based on arch/arm/include/asm/barrier.h 3 * 4 * Copyright (C) 2012 ARM Ltd. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 */ 18 #ifndef __ASM_BARRIER_H 19 #define __ASM_BARRIER_H 20 21 #ifndef __ASSEMBLY__ 22 23 #define __nops(n) ".rept " #n "\nnop\n.endr\n" 24 #define nops(n) asm volatile(__nops(n)) 25 26 #define sev() asm volatile("sev" : : : "memory") 27 #define wfe() asm volatile("wfe" : : : "memory") 28 #define wfi() asm volatile("wfi" : : : "memory") 29 30 #define isb() asm volatile("isb" : : : "memory") 31 #define dmb(opt) asm volatile("dmb " #opt : : : "memory") 32 #define dsb(opt) asm volatile("dsb " #opt : : : "memory") 33 34 #define psb_csync() asm volatile("hint #17" : : : "memory") 35 #define csdb() asm volatile("hint #20" : : : "memory") 36 37 #define spec_bar() asm volatile(ALTERNATIVE("dsb nsh\nisb\n", \ 38 SB_BARRIER_INSN"nop\n", \ 39 ARM64_HAS_SB)) 40 41 #define mb() dsb(sy) 42 #define rmb() dsb(ld) 43 #define wmb() dsb(st) 44 45 #define dma_rmb() dmb(oshld) 46 #define dma_wmb() dmb(oshst) 47 48 /* 49 * Generate a mask for array_index__nospec() that is ~0UL when 0 <= idx < sz 50 * and 0 otherwise. 51 */ 52 #define array_index_mask_nospec array_index_mask_nospec 53 static inline unsigned long array_index_mask_nospec(unsigned long idx, 54 unsigned long sz) 55 { 56 unsigned long mask; 57 58 asm volatile( 59 " cmp %1, %2\n" 60 " sbc %0, xzr, xzr\n" 61 : "=r" (mask) 62 : "r" (idx), "Ir" (sz) 63 : "cc"); 64 65 csdb(); 66 return mask; 67 } 68 69 #define __smp_mb() dmb(ish) 70 #define __smp_rmb() dmb(ishld) 71 #define __smp_wmb() dmb(ishst) 72 73 #define __smp_store_release(p, v) \ 74 do { \ 75 union { typeof(*p) __val; char __c[1]; } __u = \ 76 { .__val = (__force typeof(*p)) (v) }; \ 77 compiletime_assert_atomic_type(*p); \ 78 switch (sizeof(*p)) { \ 79 case 1: \ 80 asm volatile ("stlrb %w1, %0" \ 81 : "=Q" (*p) \ 82 : "r" (*(__u8 *)__u.__c) \ 83 : "memory"); \ 84 break; \ 85 case 2: \ 86 asm volatile ("stlrh %w1, %0" \ 87 : "=Q" (*p) \ 88 : "r" (*(__u16 *)__u.__c) \ 89 : "memory"); \ 90 break; \ 91 case 4: \ 92 asm volatile ("stlr %w1, %0" \ 93 : "=Q" (*p) \ 94 : "r" (*(__u32 *)__u.__c) \ 95 : "memory"); \ 96 break; \ 97 case 8: \ 98 asm volatile ("stlr %1, %0" \ 99 : "=Q" (*p) \ 100 : "r" (*(__u64 *)__u.__c) \ 101 : "memory"); \ 102 break; \ 103 } \ 104 } while (0) 105 106 #define __smp_load_acquire(p) \ 107 ({ \ 108 union { typeof(*p) __val; char __c[1]; } __u; \ 109 compiletime_assert_atomic_type(*p); \ 110 switch (sizeof(*p)) { \ 111 case 1: \ 112 asm volatile ("ldarb %w0, %1" \ 113 : "=r" (*(__u8 *)__u.__c) \ 114 : "Q" (*p) : "memory"); \ 115 break; \ 116 case 2: \ 117 asm volatile ("ldarh %w0, %1" \ 118 : "=r" (*(__u16 *)__u.__c) \ 119 : "Q" (*p) : "memory"); \ 120 break; \ 121 case 4: \ 122 asm volatile ("ldar %w0, %1" \ 123 : "=r" (*(__u32 *)__u.__c) \ 124 : "Q" (*p) : "memory"); \ 125 break; \ 126 case 8: \ 127 asm volatile ("ldar %0, %1" \ 128 : "=r" (*(__u64 *)__u.__c) \ 129 : "Q" (*p) : "memory"); \ 130 break; \ 131 } \ 132 __u.__val; \ 133 }) 134 135 #define smp_cond_load_relaxed(ptr, cond_expr) \ 136 ({ \ 137 typeof(ptr) __PTR = (ptr); \ 138 typeof(*ptr) VAL; \ 139 for (;;) { \ 140 VAL = READ_ONCE(*__PTR); \ 141 if (cond_expr) \ 142 break; \ 143 __cmpwait_relaxed(__PTR, VAL); \ 144 } \ 145 VAL; \ 146 }) 147 148 #define smp_cond_load_acquire(ptr, cond_expr) \ 149 ({ \ 150 typeof(ptr) __PTR = (ptr); \ 151 typeof(*ptr) VAL; \ 152 for (;;) { \ 153 VAL = smp_load_acquire(__PTR); \ 154 if (cond_expr) \ 155 break; \ 156 __cmpwait_relaxed(__PTR, VAL); \ 157 } \ 158 VAL; \ 159 }) 160 161 #include <asm-generic/barrier.h> 162 163 #endif /* __ASSEMBLY__ */ 164 165 #endif /* __ASM_BARRIER_H */ 166