1 /* 2 * Based on arch/arm/include/asm/barrier.h 3 * 4 * Copyright (C) 2012 ARM Ltd. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 */ 18 #ifndef __ASM_BARRIER_H 19 #define __ASM_BARRIER_H 20 21 #ifndef __ASSEMBLY__ 22 23 #include <linux/kasan-checks.h> 24 25 #define __nops(n) ".rept " #n "\nnop\n.endr\n" 26 #define nops(n) asm volatile(__nops(n)) 27 28 #define sev() asm volatile("sev" : : : "memory") 29 #define wfe() asm volatile("wfe" : : : "memory") 30 #define wfi() asm volatile("wfi" : : : "memory") 31 32 #define isb() asm volatile("isb" : : : "memory") 33 #define dmb(opt) asm volatile("dmb " #opt : : : "memory") 34 #define dsb(opt) asm volatile("dsb " #opt : : : "memory") 35 36 #define psb_csync() asm volatile("hint #17" : : : "memory") 37 #define csdb() asm volatile("hint #20" : : : "memory") 38 39 #define spec_bar() asm volatile(ALTERNATIVE("dsb nsh\nisb\n", \ 40 SB_BARRIER_INSN"nop\n", \ 41 ARM64_HAS_SB)) 42 43 #define mb() dsb(sy) 44 #define rmb() dsb(ld) 45 #define wmb() dsb(st) 46 47 #define dma_rmb() dmb(oshld) 48 #define dma_wmb() dmb(oshst) 49 50 /* 51 * Generate a mask for array_index__nospec() that is ~0UL when 0 <= idx < sz 52 * and 0 otherwise. 53 */ 54 #define array_index_mask_nospec array_index_mask_nospec 55 static inline unsigned long array_index_mask_nospec(unsigned long idx, 56 unsigned long sz) 57 { 58 unsigned long mask; 59 60 asm volatile( 61 " cmp %1, %2\n" 62 " sbc %0, xzr, xzr\n" 63 : "=r" (mask) 64 : "r" (idx), "Ir" (sz) 65 : "cc"); 66 67 csdb(); 68 return mask; 69 } 70 71 #define __smp_mb() dmb(ish) 72 #define __smp_rmb() dmb(ishld) 73 #define __smp_wmb() dmb(ishst) 74 75 #define __smp_store_release(p, v) \ 76 do { \ 77 typeof(p) __p = (p); \ 78 union { typeof(*p) __val; char __c[1]; } __u = \ 79 { .__val = (__force typeof(*p)) (v) }; \ 80 compiletime_assert_atomic_type(*p); \ 81 kasan_check_write(__p, sizeof(*p)); \ 82 switch (sizeof(*p)) { \ 83 case 1: \ 84 asm volatile ("stlrb %w1, %0" \ 85 : "=Q" (*__p) \ 86 : "r" (*(__u8 *)__u.__c) \ 87 : "memory"); \ 88 break; \ 89 case 2: \ 90 asm volatile ("stlrh %w1, %0" \ 91 : "=Q" (*__p) \ 92 : "r" (*(__u16 *)__u.__c) \ 93 : "memory"); \ 94 break; \ 95 case 4: \ 96 asm volatile ("stlr %w1, %0" \ 97 : "=Q" (*__p) \ 98 : "r" (*(__u32 *)__u.__c) \ 99 : "memory"); \ 100 break; \ 101 case 8: \ 102 asm volatile ("stlr %1, %0" \ 103 : "=Q" (*__p) \ 104 : "r" (*(__u64 *)__u.__c) \ 105 : "memory"); \ 106 break; \ 107 } \ 108 } while (0) 109 110 #define __smp_load_acquire(p) \ 111 ({ \ 112 union { typeof(*p) __val; char __c[1]; } __u; \ 113 typeof(p) __p = (p); \ 114 compiletime_assert_atomic_type(*p); \ 115 kasan_check_read(__p, sizeof(*p)); \ 116 switch (sizeof(*p)) { \ 117 case 1: \ 118 asm volatile ("ldarb %w0, %1" \ 119 : "=r" (*(__u8 *)__u.__c) \ 120 : "Q" (*__p) : "memory"); \ 121 break; \ 122 case 2: \ 123 asm volatile ("ldarh %w0, %1" \ 124 : "=r" (*(__u16 *)__u.__c) \ 125 : "Q" (*__p) : "memory"); \ 126 break; \ 127 case 4: \ 128 asm volatile ("ldar %w0, %1" \ 129 : "=r" (*(__u32 *)__u.__c) \ 130 : "Q" (*__p) : "memory"); \ 131 break; \ 132 case 8: \ 133 asm volatile ("ldar %0, %1" \ 134 : "=r" (*(__u64 *)__u.__c) \ 135 : "Q" (*__p) : "memory"); \ 136 break; \ 137 } \ 138 __u.__val; \ 139 }) 140 141 #define smp_cond_load_relaxed(ptr, cond_expr) \ 142 ({ \ 143 typeof(ptr) __PTR = (ptr); \ 144 typeof(*ptr) VAL; \ 145 for (;;) { \ 146 VAL = READ_ONCE(*__PTR); \ 147 if (cond_expr) \ 148 break; \ 149 __cmpwait_relaxed(__PTR, VAL); \ 150 } \ 151 VAL; \ 152 }) 153 154 #define smp_cond_load_acquire(ptr, cond_expr) \ 155 ({ \ 156 typeof(ptr) __PTR = (ptr); \ 157 typeof(*ptr) VAL; \ 158 for (;;) { \ 159 VAL = smp_load_acquire(__PTR); \ 160 if (cond_expr) \ 161 break; \ 162 __cmpwait_relaxed(__PTR, VAL); \ 163 } \ 164 VAL; \ 165 }) 166 167 #include <asm-generic/barrier.h> 168 169 #endif /* __ASSEMBLY__ */ 170 171 #endif /* __ASM_BARRIER_H */ 172