1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5 #ifndef __ASM_BARRIER_H
6 #define __ASM_BARRIER_H
7
8 /*
9 * Hint encoding:
10 *
11 * Bit4: ordering or completion (0: completion, 1: ordering)
12 * Bit3: barrier for previous read (0: true, 1: false)
13 * Bit2: barrier for previous write (0: true, 1: false)
14 * Bit1: barrier for succeeding read (0: true, 1: false)
15 * Bit0: barrier for succeeding write (0: true, 1: false)
16 *
17 * Hint 0x700: barrier for "read after read" from the same address
18 */
19
20 #define DBAR(hint) __asm__ __volatile__("dbar %0 " : : "I"(hint) : "memory")
21
22 #define crwrw 0b00000
23 #define cr_r_ 0b00101
24 #define c_w_w 0b01010
25
26 #define orwrw 0b10000
27 #define or_r_ 0b10101
28 #define o_w_w 0b11010
29
30 #define orw_w 0b10010
31 #define or_rw 0b10100
32
33 #define c_sync() DBAR(crwrw)
34 #define c_rsync() DBAR(cr_r_)
35 #define c_wsync() DBAR(c_w_w)
36
37 #define o_sync() DBAR(orwrw)
38 #define o_rsync() DBAR(or_r_)
39 #define o_wsync() DBAR(o_w_w)
40
41 #define ldacq_mb() DBAR(or_rw)
42 #define strel_mb() DBAR(orw_w)
43
44 #define mb() c_sync()
45 #define rmb() c_rsync()
46 #define wmb() c_wsync()
47 #define iob() c_sync()
48 #define wbflush() c_sync()
49
50 #define __smp_mb() o_sync()
51 #define __smp_rmb() o_rsync()
52 #define __smp_wmb() o_wsync()
53
54 #ifdef CONFIG_SMP
55 #define __WEAK_LLSC_MB " dbar 0x700 \n"
56 #else
57 #define __WEAK_LLSC_MB " \n"
58 #endif
59
60 #define __smp_mb__before_atomic() barrier()
61 #define __smp_mb__after_atomic() barrier()
62
63 /**
64 * array_index_mask_nospec() - generate a ~0 mask when index < size, 0 otherwise
65 * @index: array element index
66 * @size: number of elements in array
67 *
68 * Returns:
69 * 0 - (@index < @size)
70 */
71 #define array_index_mask_nospec array_index_mask_nospec
array_index_mask_nospec(unsigned long index,unsigned long size)72 static inline unsigned long array_index_mask_nospec(unsigned long index,
73 unsigned long size)
74 {
75 unsigned long mask;
76
77 __asm__ __volatile__(
78 "sltu %0, %1, %2\n\t"
79 #if (__SIZEOF_LONG__ == 4)
80 "sub.w %0, $zero, %0\n\t"
81 #elif (__SIZEOF_LONG__ == 8)
82 "sub.d %0, $zero, %0\n\t"
83 #endif
84 : "=r" (mask)
85 : "r" (index), "r" (size)
86 :);
87
88 return mask;
89 }
90
91 #define __smp_load_acquire(p) \
92 ({ \
93 typeof(*p) ___p1 = READ_ONCE(*p); \
94 compiletime_assert_atomic_type(*p); \
95 ldacq_mb(); \
96 ___p1; \
97 })
98
99 #define __smp_store_release(p, v) \
100 do { \
101 compiletime_assert_atomic_type(*p); \
102 strel_mb(); \
103 WRITE_ONCE(*p, v); \
104 } while (0)
105
106 #define __smp_store_mb(p, v) \
107 do { \
108 union { typeof(p) __val; char __c[1]; } __u = \
109 { .__val = (__force typeof(p)) (v) }; \
110 unsigned long __tmp; \
111 switch (sizeof(p)) { \
112 case 1: \
113 *(volatile __u8 *)&p = *(__u8 *)__u.__c; \
114 __smp_mb(); \
115 break; \
116 case 2: \
117 *(volatile __u16 *)&p = *(__u16 *)__u.__c; \
118 __smp_mb(); \
119 break; \
120 case 4: \
121 __asm__ __volatile__( \
122 "amswap_db.w %[tmp], %[val], %[mem] \n" \
123 : [mem] "+ZB" (*(u32 *)&p), [tmp] "=&r" (__tmp) \
124 : [val] "r" (*(__u32 *)__u.__c) \
125 : ); \
126 break; \
127 case 8: \
128 __asm__ __volatile__( \
129 "amswap_db.d %[tmp], %[val], %[mem] \n" \
130 : [mem] "+ZB" (*(u64 *)&p), [tmp] "=&r" (__tmp) \
131 : [val] "r" (*(__u64 *)__u.__c) \
132 : ); \
133 break; \
134 } \
135 } while (0)
136
137 #include <asm-generic/barrier.h>
138
139 #endif /* __ASM_BARRIER_H */
140