1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4  */
5 #ifndef __ASM_BARRIER_H
6 #define __ASM_BARRIER_H
7 
8 #define __sync()	__asm__ __volatile__("dbar 0" : : : "memory")
9 
10 #define fast_wmb()	__sync()
11 #define fast_rmb()	__sync()
12 #define fast_mb()	__sync()
13 #define fast_iob()	__sync()
14 #define wbflush()	__sync()
15 
16 #define wmb()		fast_wmb()
17 #define rmb()		fast_rmb()
18 #define mb()		fast_mb()
19 #define iob()		fast_iob()
20 
21 #define __smp_mb()	__asm__ __volatile__("dbar 0" : : : "memory")
22 #define __smp_rmb()	__asm__ __volatile__("dbar 0" : : : "memory")
23 #define __smp_wmb()	__asm__ __volatile__("dbar 0" : : : "memory")
24 
25 #ifdef CONFIG_SMP
26 #define __WEAK_LLSC_MB		"	dbar 0  \n"
27 #else
28 #define __WEAK_LLSC_MB		"		\n"
29 #endif
30 
31 #define __smp_mb__before_atomic()	barrier()
32 #define __smp_mb__after_atomic()	barrier()
33 
34 /**
35  * array_index_mask_nospec() - generate a ~0 mask when index < size, 0 otherwise
36  * @index: array element index
37  * @size: number of elements in array
38  *
39  * Returns:
40  *     0 - (@index < @size)
41  */
42 #define array_index_mask_nospec array_index_mask_nospec
43 static inline unsigned long array_index_mask_nospec(unsigned long index,
44 						    unsigned long size)
45 {
46 	unsigned long mask;
47 
48 	__asm__ __volatile__(
49 		"sltu	%0, %1, %2\n\t"
50 #if (__SIZEOF_LONG__ == 4)
51 		"sub.w	%0, $r0, %0\n\t"
52 #elif (__SIZEOF_LONG__ == 8)
53 		"sub.d	%0, $r0, %0\n\t"
54 #endif
55 		: "=r" (mask)
56 		: "r" (index), "r" (size)
57 		:);
58 
59 	return mask;
60 }
61 
62 #define __smp_load_acquire(p)							\
63 ({										\
64 	union { typeof(*p) __val; char __c[1]; } __u;				\
65 	unsigned long __tmp = 0;							\
66 	compiletime_assert_atomic_type(*p);					\
67 	switch (sizeof(*p)) {							\
68 	case 1:									\
69 		*(__u8 *)__u.__c = *(volatile __u8 *)p;				\
70 		__smp_mb();							\
71 		break;								\
72 	case 2:									\
73 		*(__u16 *)__u.__c = *(volatile __u16 *)p;			\
74 		__smp_mb();							\
75 		break;								\
76 	case 4:									\
77 		__asm__ __volatile__(						\
78 		"amor_db.w %[val], %[tmp], %[mem]	\n"				\
79 		: [val] "=&r" (*(__u32 *)__u.__c)				\
80 		: [mem] "ZB" (*(u32 *) p), [tmp] "r" (__tmp)			\
81 		: "memory");							\
82 		break;								\
83 	case 8:									\
84 		__asm__ __volatile__(						\
85 		"amor_db.d %[val], %[tmp], %[mem]	\n"				\
86 		: [val] "=&r" (*(__u64 *)__u.__c)				\
87 		: [mem] "ZB" (*(u64 *) p), [tmp] "r" (__tmp)			\
88 		: "memory");							\
89 		break;								\
90 	}									\
91 	(typeof(*p))__u.__val;								\
92 })
93 
94 #define __smp_store_release(p, v)						\
95 do {										\
96 	union { typeof(*p) __val; char __c[1]; } __u =				\
97 		{ .__val = (__force typeof(*p)) (v) };				\
98 	unsigned long __tmp;							\
99 	compiletime_assert_atomic_type(*p);					\
100 	switch (sizeof(*p)) {							\
101 	case 1:									\
102 		__smp_mb();							\
103 		*(volatile __u8 *)p = *(__u8 *)__u.__c;				\
104 		break;								\
105 	case 2:									\
106 		__smp_mb();							\
107 		*(volatile __u16 *)p = *(__u16 *)__u.__c;			\
108 		break;								\
109 	case 4:									\
110 		__asm__ __volatile__(						\
111 		"amswap_db.w %[tmp], %[val], %[mem]	\n"			\
112 		: [mem] "+ZB" (*(u32 *)p), [tmp] "=&r" (__tmp)			\
113 		: [val] "r" (*(__u32 *)__u.__c)					\
114 		: );								\
115 		break;								\
116 	case 8:									\
117 		__asm__ __volatile__(						\
118 		"amswap_db.d %[tmp], %[val], %[mem]	\n"			\
119 		: [mem] "+ZB" (*(u64 *)p), [tmp] "=&r" (__tmp)			\
120 		: [val] "r" (*(__u64 *)__u.__c)					\
121 		: );								\
122 		break;								\
123 	}									\
124 } while (0)
125 
126 #define __smp_store_mb(p, v)							\
127 do {										\
128 	union { typeof(p) __val; char __c[1]; } __u =				\
129 		{ .__val = (__force typeof(p)) (v) };				\
130 	unsigned long __tmp;							\
131 	switch (sizeof(p)) {							\
132 	case 1:									\
133 		*(volatile __u8 *)&p = *(__u8 *)__u.__c;			\
134 		__smp_mb();							\
135 		break;								\
136 	case 2:									\
137 		*(volatile __u16 *)&p = *(__u16 *)__u.__c;			\
138 		__smp_mb();							\
139 		break;								\
140 	case 4:									\
141 		__asm__ __volatile__(						\
142 		"amswap_db.w %[tmp], %[val], %[mem]	\n"			\
143 		: [mem] "+ZB" (*(u32 *)&p), [tmp] "=&r" (__tmp)			\
144 		: [val] "r" (*(__u32 *)__u.__c)					\
145 		: );								\
146 		break;								\
147 	case 8:									\
148 		__asm__ __volatile__(						\
149 		"amswap_db.d %[tmp], %[val], %[mem]	\n"			\
150 		: [mem] "+ZB" (*(u64 *)&p), [tmp] "=&r" (__tmp)			\
151 		: [val] "r" (*(__u64 *)__u.__c)					\
152 		: );								\
153 		break;								\
154 	}									\
155 } while (0)
156 
157 #include <asm-generic/barrier.h>
158 
159 #endif /* __ASM_BARRIER_H */
160