1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4  */
5 #ifndef __ASM_BARRIER_H
6 #define __ASM_BARRIER_H
7 
8 /*
9  * Hint encoding:
10  *
11  * Bit4: ordering or completion (0: completion, 1: ordering)
12  * Bit3: barrier for previous read (0: true, 1: false)
13  * Bit2: barrier for previous write (0: true, 1: false)
14  * Bit1: barrier for succeeding read (0: true, 1: false)
15  * Bit0: barrier for succeeding write (0: true, 1: false)
16  *
17  * Hint 0x700: barrier for "read after read" from the same address
18  */
19 
20 #define DBAR(hint) __asm__ __volatile__("dbar %0 " : : "I"(hint) : "memory")
21 
22 #define crwrw		0b00000
23 #define cr_r_		0b00101
24 #define c_w_w		0b01010
25 
26 #define orwrw		0b10000
27 #define or_r_		0b10101
28 #define o_w_w		0b11010
29 
30 #define orw_w		0b10010
31 #define or_rw		0b10100
32 
33 #define c_sync()	DBAR(crwrw)
34 #define c_rsync()	DBAR(cr_r_)
35 #define c_wsync()	DBAR(c_w_w)
36 
37 #define o_sync()	DBAR(orwrw)
38 #define o_rsync()	DBAR(or_r_)
39 #define o_wsync()	DBAR(o_w_w)
40 
41 #define ldacq_mb()	DBAR(or_rw)
42 #define strel_mb()	DBAR(orw_w)
43 
44 #define mb()		c_sync()
45 #define rmb()		c_rsync()
46 #define wmb()		c_wsync()
47 #define iob()		c_sync()
48 #define wbflush()	c_sync()
49 
50 #define __smp_mb()	o_sync()
51 #define __smp_rmb()	o_rsync()
52 #define __smp_wmb()	o_wsync()
53 
54 #ifdef CONFIG_SMP
55 #define __WEAK_LLSC_MB		"	dbar 0x700	\n"
56 #else
57 #define __WEAK_LLSC_MB		"			\n"
58 #endif
59 
60 #define __smp_mb__before_atomic()	barrier()
61 #define __smp_mb__after_atomic()	barrier()
62 
63 /**
64  * array_index_mask_nospec() - generate a ~0 mask when index < size, 0 otherwise
65  * @index: array element index
66  * @size: number of elements in array
67  *
68  * Returns:
69  *     0 - (@index < @size)
70  */
71 #define array_index_mask_nospec array_index_mask_nospec
array_index_mask_nospec(unsigned long index,unsigned long size)72 static inline unsigned long array_index_mask_nospec(unsigned long index,
73 						    unsigned long size)
74 {
75 	unsigned long mask;
76 
77 	__asm__ __volatile__(
78 		"sltu	%0, %1, %2\n\t"
79 #if (__SIZEOF_LONG__ == 4)
80 		"sub.w	%0, $zero, %0\n\t"
81 #elif (__SIZEOF_LONG__ == 8)
82 		"sub.d	%0, $zero, %0\n\t"
83 #endif
84 		: "=r" (mask)
85 		: "r" (index), "r" (size)
86 		:);
87 
88 	return mask;
89 }
90 
91 #define __smp_load_acquire(p)				\
92 ({							\
93 	typeof(*p) ___p1 = READ_ONCE(*p);		\
94 	compiletime_assert_atomic_type(*p);		\
95 	ldacq_mb();					\
96 	___p1;						\
97 })
98 
99 #define __smp_store_release(p, v)			\
100 do {							\
101 	compiletime_assert_atomic_type(*p);		\
102 	strel_mb();					\
103 	WRITE_ONCE(*p, v);				\
104 } while (0)
105 
106 #define __smp_store_mb(p, v)							\
107 do {										\
108 	union { typeof(p) __val; char __c[1]; } __u =				\
109 		{ .__val = (__force typeof(p)) (v) };				\
110 	unsigned long __tmp;							\
111 	switch (sizeof(p)) {							\
112 	case 1:									\
113 		*(volatile __u8 *)&p = *(__u8 *)__u.__c;			\
114 		__smp_mb();							\
115 		break;								\
116 	case 2:									\
117 		*(volatile __u16 *)&p = *(__u16 *)__u.__c;			\
118 		__smp_mb();							\
119 		break;								\
120 	case 4:									\
121 		__asm__ __volatile__(						\
122 		"amswap_db.w %[tmp], %[val], %[mem]	\n"			\
123 		: [mem] "+ZB" (*(u32 *)&p), [tmp] "=&r" (__tmp)			\
124 		: [val] "r" (*(__u32 *)__u.__c)					\
125 		: );								\
126 		break;								\
127 	case 8:									\
128 		__asm__ __volatile__(						\
129 		"amswap_db.d %[tmp], %[val], %[mem]	\n"			\
130 		: [mem] "+ZB" (*(u64 *)&p), [tmp] "=&r" (__tmp)			\
131 		: [val] "r" (*(__u64 *)__u.__c)					\
132 		: );								\
133 		break;								\
134 	}									\
135 } while (0)
136 
137 #include <asm-generic/barrier.h>
138 
139 #endif /* __ASM_BARRIER_H */
140