1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4  */
5 #ifndef __ASM_CMPXCHG_H
6 #define __ASM_CMPXCHG_H
7 
8 #include <linux/bits.h>
9 #include <linux/build_bug.h>
10 #include <asm/barrier.h>
11 
12 #define __xchg_asm(amswap_db, m, val)		\
13 ({						\
14 		__typeof(val) __ret;		\
15 						\
16 		__asm__ __volatile__ (		\
17 		" "amswap_db" %1, %z2, %0 \n"	\
18 		: "+ZB" (*m), "=&r" (__ret)	\
19 		: "Jr" (val)			\
20 		: "memory");			\
21 						\
22 		__ret;				\
23 })
24 
25 static inline unsigned int __xchg_small(volatile void *ptr, unsigned int val,
26 					unsigned int size)
27 {
28 	unsigned int shift;
29 	u32 old32, mask, temp;
30 	volatile u32 *ptr32;
31 
32 	/* Mask value to the correct size. */
33 	mask = GENMASK((size * BITS_PER_BYTE) - 1, 0);
34 	val &= mask;
35 
36 	/*
37 	 * Calculate a shift & mask that correspond to the value we wish to
38 	 * exchange within the naturally aligned 4 byte integerthat includes
39 	 * it.
40 	 */
41 	shift = (unsigned long)ptr & 0x3;
42 	shift *= BITS_PER_BYTE;
43 	mask <<= shift;
44 
45 	/*
46 	 * Calculate a pointer to the naturally aligned 4 byte integer that
47 	 * includes our byte of interest, and load its value.
48 	 */
49 	ptr32 = (volatile u32 *)((unsigned long)ptr & ~0x3);
50 
51 	asm volatile (
52 	"1:	ll.w		%0, %3		\n"
53 	"	andn		%1, %0, %z4	\n"
54 	"	or		%1, %1, %z5	\n"
55 	"	sc.w		%1, %2		\n"
56 	"	beqz		%1, 1b		\n"
57 	: "=&r" (old32), "=&r" (temp), "=ZC" (*ptr32)
58 	: "ZC" (*ptr32), "Jr" (mask), "Jr" (val << shift)
59 	: "memory");
60 
61 	return (old32 & mask) >> shift;
62 }
63 
64 static __always_inline unsigned long
65 __arch_xchg(volatile void *ptr, unsigned long x, int size)
66 {
67 	switch (size) {
68 	case 1:
69 	case 2:
70 		return __xchg_small(ptr, x, size);
71 
72 	case 4:
73 		return __xchg_asm("amswap_db.w", (volatile u32 *)ptr, (u32)x);
74 
75 	case 8:
76 		return __xchg_asm("amswap_db.d", (volatile u64 *)ptr, (u64)x);
77 
78 	default:
79 		BUILD_BUG();
80 	}
81 
82 	return 0;
83 }
84 
85 #define arch_xchg(ptr, x)						\
86 ({									\
87 	__typeof__(*(ptr)) __res;					\
88 									\
89 	__res = (__typeof__(*(ptr)))					\
90 		__arch_xchg((ptr), (unsigned long)(x), sizeof(*(ptr)));	\
91 									\
92 	__res;								\
93 })
94 
95 #define __cmpxchg_asm(ld, st, m, old, new)				\
96 ({									\
97 	__typeof(old) __ret;						\
98 									\
99 	__asm__ __volatile__(						\
100 	"1:	" ld "	%0, %2		# __cmpxchg_asm \n"		\
101 	"	bne	%0, %z3, 2f			\n"		\
102 	"	move	$t0, %z4			\n"		\
103 	"	" st "	$t0, %1				\n"		\
104 	"	beqz	$t0, 1b				\n"		\
105 	"2:						\n"		\
106 	__WEAK_LLSC_MB							\
107 	: "=&r" (__ret), "=ZB"(*m)					\
108 	: "ZB"(*m), "Jr" (old), "Jr" (new)				\
109 	: "t0", "memory");						\
110 									\
111 	__ret;								\
112 })
113 
114 static inline unsigned int __cmpxchg_small(volatile void *ptr, unsigned int old,
115 					   unsigned int new, unsigned int size)
116 {
117 	unsigned int shift;
118 	u32 old32, mask, temp;
119 	volatile u32 *ptr32;
120 
121 	/* Mask inputs to the correct size. */
122 	mask = GENMASK((size * BITS_PER_BYTE) - 1, 0);
123 	old &= mask;
124 	new &= mask;
125 
126 	/*
127 	 * Calculate a shift & mask that correspond to the value we wish to
128 	 * compare & exchange within the naturally aligned 4 byte integer
129 	 * that includes it.
130 	 */
131 	shift = (unsigned long)ptr & 0x3;
132 	shift *= BITS_PER_BYTE;
133 	old <<= shift;
134 	new <<= shift;
135 	mask <<= shift;
136 
137 	/*
138 	 * Calculate a pointer to the naturally aligned 4 byte integer that
139 	 * includes our byte of interest, and load its value.
140 	 */
141 	ptr32 = (volatile u32 *)((unsigned long)ptr & ~0x3);
142 
143 	asm volatile (
144 	"1:	ll.w		%0, %3		\n"
145 	"	and		%1, %0, %z4	\n"
146 	"	bne		%1, %z5, 2f	\n"
147 	"	andn		%1, %0, %z4	\n"
148 	"	or		%1, %1, %z6	\n"
149 	"	sc.w		%1, %2		\n"
150 	"	beqz		%1, 1b		\n"
151 	"	b		3f		\n"
152 	"2:					\n"
153 	__WEAK_LLSC_MB
154 	"3:					\n"
155 	: "=&r" (old32), "=&r" (temp), "=ZC" (*ptr32)
156 	: "ZC" (*ptr32), "Jr" (mask), "Jr" (old), "Jr" (new)
157 	: "memory");
158 
159 	return (old32 & mask) >> shift;
160 }
161 
162 static __always_inline unsigned long
163 __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, unsigned int size)
164 {
165 	switch (size) {
166 	case 1:
167 	case 2:
168 		return __cmpxchg_small(ptr, old, new, size);
169 
170 	case 4:
171 		return __cmpxchg_asm("ll.w", "sc.w", (volatile u32 *)ptr,
172 				     (u32)old, new);
173 
174 	case 8:
175 		return __cmpxchg_asm("ll.d", "sc.d", (volatile u64 *)ptr,
176 				     (u64)old, new);
177 
178 	default:
179 		BUILD_BUG();
180 	}
181 
182 	return 0;
183 }
184 
185 #define arch_cmpxchg_local(ptr, old, new)				\
186 	((__typeof__(*(ptr)))						\
187 		__cmpxchg((ptr),					\
188 			  (unsigned long)(__typeof__(*(ptr)))(old),	\
189 			  (unsigned long)(__typeof__(*(ptr)))(new),	\
190 			  sizeof(*(ptr))))
191 
192 #define arch_cmpxchg(ptr, old, new)					\
193 ({									\
194 	__typeof__(*(ptr)) __res;					\
195 									\
196 	__res = arch_cmpxchg_local((ptr), (old), (new));		\
197 									\
198 	__res;								\
199 })
200 
201 #ifdef CONFIG_64BIT
202 #define arch_cmpxchg64_local(ptr, o, n)					\
203   ({									\
204 	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
205 	arch_cmpxchg_local((ptr), (o), (n));				\
206   })
207 
208 #define arch_cmpxchg64(ptr, o, n)					\
209   ({									\
210 	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
211 	arch_cmpxchg((ptr), (o), (n));					\
212   })
213 #else
214 #include <asm-generic/cmpxchg-local.h>
215 #define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n))
216 #define arch_cmpxchg64(ptr, o, n) arch_cmpxchg64_local((ptr), (o), (n))
217 #endif
218 
219 #endif /* __ASM_CMPXCHG_H */
220