1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4  */
5 #ifndef __ASM_PERCPU_H
6 #define __ASM_PERCPU_H
7 
8 #include <asm/cmpxchg.h>
9 #include <asm/loongarch.h>
10 
11 /*
12  * The "address" (in fact, offset from $r21) of a per-CPU variable is close to
13  * the loading address of main kernel image, but far from where the modules are
14  * loaded. Tell the compiler this fact when using explicit relocs.
15  */
16 #if defined(MODULE) && defined(CONFIG_AS_HAS_EXPLICIT_RELOCS)
17 # if __has_attribute(model)
18 #  define PER_CPU_ATTRIBUTES __attribute__((model("extreme")))
19 # else
20 #  error compiler support for the model attribute is necessary when a recent assembler is used
21 # endif
22 #endif
23 
24 /* Use r21 for fast access */
25 register unsigned long __my_cpu_offset __asm__("$r21");
26 
27 static inline void set_my_cpu_offset(unsigned long off)
28 {
29 	__my_cpu_offset = off;
30 	csr_write64(off, PERCPU_BASE_KS);
31 }
32 
33 #define __my_cpu_offset					\
34 ({							\
35 	__asm__ __volatile__("":"+r"(__my_cpu_offset));	\
36 	__my_cpu_offset;				\
37 })
38 
39 #define PERCPU_OP(op, asm_op, c_op)					\
40 static __always_inline unsigned long __percpu_##op(void *ptr,		\
41 			unsigned long val, int size)			\
42 {									\
43 	unsigned long ret;						\
44 									\
45 	switch (size) {							\
46 	case 4:								\
47 		__asm__ __volatile__(					\
48 		"am"#asm_op".w"	" %[ret], %[val], %[ptr]	\n"		\
49 		: [ret] "=&r" (ret), [ptr] "+ZB"(*(u32 *)ptr)		\
50 		: [val] "r" (val));					\
51 		break;							\
52 	case 8:								\
53 		__asm__ __volatile__(					\
54 		"am"#asm_op".d" " %[ret], %[val], %[ptr]	\n"		\
55 		: [ret] "=&r" (ret), [ptr] "+ZB"(*(u64 *)ptr)		\
56 		: [val] "r" (val));					\
57 		break;							\
58 	default:							\
59 		ret = 0;						\
60 		BUILD_BUG();						\
61 	}								\
62 									\
63 	return ret c_op val;						\
64 }
65 
66 PERCPU_OP(add, add, +)
67 PERCPU_OP(and, and, &)
68 PERCPU_OP(or, or, |)
69 #undef PERCPU_OP
70 
71 static __always_inline unsigned long __percpu_read(void *ptr, int size)
72 {
73 	unsigned long ret;
74 
75 	switch (size) {
76 	case 1:
77 		__asm__ __volatile__ ("ldx.b %[ret], $r21, %[ptr]	\n"
78 		: [ret] "=&r"(ret)
79 		: [ptr] "r"(ptr)
80 		: "memory");
81 		break;
82 	case 2:
83 		__asm__ __volatile__ ("ldx.h %[ret], $r21, %[ptr]	\n"
84 		: [ret] "=&r"(ret)
85 		: [ptr] "r"(ptr)
86 		: "memory");
87 		break;
88 	case 4:
89 		__asm__ __volatile__ ("ldx.w %[ret], $r21, %[ptr]	\n"
90 		: [ret] "=&r"(ret)
91 		: [ptr] "r"(ptr)
92 		: "memory");
93 		break;
94 	case 8:
95 		__asm__ __volatile__ ("ldx.d %[ret], $r21, %[ptr]	\n"
96 		: [ret] "=&r"(ret)
97 		: [ptr] "r"(ptr)
98 		: "memory");
99 		break;
100 	default:
101 		ret = 0;
102 		BUILD_BUG();
103 	}
104 
105 	return ret;
106 }
107 
108 static __always_inline void __percpu_write(void *ptr, unsigned long val, int size)
109 {
110 	switch (size) {
111 	case 1:
112 		__asm__ __volatile__("stx.b %[val], $r21, %[ptr]	\n"
113 		:
114 		: [val] "r" (val), [ptr] "r" (ptr)
115 		: "memory");
116 		break;
117 	case 2:
118 		__asm__ __volatile__("stx.h %[val], $r21, %[ptr]	\n"
119 		:
120 		: [val] "r" (val), [ptr] "r" (ptr)
121 		: "memory");
122 		break;
123 	case 4:
124 		__asm__ __volatile__("stx.w %[val], $r21, %[ptr]	\n"
125 		:
126 		: [val] "r" (val), [ptr] "r" (ptr)
127 		: "memory");
128 		break;
129 	case 8:
130 		__asm__ __volatile__("stx.d %[val], $r21, %[ptr]	\n"
131 		:
132 		: [val] "r" (val), [ptr] "r" (ptr)
133 		: "memory");
134 		break;
135 	default:
136 		BUILD_BUG();
137 	}
138 }
139 
140 static __always_inline unsigned long __percpu_xchg(void *ptr, unsigned long val,
141 						   int size)
142 {
143 	switch (size) {
144 	case 1:
145 	case 2:
146 		return __xchg_small((volatile void *)ptr, val, size);
147 
148 	case 4:
149 		return __xchg_asm("amswap.w", (volatile u32 *)ptr, (u32)val);
150 
151 	case 8:
152 		return __xchg_asm("amswap.d", (volatile u64 *)ptr, (u64)val);
153 
154 	default:
155 		BUILD_BUG();
156 	}
157 
158 	return 0;
159 }
160 
161 /* this_cpu_cmpxchg */
162 #define _protect_cmpxchg_local(pcp, o, n)			\
163 ({								\
164 	typeof(*raw_cpu_ptr(&(pcp))) __ret;			\
165 	preempt_disable_notrace();				\
166 	__ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n);	\
167 	preempt_enable_notrace();				\
168 	__ret;							\
169 })
170 
171 #define _percpu_read(pcp)						\
172 ({									\
173 	typeof(pcp) __retval;						\
174 	__retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp));	\
175 	__retval;							\
176 })
177 
178 #define _percpu_write(pcp, val)						\
179 do {									\
180 	__percpu_write(&(pcp), (unsigned long)(val), sizeof(pcp));	\
181 } while (0)								\
182 
183 #define _pcp_protect(operation, pcp, val)			\
184 ({								\
185 	typeof(pcp) __retval;					\
186 	preempt_disable_notrace();				\
187 	__retval = (typeof(pcp))operation(raw_cpu_ptr(&(pcp)),	\
188 					  (val), sizeof(pcp));	\
189 	preempt_enable_notrace();				\
190 	__retval;						\
191 })
192 
193 #define _percpu_add(pcp, val) \
194 	_pcp_protect(__percpu_add, pcp, val)
195 
196 #define _percpu_add_return(pcp, val) _percpu_add(pcp, val)
197 
198 #define _percpu_and(pcp, val) \
199 	_pcp_protect(__percpu_and, pcp, val)
200 
201 #define _percpu_or(pcp, val) \
202 	_pcp_protect(__percpu_or, pcp, val)
203 
204 #define _percpu_xchg(pcp, val) ((typeof(pcp)) \
205 	_pcp_protect(__percpu_xchg, pcp, (unsigned long)(val)))
206 
207 #define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
208 #define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
209 
210 #define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
211 #define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
212 
213 #define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
214 #define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
215 
216 #define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
217 #define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
218 
219 #define this_cpu_read_1(pcp) _percpu_read(pcp)
220 #define this_cpu_read_2(pcp) _percpu_read(pcp)
221 #define this_cpu_read_4(pcp) _percpu_read(pcp)
222 #define this_cpu_read_8(pcp) _percpu_read(pcp)
223 
224 #define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
225 #define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
226 #define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
227 #define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)
228 
229 #define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
230 #define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
231 #define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
232 #define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
233 
234 #define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
235 #define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
236 #define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
237 #define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
238 
239 #include <asm-generic/percpu.h>
240 
241 #endif /* __ASM_PERCPU_H */
242