xref: /openbmc/linux/arch/s390/include/asm/percpu.h (revision b34e08d5)
1 #ifndef __ARCH_S390_PERCPU__
2 #define __ARCH_S390_PERCPU__
3 
4 #include <linux/preempt.h>
5 #include <asm/cmpxchg.h>
6 
7 /*
8  * s390 uses its own implementation for per cpu data, the offset of
9  * the cpu local data area is cached in the cpu's lowcore memory.
10  */
11 #define __my_cpu_offset S390_lowcore.percpu_offset
12 
13 #ifdef CONFIG_64BIT
14 
15 /*
16  * For 64 bit module code, the module may be more than 4G above the
17  * per cpu area, use weak definitions to force the compiler to
18  * generate external references.
19  */
20 #if defined(CONFIG_SMP) && defined(MODULE)
21 #define ARCH_NEEDS_WEAK_PER_CPU
22 #endif
23 
24 /*
25  * We use a compare-and-swap loop since that uses less cpu cycles than
26  * disabling and enabling interrupts like the generic variant would do.
27  */
28 #define arch_this_cpu_to_op_simple(pcp, val, op)			\
29 ({									\
30 	typedef typeof(pcp) pcp_op_T__;					\
31 	pcp_op_T__ old__, new__, prev__;				\
32 	pcp_op_T__ *ptr__;						\
33 	preempt_disable();						\
34 	ptr__ = __this_cpu_ptr(&(pcp));					\
35 	prev__ = *ptr__;						\
36 	do {								\
37 		old__ = prev__;						\
38 		new__ = old__ op (val);					\
39 		prev__ = cmpxchg(ptr__, old__, new__);			\
40 	} while (prev__ != old__);					\
41 	preempt_enable();						\
42 	new__;								\
43 })
44 
45 #define this_cpu_add_1(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, +)
46 #define this_cpu_add_2(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, +)
47 #define this_cpu_add_return_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
48 #define this_cpu_add_return_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
49 #define this_cpu_and_1(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, &)
50 #define this_cpu_and_2(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, &)
51 #define this_cpu_or_1(pcp, val)		arch_this_cpu_to_op_simple(pcp, val, |)
52 #define this_cpu_or_2(pcp, val)		arch_this_cpu_to_op_simple(pcp, val, |)
53 
54 #ifndef CONFIG_HAVE_MARCH_Z196_FEATURES
55 
56 #define this_cpu_add_4(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, +)
57 #define this_cpu_add_8(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, +)
58 #define this_cpu_add_return_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
59 #define this_cpu_add_return_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
60 #define this_cpu_and_4(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, &)
61 #define this_cpu_and_8(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, &)
62 #define this_cpu_or_4(pcp, val)		arch_this_cpu_to_op_simple(pcp, val, |)
63 #define this_cpu_or_8(pcp, val)		arch_this_cpu_to_op_simple(pcp, val, |)
64 
65 #else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
66 
67 #define arch_this_cpu_add(pcp, val, op1, op2, szcast)			\
68 {									\
69 	typedef typeof(pcp) pcp_op_T__; 				\
70 	pcp_op_T__ val__ = (val);					\
71 	pcp_op_T__ old__, *ptr__;					\
72 	preempt_disable();						\
73 	ptr__ = __this_cpu_ptr(&(pcp)); 				\
74 	if (__builtin_constant_p(val__) &&				\
75 	    ((szcast)val__ > -129) && ((szcast)val__ < 128)) {		\
76 		asm volatile(						\
77 			op2 "   %[ptr__],%[val__]\n"			\
78 			: [ptr__] "+Q" (*ptr__) 			\
79 			: [val__] "i" ((szcast)val__)			\
80 			: "cc");					\
81 	} else {							\
82 		asm volatile(						\
83 			op1 "   %[old__],%[val__],%[ptr__]\n"		\
84 			: [old__] "=d" (old__), [ptr__] "+Q" (*ptr__)	\
85 			: [val__] "d" (val__)				\
86 			: "cc");					\
87 	}								\
88 	preempt_enable();						\
89 }
90 
91 #define this_cpu_add_4(pcp, val) arch_this_cpu_add(pcp, val, "laa", "asi", int)
92 #define this_cpu_add_8(pcp, val) arch_this_cpu_add(pcp, val, "laag", "agsi", long)
93 
94 #define arch_this_cpu_add_return(pcp, val, op)				\
95 ({									\
96 	typedef typeof(pcp) pcp_op_T__; 				\
97 	pcp_op_T__ val__ = (val);					\
98 	pcp_op_T__ old__, *ptr__;					\
99 	preempt_disable();						\
100 	ptr__ = __this_cpu_ptr(&(pcp)); 				\
101 	asm volatile(							\
102 		op "    %[old__],%[val__],%[ptr__]\n"			\
103 		: [old__] "=d" (old__), [ptr__] "+Q" (*ptr__)		\
104 		: [val__] "d" (val__)					\
105 		: "cc");						\
106 	preempt_enable();						\
107 	old__ + val__;							\
108 })
109 
110 #define this_cpu_add_return_4(pcp, val) arch_this_cpu_add_return(pcp, val, "laa")
111 #define this_cpu_add_return_8(pcp, val) arch_this_cpu_add_return(pcp, val, "laag")
112 
113 #define arch_this_cpu_to_op(pcp, val, op)				\
114 {									\
115 	typedef typeof(pcp) pcp_op_T__; 				\
116 	pcp_op_T__ val__ = (val);					\
117 	pcp_op_T__ old__, *ptr__;					\
118 	preempt_disable();						\
119 	ptr__ = __this_cpu_ptr(&(pcp)); 				\
120 	asm volatile(							\
121 		op "    %[old__],%[val__],%[ptr__]\n"			\
122 		: [old__] "=d" (old__), [ptr__] "+Q" (*ptr__)		\
123 		: [val__] "d" (val__)					\
124 		: "cc");						\
125 	preempt_enable();						\
126 }
127 
128 #define this_cpu_and_4(pcp, val)	arch_this_cpu_to_op(pcp, val, "lan")
129 #define this_cpu_and_8(pcp, val)	arch_this_cpu_to_op(pcp, val, "lang")
130 #define this_cpu_or_4(pcp, val)		arch_this_cpu_to_op(pcp, val, "lao")
131 #define this_cpu_or_8(pcp, val)		arch_this_cpu_to_op(pcp, val, "laog")
132 
133 #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
134 
135 #define arch_this_cpu_cmpxchg(pcp, oval, nval)				\
136 ({									\
137 	typedef typeof(pcp) pcp_op_T__;					\
138 	pcp_op_T__ ret__;						\
139 	pcp_op_T__ *ptr__;						\
140 	preempt_disable();						\
141 	ptr__ = __this_cpu_ptr(&(pcp));					\
142 	ret__ = cmpxchg(ptr__, oval, nval);				\
143 	preempt_enable();						\
144 	ret__;								\
145 })
146 
147 #define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
148 #define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
149 #define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
150 #define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
151 
152 #define arch_this_cpu_xchg(pcp, nval)					\
153 ({									\
154 	typeof(pcp) *ptr__;						\
155 	typeof(pcp) ret__;						\
156 	preempt_disable();						\
157 	ptr__ = __this_cpu_ptr(&(pcp));					\
158 	ret__ = xchg(ptr__, nval);					\
159 	preempt_enable();						\
160 	ret__;								\
161 })
162 
163 #define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg(pcp, nval)
164 #define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg(pcp, nval)
165 #define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval)
166 #define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval)
167 
168 #define arch_this_cpu_cmpxchg_double(pcp1, pcp2, o1, o2, n1, n2)	\
169 ({									\
170 	typeof(pcp1) o1__ = (o1), n1__ = (n1);				\
171 	typeof(pcp2) o2__ = (o2), n2__ = (n2);				\
172 	typeof(pcp1) *p1__;						\
173 	typeof(pcp2) *p2__;						\
174 	int ret__;							\
175 	preempt_disable();						\
176 	p1__ = __this_cpu_ptr(&(pcp1));					\
177 	p2__ = __this_cpu_ptr(&(pcp2));					\
178 	ret__ = __cmpxchg_double(p1__, p2__, o1__, o2__, n1__, n2__);	\
179 	preempt_enable();						\
180 	ret__;								\
181 })
182 
183 #define this_cpu_cmpxchg_double_4 arch_this_cpu_cmpxchg_double
184 #define this_cpu_cmpxchg_double_8 arch_this_cpu_cmpxchg_double
185 
186 #endif /* CONFIG_64BIT */
187 
188 #include <asm-generic/percpu.h>
189 
190 #endif /* __ARCH_S390_PERCPU__ */
191