1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __ARCH_S390_PERCPU__ 3 #define __ARCH_S390_PERCPU__ 4 5 #include <linux/preempt.h> 6 #include <asm/cmpxchg.h> 7 8 /* 9 * s390 uses its own implementation for per cpu data, the offset of 10 * the cpu local data area is cached in the cpu's lowcore memory. 11 */ 12 #define __my_cpu_offset S390_lowcore.percpu_offset 13 14 /* 15 * For 64 bit module code, the module may be more than 4G above the 16 * per cpu area, use weak definitions to force the compiler to 17 * generate external references. 18 */ 19 #if defined(MODULE) 20 #define ARCH_NEEDS_WEAK_PER_CPU 21 #endif 22 23 /* 24 * We use a compare-and-swap loop since that uses less cpu cycles than 25 * disabling and enabling interrupts like the generic variant would do. 26 */ 27 #define arch_this_cpu_to_op_simple(pcp, val, op) \ 28 ({ \ 29 typedef typeof(pcp) pcp_op_T__; \ 30 pcp_op_T__ old__, new__, prev__; \ 31 pcp_op_T__ *ptr__; \ 32 preempt_disable_notrace(); \ 33 ptr__ = raw_cpu_ptr(&(pcp)); \ 34 prev__ = READ_ONCE(*ptr__); \ 35 do { \ 36 old__ = prev__; \ 37 new__ = old__ op (val); \ 38 prev__ = cmpxchg(ptr__, old__, new__); \ 39 } while (prev__ != old__); \ 40 preempt_enable_notrace(); \ 41 new__; \ 42 }) 43 44 #define this_cpu_add_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 45 #define this_cpu_add_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 46 #define this_cpu_add_return_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 47 #define this_cpu_add_return_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 48 #define this_cpu_and_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) 49 #define this_cpu_and_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) 50 #define this_cpu_or_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) 51 #define this_cpu_or_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) 52 53 #ifndef CONFIG_HAVE_MARCH_Z196_FEATURES 54 55 #define this_cpu_add_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 56 #define this_cpu_add_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 57 #define this_cpu_add_return_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 58 #define this_cpu_add_return_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 59 #define this_cpu_and_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) 60 #define this_cpu_and_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) 61 #define this_cpu_or_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) 62 #define this_cpu_or_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) 63 64 #else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ 65 66 #define arch_this_cpu_add(pcp, val, op1, op2, szcast) \ 67 { \ 68 typedef typeof(pcp) pcp_op_T__; \ 69 pcp_op_T__ val__ = (val); \ 70 pcp_op_T__ old__, *ptr__; \ 71 preempt_disable_notrace(); \ 72 ptr__ = raw_cpu_ptr(&(pcp)); \ 73 if (__builtin_constant_p(val__) && \ 74 ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ 75 asm volatile( \ 76 op2 " %[ptr__],%[val__]\n" \ 77 : [ptr__] "+Q" (*ptr__) \ 78 : [val__] "i" ((szcast)val__) \ 79 : "cc"); \ 80 } else { \ 81 asm volatile( \ 82 op1 " %[old__],%[val__],%[ptr__]\n" \ 83 : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ 84 : [val__] "d" (val__) \ 85 : "cc"); \ 86 } \ 87 preempt_enable_notrace(); \ 88 } 89 90 #define this_cpu_add_4(pcp, val) arch_this_cpu_add(pcp, val, "laa", "asi", int) 91 #define this_cpu_add_8(pcp, val) arch_this_cpu_add(pcp, val, "laag", "agsi", long) 92 93 #define arch_this_cpu_add_return(pcp, val, op) \ 94 ({ \ 95 typedef typeof(pcp) pcp_op_T__; \ 96 pcp_op_T__ val__ = (val); \ 97 pcp_op_T__ old__, *ptr__; \ 98 preempt_disable_notrace(); \ 99 ptr__ = raw_cpu_ptr(&(pcp)); \ 100 asm volatile( \ 101 op " %[old__],%[val__],%[ptr__]\n" \ 102 : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ 103 : [val__] "d" (val__) \ 104 : "cc"); \ 105 preempt_enable_notrace(); \ 106 old__ + val__; \ 107 }) 108 109 #define this_cpu_add_return_4(pcp, val) arch_this_cpu_add_return(pcp, val, "laa") 110 #define this_cpu_add_return_8(pcp, val) arch_this_cpu_add_return(pcp, val, "laag") 111 112 #define arch_this_cpu_to_op(pcp, val, op) \ 113 { \ 114 typedef typeof(pcp) pcp_op_T__; \ 115 pcp_op_T__ val__ = (val); \ 116 pcp_op_T__ old__, *ptr__; \ 117 preempt_disable_notrace(); \ 118 ptr__ = raw_cpu_ptr(&(pcp)); \ 119 asm volatile( \ 120 op " %[old__],%[val__],%[ptr__]\n" \ 121 : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ 122 : [val__] "d" (val__) \ 123 : "cc"); \ 124 preempt_enable_notrace(); \ 125 } 126 127 #define this_cpu_and_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lan") 128 #define this_cpu_and_8(pcp, val) arch_this_cpu_to_op(pcp, val, "lang") 129 #define this_cpu_or_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lao") 130 #define this_cpu_or_8(pcp, val) arch_this_cpu_to_op(pcp, val, "laog") 131 132 #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ 133 134 #define arch_this_cpu_cmpxchg(pcp, oval, nval) \ 135 ({ \ 136 typedef typeof(pcp) pcp_op_T__; \ 137 pcp_op_T__ ret__; \ 138 pcp_op_T__ *ptr__; \ 139 preempt_disable_notrace(); \ 140 ptr__ = raw_cpu_ptr(&(pcp)); \ 141 ret__ = cmpxchg(ptr__, oval, nval); \ 142 preempt_enable_notrace(); \ 143 ret__; \ 144 }) 145 146 #define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) 147 #define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) 148 #define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) 149 #define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) 150 151 #define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n) 152 153 #define this_cpu_cmpxchg128(pcp, oval, nval) \ 154 ({ \ 155 typedef typeof(pcp) pcp_op_T__; \ 156 u128 old__, new__, ret__; \ 157 pcp_op_T__ *ptr__; \ 158 old__ = oval; \ 159 new__ = nval; \ 160 preempt_disable_notrace(); \ 161 ptr__ = raw_cpu_ptr(&(pcp)); \ 162 ret__ = cmpxchg128((void *)ptr__, old__, new__); \ 163 preempt_enable_notrace(); \ 164 ret__; \ 165 }) 166 167 #define arch_this_cpu_xchg(pcp, nval) \ 168 ({ \ 169 typeof(pcp) *ptr__; \ 170 typeof(pcp) ret__; \ 171 preempt_disable_notrace(); \ 172 ptr__ = raw_cpu_ptr(&(pcp)); \ 173 ret__ = xchg(ptr__, nval); \ 174 preempt_enable_notrace(); \ 175 ret__; \ 176 }) 177 178 #define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg(pcp, nval) 179 #define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg(pcp, nval) 180 #define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval) 181 #define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval) 182 183 #include <asm-generic/percpu.h> 184 185 #endif /* __ARCH_S390_PERCPU__ */ 186