1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_GENERIC_PERCPU_H_ 3 #define _ASM_GENERIC_PERCPU_H_ 4 5 #include <linux/compiler.h> 6 #include <linux/threads.h> 7 #include <linux/percpu-defs.h> 8 9 #ifdef CONFIG_SMP 10 11 /* 12 * per_cpu_offset() is the offset that has to be added to a 13 * percpu variable to get to the instance for a certain processor. 14 * 15 * Most arches use the __per_cpu_offset array for those offsets but 16 * some arches have their own ways of determining the offset (x86_64, s390). 17 */ 18 #ifndef __per_cpu_offset 19 extern unsigned long __per_cpu_offset[NR_CPUS]; 20 21 #define per_cpu_offset(x) (__per_cpu_offset[x]) 22 #endif 23 24 /* 25 * Determine the offset for the currently active processor. 26 * An arch may define __my_cpu_offset to provide a more effective 27 * means of obtaining the offset to the per cpu variables of the 28 * current processor. 29 */ 30 #ifndef __my_cpu_offset 31 #define __my_cpu_offset per_cpu_offset(raw_smp_processor_id()) 32 #endif 33 #ifdef CONFIG_DEBUG_PREEMPT 34 #define my_cpu_offset per_cpu_offset(smp_processor_id()) 35 #else 36 #define my_cpu_offset __my_cpu_offset 37 #endif 38 39 /* 40 * Arch may define arch_raw_cpu_ptr() to provide more efficient address 41 * translations for raw_cpu_ptr(). 42 */ 43 #ifndef arch_raw_cpu_ptr 44 #define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset) 45 #endif 46 47 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA 48 extern void setup_per_cpu_areas(void); 49 #endif 50 51 #endif /* SMP */ 52 53 #ifndef PER_CPU_BASE_SECTION 54 #ifdef CONFIG_SMP 55 #define PER_CPU_BASE_SECTION ".data..percpu" 56 #else 57 #define PER_CPU_BASE_SECTION ".data" 58 #endif 59 #endif 60 61 #ifndef PER_CPU_ATTRIBUTES 62 #define PER_CPU_ATTRIBUTES 63 #endif 64 65 #ifndef PER_CPU_DEF_ATTRIBUTES 66 #define PER_CPU_DEF_ATTRIBUTES 67 #endif 68 69 #define raw_cpu_generic_read(pcp) \ 70 ({ \ 71 *raw_cpu_ptr(&(pcp)); \ 72 }) 73 74 #define raw_cpu_generic_to_op(pcp, val, op) \ 75 do { \ 76 *raw_cpu_ptr(&(pcp)) op val; \ 77 } while (0) 78 79 #define raw_cpu_generic_add_return(pcp, val) \ 80 ({ \ 81 typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp)); \ 82 \ 83 *__p += val; \ 84 *__p; \ 85 }) 86 87 #define raw_cpu_generic_xchg(pcp, nval) \ 88 ({ \ 89 typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp)); \ 90 typeof(pcp) __ret; \ 91 __ret = *__p; \ 92 *__p = nval; \ 93 __ret; \ 94 }) 95 96 #define raw_cpu_generic_cmpxchg(pcp, oval, nval) \ 97 ({ \ 98 typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp)); \ 99 typeof(pcp) __ret; \ 100 __ret = *__p; \ 101 if (__ret == (oval)) \ 102 *__p = nval; \ 103 __ret; \ 104 }) 105 106 #define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 107 ({ \ 108 typeof(&(pcp1)) __p1 = raw_cpu_ptr(&(pcp1)); \ 109 typeof(&(pcp2)) __p2 = raw_cpu_ptr(&(pcp2)); \ 110 int __ret = 0; \ 111 if (*__p1 == (oval1) && *__p2 == (oval2)) { \ 112 *__p1 = nval1; \ 113 *__p2 = nval2; \ 114 __ret = 1; \ 115 } \ 116 (__ret); \ 117 }) 118 119 #define __this_cpu_generic_read_nopreempt(pcp) \ 120 ({ \ 121 typeof(pcp) __ret; \ 122 preempt_disable_notrace(); \ 123 __ret = READ_ONCE(*raw_cpu_ptr(&(pcp))); \ 124 preempt_enable_notrace(); \ 125 __ret; \ 126 }) 127 128 #define __this_cpu_generic_read_noirq(pcp) \ 129 ({ \ 130 typeof(pcp) __ret; \ 131 unsigned long __flags; \ 132 raw_local_irq_save(__flags); \ 133 __ret = raw_cpu_generic_read(pcp); \ 134 raw_local_irq_restore(__flags); \ 135 __ret; \ 136 }) 137 138 #define this_cpu_generic_read(pcp) \ 139 ({ \ 140 typeof(pcp) __ret; \ 141 if (__native_word(pcp)) \ 142 __ret = __this_cpu_generic_read_nopreempt(pcp); \ 143 else \ 144 __ret = __this_cpu_generic_read_noirq(pcp); \ 145 __ret; \ 146 }) 147 148 #define this_cpu_generic_to_op(pcp, val, op) \ 149 do { \ 150 unsigned long __flags; \ 151 raw_local_irq_save(__flags); \ 152 raw_cpu_generic_to_op(pcp, val, op); \ 153 raw_local_irq_restore(__flags); \ 154 } while (0) 155 156 157 #define this_cpu_generic_add_return(pcp, val) \ 158 ({ \ 159 typeof(pcp) __ret; \ 160 unsigned long __flags; \ 161 raw_local_irq_save(__flags); \ 162 __ret = raw_cpu_generic_add_return(pcp, val); \ 163 raw_local_irq_restore(__flags); \ 164 __ret; \ 165 }) 166 167 #define this_cpu_generic_xchg(pcp, nval) \ 168 ({ \ 169 typeof(pcp) __ret; \ 170 unsigned long __flags; \ 171 raw_local_irq_save(__flags); \ 172 __ret = raw_cpu_generic_xchg(pcp, nval); \ 173 raw_local_irq_restore(__flags); \ 174 __ret; \ 175 }) 176 177 #define this_cpu_generic_cmpxchg(pcp, oval, nval) \ 178 ({ \ 179 typeof(pcp) __ret; \ 180 unsigned long __flags; \ 181 raw_local_irq_save(__flags); \ 182 __ret = raw_cpu_generic_cmpxchg(pcp, oval, nval); \ 183 raw_local_irq_restore(__flags); \ 184 __ret; \ 185 }) 186 187 #define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 188 ({ \ 189 int __ret; \ 190 unsigned long __flags; \ 191 raw_local_irq_save(__flags); \ 192 __ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \ 193 oval1, oval2, nval1, nval2); \ 194 raw_local_irq_restore(__flags); \ 195 __ret; \ 196 }) 197 198 #ifndef raw_cpu_read_1 199 #define raw_cpu_read_1(pcp) raw_cpu_generic_read(pcp) 200 #endif 201 #ifndef raw_cpu_read_2 202 #define raw_cpu_read_2(pcp) raw_cpu_generic_read(pcp) 203 #endif 204 #ifndef raw_cpu_read_4 205 #define raw_cpu_read_4(pcp) raw_cpu_generic_read(pcp) 206 #endif 207 #ifndef raw_cpu_read_8 208 #define raw_cpu_read_8(pcp) raw_cpu_generic_read(pcp) 209 #endif 210 211 #ifndef raw_cpu_write_1 212 #define raw_cpu_write_1(pcp, val) raw_cpu_generic_to_op(pcp, val, =) 213 #endif 214 #ifndef raw_cpu_write_2 215 #define raw_cpu_write_2(pcp, val) raw_cpu_generic_to_op(pcp, val, =) 216 #endif 217 #ifndef raw_cpu_write_4 218 #define raw_cpu_write_4(pcp, val) raw_cpu_generic_to_op(pcp, val, =) 219 #endif 220 #ifndef raw_cpu_write_8 221 #define raw_cpu_write_8(pcp, val) raw_cpu_generic_to_op(pcp, val, =) 222 #endif 223 224 #ifndef raw_cpu_add_1 225 #define raw_cpu_add_1(pcp, val) raw_cpu_generic_to_op(pcp, val, +=) 226 #endif 227 #ifndef raw_cpu_add_2 228 #define raw_cpu_add_2(pcp, val) raw_cpu_generic_to_op(pcp, val, +=) 229 #endif 230 #ifndef raw_cpu_add_4 231 #define raw_cpu_add_4(pcp, val) raw_cpu_generic_to_op(pcp, val, +=) 232 #endif 233 #ifndef raw_cpu_add_8 234 #define raw_cpu_add_8(pcp, val) raw_cpu_generic_to_op(pcp, val, +=) 235 #endif 236 237 #ifndef raw_cpu_and_1 238 #define raw_cpu_and_1(pcp, val) raw_cpu_generic_to_op(pcp, val, &=) 239 #endif 240 #ifndef raw_cpu_and_2 241 #define raw_cpu_and_2(pcp, val) raw_cpu_generic_to_op(pcp, val, &=) 242 #endif 243 #ifndef raw_cpu_and_4 244 #define raw_cpu_and_4(pcp, val) raw_cpu_generic_to_op(pcp, val, &=) 245 #endif 246 #ifndef raw_cpu_and_8 247 #define raw_cpu_and_8(pcp, val) raw_cpu_generic_to_op(pcp, val, &=) 248 #endif 249 250 #ifndef raw_cpu_or_1 251 #define raw_cpu_or_1(pcp, val) raw_cpu_generic_to_op(pcp, val, |=) 252 #endif 253 #ifndef raw_cpu_or_2 254 #define raw_cpu_or_2(pcp, val) raw_cpu_generic_to_op(pcp, val, |=) 255 #endif 256 #ifndef raw_cpu_or_4 257 #define raw_cpu_or_4(pcp, val) raw_cpu_generic_to_op(pcp, val, |=) 258 #endif 259 #ifndef raw_cpu_or_8 260 #define raw_cpu_or_8(pcp, val) raw_cpu_generic_to_op(pcp, val, |=) 261 #endif 262 263 #ifndef raw_cpu_add_return_1 264 #define raw_cpu_add_return_1(pcp, val) raw_cpu_generic_add_return(pcp, val) 265 #endif 266 #ifndef raw_cpu_add_return_2 267 #define raw_cpu_add_return_2(pcp, val) raw_cpu_generic_add_return(pcp, val) 268 #endif 269 #ifndef raw_cpu_add_return_4 270 #define raw_cpu_add_return_4(pcp, val) raw_cpu_generic_add_return(pcp, val) 271 #endif 272 #ifndef raw_cpu_add_return_8 273 #define raw_cpu_add_return_8(pcp, val) raw_cpu_generic_add_return(pcp, val) 274 #endif 275 276 #ifndef raw_cpu_xchg_1 277 #define raw_cpu_xchg_1(pcp, nval) raw_cpu_generic_xchg(pcp, nval) 278 #endif 279 #ifndef raw_cpu_xchg_2 280 #define raw_cpu_xchg_2(pcp, nval) raw_cpu_generic_xchg(pcp, nval) 281 #endif 282 #ifndef raw_cpu_xchg_4 283 #define raw_cpu_xchg_4(pcp, nval) raw_cpu_generic_xchg(pcp, nval) 284 #endif 285 #ifndef raw_cpu_xchg_8 286 #define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval) 287 #endif 288 289 #ifndef raw_cpu_cmpxchg_1 290 #define raw_cpu_cmpxchg_1(pcp, oval, nval) \ 291 raw_cpu_generic_cmpxchg(pcp, oval, nval) 292 #endif 293 #ifndef raw_cpu_cmpxchg_2 294 #define raw_cpu_cmpxchg_2(pcp, oval, nval) \ 295 raw_cpu_generic_cmpxchg(pcp, oval, nval) 296 #endif 297 #ifndef raw_cpu_cmpxchg_4 298 #define raw_cpu_cmpxchg_4(pcp, oval, nval) \ 299 raw_cpu_generic_cmpxchg(pcp, oval, nval) 300 #endif 301 #ifndef raw_cpu_cmpxchg_8 302 #define raw_cpu_cmpxchg_8(pcp, oval, nval) \ 303 raw_cpu_generic_cmpxchg(pcp, oval, nval) 304 #endif 305 306 #ifndef raw_cpu_cmpxchg_double_1 307 #define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 308 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 309 #endif 310 #ifndef raw_cpu_cmpxchg_double_2 311 #define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 312 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 313 #endif 314 #ifndef raw_cpu_cmpxchg_double_4 315 #define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 316 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 317 #endif 318 #ifndef raw_cpu_cmpxchg_double_8 319 #define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 320 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 321 #endif 322 323 #ifndef this_cpu_read_1 324 #define this_cpu_read_1(pcp) this_cpu_generic_read(pcp) 325 #endif 326 #ifndef this_cpu_read_2 327 #define this_cpu_read_2(pcp) this_cpu_generic_read(pcp) 328 #endif 329 #ifndef this_cpu_read_4 330 #define this_cpu_read_4(pcp) this_cpu_generic_read(pcp) 331 #endif 332 #ifndef this_cpu_read_8 333 #define this_cpu_read_8(pcp) this_cpu_generic_read(pcp) 334 #endif 335 336 #ifndef this_cpu_write_1 337 #define this_cpu_write_1(pcp, val) this_cpu_generic_to_op(pcp, val, =) 338 #endif 339 #ifndef this_cpu_write_2 340 #define this_cpu_write_2(pcp, val) this_cpu_generic_to_op(pcp, val, =) 341 #endif 342 #ifndef this_cpu_write_4 343 #define this_cpu_write_4(pcp, val) this_cpu_generic_to_op(pcp, val, =) 344 #endif 345 #ifndef this_cpu_write_8 346 #define this_cpu_write_8(pcp, val) this_cpu_generic_to_op(pcp, val, =) 347 #endif 348 349 #ifndef this_cpu_add_1 350 #define this_cpu_add_1(pcp, val) this_cpu_generic_to_op(pcp, val, +=) 351 #endif 352 #ifndef this_cpu_add_2 353 #define this_cpu_add_2(pcp, val) this_cpu_generic_to_op(pcp, val, +=) 354 #endif 355 #ifndef this_cpu_add_4 356 #define this_cpu_add_4(pcp, val) this_cpu_generic_to_op(pcp, val, +=) 357 #endif 358 #ifndef this_cpu_add_8 359 #define this_cpu_add_8(pcp, val) this_cpu_generic_to_op(pcp, val, +=) 360 #endif 361 362 #ifndef this_cpu_and_1 363 #define this_cpu_and_1(pcp, val) this_cpu_generic_to_op(pcp, val, &=) 364 #endif 365 #ifndef this_cpu_and_2 366 #define this_cpu_and_2(pcp, val) this_cpu_generic_to_op(pcp, val, &=) 367 #endif 368 #ifndef this_cpu_and_4 369 #define this_cpu_and_4(pcp, val) this_cpu_generic_to_op(pcp, val, &=) 370 #endif 371 #ifndef this_cpu_and_8 372 #define this_cpu_and_8(pcp, val) this_cpu_generic_to_op(pcp, val, &=) 373 #endif 374 375 #ifndef this_cpu_or_1 376 #define this_cpu_or_1(pcp, val) this_cpu_generic_to_op(pcp, val, |=) 377 #endif 378 #ifndef this_cpu_or_2 379 #define this_cpu_or_2(pcp, val) this_cpu_generic_to_op(pcp, val, |=) 380 #endif 381 #ifndef this_cpu_or_4 382 #define this_cpu_or_4(pcp, val) this_cpu_generic_to_op(pcp, val, |=) 383 #endif 384 #ifndef this_cpu_or_8 385 #define this_cpu_or_8(pcp, val) this_cpu_generic_to_op(pcp, val, |=) 386 #endif 387 388 #ifndef this_cpu_add_return_1 389 #define this_cpu_add_return_1(pcp, val) this_cpu_generic_add_return(pcp, val) 390 #endif 391 #ifndef this_cpu_add_return_2 392 #define this_cpu_add_return_2(pcp, val) this_cpu_generic_add_return(pcp, val) 393 #endif 394 #ifndef this_cpu_add_return_4 395 #define this_cpu_add_return_4(pcp, val) this_cpu_generic_add_return(pcp, val) 396 #endif 397 #ifndef this_cpu_add_return_8 398 #define this_cpu_add_return_8(pcp, val) this_cpu_generic_add_return(pcp, val) 399 #endif 400 401 #ifndef this_cpu_xchg_1 402 #define this_cpu_xchg_1(pcp, nval) this_cpu_generic_xchg(pcp, nval) 403 #endif 404 #ifndef this_cpu_xchg_2 405 #define this_cpu_xchg_2(pcp, nval) this_cpu_generic_xchg(pcp, nval) 406 #endif 407 #ifndef this_cpu_xchg_4 408 #define this_cpu_xchg_4(pcp, nval) this_cpu_generic_xchg(pcp, nval) 409 #endif 410 #ifndef this_cpu_xchg_8 411 #define this_cpu_xchg_8(pcp, nval) this_cpu_generic_xchg(pcp, nval) 412 #endif 413 414 #ifndef this_cpu_cmpxchg_1 415 #define this_cpu_cmpxchg_1(pcp, oval, nval) \ 416 this_cpu_generic_cmpxchg(pcp, oval, nval) 417 #endif 418 #ifndef this_cpu_cmpxchg_2 419 #define this_cpu_cmpxchg_2(pcp, oval, nval) \ 420 this_cpu_generic_cmpxchg(pcp, oval, nval) 421 #endif 422 #ifndef this_cpu_cmpxchg_4 423 #define this_cpu_cmpxchg_4(pcp, oval, nval) \ 424 this_cpu_generic_cmpxchg(pcp, oval, nval) 425 #endif 426 #ifndef this_cpu_cmpxchg_8 427 #define this_cpu_cmpxchg_8(pcp, oval, nval) \ 428 this_cpu_generic_cmpxchg(pcp, oval, nval) 429 #endif 430 431 #ifndef this_cpu_cmpxchg_double_1 432 #define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 433 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 434 #endif 435 #ifndef this_cpu_cmpxchg_double_2 436 #define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 437 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 438 #endif 439 #ifndef this_cpu_cmpxchg_double_4 440 #define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 441 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 442 #endif 443 #ifndef this_cpu_cmpxchg_double_8 444 #define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 445 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 446 #endif 447 448 #endif /* _ASM_GENERIC_PERCPU_H_ */ 449