1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 */ 5 #ifndef __ASM_PERCPU_H 6 #define __ASM_PERCPU_H 7 8 #include <asm/cmpxchg.h> 9 #include <asm/loongarch.h> 10 11 /* 12 * The "address" (in fact, offset from $r21) of a per-CPU variable is close to 13 * the loading address of main kernel image, but far from where the modules are 14 * loaded. Tell the compiler this fact when using explicit relocs. 15 */ 16 #if defined(MODULE) && defined(CONFIG_AS_HAS_EXPLICIT_RELOCS) 17 # if __has_attribute(model) 18 # define PER_CPU_ATTRIBUTES __attribute__((model("extreme"))) 19 # else 20 # error compiler support for the model attribute is necessary when a recent assembler is used 21 # endif 22 #endif 23 24 /* Use r21 for fast access */ 25 register unsigned long __my_cpu_offset __asm__("$r21"); 26 27 static inline void set_my_cpu_offset(unsigned long off) 28 { 29 __my_cpu_offset = off; 30 csr_write64(off, PERCPU_BASE_KS); 31 } 32 #define __my_cpu_offset __my_cpu_offset 33 34 #define PERCPU_OP(op, asm_op, c_op) \ 35 static inline unsigned long __percpu_##op(void *ptr, \ 36 unsigned long val, int size) \ 37 { \ 38 unsigned long ret; \ 39 \ 40 switch (size) { \ 41 case 4: \ 42 __asm__ __volatile__( \ 43 "am"#asm_op".w" " %[ret], %[val], %[ptr] \n" \ 44 : [ret] "=&r" (ret), [ptr] "+ZB"(*(u32 *)ptr) \ 45 : [val] "r" (val)); \ 46 break; \ 47 case 8: \ 48 __asm__ __volatile__( \ 49 "am"#asm_op".d" " %[ret], %[val], %[ptr] \n" \ 50 : [ret] "=&r" (ret), [ptr] "+ZB"(*(u64 *)ptr) \ 51 : [val] "r" (val)); \ 52 break; \ 53 default: \ 54 ret = 0; \ 55 BUILD_BUG(); \ 56 } \ 57 \ 58 return ret c_op val; \ 59 } 60 61 PERCPU_OP(add, add, +) 62 PERCPU_OP(and, and, &) 63 PERCPU_OP(or, or, |) 64 #undef PERCPU_OP 65 66 static inline unsigned long __percpu_read(void *ptr, int size) 67 { 68 unsigned long ret; 69 70 switch (size) { 71 case 1: 72 __asm__ __volatile__ ("ldx.b %[ret], $r21, %[ptr] \n" 73 : [ret] "=&r"(ret) 74 : [ptr] "r"(ptr) 75 : "memory"); 76 break; 77 case 2: 78 __asm__ __volatile__ ("ldx.h %[ret], $r21, %[ptr] \n" 79 : [ret] "=&r"(ret) 80 : [ptr] "r"(ptr) 81 : "memory"); 82 break; 83 case 4: 84 __asm__ __volatile__ ("ldx.w %[ret], $r21, %[ptr] \n" 85 : [ret] "=&r"(ret) 86 : [ptr] "r"(ptr) 87 : "memory"); 88 break; 89 case 8: 90 __asm__ __volatile__ ("ldx.d %[ret], $r21, %[ptr] \n" 91 : [ret] "=&r"(ret) 92 : [ptr] "r"(ptr) 93 : "memory"); 94 break; 95 default: 96 ret = 0; 97 BUILD_BUG(); 98 } 99 100 return ret; 101 } 102 103 static inline void __percpu_write(void *ptr, unsigned long val, int size) 104 { 105 switch (size) { 106 case 1: 107 __asm__ __volatile__("stx.b %[val], $r21, %[ptr] \n" 108 : 109 : [val] "r" (val), [ptr] "r" (ptr) 110 : "memory"); 111 break; 112 case 2: 113 __asm__ __volatile__("stx.h %[val], $r21, %[ptr] \n" 114 : 115 : [val] "r" (val), [ptr] "r" (ptr) 116 : "memory"); 117 break; 118 case 4: 119 __asm__ __volatile__("stx.w %[val], $r21, %[ptr] \n" 120 : 121 : [val] "r" (val), [ptr] "r" (ptr) 122 : "memory"); 123 break; 124 case 8: 125 __asm__ __volatile__("stx.d %[val], $r21, %[ptr] \n" 126 : 127 : [val] "r" (val), [ptr] "r" (ptr) 128 : "memory"); 129 break; 130 default: 131 BUILD_BUG(); 132 } 133 } 134 135 static inline unsigned long __percpu_xchg(void *ptr, unsigned long val, 136 int size) 137 { 138 switch (size) { 139 case 1: 140 case 2: 141 return __xchg_small((volatile void *)ptr, val, size); 142 143 case 4: 144 return __xchg_asm("amswap.w", (volatile u32 *)ptr, (u32)val); 145 146 case 8: 147 return __xchg_asm("amswap.d", (volatile u64 *)ptr, (u64)val); 148 149 default: 150 BUILD_BUG(); 151 } 152 153 return 0; 154 } 155 156 /* this_cpu_cmpxchg */ 157 #define _protect_cmpxchg_local(pcp, o, n) \ 158 ({ \ 159 typeof(*raw_cpu_ptr(&(pcp))) __ret; \ 160 preempt_disable_notrace(); \ 161 __ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n); \ 162 preempt_enable_notrace(); \ 163 __ret; \ 164 }) 165 166 #define _percpu_read(pcp) \ 167 ({ \ 168 typeof(pcp) __retval; \ 169 __retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp)); \ 170 __retval; \ 171 }) 172 173 #define _percpu_write(pcp, val) \ 174 do { \ 175 __percpu_write(&(pcp), (unsigned long)(val), sizeof(pcp)); \ 176 } while (0) \ 177 178 #define _pcp_protect(operation, pcp, val) \ 179 ({ \ 180 typeof(pcp) __retval; \ 181 preempt_disable_notrace(); \ 182 __retval = (typeof(pcp))operation(raw_cpu_ptr(&(pcp)), \ 183 (val), sizeof(pcp)); \ 184 preempt_enable_notrace(); \ 185 __retval; \ 186 }) 187 188 #define _percpu_add(pcp, val) \ 189 _pcp_protect(__percpu_add, pcp, val) 190 191 #define _percpu_add_return(pcp, val) _percpu_add(pcp, val) 192 193 #define _percpu_and(pcp, val) \ 194 _pcp_protect(__percpu_and, pcp, val) 195 196 #define _percpu_or(pcp, val) \ 197 _pcp_protect(__percpu_or, pcp, val) 198 199 #define _percpu_xchg(pcp, val) ((typeof(pcp)) \ 200 _pcp_protect(__percpu_xchg, pcp, (unsigned long)(val))) 201 202 #define this_cpu_add_4(pcp, val) _percpu_add(pcp, val) 203 #define this_cpu_add_8(pcp, val) _percpu_add(pcp, val) 204 205 #define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val) 206 #define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val) 207 208 #define this_cpu_and_4(pcp, val) _percpu_and(pcp, val) 209 #define this_cpu_and_8(pcp, val) _percpu_and(pcp, val) 210 211 #define this_cpu_or_4(pcp, val) _percpu_or(pcp, val) 212 #define this_cpu_or_8(pcp, val) _percpu_or(pcp, val) 213 214 #define this_cpu_read_1(pcp) _percpu_read(pcp) 215 #define this_cpu_read_2(pcp) _percpu_read(pcp) 216 #define this_cpu_read_4(pcp) _percpu_read(pcp) 217 #define this_cpu_read_8(pcp) _percpu_read(pcp) 218 219 #define this_cpu_write_1(pcp, val) _percpu_write(pcp, val) 220 #define this_cpu_write_2(pcp, val) _percpu_write(pcp, val) 221 #define this_cpu_write_4(pcp, val) _percpu_write(pcp, val) 222 #define this_cpu_write_8(pcp, val) _percpu_write(pcp, val) 223 224 #define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val) 225 #define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val) 226 #define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val) 227 #define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val) 228 229 #define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) 230 #define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) 231 #define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) 232 #define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) 233 234 #include <asm-generic/percpu.h> 235 236 #endif /* __ASM_PERCPU_H */ 237