1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 */ 5 #ifndef __ASM_PERCPU_H 6 #define __ASM_PERCPU_H 7 8 #include <asm/cmpxchg.h> 9 #include <asm/loongarch.h> 10 11 /* Use r21 for fast access */ 12 register unsigned long __my_cpu_offset __asm__("$r21"); 13 14 static inline void set_my_cpu_offset(unsigned long off) 15 { 16 __my_cpu_offset = off; 17 csr_write64(off, PERCPU_BASE_KS); 18 } 19 #define __my_cpu_offset __my_cpu_offset 20 21 #define PERCPU_OP(op, asm_op, c_op) \ 22 static inline unsigned long __percpu_##op(void *ptr, \ 23 unsigned long val, int size) \ 24 { \ 25 unsigned long ret; \ 26 \ 27 switch (size) { \ 28 case 4: \ 29 __asm__ __volatile__( \ 30 "am"#asm_op".w" " %[ret], %[val], %[ptr] \n" \ 31 : [ret] "=&r" (ret), [ptr] "+ZB"(*(u32 *)ptr) \ 32 : [val] "r" (val)); \ 33 break; \ 34 case 8: \ 35 __asm__ __volatile__( \ 36 "am"#asm_op".d" " %[ret], %[val], %[ptr] \n" \ 37 : [ret] "=&r" (ret), [ptr] "+ZB"(*(u64 *)ptr) \ 38 : [val] "r" (val)); \ 39 break; \ 40 default: \ 41 ret = 0; \ 42 BUILD_BUG(); \ 43 } \ 44 \ 45 return ret c_op val; \ 46 } 47 48 PERCPU_OP(add, add, +) 49 PERCPU_OP(and, and, &) 50 PERCPU_OP(or, or, |) 51 #undef PERCPU_OP 52 53 static inline unsigned long __percpu_read(void *ptr, int size) 54 { 55 unsigned long ret; 56 57 switch (size) { 58 case 1: 59 __asm__ __volatile__ ("ldx.b %[ret], $r21, %[ptr] \n" 60 : [ret] "=&r"(ret) 61 : [ptr] "r"(ptr) 62 : "memory"); 63 break; 64 case 2: 65 __asm__ __volatile__ ("ldx.h %[ret], $r21, %[ptr] \n" 66 : [ret] "=&r"(ret) 67 : [ptr] "r"(ptr) 68 : "memory"); 69 break; 70 case 4: 71 __asm__ __volatile__ ("ldx.w %[ret], $r21, %[ptr] \n" 72 : [ret] "=&r"(ret) 73 : [ptr] "r"(ptr) 74 : "memory"); 75 break; 76 case 8: 77 __asm__ __volatile__ ("ldx.d %[ret], $r21, %[ptr] \n" 78 : [ret] "=&r"(ret) 79 : [ptr] "r"(ptr) 80 : "memory"); 81 break; 82 default: 83 ret = 0; 84 BUILD_BUG(); 85 } 86 87 return ret; 88 } 89 90 static inline void __percpu_write(void *ptr, unsigned long val, int size) 91 { 92 switch (size) { 93 case 1: 94 __asm__ __volatile__("stx.b %[val], $r21, %[ptr] \n" 95 : 96 : [val] "r" (val), [ptr] "r" (ptr) 97 : "memory"); 98 break; 99 case 2: 100 __asm__ __volatile__("stx.h %[val], $r21, %[ptr] \n" 101 : 102 : [val] "r" (val), [ptr] "r" (ptr) 103 : "memory"); 104 break; 105 case 4: 106 __asm__ __volatile__("stx.w %[val], $r21, %[ptr] \n" 107 : 108 : [val] "r" (val), [ptr] "r" (ptr) 109 : "memory"); 110 break; 111 case 8: 112 __asm__ __volatile__("stx.d %[val], $r21, %[ptr] \n" 113 : 114 : [val] "r" (val), [ptr] "r" (ptr) 115 : "memory"); 116 break; 117 default: 118 BUILD_BUG(); 119 } 120 } 121 122 static inline unsigned long __percpu_xchg(void *ptr, unsigned long val, 123 int size) 124 { 125 switch (size) { 126 case 1: 127 case 2: 128 return __xchg_small((volatile void *)ptr, val, size); 129 130 case 4: 131 return __xchg_asm("amswap.w", (volatile u32 *)ptr, (u32)val); 132 133 case 8: 134 return __xchg_asm("amswap.d", (volatile u64 *)ptr, (u64)val); 135 136 default: 137 BUILD_BUG(); 138 } 139 140 return 0; 141 } 142 143 /* this_cpu_cmpxchg */ 144 #define _protect_cmpxchg_local(pcp, o, n) \ 145 ({ \ 146 typeof(*raw_cpu_ptr(&(pcp))) __ret; \ 147 preempt_disable_notrace(); \ 148 __ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n); \ 149 preempt_enable_notrace(); \ 150 __ret; \ 151 }) 152 153 #define _percpu_read(pcp) \ 154 ({ \ 155 typeof(pcp) __retval; \ 156 __retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp)); \ 157 __retval; \ 158 }) 159 160 #define _percpu_write(pcp, val) \ 161 do { \ 162 __percpu_write(&(pcp), (unsigned long)(val), sizeof(pcp)); \ 163 } while (0) \ 164 165 #define _pcp_protect(operation, pcp, val) \ 166 ({ \ 167 typeof(pcp) __retval; \ 168 preempt_disable_notrace(); \ 169 __retval = (typeof(pcp))operation(raw_cpu_ptr(&(pcp)), \ 170 (val), sizeof(pcp)); \ 171 preempt_enable_notrace(); \ 172 __retval; \ 173 }) 174 175 #define _percpu_add(pcp, val) \ 176 _pcp_protect(__percpu_add, pcp, val) 177 178 #define _percpu_add_return(pcp, val) _percpu_add(pcp, val) 179 180 #define _percpu_and(pcp, val) \ 181 _pcp_protect(__percpu_and, pcp, val) 182 183 #define _percpu_or(pcp, val) \ 184 _pcp_protect(__percpu_or, pcp, val) 185 186 #define _percpu_xchg(pcp, val) ((typeof(pcp)) \ 187 _pcp_protect(__percpu_xchg, pcp, (unsigned long)(val))) 188 189 #define this_cpu_add_4(pcp, val) _percpu_add(pcp, val) 190 #define this_cpu_add_8(pcp, val) _percpu_add(pcp, val) 191 192 #define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val) 193 #define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val) 194 195 #define this_cpu_and_4(pcp, val) _percpu_and(pcp, val) 196 #define this_cpu_and_8(pcp, val) _percpu_and(pcp, val) 197 198 #define this_cpu_or_4(pcp, val) _percpu_or(pcp, val) 199 #define this_cpu_or_8(pcp, val) _percpu_or(pcp, val) 200 201 #define this_cpu_read_1(pcp) _percpu_read(pcp) 202 #define this_cpu_read_2(pcp) _percpu_read(pcp) 203 #define this_cpu_read_4(pcp) _percpu_read(pcp) 204 #define this_cpu_read_8(pcp) _percpu_read(pcp) 205 206 #define this_cpu_write_1(pcp, val) _percpu_write(pcp, val) 207 #define this_cpu_write_2(pcp, val) _percpu_write(pcp, val) 208 #define this_cpu_write_4(pcp, val) _percpu_write(pcp, val) 209 #define this_cpu_write_8(pcp, val) _percpu_write(pcp, val) 210 211 #define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val) 212 #define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val) 213 #define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val) 214 #define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val) 215 216 #define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) 217 #define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) 218 #define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) 219 #define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) 220 221 #include <asm-generic/percpu.h> 222 223 #endif /* __ASM_PERCPU_H */ 224