1 #ifndef _ASM_X86_MSR_H 2 #define _ASM_X86_MSR_H 3 4 #include <uapi/asm/msr.h> 5 6 #ifndef __ASSEMBLY__ 7 8 #include <asm/asm.h> 9 #include <asm/errno.h> 10 #include <asm/cpumask.h> 11 12 struct msr { 13 union { 14 struct { 15 u32 l; 16 u32 h; 17 }; 18 u64 q; 19 }; 20 }; 21 22 struct msr_info { 23 u32 msr_no; 24 struct msr reg; 25 struct msr *msrs; 26 int err; 27 }; 28 29 struct msr_regs_info { 30 u32 *regs; 31 int err; 32 }; 33 34 static inline unsigned long long native_read_tscp(unsigned int *aux) 35 { 36 unsigned long low, high; 37 asm volatile(".byte 0x0f,0x01,0xf9" 38 : "=a" (low), "=d" (high), "=c" (*aux)); 39 return low | ((u64)high << 32); 40 } 41 42 /* 43 * both i386 and x86_64 returns 64-bit value in edx:eax, but gcc's "A" 44 * constraint has different meanings. For i386, "A" means exactly 45 * edx:eax, while for x86_64 it doesn't mean rdx:rax or edx:eax. Instead, 46 * it means rax *or* rdx. 47 */ 48 #ifdef CONFIG_X86_64 49 #define DECLARE_ARGS(val, low, high) unsigned low, high 50 #define EAX_EDX_VAL(val, low, high) ((low) | ((u64)(high) << 32)) 51 #define EAX_EDX_ARGS(val, low, high) "a" (low), "d" (high) 52 #define EAX_EDX_RET(val, low, high) "=a" (low), "=d" (high) 53 #else 54 #define DECLARE_ARGS(val, low, high) unsigned long long val 55 #define EAX_EDX_VAL(val, low, high) (val) 56 #define EAX_EDX_ARGS(val, low, high) "A" (val) 57 #define EAX_EDX_RET(val, low, high) "=A" (val) 58 #endif 59 60 static inline unsigned long long native_read_msr(unsigned int msr) 61 { 62 DECLARE_ARGS(val, low, high); 63 64 asm volatile("rdmsr" : EAX_EDX_RET(val, low, high) : "c" (msr)); 65 return EAX_EDX_VAL(val, low, high); 66 } 67 68 static inline unsigned long long native_read_msr_safe(unsigned int msr, 69 int *err) 70 { 71 DECLARE_ARGS(val, low, high); 72 73 asm volatile("2: rdmsr ; xor %[err],%[err]\n" 74 "1:\n\t" 75 ".section .fixup,\"ax\"\n\t" 76 "3: mov %[fault],%[err] ; jmp 1b\n\t" 77 ".previous\n\t" 78 _ASM_EXTABLE(2b, 3b) 79 : [err] "=r" (*err), EAX_EDX_RET(val, low, high) 80 : "c" (msr), [fault] "i" (-EIO)); 81 return EAX_EDX_VAL(val, low, high); 82 } 83 84 static inline void native_write_msr(unsigned int msr, 85 unsigned low, unsigned high) 86 { 87 asm volatile("wrmsr" : : "c" (msr), "a"(low), "d" (high) : "memory"); 88 } 89 90 /* Can be uninlined because referenced by paravirt */ 91 notrace static inline int native_write_msr_safe(unsigned int msr, 92 unsigned low, unsigned high) 93 { 94 int err; 95 asm volatile("2: wrmsr ; xor %[err],%[err]\n" 96 "1:\n\t" 97 ".section .fixup,\"ax\"\n\t" 98 "3: mov %[fault],%[err] ; jmp 1b\n\t" 99 ".previous\n\t" 100 _ASM_EXTABLE(2b, 3b) 101 : [err] "=a" (err) 102 : "c" (msr), "0" (low), "d" (high), 103 [fault] "i" (-EIO) 104 : "memory"); 105 return err; 106 } 107 108 extern unsigned long long native_read_tsc(void); 109 110 extern int rdmsr_safe_regs(u32 regs[8]); 111 extern int wrmsr_safe_regs(u32 regs[8]); 112 113 static __always_inline unsigned long long __native_read_tsc(void) 114 { 115 DECLARE_ARGS(val, low, high); 116 117 asm volatile("rdtsc" : EAX_EDX_RET(val, low, high)); 118 119 return EAX_EDX_VAL(val, low, high); 120 } 121 122 static inline unsigned long long native_read_pmc(int counter) 123 { 124 DECLARE_ARGS(val, low, high); 125 126 asm volatile("rdpmc" : EAX_EDX_RET(val, low, high) : "c" (counter)); 127 return EAX_EDX_VAL(val, low, high); 128 } 129 130 #ifdef CONFIG_PARAVIRT 131 #include <asm/paravirt.h> 132 #else 133 #include <linux/errno.h> 134 /* 135 * Access to machine-specific registers (available on 586 and better only) 136 * Note: the rd* operations modify the parameters directly (without using 137 * pointer indirection), this allows gcc to optimize better 138 */ 139 140 #define rdmsr(msr, low, high) \ 141 do { \ 142 u64 __val = native_read_msr((msr)); \ 143 (void)((low) = (u32)__val); \ 144 (void)((high) = (u32)(__val >> 32)); \ 145 } while (0) 146 147 static inline void wrmsr(unsigned msr, unsigned low, unsigned high) 148 { 149 native_write_msr(msr, low, high); 150 } 151 152 #define rdmsrl(msr, val) \ 153 ((val) = native_read_msr((msr))) 154 155 #define wrmsrl(msr, val) \ 156 native_write_msr((msr), (u32)((u64)(val)), (u32)((u64)(val) >> 32)) 157 158 /* wrmsr with exception handling */ 159 static inline int wrmsr_safe(unsigned msr, unsigned low, unsigned high) 160 { 161 return native_write_msr_safe(msr, low, high); 162 } 163 164 /* rdmsr with exception handling */ 165 #define rdmsr_safe(msr, low, high) \ 166 ({ \ 167 int __err; \ 168 u64 __val = native_read_msr_safe((msr), &__err); \ 169 (*low) = (u32)__val; \ 170 (*high) = (u32)(__val >> 32); \ 171 __err; \ 172 }) 173 174 static inline int rdmsrl_safe(unsigned msr, unsigned long long *p) 175 { 176 int err; 177 178 *p = native_read_msr_safe(msr, &err); 179 return err; 180 } 181 182 #define rdtscl(low) \ 183 ((low) = (u32)__native_read_tsc()) 184 185 #define rdtscll(val) \ 186 ((val) = __native_read_tsc()) 187 188 #define rdpmc(counter, low, high) \ 189 do { \ 190 u64 _l = native_read_pmc((counter)); \ 191 (low) = (u32)_l; \ 192 (high) = (u32)(_l >> 32); \ 193 } while (0) 194 195 #define rdpmcl(counter, val) ((val) = native_read_pmc(counter)) 196 197 #define rdtscp(low, high, aux) \ 198 do { \ 199 unsigned long long _val = native_read_tscp(&(aux)); \ 200 (low) = (u32)_val; \ 201 (high) = (u32)(_val >> 32); \ 202 } while (0) 203 204 #define rdtscpll(val, aux) (val) = native_read_tscp(&(aux)) 205 206 #endif /* !CONFIG_PARAVIRT */ 207 208 #define wrmsrl_safe(msr, val) wrmsr_safe((msr), (u32)(val), \ 209 (u32)((val) >> 32)) 210 211 #define write_tsc(low, high) wrmsr(MSR_IA32_TSC, (low), (high)) 212 213 #define write_rdtscp_aux(val) wrmsr(MSR_TSC_AUX, (val), 0) 214 215 struct msr *msrs_alloc(void); 216 void msrs_free(struct msr *msrs); 217 218 #ifdef CONFIG_SMP 219 int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); 220 int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); 221 int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q); 222 int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q); 223 void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs); 224 void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs); 225 int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); 226 int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); 227 int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q); 228 int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q); 229 int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]); 230 int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]); 231 #else /* CONFIG_SMP */ 232 static inline int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h) 233 { 234 rdmsr(msr_no, *l, *h); 235 return 0; 236 } 237 static inline int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) 238 { 239 wrmsr(msr_no, l, h); 240 return 0; 241 } 242 static inline int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) 243 { 244 rdmsrl(msr_no, *q); 245 return 0; 246 } 247 static inline int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q) 248 { 249 wrmsrl(msr_no, q); 250 return 0; 251 } 252 static inline void rdmsr_on_cpus(const struct cpumask *m, u32 msr_no, 253 struct msr *msrs) 254 { 255 rdmsr_on_cpu(0, msr_no, &(msrs[0].l), &(msrs[0].h)); 256 } 257 static inline void wrmsr_on_cpus(const struct cpumask *m, u32 msr_no, 258 struct msr *msrs) 259 { 260 wrmsr_on_cpu(0, msr_no, msrs[0].l, msrs[0].h); 261 } 262 static inline int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, 263 u32 *l, u32 *h) 264 { 265 return rdmsr_safe(msr_no, l, h); 266 } 267 static inline int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) 268 { 269 return wrmsr_safe(msr_no, l, h); 270 } 271 static inline int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) 272 { 273 return rdmsrl_safe(msr_no, q); 274 } 275 static inline int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q) 276 { 277 return wrmsrl_safe(msr_no, q); 278 } 279 static inline int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]) 280 { 281 return rdmsr_safe_regs(regs); 282 } 283 static inline int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]) 284 { 285 return wrmsr_safe_regs(regs); 286 } 287 #endif /* CONFIG_SMP */ 288 #endif /* __ASSEMBLY__ */ 289 #endif /* _ASM_X86_MSR_H */ 290