1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_MSHYPER_H 3 #define _ASM_X86_MSHYPER_H 4 5 #include <linux/types.h> 6 #include <linux/atomic.h> 7 #include <linux/nmi.h> 8 #include <asm/io.h> 9 #include <asm/hyperv-tlfs.h> 10 #include <asm/nospec-branch.h> 11 12 #define VP_INVAL U32_MAX 13 14 struct ms_hyperv_info { 15 u32 features; 16 u32 misc_features; 17 u32 hints; 18 u32 nested_features; 19 u32 max_vp_index; 20 u32 max_lp_index; 21 }; 22 23 extern struct ms_hyperv_info ms_hyperv; 24 25 /* 26 * Generate the guest ID. 27 */ 28 29 static inline __u64 generate_guest_id(__u64 d_info1, __u64 kernel_version, 30 __u64 d_info2) 31 { 32 __u64 guest_id = 0; 33 34 guest_id = (((__u64)HV_LINUX_VENDOR_ID) << 48); 35 guest_id |= (d_info1 << 48); 36 guest_id |= (kernel_version << 16); 37 guest_id |= d_info2; 38 39 return guest_id; 40 } 41 42 43 /* Free the message slot and signal end-of-message if required */ 44 static inline void vmbus_signal_eom(struct hv_message *msg, u32 old_msg_type) 45 { 46 /* 47 * On crash we're reading some other CPU's message page and we need 48 * to be careful: this other CPU may already had cleared the header 49 * and the host may already had delivered some other message there. 50 * In case we blindly write msg->header.message_type we're going 51 * to lose it. We can still lose a message of the same type but 52 * we count on the fact that there can only be one 53 * CHANNELMSG_UNLOAD_RESPONSE and we don't care about other messages 54 * on crash. 55 */ 56 if (cmpxchg(&msg->header.message_type, old_msg_type, 57 HVMSG_NONE) != old_msg_type) 58 return; 59 60 /* 61 * Make sure the write to MessageType (ie set to 62 * HVMSG_NONE) happens before we read the 63 * MessagePending and EOMing. Otherwise, the EOMing 64 * will not deliver any more messages since there is 65 * no empty slot 66 */ 67 mb(); 68 69 if (msg->header.message_flags.msg_pending) { 70 /* 71 * This will cause message queue rescan to 72 * possibly deliver another msg from the 73 * hypervisor 74 */ 75 wrmsrl(HV_X64_MSR_EOM, 0); 76 } 77 } 78 79 #define hv_init_timer(timer, tick) wrmsrl(timer, tick) 80 #define hv_init_timer_config(config, val) wrmsrl(config, val) 81 82 #define hv_get_simp(val) rdmsrl(HV_X64_MSR_SIMP, val) 83 #define hv_set_simp(val) wrmsrl(HV_X64_MSR_SIMP, val) 84 85 #define hv_get_siefp(val) rdmsrl(HV_X64_MSR_SIEFP, val) 86 #define hv_set_siefp(val) wrmsrl(HV_X64_MSR_SIEFP, val) 87 88 #define hv_get_synic_state(val) rdmsrl(HV_X64_MSR_SCONTROL, val) 89 #define hv_set_synic_state(val) wrmsrl(HV_X64_MSR_SCONTROL, val) 90 91 #define hv_get_vp_index(index) rdmsrl(HV_X64_MSR_VP_INDEX, index) 92 93 #define hv_get_synint_state(int_num, val) rdmsrl(int_num, val) 94 #define hv_set_synint_state(int_num, val) wrmsrl(int_num, val) 95 96 void hyperv_callback_vector(void); 97 void hyperv_reenlightenment_vector(void); 98 #ifdef CONFIG_TRACING 99 #define trace_hyperv_callback_vector hyperv_callback_vector 100 #endif 101 void hyperv_vector_handler(struct pt_regs *regs); 102 void hv_setup_vmbus_irq(void (*handler)(void)); 103 void hv_remove_vmbus_irq(void); 104 105 void hv_setup_kexec_handler(void (*handler)(void)); 106 void hv_remove_kexec_handler(void); 107 void hv_setup_crash_handler(void (*handler)(struct pt_regs *regs)); 108 void hv_remove_crash_handler(void); 109 110 /* 111 * Routines for stimer0 Direct Mode handling. 112 * On x86/x64, there are no percpu actions to take. 113 */ 114 void hv_stimer0_vector_handler(struct pt_regs *regs); 115 void hv_stimer0_callback_vector(void); 116 int hv_setup_stimer0_irq(int *irq, int *vector, void (*handler)(void)); 117 void hv_remove_stimer0_irq(int irq); 118 119 static inline void hv_enable_stimer0_percpu_irq(int irq) {} 120 static inline void hv_disable_stimer0_percpu_irq(int irq) {} 121 122 123 #if IS_ENABLED(CONFIG_HYPERV) 124 extern struct clocksource *hyperv_cs; 125 extern void *hv_hypercall_pg; 126 extern void __percpu **hyperv_pcpu_input_arg; 127 128 static inline u64 hv_do_hypercall(u64 control, void *input, void *output) 129 { 130 u64 input_address = input ? virt_to_phys(input) : 0; 131 u64 output_address = output ? virt_to_phys(output) : 0; 132 u64 hv_status; 133 134 #ifdef CONFIG_X86_64 135 if (!hv_hypercall_pg) 136 return U64_MAX; 137 138 __asm__ __volatile__("mov %4, %%r8\n" 139 CALL_NOSPEC 140 : "=a" (hv_status), ASM_CALL_CONSTRAINT, 141 "+c" (control), "+d" (input_address) 142 : "r" (output_address), 143 THUNK_TARGET(hv_hypercall_pg) 144 : "cc", "memory", "r8", "r9", "r10", "r11"); 145 #else 146 u32 input_address_hi = upper_32_bits(input_address); 147 u32 input_address_lo = lower_32_bits(input_address); 148 u32 output_address_hi = upper_32_bits(output_address); 149 u32 output_address_lo = lower_32_bits(output_address); 150 151 if (!hv_hypercall_pg) 152 return U64_MAX; 153 154 __asm__ __volatile__(CALL_NOSPEC 155 : "=A" (hv_status), 156 "+c" (input_address_lo), ASM_CALL_CONSTRAINT 157 : "A" (control), 158 "b" (input_address_hi), 159 "D"(output_address_hi), "S"(output_address_lo), 160 THUNK_TARGET(hv_hypercall_pg) 161 : "cc", "memory"); 162 #endif /* !x86_64 */ 163 return hv_status; 164 } 165 166 /* Fast hypercall with 8 bytes of input and no output */ 167 static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1) 168 { 169 u64 hv_status, control = (u64)code | HV_HYPERCALL_FAST_BIT; 170 171 #ifdef CONFIG_X86_64 172 { 173 __asm__ __volatile__(CALL_NOSPEC 174 : "=a" (hv_status), ASM_CALL_CONSTRAINT, 175 "+c" (control), "+d" (input1) 176 : THUNK_TARGET(hv_hypercall_pg) 177 : "cc", "r8", "r9", "r10", "r11"); 178 } 179 #else 180 { 181 u32 input1_hi = upper_32_bits(input1); 182 u32 input1_lo = lower_32_bits(input1); 183 184 __asm__ __volatile__ (CALL_NOSPEC 185 : "=A"(hv_status), 186 "+c"(input1_lo), 187 ASM_CALL_CONSTRAINT 188 : "A" (control), 189 "b" (input1_hi), 190 THUNK_TARGET(hv_hypercall_pg) 191 : "cc", "edi", "esi"); 192 } 193 #endif 194 return hv_status; 195 } 196 197 /* 198 * Rep hypercalls. Callers of this functions are supposed to ensure that 199 * rep_count and varhead_size comply with Hyper-V hypercall definition. 200 */ 201 static inline u64 hv_do_rep_hypercall(u16 code, u16 rep_count, u16 varhead_size, 202 void *input, void *output) 203 { 204 u64 control = code; 205 u64 status; 206 u16 rep_comp; 207 208 control |= (u64)varhead_size << HV_HYPERCALL_VARHEAD_OFFSET; 209 control |= (u64)rep_count << HV_HYPERCALL_REP_COMP_OFFSET; 210 211 do { 212 status = hv_do_hypercall(control, input, output); 213 if ((status & HV_HYPERCALL_RESULT_MASK) != HV_STATUS_SUCCESS) 214 return status; 215 216 /* Bits 32-43 of status have 'Reps completed' data. */ 217 rep_comp = (status & HV_HYPERCALL_REP_COMP_MASK) >> 218 HV_HYPERCALL_REP_COMP_OFFSET; 219 220 control &= ~HV_HYPERCALL_REP_START_MASK; 221 control |= (u64)rep_comp << HV_HYPERCALL_REP_START_OFFSET; 222 223 touch_nmi_watchdog(); 224 } while (rep_comp < rep_count); 225 226 return status; 227 } 228 229 /* 230 * Hypervisor's notion of virtual processor ID is different from 231 * Linux' notion of CPU ID. This information can only be retrieved 232 * in the context of the calling CPU. Setup a map for easy access 233 * to this information. 234 */ 235 extern u32 *hv_vp_index; 236 extern u32 hv_max_vp_index; 237 extern struct hv_vp_assist_page **hv_vp_assist_page; 238 239 static inline struct hv_vp_assist_page *hv_get_vp_assist_page(unsigned int cpu) 240 { 241 if (!hv_vp_assist_page) 242 return NULL; 243 244 return hv_vp_assist_page[cpu]; 245 } 246 247 /** 248 * hv_cpu_number_to_vp_number() - Map CPU to VP. 249 * @cpu_number: CPU number in Linux terms 250 * 251 * This function returns the mapping between the Linux processor 252 * number and the hypervisor's virtual processor number, useful 253 * in making hypercalls and such that talk about specific 254 * processors. 255 * 256 * Return: Virtual processor number in Hyper-V terms 257 */ 258 static inline int hv_cpu_number_to_vp_number(int cpu_number) 259 { 260 return hv_vp_index[cpu_number]; 261 } 262 263 static inline int cpumask_to_vpset(struct hv_vpset *vpset, 264 const struct cpumask *cpus) 265 { 266 int cpu, vcpu, vcpu_bank, vcpu_offset, nr_bank = 1; 267 268 /* valid_bank_mask can represent up to 64 banks */ 269 if (hv_max_vp_index / 64 >= 64) 270 return 0; 271 272 /* 273 * Clear all banks up to the maximum possible bank as hv_tlb_flush_ex 274 * structs are not cleared between calls, we risk flushing unneeded 275 * vCPUs otherwise. 276 */ 277 for (vcpu_bank = 0; vcpu_bank <= hv_max_vp_index / 64; vcpu_bank++) 278 vpset->bank_contents[vcpu_bank] = 0; 279 280 /* 281 * Some banks may end up being empty but this is acceptable. 282 */ 283 for_each_cpu(cpu, cpus) { 284 vcpu = hv_cpu_number_to_vp_number(cpu); 285 if (vcpu == VP_INVAL) 286 return -1; 287 vcpu_bank = vcpu / 64; 288 vcpu_offset = vcpu % 64; 289 __set_bit(vcpu_offset, (unsigned long *) 290 &vpset->bank_contents[vcpu_bank]); 291 if (vcpu_bank >= nr_bank) 292 nr_bank = vcpu_bank + 1; 293 } 294 vpset->valid_bank_mask = GENMASK_ULL(nr_bank - 1, 0); 295 return nr_bank; 296 } 297 298 void __init hyperv_init(void); 299 void hyperv_setup_mmu_ops(void); 300 void hyperv_report_panic(struct pt_regs *regs, long err); 301 bool hv_is_hyperv_initialized(void); 302 void hyperv_cleanup(void); 303 304 void hyperv_reenlightenment_intr(struct pt_regs *regs); 305 void set_hv_tscchange_cb(void (*cb)(void)); 306 void clear_hv_tscchange_cb(void); 307 void hyperv_stop_tsc_emulation(void); 308 309 #ifdef CONFIG_X86_64 310 void hv_apic_init(void); 311 #else 312 static inline void hv_apic_init(void) {} 313 #endif 314 315 #else /* CONFIG_HYPERV */ 316 static inline void hyperv_init(void) {} 317 static inline bool hv_is_hyperv_initialized(void) { return false; } 318 static inline void hyperv_cleanup(void) {} 319 static inline void hyperv_setup_mmu_ops(void) {} 320 static inline void set_hv_tscchange_cb(void (*cb)(void)) {} 321 static inline void clear_hv_tscchange_cb(void) {} 322 static inline void hyperv_stop_tsc_emulation(void) {}; 323 static inline struct hv_vp_assist_page *hv_get_vp_assist_page(unsigned int cpu) 324 { 325 return NULL; 326 } 327 #endif /* CONFIG_HYPERV */ 328 329 #ifdef CONFIG_HYPERV_TSCPAGE 330 struct ms_hyperv_tsc_page *hv_get_tsc_page(void); 331 static inline u64 hv_read_tsc_page_tsc(const struct ms_hyperv_tsc_page *tsc_pg, 332 u64 *cur_tsc) 333 { 334 u64 scale, offset; 335 u32 sequence; 336 337 /* 338 * The protocol for reading Hyper-V TSC page is specified in Hypervisor 339 * Top-Level Functional Specification ver. 3.0 and above. To get the 340 * reference time we must do the following: 341 * - READ ReferenceTscSequence 342 * A special '0' value indicates the time source is unreliable and we 343 * need to use something else. The currently published specification 344 * versions (up to 4.0b) contain a mistake and wrongly claim '-1' 345 * instead of '0' as the special value, see commit c35b82ef0294. 346 * - ReferenceTime = 347 * ((RDTSC() * ReferenceTscScale) >> 64) + ReferenceTscOffset 348 * - READ ReferenceTscSequence again. In case its value has changed 349 * since our first reading we need to discard ReferenceTime and repeat 350 * the whole sequence as the hypervisor was updating the page in 351 * between. 352 */ 353 do { 354 sequence = READ_ONCE(tsc_pg->tsc_sequence); 355 if (!sequence) 356 return U64_MAX; 357 /* 358 * Make sure we read sequence before we read other values from 359 * TSC page. 360 */ 361 smp_rmb(); 362 363 scale = READ_ONCE(tsc_pg->tsc_scale); 364 offset = READ_ONCE(tsc_pg->tsc_offset); 365 *cur_tsc = rdtsc_ordered(); 366 367 /* 368 * Make sure we read sequence after we read all other values 369 * from TSC page. 370 */ 371 smp_rmb(); 372 373 } while (READ_ONCE(tsc_pg->tsc_sequence) != sequence); 374 375 return mul_u64_u64_shr(*cur_tsc, scale, 64) + offset; 376 } 377 378 static inline u64 hv_read_tsc_page(const struct ms_hyperv_tsc_page *tsc_pg) 379 { 380 u64 cur_tsc; 381 382 return hv_read_tsc_page_tsc(tsc_pg, &cur_tsc); 383 } 384 385 #else 386 static inline struct ms_hyperv_tsc_page *hv_get_tsc_page(void) 387 { 388 return NULL; 389 } 390 391 static inline u64 hv_read_tsc_page_tsc(const struct ms_hyperv_tsc_page *tsc_pg, 392 u64 *cur_tsc) 393 { 394 BUG(); 395 return U64_MAX; 396 } 397 #endif 398 #endif 399