1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_MSHYPER_H 3 #define _ASM_X86_MSHYPER_H 4 5 #include <linux/types.h> 6 #include <linux/atomic.h> 7 #include <linux/nmi.h> 8 #include <asm/io.h> 9 #include <asm/hyperv-tlfs.h> 10 #include <asm/nospec-branch.h> 11 12 struct ms_hyperv_info { 13 u32 features; 14 u32 misc_features; 15 u32 hints; 16 u32 nested_features; 17 u32 max_vp_index; 18 u32 max_lp_index; 19 }; 20 21 extern struct ms_hyperv_info ms_hyperv; 22 23 24 /* 25 * Generate the guest ID. 26 */ 27 28 static inline __u64 generate_guest_id(__u64 d_info1, __u64 kernel_version, 29 __u64 d_info2) 30 { 31 __u64 guest_id = 0; 32 33 guest_id = (((__u64)HV_LINUX_VENDOR_ID) << 48); 34 guest_id |= (d_info1 << 48); 35 guest_id |= (kernel_version << 16); 36 guest_id |= d_info2; 37 38 return guest_id; 39 } 40 41 42 /* Free the message slot and signal end-of-message if required */ 43 static inline void vmbus_signal_eom(struct hv_message *msg, u32 old_msg_type) 44 { 45 /* 46 * On crash we're reading some other CPU's message page and we need 47 * to be careful: this other CPU may already had cleared the header 48 * and the host may already had delivered some other message there. 49 * In case we blindly write msg->header.message_type we're going 50 * to lose it. We can still lose a message of the same type but 51 * we count on the fact that there can only be one 52 * CHANNELMSG_UNLOAD_RESPONSE and we don't care about other messages 53 * on crash. 54 */ 55 if (cmpxchg(&msg->header.message_type, old_msg_type, 56 HVMSG_NONE) != old_msg_type) 57 return; 58 59 /* 60 * Make sure the write to MessageType (ie set to 61 * HVMSG_NONE) happens before we read the 62 * MessagePending and EOMing. Otherwise, the EOMing 63 * will not deliver any more messages since there is 64 * no empty slot 65 */ 66 mb(); 67 68 if (msg->header.message_flags.msg_pending) { 69 /* 70 * This will cause message queue rescan to 71 * possibly deliver another msg from the 72 * hypervisor 73 */ 74 wrmsrl(HV_X64_MSR_EOM, 0); 75 } 76 } 77 78 #define hv_init_timer(timer, tick) wrmsrl(timer, tick) 79 #define hv_init_timer_config(config, val) wrmsrl(config, val) 80 81 #define hv_get_simp(val) rdmsrl(HV_X64_MSR_SIMP, val) 82 #define hv_set_simp(val) wrmsrl(HV_X64_MSR_SIMP, val) 83 84 #define hv_get_siefp(val) rdmsrl(HV_X64_MSR_SIEFP, val) 85 #define hv_set_siefp(val) wrmsrl(HV_X64_MSR_SIEFP, val) 86 87 #define hv_get_synic_state(val) rdmsrl(HV_X64_MSR_SCONTROL, val) 88 #define hv_set_synic_state(val) wrmsrl(HV_X64_MSR_SCONTROL, val) 89 90 #define hv_get_vp_index(index) rdmsrl(HV_X64_MSR_VP_INDEX, index) 91 92 #define hv_get_synint_state(int_num, val) rdmsrl(int_num, val) 93 #define hv_set_synint_state(int_num, val) wrmsrl(int_num, val) 94 95 void hyperv_callback_vector(void); 96 void hyperv_reenlightenment_vector(void); 97 #ifdef CONFIG_TRACING 98 #define trace_hyperv_callback_vector hyperv_callback_vector 99 #endif 100 void hyperv_vector_handler(struct pt_regs *regs); 101 void hv_setup_vmbus_irq(void (*handler)(void)); 102 void hv_remove_vmbus_irq(void); 103 104 void hv_setup_kexec_handler(void (*handler)(void)); 105 void hv_remove_kexec_handler(void); 106 void hv_setup_crash_handler(void (*handler)(struct pt_regs *regs)); 107 void hv_remove_crash_handler(void); 108 109 /* 110 * Routines for stimer0 Direct Mode handling. 111 * On x86/x64, there are no percpu actions to take. 112 */ 113 void hv_stimer0_vector_handler(struct pt_regs *regs); 114 void hv_stimer0_callback_vector(void); 115 int hv_setup_stimer0_irq(int *irq, int *vector, void (*handler)(void)); 116 void hv_remove_stimer0_irq(int irq); 117 118 static inline void hv_enable_stimer0_percpu_irq(int irq) {} 119 static inline void hv_disable_stimer0_percpu_irq(int irq) {} 120 121 122 #if IS_ENABLED(CONFIG_HYPERV) 123 extern struct clocksource *hyperv_cs; 124 extern void *hv_hypercall_pg; 125 extern void __percpu **hyperv_pcpu_input_arg; 126 127 static inline u64 hv_do_hypercall(u64 control, void *input, void *output) 128 { 129 u64 input_address = input ? virt_to_phys(input) : 0; 130 u64 output_address = output ? virt_to_phys(output) : 0; 131 u64 hv_status; 132 133 #ifdef CONFIG_X86_64 134 if (!hv_hypercall_pg) 135 return U64_MAX; 136 137 __asm__ __volatile__("mov %4, %%r8\n" 138 CALL_NOSPEC 139 : "=a" (hv_status), ASM_CALL_CONSTRAINT, 140 "+c" (control), "+d" (input_address) 141 : "r" (output_address), 142 THUNK_TARGET(hv_hypercall_pg) 143 : "cc", "memory", "r8", "r9", "r10", "r11"); 144 #else 145 u32 input_address_hi = upper_32_bits(input_address); 146 u32 input_address_lo = lower_32_bits(input_address); 147 u32 output_address_hi = upper_32_bits(output_address); 148 u32 output_address_lo = lower_32_bits(output_address); 149 150 if (!hv_hypercall_pg) 151 return U64_MAX; 152 153 __asm__ __volatile__(CALL_NOSPEC 154 : "=A" (hv_status), 155 "+c" (input_address_lo), ASM_CALL_CONSTRAINT 156 : "A" (control), 157 "b" (input_address_hi), 158 "D"(output_address_hi), "S"(output_address_lo), 159 THUNK_TARGET(hv_hypercall_pg) 160 : "cc", "memory"); 161 #endif /* !x86_64 */ 162 return hv_status; 163 } 164 165 /* Fast hypercall with 8 bytes of input and no output */ 166 static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1) 167 { 168 u64 hv_status, control = (u64)code | HV_HYPERCALL_FAST_BIT; 169 170 #ifdef CONFIG_X86_64 171 { 172 __asm__ __volatile__(CALL_NOSPEC 173 : "=a" (hv_status), ASM_CALL_CONSTRAINT, 174 "+c" (control), "+d" (input1) 175 : THUNK_TARGET(hv_hypercall_pg) 176 : "cc", "r8", "r9", "r10", "r11"); 177 } 178 #else 179 { 180 u32 input1_hi = upper_32_bits(input1); 181 u32 input1_lo = lower_32_bits(input1); 182 183 __asm__ __volatile__ (CALL_NOSPEC 184 : "=A"(hv_status), 185 "+c"(input1_lo), 186 ASM_CALL_CONSTRAINT 187 : "A" (control), 188 "b" (input1_hi), 189 THUNK_TARGET(hv_hypercall_pg) 190 : "cc", "edi", "esi"); 191 } 192 #endif 193 return hv_status; 194 } 195 196 /* 197 * Rep hypercalls. Callers of this functions are supposed to ensure that 198 * rep_count and varhead_size comply with Hyper-V hypercall definition. 199 */ 200 static inline u64 hv_do_rep_hypercall(u16 code, u16 rep_count, u16 varhead_size, 201 void *input, void *output) 202 { 203 u64 control = code; 204 u64 status; 205 u16 rep_comp; 206 207 control |= (u64)varhead_size << HV_HYPERCALL_VARHEAD_OFFSET; 208 control |= (u64)rep_count << HV_HYPERCALL_REP_COMP_OFFSET; 209 210 do { 211 status = hv_do_hypercall(control, input, output); 212 if ((status & HV_HYPERCALL_RESULT_MASK) != HV_STATUS_SUCCESS) 213 return status; 214 215 /* Bits 32-43 of status have 'Reps completed' data. */ 216 rep_comp = (status & HV_HYPERCALL_REP_COMP_MASK) >> 217 HV_HYPERCALL_REP_COMP_OFFSET; 218 219 control &= ~HV_HYPERCALL_REP_START_MASK; 220 control |= (u64)rep_comp << HV_HYPERCALL_REP_START_OFFSET; 221 222 touch_nmi_watchdog(); 223 } while (rep_comp < rep_count); 224 225 return status; 226 } 227 228 /* 229 * Hypervisor's notion of virtual processor ID is different from 230 * Linux' notion of CPU ID. This information can only be retrieved 231 * in the context of the calling CPU. Setup a map for easy access 232 * to this information. 233 */ 234 extern u32 *hv_vp_index; 235 extern u32 hv_max_vp_index; 236 extern struct hv_vp_assist_page **hv_vp_assist_page; 237 238 static inline struct hv_vp_assist_page *hv_get_vp_assist_page(unsigned int cpu) 239 { 240 if (!hv_vp_assist_page) 241 return NULL; 242 243 return hv_vp_assist_page[cpu]; 244 } 245 246 /** 247 * hv_cpu_number_to_vp_number() - Map CPU to VP. 248 * @cpu_number: CPU number in Linux terms 249 * 250 * This function returns the mapping between the Linux processor 251 * number and the hypervisor's virtual processor number, useful 252 * in making hypercalls and such that talk about specific 253 * processors. 254 * 255 * Return: Virtual processor number in Hyper-V terms 256 */ 257 static inline int hv_cpu_number_to_vp_number(int cpu_number) 258 { 259 return hv_vp_index[cpu_number]; 260 } 261 262 static inline int cpumask_to_vpset(struct hv_vpset *vpset, 263 const struct cpumask *cpus) 264 { 265 int cpu, vcpu, vcpu_bank, vcpu_offset, nr_bank = 1; 266 267 /* valid_bank_mask can represent up to 64 banks */ 268 if (hv_max_vp_index / 64 >= 64) 269 return 0; 270 271 /* 272 * Clear all banks up to the maximum possible bank as hv_tlb_flush_ex 273 * structs are not cleared between calls, we risk flushing unneeded 274 * vCPUs otherwise. 275 */ 276 for (vcpu_bank = 0; vcpu_bank <= hv_max_vp_index / 64; vcpu_bank++) 277 vpset->bank_contents[vcpu_bank] = 0; 278 279 /* 280 * Some banks may end up being empty but this is acceptable. 281 */ 282 for_each_cpu(cpu, cpus) { 283 vcpu = hv_cpu_number_to_vp_number(cpu); 284 vcpu_bank = vcpu / 64; 285 vcpu_offset = vcpu % 64; 286 __set_bit(vcpu_offset, (unsigned long *) 287 &vpset->bank_contents[vcpu_bank]); 288 if (vcpu_bank >= nr_bank) 289 nr_bank = vcpu_bank + 1; 290 } 291 vpset->valid_bank_mask = GENMASK_ULL(nr_bank - 1, 0); 292 return nr_bank; 293 } 294 295 void __init hyperv_init(void); 296 void hyperv_setup_mmu_ops(void); 297 void hyperv_report_panic(struct pt_regs *regs, long err); 298 bool hv_is_hyperv_initialized(void); 299 void hyperv_cleanup(void); 300 301 void hyperv_reenlightenment_intr(struct pt_regs *regs); 302 void set_hv_tscchange_cb(void (*cb)(void)); 303 void clear_hv_tscchange_cb(void); 304 void hyperv_stop_tsc_emulation(void); 305 306 #ifdef CONFIG_X86_64 307 void hv_apic_init(void); 308 #else 309 static inline void hv_apic_init(void) {} 310 #endif 311 312 #else /* CONFIG_HYPERV */ 313 static inline void hyperv_init(void) {} 314 static inline bool hv_is_hyperv_initialized(void) { return false; } 315 static inline void hyperv_cleanup(void) {} 316 static inline void hyperv_setup_mmu_ops(void) {} 317 static inline void set_hv_tscchange_cb(void (*cb)(void)) {} 318 static inline void clear_hv_tscchange_cb(void) {} 319 static inline void hyperv_stop_tsc_emulation(void) {}; 320 static inline struct hv_vp_assist_page *hv_get_vp_assist_page(unsigned int cpu) 321 { 322 return NULL; 323 } 324 #endif /* CONFIG_HYPERV */ 325 326 #ifdef CONFIG_HYPERV_TSCPAGE 327 struct ms_hyperv_tsc_page *hv_get_tsc_page(void); 328 static inline u64 hv_read_tsc_page_tsc(const struct ms_hyperv_tsc_page *tsc_pg, 329 u64 *cur_tsc) 330 { 331 u64 scale, offset; 332 u32 sequence; 333 334 /* 335 * The protocol for reading Hyper-V TSC page is specified in Hypervisor 336 * Top-Level Functional Specification ver. 3.0 and above. To get the 337 * reference time we must do the following: 338 * - READ ReferenceTscSequence 339 * A special '0' value indicates the time source is unreliable and we 340 * need to use something else. The currently published specification 341 * versions (up to 4.0b) contain a mistake and wrongly claim '-1' 342 * instead of '0' as the special value, see commit c35b82ef0294. 343 * - ReferenceTime = 344 * ((RDTSC() * ReferenceTscScale) >> 64) + ReferenceTscOffset 345 * - READ ReferenceTscSequence again. In case its value has changed 346 * since our first reading we need to discard ReferenceTime and repeat 347 * the whole sequence as the hypervisor was updating the page in 348 * between. 349 */ 350 do { 351 sequence = READ_ONCE(tsc_pg->tsc_sequence); 352 if (!sequence) 353 return U64_MAX; 354 /* 355 * Make sure we read sequence before we read other values from 356 * TSC page. 357 */ 358 smp_rmb(); 359 360 scale = READ_ONCE(tsc_pg->tsc_scale); 361 offset = READ_ONCE(tsc_pg->tsc_offset); 362 *cur_tsc = rdtsc_ordered(); 363 364 /* 365 * Make sure we read sequence after we read all other values 366 * from TSC page. 367 */ 368 smp_rmb(); 369 370 } while (READ_ONCE(tsc_pg->tsc_sequence) != sequence); 371 372 return mul_u64_u64_shr(*cur_tsc, scale, 64) + offset; 373 } 374 375 static inline u64 hv_read_tsc_page(const struct ms_hyperv_tsc_page *tsc_pg) 376 { 377 u64 cur_tsc; 378 379 return hv_read_tsc_page_tsc(tsc_pg, &cur_tsc); 380 } 381 382 #else 383 static inline struct ms_hyperv_tsc_page *hv_get_tsc_page(void) 384 { 385 return NULL; 386 } 387 388 static inline u64 hv_read_tsc_page_tsc(const struct ms_hyperv_tsc_page *tsc_pg, 389 u64 *cur_tsc) 390 { 391 BUG(); 392 return U64_MAX; 393 } 394 #endif 395 #endif 396