1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * AMD SVM support 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Yaniv Kamay <yaniv@qumranet.com> 12 * Avi Kivity <avi@qumranet.com> 13 */ 14 15 #define pr_fmt(fmt) "SVM: " fmt 16 17 #include <linux/kvm_types.h> 18 #include <linux/hashtable.h> 19 #include <linux/amd-iommu.h> 20 #include <linux/kvm_host.h> 21 22 #include <asm/irq_remapping.h> 23 24 #include "trace.h" 25 #include "lapic.h" 26 #include "x86.h" 27 #include "irq.h" 28 #include "svm.h" 29 30 /* AVIC GATAG is encoded using VM and VCPU IDs */ 31 #define AVIC_VCPU_ID_BITS 8 32 #define AVIC_VCPU_ID_MASK ((1 << AVIC_VCPU_ID_BITS) - 1) 33 34 #define AVIC_VM_ID_BITS 24 35 #define AVIC_VM_ID_NR (1 << AVIC_VM_ID_BITS) 36 #define AVIC_VM_ID_MASK ((1 << AVIC_VM_ID_BITS) - 1) 37 38 #define AVIC_GATAG(x, y) (((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \ 39 (y & AVIC_VCPU_ID_MASK)) 40 #define AVIC_GATAG_TO_VMID(x) ((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK) 41 #define AVIC_GATAG_TO_VCPUID(x) (x & AVIC_VCPU_ID_MASK) 42 43 /* Note: 44 * This hash table is used to map VM_ID to a struct kvm_svm, 45 * when handling AMD IOMMU GALOG notification to schedule in 46 * a particular vCPU. 47 */ 48 #define SVM_VM_DATA_HASH_BITS 8 49 static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS); 50 static u32 next_vm_id = 0; 51 static bool next_vm_id_wrapped = 0; 52 static DEFINE_SPINLOCK(svm_vm_data_hash_lock); 53 54 /* 55 * This is a wrapper of struct amd_iommu_ir_data. 56 */ 57 struct amd_svm_iommu_ir { 58 struct list_head node; /* Used by SVM for per-vcpu ir_list */ 59 void *data; /* Storing pointer to struct amd_ir_data */ 60 }; 61 62 63 /* Note: 64 * This function is called from IOMMU driver to notify 65 * SVM to schedule in a particular vCPU of a particular VM. 66 */ 67 int avic_ga_log_notifier(u32 ga_tag) 68 { 69 unsigned long flags; 70 struct kvm_svm *kvm_svm; 71 struct kvm_vcpu *vcpu = NULL; 72 u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag); 73 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag); 74 75 pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id); 76 trace_kvm_avic_ga_log(vm_id, vcpu_id); 77 78 spin_lock_irqsave(&svm_vm_data_hash_lock, flags); 79 hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) { 80 if (kvm_svm->avic_vm_id != vm_id) 81 continue; 82 vcpu = kvm_get_vcpu_by_id(&kvm_svm->kvm, vcpu_id); 83 break; 84 } 85 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); 86 87 /* Note: 88 * At this point, the IOMMU should have already set the pending 89 * bit in the vAPIC backing page. So, we just need to schedule 90 * in the vcpu. 91 */ 92 if (vcpu) 93 kvm_vcpu_wake_up(vcpu); 94 95 return 0; 96 } 97 98 void avic_vm_destroy(struct kvm *kvm) 99 { 100 unsigned long flags; 101 struct kvm_svm *kvm_svm = to_kvm_svm(kvm); 102 103 if (!enable_apicv) 104 return; 105 106 if (kvm_svm->avic_logical_id_table_page) 107 __free_page(kvm_svm->avic_logical_id_table_page); 108 if (kvm_svm->avic_physical_id_table_page) 109 __free_page(kvm_svm->avic_physical_id_table_page); 110 111 spin_lock_irqsave(&svm_vm_data_hash_lock, flags); 112 hash_del(&kvm_svm->hnode); 113 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); 114 } 115 116 int avic_vm_init(struct kvm *kvm) 117 { 118 unsigned long flags; 119 int err = -ENOMEM; 120 struct kvm_svm *kvm_svm = to_kvm_svm(kvm); 121 struct kvm_svm *k2; 122 struct page *p_page; 123 struct page *l_page; 124 u32 vm_id; 125 126 if (!enable_apicv) 127 return 0; 128 129 /* Allocating physical APIC ID table (4KB) */ 130 p_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 131 if (!p_page) 132 goto free_avic; 133 134 kvm_svm->avic_physical_id_table_page = p_page; 135 136 /* Allocating logical APIC ID table (4KB) */ 137 l_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 138 if (!l_page) 139 goto free_avic; 140 141 kvm_svm->avic_logical_id_table_page = l_page; 142 143 spin_lock_irqsave(&svm_vm_data_hash_lock, flags); 144 again: 145 vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK; 146 if (vm_id == 0) { /* id is 1-based, zero is not okay */ 147 next_vm_id_wrapped = 1; 148 goto again; 149 } 150 /* Is it still in use? Only possible if wrapped at least once */ 151 if (next_vm_id_wrapped) { 152 hash_for_each_possible(svm_vm_data_hash, k2, hnode, vm_id) { 153 if (k2->avic_vm_id == vm_id) 154 goto again; 155 } 156 } 157 kvm_svm->avic_vm_id = vm_id; 158 hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id); 159 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); 160 161 return 0; 162 163 free_avic: 164 avic_vm_destroy(kvm); 165 return err; 166 } 167 168 void avic_init_vmcb(struct vcpu_svm *svm, struct vmcb *vmcb) 169 { 170 struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm); 171 phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page)); 172 phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page)); 173 phys_addr_t ppa = __sme_set(page_to_phys(kvm_svm->avic_physical_id_table_page)); 174 175 vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK; 176 vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK; 177 vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK; 178 vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT; 179 vmcb->control.avic_vapic_bar = APIC_DEFAULT_PHYS_BASE & VMCB_AVIC_APIC_BAR_MASK; 180 181 if (kvm_apicv_activated(svm->vcpu.kvm)) 182 vmcb->control.int_ctl |= AVIC_ENABLE_MASK; 183 else 184 vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK; 185 } 186 187 static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu, 188 unsigned int index) 189 { 190 u64 *avic_physical_id_table; 191 struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); 192 193 if (index >= AVIC_MAX_PHYSICAL_ID_COUNT) 194 return NULL; 195 196 avic_physical_id_table = page_address(kvm_svm->avic_physical_id_table_page); 197 198 return &avic_physical_id_table[index]; 199 } 200 201 /* 202 * Note: 203 * AVIC hardware walks the nested page table to check permissions, 204 * but does not use the SPA address specified in the leaf page 205 * table entry since it uses address in the AVIC_BACKING_PAGE pointer 206 * field of the VMCB. Therefore, we set up the 207 * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here. 208 */ 209 static int avic_alloc_access_page(struct kvm *kvm) 210 { 211 void __user *ret; 212 int r = 0; 213 214 mutex_lock(&kvm->slots_lock); 215 216 if (kvm->arch.apic_access_memslot_enabled) 217 goto out; 218 219 ret = __x86_set_memory_region(kvm, 220 APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 221 APIC_DEFAULT_PHYS_BASE, 222 PAGE_SIZE); 223 if (IS_ERR(ret)) { 224 r = PTR_ERR(ret); 225 goto out; 226 } 227 228 kvm->arch.apic_access_memslot_enabled = true; 229 out: 230 mutex_unlock(&kvm->slots_lock); 231 return r; 232 } 233 234 static int avic_init_backing_page(struct kvm_vcpu *vcpu) 235 { 236 u64 *entry, new_entry; 237 int id = vcpu->vcpu_id; 238 struct vcpu_svm *svm = to_svm(vcpu); 239 240 if (id >= AVIC_MAX_PHYSICAL_ID_COUNT) 241 return -EINVAL; 242 243 if (!vcpu->arch.apic->regs) 244 return -EINVAL; 245 246 if (kvm_apicv_activated(vcpu->kvm)) { 247 int ret; 248 249 ret = avic_alloc_access_page(vcpu->kvm); 250 if (ret) 251 return ret; 252 } 253 254 svm->avic_backing_page = virt_to_page(vcpu->arch.apic->regs); 255 256 /* Setting AVIC backing page address in the phy APIC ID table */ 257 entry = avic_get_physical_id_entry(vcpu, id); 258 if (!entry) 259 return -EINVAL; 260 261 new_entry = __sme_set((page_to_phys(svm->avic_backing_page) & 262 AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) | 263 AVIC_PHYSICAL_ID_ENTRY_VALID_MASK); 264 WRITE_ONCE(*entry, new_entry); 265 266 svm->avic_physical_id_cache = entry; 267 268 return 0; 269 } 270 271 void avic_ring_doorbell(struct kvm_vcpu *vcpu) 272 { 273 /* 274 * Note, the vCPU could get migrated to a different pCPU at any point, 275 * which could result in signalling the wrong/previous pCPU. But if 276 * that happens the vCPU is guaranteed to do a VMRUN (after being 277 * migrated) and thus will process pending interrupts, i.e. a doorbell 278 * is not needed (and the spurious one is harmless). 279 */ 280 int cpu = READ_ONCE(vcpu->cpu); 281 282 if (cpu != get_cpu()) 283 wrmsrl(MSR_AMD64_SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpu)); 284 put_cpu(); 285 } 286 287 /* 288 * A fast-path version of avic_kick_target_vcpus(), which attempts to match 289 * destination APIC ID to vCPU without looping through all vCPUs. 290 */ 291 static int avic_kick_target_vcpus_fast(struct kvm *kvm, struct kvm_lapic *source, 292 u32 icrl, u32 icrh, u32 index) 293 { 294 u32 dest, apic_id; 295 struct kvm_vcpu *vcpu; 296 int dest_mode = icrl & APIC_DEST_MASK; 297 int shorthand = icrl & APIC_SHORT_MASK; 298 struct kvm_svm *kvm_svm = to_kvm_svm(kvm); 299 u32 *avic_logical_id_table = page_address(kvm_svm->avic_logical_id_table_page); 300 301 if (shorthand != APIC_DEST_NOSHORT) 302 return -EINVAL; 303 304 /* 305 * The AVIC incomplete IPI #vmexit info provides index into 306 * the physical APIC ID table, which can be used to derive 307 * guest physical APIC ID. 308 */ 309 if (dest_mode == APIC_DEST_PHYSICAL) { 310 apic_id = index; 311 } else { 312 if (!apic_x2apic_mode(source)) { 313 /* For xAPIC logical mode, the index is for logical APIC table. */ 314 apic_id = avic_logical_id_table[index] & 0x1ff; 315 } else { 316 return -EINVAL; 317 } 318 } 319 320 /* 321 * Assuming vcpu ID is the same as physical apic ID, 322 * and use it to retrieve the target vCPU. 323 */ 324 vcpu = kvm_get_vcpu_by_id(kvm, apic_id); 325 if (!vcpu) 326 return -EINVAL; 327 328 if (apic_x2apic_mode(vcpu->arch.apic)) 329 dest = icrh; 330 else 331 dest = GET_APIC_DEST_FIELD(icrh); 332 333 /* 334 * Try matching the destination APIC ID with the vCPU. 335 */ 336 if (kvm_apic_match_dest(vcpu, source, shorthand, dest, dest_mode)) { 337 vcpu->arch.apic->irr_pending = true; 338 svm_complete_interrupt_delivery(vcpu, 339 icrl & APIC_MODE_MASK, 340 icrl & APIC_INT_LEVELTRIG, 341 icrl & APIC_VECTOR_MASK); 342 return 0; 343 } 344 345 return -EINVAL; 346 } 347 348 static void avic_kick_target_vcpus(struct kvm *kvm, struct kvm_lapic *source, 349 u32 icrl, u32 icrh, u32 index) 350 { 351 unsigned long i; 352 struct kvm_vcpu *vcpu; 353 354 if (!avic_kick_target_vcpus_fast(kvm, source, icrl, icrh, index)) 355 return; 356 357 trace_kvm_avic_kick_vcpu_slowpath(icrh, icrl, index); 358 359 /* 360 * Wake any target vCPUs that are blocking, i.e. waiting for a wake 361 * event. There's no need to signal doorbells, as hardware has handled 362 * vCPUs that were in guest at the time of the IPI, and vCPUs that have 363 * since entered the guest will have processed pending IRQs at VMRUN. 364 */ 365 kvm_for_each_vcpu(i, vcpu, kvm) { 366 if (kvm_apic_match_dest(vcpu, source, icrl & APIC_SHORT_MASK, 367 GET_APIC_DEST_FIELD(icrh), 368 icrl & APIC_DEST_MASK)) { 369 vcpu->arch.apic->irr_pending = true; 370 svm_complete_interrupt_delivery(vcpu, 371 icrl & APIC_MODE_MASK, 372 icrl & APIC_INT_LEVELTRIG, 373 icrl & APIC_VECTOR_MASK); 374 } 375 } 376 } 377 378 int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu) 379 { 380 struct vcpu_svm *svm = to_svm(vcpu); 381 u32 icrh = svm->vmcb->control.exit_info_1 >> 32; 382 u32 icrl = svm->vmcb->control.exit_info_1; 383 u32 id = svm->vmcb->control.exit_info_2 >> 32; 384 u32 index = svm->vmcb->control.exit_info_2 & 0x1FF; 385 struct kvm_lapic *apic = vcpu->arch.apic; 386 387 trace_kvm_avic_incomplete_ipi(vcpu->vcpu_id, icrh, icrl, id, index); 388 389 switch (id) { 390 case AVIC_IPI_FAILURE_INVALID_INT_TYPE: 391 /* 392 * Emulate IPIs that are not handled by AVIC hardware, which 393 * only virtualizes Fixed, Edge-Triggered INTRs. The exit is 394 * a trap, e.g. ICR holds the correct value and RIP has been 395 * advanced, KVM is responsible only for emulating the IPI. 396 * Sadly, hardware may sometimes leave the BUSY flag set, in 397 * which case KVM needs to emulate the ICR write as well in 398 * order to clear the BUSY flag. 399 */ 400 if (icrl & APIC_ICR_BUSY) 401 kvm_apic_write_nodecode(vcpu, APIC_ICR); 402 else 403 kvm_apic_send_ipi(apic, icrl, icrh); 404 break; 405 case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING: 406 /* 407 * At this point, we expect that the AVIC HW has already 408 * set the appropriate IRR bits on the valid target 409 * vcpus. So, we just need to kick the appropriate vcpu. 410 */ 411 avic_kick_target_vcpus(vcpu->kvm, apic, icrl, icrh, index); 412 break; 413 case AVIC_IPI_FAILURE_INVALID_TARGET: 414 break; 415 case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE: 416 WARN_ONCE(1, "Invalid backing page\n"); 417 break; 418 default: 419 pr_err("Unknown IPI interception\n"); 420 } 421 422 return 1; 423 } 424 425 unsigned long avic_vcpu_get_apicv_inhibit_reasons(struct kvm_vcpu *vcpu) 426 { 427 if (is_guest_mode(vcpu)) 428 return APICV_INHIBIT_REASON_NESTED; 429 return 0; 430 } 431 432 static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat) 433 { 434 struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); 435 int index; 436 u32 *logical_apic_id_table; 437 int dlid = GET_APIC_LOGICAL_ID(ldr); 438 439 if (!dlid) 440 return NULL; 441 442 if (flat) { /* flat */ 443 index = ffs(dlid) - 1; 444 if (index > 7) 445 return NULL; 446 } else { /* cluster */ 447 int cluster = (dlid & 0xf0) >> 4; 448 int apic = ffs(dlid & 0x0f) - 1; 449 450 if ((apic < 0) || (apic > 7) || 451 (cluster >= 0xf)) 452 return NULL; 453 index = (cluster << 2) + apic; 454 } 455 456 logical_apic_id_table = (u32 *) page_address(kvm_svm->avic_logical_id_table_page); 457 458 return &logical_apic_id_table[index]; 459 } 460 461 static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr) 462 { 463 bool flat; 464 u32 *entry, new_entry; 465 466 flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT; 467 entry = avic_get_logical_id_entry(vcpu, ldr, flat); 468 if (!entry) 469 return -EINVAL; 470 471 new_entry = READ_ONCE(*entry); 472 new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK; 473 new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK); 474 new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK; 475 WRITE_ONCE(*entry, new_entry); 476 477 return 0; 478 } 479 480 static void avic_invalidate_logical_id_entry(struct kvm_vcpu *vcpu) 481 { 482 struct vcpu_svm *svm = to_svm(vcpu); 483 bool flat = svm->dfr_reg == APIC_DFR_FLAT; 484 u32 *entry = avic_get_logical_id_entry(vcpu, svm->ldr_reg, flat); 485 486 if (entry) 487 clear_bit(AVIC_LOGICAL_ID_ENTRY_VALID_BIT, (unsigned long *)entry); 488 } 489 490 static int avic_handle_ldr_update(struct kvm_vcpu *vcpu) 491 { 492 int ret = 0; 493 struct vcpu_svm *svm = to_svm(vcpu); 494 u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR); 495 u32 id = kvm_xapic_id(vcpu->arch.apic); 496 497 if (ldr == svm->ldr_reg) 498 return 0; 499 500 avic_invalidate_logical_id_entry(vcpu); 501 502 if (ldr) 503 ret = avic_ldr_write(vcpu, id, ldr); 504 505 if (!ret) 506 svm->ldr_reg = ldr; 507 508 return ret; 509 } 510 511 static int avic_handle_apic_id_update(struct kvm_vcpu *vcpu) 512 { 513 u64 *old, *new; 514 struct vcpu_svm *svm = to_svm(vcpu); 515 u32 id = kvm_xapic_id(vcpu->arch.apic); 516 517 if (vcpu->vcpu_id == id) 518 return 0; 519 520 old = avic_get_physical_id_entry(vcpu, vcpu->vcpu_id); 521 new = avic_get_physical_id_entry(vcpu, id); 522 if (!new || !old) 523 return 1; 524 525 /* We need to move physical_id_entry to new offset */ 526 *new = *old; 527 *old = 0ULL; 528 to_svm(vcpu)->avic_physical_id_cache = new; 529 530 /* 531 * Also update the guest physical APIC ID in the logical 532 * APIC ID table entry if already setup the LDR. 533 */ 534 if (svm->ldr_reg) 535 avic_handle_ldr_update(vcpu); 536 537 return 0; 538 } 539 540 static void avic_handle_dfr_update(struct kvm_vcpu *vcpu) 541 { 542 struct vcpu_svm *svm = to_svm(vcpu); 543 u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR); 544 545 if (svm->dfr_reg == dfr) 546 return; 547 548 avic_invalidate_logical_id_entry(vcpu); 549 svm->dfr_reg = dfr; 550 } 551 552 static int avic_unaccel_trap_write(struct kvm_vcpu *vcpu) 553 { 554 u32 offset = to_svm(vcpu)->vmcb->control.exit_info_1 & 555 AVIC_UNACCEL_ACCESS_OFFSET_MASK; 556 557 switch (offset) { 558 case APIC_ID: 559 if (avic_handle_apic_id_update(vcpu)) 560 return 0; 561 break; 562 case APIC_LDR: 563 if (avic_handle_ldr_update(vcpu)) 564 return 0; 565 break; 566 case APIC_DFR: 567 avic_handle_dfr_update(vcpu); 568 break; 569 default: 570 break; 571 } 572 573 kvm_apic_write_nodecode(vcpu, offset); 574 return 1; 575 } 576 577 static bool is_avic_unaccelerated_access_trap(u32 offset) 578 { 579 bool ret = false; 580 581 switch (offset) { 582 case APIC_ID: 583 case APIC_EOI: 584 case APIC_RRR: 585 case APIC_LDR: 586 case APIC_DFR: 587 case APIC_SPIV: 588 case APIC_ESR: 589 case APIC_ICR: 590 case APIC_LVTT: 591 case APIC_LVTTHMR: 592 case APIC_LVTPC: 593 case APIC_LVT0: 594 case APIC_LVT1: 595 case APIC_LVTERR: 596 case APIC_TMICT: 597 case APIC_TDCR: 598 ret = true; 599 break; 600 default: 601 break; 602 } 603 return ret; 604 } 605 606 int avic_unaccelerated_access_interception(struct kvm_vcpu *vcpu) 607 { 608 struct vcpu_svm *svm = to_svm(vcpu); 609 int ret = 0; 610 u32 offset = svm->vmcb->control.exit_info_1 & 611 AVIC_UNACCEL_ACCESS_OFFSET_MASK; 612 u32 vector = svm->vmcb->control.exit_info_2 & 613 AVIC_UNACCEL_ACCESS_VECTOR_MASK; 614 bool write = (svm->vmcb->control.exit_info_1 >> 32) & 615 AVIC_UNACCEL_ACCESS_WRITE_MASK; 616 bool trap = is_avic_unaccelerated_access_trap(offset); 617 618 trace_kvm_avic_unaccelerated_access(vcpu->vcpu_id, offset, 619 trap, write, vector); 620 if (trap) { 621 /* Handling Trap */ 622 WARN_ONCE(!write, "svm: Handling trap read.\n"); 623 ret = avic_unaccel_trap_write(vcpu); 624 } else { 625 /* Handling Fault */ 626 ret = kvm_emulate_instruction(vcpu, 0); 627 } 628 629 return ret; 630 } 631 632 int avic_init_vcpu(struct vcpu_svm *svm) 633 { 634 int ret; 635 struct kvm_vcpu *vcpu = &svm->vcpu; 636 637 if (!enable_apicv || !irqchip_in_kernel(vcpu->kvm)) 638 return 0; 639 640 ret = avic_init_backing_page(vcpu); 641 if (ret) 642 return ret; 643 644 INIT_LIST_HEAD(&svm->ir_list); 645 spin_lock_init(&svm->ir_list_lock); 646 svm->dfr_reg = APIC_DFR_FLAT; 647 648 return ret; 649 } 650 651 void avic_apicv_post_state_restore(struct kvm_vcpu *vcpu) 652 { 653 if (avic_handle_apic_id_update(vcpu) != 0) 654 return; 655 avic_handle_dfr_update(vcpu); 656 avic_handle_ldr_update(vcpu); 657 } 658 659 static int avic_set_pi_irte_mode(struct kvm_vcpu *vcpu, bool activate) 660 { 661 int ret = 0; 662 unsigned long flags; 663 struct amd_svm_iommu_ir *ir; 664 struct vcpu_svm *svm = to_svm(vcpu); 665 666 if (!kvm_arch_has_assigned_device(vcpu->kvm)) 667 return 0; 668 669 /* 670 * Here, we go through the per-vcpu ir_list to update all existing 671 * interrupt remapping table entry targeting this vcpu. 672 */ 673 spin_lock_irqsave(&svm->ir_list_lock, flags); 674 675 if (list_empty(&svm->ir_list)) 676 goto out; 677 678 list_for_each_entry(ir, &svm->ir_list, node) { 679 if (activate) 680 ret = amd_iommu_activate_guest_mode(ir->data); 681 else 682 ret = amd_iommu_deactivate_guest_mode(ir->data); 683 if (ret) 684 break; 685 } 686 out: 687 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 688 return ret; 689 } 690 691 static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi) 692 { 693 unsigned long flags; 694 struct amd_svm_iommu_ir *cur; 695 696 spin_lock_irqsave(&svm->ir_list_lock, flags); 697 list_for_each_entry(cur, &svm->ir_list, node) { 698 if (cur->data != pi->ir_data) 699 continue; 700 list_del(&cur->node); 701 kfree(cur); 702 break; 703 } 704 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 705 } 706 707 static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi) 708 { 709 int ret = 0; 710 unsigned long flags; 711 struct amd_svm_iommu_ir *ir; 712 713 /** 714 * In some cases, the existing irte is updated and re-set, 715 * so we need to check here if it's already been * added 716 * to the ir_list. 717 */ 718 if (pi->ir_data && (pi->prev_ga_tag != 0)) { 719 struct kvm *kvm = svm->vcpu.kvm; 720 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag); 721 struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id); 722 struct vcpu_svm *prev_svm; 723 724 if (!prev_vcpu) { 725 ret = -EINVAL; 726 goto out; 727 } 728 729 prev_svm = to_svm(prev_vcpu); 730 svm_ir_list_del(prev_svm, pi); 731 } 732 733 /** 734 * Allocating new amd_iommu_pi_data, which will get 735 * add to the per-vcpu ir_list. 736 */ 737 ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL_ACCOUNT); 738 if (!ir) { 739 ret = -ENOMEM; 740 goto out; 741 } 742 ir->data = pi->ir_data; 743 744 spin_lock_irqsave(&svm->ir_list_lock, flags); 745 list_add(&ir->node, &svm->ir_list); 746 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 747 out: 748 return ret; 749 } 750 751 /* 752 * Note: 753 * The HW cannot support posting multicast/broadcast 754 * interrupts to a vCPU. So, we still use legacy interrupt 755 * remapping for these kind of interrupts. 756 * 757 * For lowest-priority interrupts, we only support 758 * those with single CPU as the destination, e.g. user 759 * configures the interrupts via /proc/irq or uses 760 * irqbalance to make the interrupts single-CPU. 761 */ 762 static int 763 get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, 764 struct vcpu_data *vcpu_info, struct vcpu_svm **svm) 765 { 766 struct kvm_lapic_irq irq; 767 struct kvm_vcpu *vcpu = NULL; 768 769 kvm_set_msi_irq(kvm, e, &irq); 770 771 if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) || 772 !kvm_irq_is_postable(&irq)) { 773 pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n", 774 __func__, irq.vector); 775 return -1; 776 } 777 778 pr_debug("SVM: %s: use GA mode for irq %u\n", __func__, 779 irq.vector); 780 *svm = to_svm(vcpu); 781 vcpu_info->pi_desc_addr = __sme_set(page_to_phys((*svm)->avic_backing_page)); 782 vcpu_info->vector = irq.vector; 783 784 return 0; 785 } 786 787 /* 788 * avic_pi_update_irte - set IRTE for Posted-Interrupts 789 * 790 * @kvm: kvm 791 * @host_irq: host irq of the interrupt 792 * @guest_irq: gsi of the interrupt 793 * @set: set or unset PI 794 * returns 0 on success, < 0 on failure 795 */ 796 int avic_pi_update_irte(struct kvm *kvm, unsigned int host_irq, 797 uint32_t guest_irq, bool set) 798 { 799 struct kvm_kernel_irq_routing_entry *e; 800 struct kvm_irq_routing_table *irq_rt; 801 int idx, ret = 0; 802 803 if (!kvm_arch_has_assigned_device(kvm) || 804 !irq_remapping_cap(IRQ_POSTING_CAP)) 805 return 0; 806 807 pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n", 808 __func__, host_irq, guest_irq, set); 809 810 idx = srcu_read_lock(&kvm->irq_srcu); 811 irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu); 812 813 if (guest_irq >= irq_rt->nr_rt_entries || 814 hlist_empty(&irq_rt->map[guest_irq])) { 815 pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n", 816 guest_irq, irq_rt->nr_rt_entries); 817 goto out; 818 } 819 820 hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) { 821 struct vcpu_data vcpu_info; 822 struct vcpu_svm *svm = NULL; 823 824 if (e->type != KVM_IRQ_ROUTING_MSI) 825 continue; 826 827 /** 828 * Here, we setup with legacy mode in the following cases: 829 * 1. When cannot target interrupt to a specific vcpu. 830 * 2. Unsetting posted interrupt. 831 * 3. APIC virtualization is disabled for the vcpu. 832 * 4. IRQ has incompatible delivery mode (SMI, INIT, etc) 833 */ 834 if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set && 835 kvm_vcpu_apicv_active(&svm->vcpu)) { 836 struct amd_iommu_pi_data pi; 837 838 /* Try to enable guest_mode in IRTE */ 839 pi.base = __sme_set(page_to_phys(svm->avic_backing_page) & 840 AVIC_HPA_MASK); 841 pi.ga_tag = AVIC_GATAG(to_kvm_svm(kvm)->avic_vm_id, 842 svm->vcpu.vcpu_id); 843 pi.is_guest_mode = true; 844 pi.vcpu_data = &vcpu_info; 845 ret = irq_set_vcpu_affinity(host_irq, &pi); 846 847 /** 848 * Here, we successfully setting up vcpu affinity in 849 * IOMMU guest mode. Now, we need to store the posted 850 * interrupt information in a per-vcpu ir_list so that 851 * we can reference to them directly when we update vcpu 852 * scheduling information in IOMMU irte. 853 */ 854 if (!ret && pi.is_guest_mode) 855 svm_ir_list_add(svm, &pi); 856 } else { 857 /* Use legacy mode in IRTE */ 858 struct amd_iommu_pi_data pi; 859 860 /** 861 * Here, pi is used to: 862 * - Tell IOMMU to use legacy mode for this interrupt. 863 * - Retrieve ga_tag of prior interrupt remapping data. 864 */ 865 pi.prev_ga_tag = 0; 866 pi.is_guest_mode = false; 867 ret = irq_set_vcpu_affinity(host_irq, &pi); 868 869 /** 870 * Check if the posted interrupt was previously 871 * setup with the guest_mode by checking if the ga_tag 872 * was cached. If so, we need to clean up the per-vcpu 873 * ir_list. 874 */ 875 if (!ret && pi.prev_ga_tag) { 876 int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag); 877 struct kvm_vcpu *vcpu; 878 879 vcpu = kvm_get_vcpu_by_id(kvm, id); 880 if (vcpu) 881 svm_ir_list_del(to_svm(vcpu), &pi); 882 } 883 } 884 885 if (!ret && svm) { 886 trace_kvm_pi_irte_update(host_irq, svm->vcpu.vcpu_id, 887 e->gsi, vcpu_info.vector, 888 vcpu_info.pi_desc_addr, set); 889 } 890 891 if (ret < 0) { 892 pr_err("%s: failed to update PI IRTE\n", __func__); 893 goto out; 894 } 895 } 896 897 ret = 0; 898 out: 899 srcu_read_unlock(&kvm->irq_srcu, idx); 900 return ret; 901 } 902 903 bool avic_check_apicv_inhibit_reasons(enum kvm_apicv_inhibit reason) 904 { 905 ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) | 906 BIT(APICV_INHIBIT_REASON_ABSENT) | 907 BIT(APICV_INHIBIT_REASON_HYPERV) | 908 BIT(APICV_INHIBIT_REASON_NESTED) | 909 BIT(APICV_INHIBIT_REASON_IRQWIN) | 910 BIT(APICV_INHIBIT_REASON_PIT_REINJ) | 911 BIT(APICV_INHIBIT_REASON_X2APIC) | 912 BIT(APICV_INHIBIT_REASON_BLOCKIRQ) | 913 BIT(APICV_INHIBIT_REASON_SEV); 914 915 return supported & BIT(reason); 916 } 917 918 919 static inline int 920 avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r) 921 { 922 int ret = 0; 923 unsigned long flags; 924 struct amd_svm_iommu_ir *ir; 925 struct vcpu_svm *svm = to_svm(vcpu); 926 927 if (!kvm_arch_has_assigned_device(vcpu->kvm)) 928 return 0; 929 930 /* 931 * Here, we go through the per-vcpu ir_list to update all existing 932 * interrupt remapping table entry targeting this vcpu. 933 */ 934 spin_lock_irqsave(&svm->ir_list_lock, flags); 935 936 if (list_empty(&svm->ir_list)) 937 goto out; 938 939 list_for_each_entry(ir, &svm->ir_list, node) { 940 ret = amd_iommu_update_ga(cpu, r, ir->data); 941 if (ret) 942 break; 943 } 944 out: 945 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 946 return ret; 947 } 948 949 void __avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 950 { 951 u64 entry; 952 int h_physical_id = kvm_cpu_get_apicid(cpu); 953 struct vcpu_svm *svm = to_svm(vcpu); 954 955 lockdep_assert_preemption_disabled(); 956 957 if (WARN_ON(h_physical_id & ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK)) 958 return; 959 960 /* 961 * No need to update anything if the vCPU is blocking, i.e. if the vCPU 962 * is being scheduled in after being preempted. The CPU entries in the 963 * Physical APIC table and IRTE are consumed iff IsRun{ning} is '1'. 964 * If the vCPU was migrated, its new CPU value will be stuffed when the 965 * vCPU unblocks. 966 */ 967 if (kvm_vcpu_is_blocking(vcpu)) 968 return; 969 970 entry = READ_ONCE(*(svm->avic_physical_id_cache)); 971 WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK); 972 973 entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK; 974 entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK); 975 entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; 976 977 WRITE_ONCE(*(svm->avic_physical_id_cache), entry); 978 avic_update_iommu_vcpu_affinity(vcpu, h_physical_id, true); 979 } 980 981 void __avic_vcpu_put(struct kvm_vcpu *vcpu) 982 { 983 u64 entry; 984 struct vcpu_svm *svm = to_svm(vcpu); 985 986 lockdep_assert_preemption_disabled(); 987 988 entry = READ_ONCE(*(svm->avic_physical_id_cache)); 989 990 /* Nothing to do if IsRunning == '0' due to vCPU blocking. */ 991 if (!(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK)) 992 return; 993 994 avic_update_iommu_vcpu_affinity(vcpu, -1, 0); 995 996 entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; 997 WRITE_ONCE(*(svm->avic_physical_id_cache), entry); 998 } 999 1000 static void avic_vcpu_load(struct kvm_vcpu *vcpu) 1001 { 1002 int cpu = get_cpu(); 1003 1004 WARN_ON(cpu != vcpu->cpu); 1005 1006 __avic_vcpu_load(vcpu, cpu); 1007 1008 put_cpu(); 1009 } 1010 1011 static void avic_vcpu_put(struct kvm_vcpu *vcpu) 1012 { 1013 preempt_disable(); 1014 1015 __avic_vcpu_put(vcpu); 1016 1017 preempt_enable(); 1018 } 1019 1020 void avic_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) 1021 { 1022 struct vcpu_svm *svm = to_svm(vcpu); 1023 struct vmcb *vmcb = svm->vmcb01.ptr; 1024 bool activated = kvm_vcpu_apicv_active(vcpu); 1025 1026 if (!enable_apicv) 1027 return; 1028 1029 if (activated) { 1030 /** 1031 * During AVIC temporary deactivation, guest could update 1032 * APIC ID, DFR and LDR registers, which would not be trapped 1033 * by avic_unaccelerated_access_interception(). In this case, 1034 * we need to check and update the AVIC logical APIC ID table 1035 * accordingly before re-activating. 1036 */ 1037 avic_apicv_post_state_restore(vcpu); 1038 vmcb->control.int_ctl |= AVIC_ENABLE_MASK; 1039 } else { 1040 vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK; 1041 } 1042 vmcb_mark_dirty(vmcb, VMCB_AVIC); 1043 1044 if (activated) 1045 avic_vcpu_load(vcpu); 1046 else 1047 avic_vcpu_put(vcpu); 1048 1049 avic_set_pi_irte_mode(vcpu, activated); 1050 } 1051 1052 void avic_vcpu_blocking(struct kvm_vcpu *vcpu) 1053 { 1054 if (!kvm_vcpu_apicv_active(vcpu)) 1055 return; 1056 1057 /* 1058 * Unload the AVIC when the vCPU is about to block, _before_ 1059 * the vCPU actually blocks. 1060 * 1061 * Any IRQs that arrive before IsRunning=0 will not cause an 1062 * incomplete IPI vmexit on the source, therefore vIRR will also 1063 * be checked by kvm_vcpu_check_block() before blocking. The 1064 * memory barrier implicit in set_current_state orders writing 1065 * IsRunning=0 before reading the vIRR. The processor needs a 1066 * matching memory barrier on interrupt delivery between writing 1067 * IRR and reading IsRunning; the lack of this barrier might be 1068 * the cause of errata #1235). 1069 */ 1070 avic_vcpu_put(vcpu); 1071 } 1072 1073 void avic_vcpu_unblocking(struct kvm_vcpu *vcpu) 1074 { 1075 if (!kvm_vcpu_apicv_active(vcpu)) 1076 return; 1077 1078 avic_vcpu_load(vcpu); 1079 } 1080