1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * AMD SVM support 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Yaniv Kamay <yaniv@qumranet.com> 12 * Avi Kivity <avi@qumranet.com> 13 */ 14 15 #define pr_fmt(fmt) "SVM: " fmt 16 17 #include <linux/kvm_types.h> 18 #include <linux/hashtable.h> 19 #include <linux/amd-iommu.h> 20 #include <linux/kvm_host.h> 21 22 #include <asm/irq_remapping.h> 23 24 #include "trace.h" 25 #include "lapic.h" 26 #include "x86.h" 27 #include "irq.h" 28 #include "svm.h" 29 30 #define SVM_AVIC_DOORBELL 0xc001011b 31 32 #define AVIC_HPA_MASK ~((0xFFFULL << 52) | 0xFFF) 33 34 /* 35 * 0xff is broadcast, so the max index allowed for physical APIC ID 36 * table is 0xfe. APIC IDs above 0xff are reserved. 37 */ 38 #define AVIC_MAX_PHYSICAL_ID_COUNT 255 39 40 #define AVIC_UNACCEL_ACCESS_WRITE_MASK 1 41 #define AVIC_UNACCEL_ACCESS_OFFSET_MASK 0xFF0 42 #define AVIC_UNACCEL_ACCESS_VECTOR_MASK 0xFFFFFFFF 43 44 /* AVIC GATAG is encoded using VM and VCPU IDs */ 45 #define AVIC_VCPU_ID_BITS 8 46 #define AVIC_VCPU_ID_MASK ((1 << AVIC_VCPU_ID_BITS) - 1) 47 48 #define AVIC_VM_ID_BITS 24 49 #define AVIC_VM_ID_NR (1 << AVIC_VM_ID_BITS) 50 #define AVIC_VM_ID_MASK ((1 << AVIC_VM_ID_BITS) - 1) 51 52 #define AVIC_GATAG(x, y) (((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \ 53 (y & AVIC_VCPU_ID_MASK)) 54 #define AVIC_GATAG_TO_VMID(x) ((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK) 55 #define AVIC_GATAG_TO_VCPUID(x) (x & AVIC_VCPU_ID_MASK) 56 57 /* Note: 58 * This hash table is used to map VM_ID to a struct kvm_svm, 59 * when handling AMD IOMMU GALOG notification to schedule in 60 * a particular vCPU. 61 */ 62 #define SVM_VM_DATA_HASH_BITS 8 63 static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS); 64 static u32 next_vm_id = 0; 65 static bool next_vm_id_wrapped = 0; 66 static DEFINE_SPINLOCK(svm_vm_data_hash_lock); 67 68 /* 69 * This is a wrapper of struct amd_iommu_ir_data. 70 */ 71 struct amd_svm_iommu_ir { 72 struct list_head node; /* Used by SVM for per-vcpu ir_list */ 73 void *data; /* Storing pointer to struct amd_ir_data */ 74 }; 75 76 enum avic_ipi_failure_cause { 77 AVIC_IPI_FAILURE_INVALID_INT_TYPE, 78 AVIC_IPI_FAILURE_TARGET_NOT_RUNNING, 79 AVIC_IPI_FAILURE_INVALID_TARGET, 80 AVIC_IPI_FAILURE_INVALID_BACKING_PAGE, 81 }; 82 83 /* Note: 84 * This function is called from IOMMU driver to notify 85 * SVM to schedule in a particular vCPU of a particular VM. 86 */ 87 int avic_ga_log_notifier(u32 ga_tag) 88 { 89 unsigned long flags; 90 struct kvm_svm *kvm_svm; 91 struct kvm_vcpu *vcpu = NULL; 92 u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag); 93 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag); 94 95 pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id); 96 trace_kvm_avic_ga_log(vm_id, vcpu_id); 97 98 spin_lock_irqsave(&svm_vm_data_hash_lock, flags); 99 hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) { 100 if (kvm_svm->avic_vm_id != vm_id) 101 continue; 102 vcpu = kvm_get_vcpu_by_id(&kvm_svm->kvm, vcpu_id); 103 break; 104 } 105 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); 106 107 /* Note: 108 * At this point, the IOMMU should have already set the pending 109 * bit in the vAPIC backing page. So, we just need to schedule 110 * in the vcpu. 111 */ 112 if (vcpu) 113 kvm_vcpu_wake_up(vcpu); 114 115 return 0; 116 } 117 118 void avic_vm_destroy(struct kvm *kvm) 119 { 120 unsigned long flags; 121 struct kvm_svm *kvm_svm = to_kvm_svm(kvm); 122 123 if (!enable_apicv) 124 return; 125 126 if (kvm_svm->avic_logical_id_table_page) 127 __free_page(kvm_svm->avic_logical_id_table_page); 128 if (kvm_svm->avic_physical_id_table_page) 129 __free_page(kvm_svm->avic_physical_id_table_page); 130 131 spin_lock_irqsave(&svm_vm_data_hash_lock, flags); 132 hash_del(&kvm_svm->hnode); 133 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); 134 } 135 136 int avic_vm_init(struct kvm *kvm) 137 { 138 unsigned long flags; 139 int err = -ENOMEM; 140 struct kvm_svm *kvm_svm = to_kvm_svm(kvm); 141 struct kvm_svm *k2; 142 struct page *p_page; 143 struct page *l_page; 144 u32 vm_id; 145 146 if (!enable_apicv) 147 return 0; 148 149 /* Allocating physical APIC ID table (4KB) */ 150 p_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 151 if (!p_page) 152 goto free_avic; 153 154 kvm_svm->avic_physical_id_table_page = p_page; 155 156 /* Allocating logical APIC ID table (4KB) */ 157 l_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 158 if (!l_page) 159 goto free_avic; 160 161 kvm_svm->avic_logical_id_table_page = l_page; 162 163 spin_lock_irqsave(&svm_vm_data_hash_lock, flags); 164 again: 165 vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK; 166 if (vm_id == 0) { /* id is 1-based, zero is not okay */ 167 next_vm_id_wrapped = 1; 168 goto again; 169 } 170 /* Is it still in use? Only possible if wrapped at least once */ 171 if (next_vm_id_wrapped) { 172 hash_for_each_possible(svm_vm_data_hash, k2, hnode, vm_id) { 173 if (k2->avic_vm_id == vm_id) 174 goto again; 175 } 176 } 177 kvm_svm->avic_vm_id = vm_id; 178 hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id); 179 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); 180 181 return 0; 182 183 free_avic: 184 avic_vm_destroy(kvm); 185 return err; 186 } 187 188 void avic_init_vmcb(struct vcpu_svm *svm) 189 { 190 struct vmcb *vmcb = svm->vmcb; 191 struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm); 192 phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page)); 193 phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page)); 194 phys_addr_t ppa = __sme_set(page_to_phys(kvm_svm->avic_physical_id_table_page)); 195 196 vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK; 197 vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK; 198 vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK; 199 vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT; 200 vmcb->control.avic_vapic_bar = APIC_DEFAULT_PHYS_BASE & VMCB_AVIC_APIC_BAR_MASK; 201 202 if (kvm_apicv_activated(svm->vcpu.kvm)) 203 vmcb->control.int_ctl |= AVIC_ENABLE_MASK; 204 else 205 vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK; 206 } 207 208 static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu, 209 unsigned int index) 210 { 211 u64 *avic_physical_id_table; 212 struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); 213 214 if (index >= AVIC_MAX_PHYSICAL_ID_COUNT) 215 return NULL; 216 217 avic_physical_id_table = page_address(kvm_svm->avic_physical_id_table_page); 218 219 return &avic_physical_id_table[index]; 220 } 221 222 /* 223 * Note: 224 * AVIC hardware walks the nested page table to check permissions, 225 * but does not use the SPA address specified in the leaf page 226 * table entry since it uses address in the AVIC_BACKING_PAGE pointer 227 * field of the VMCB. Therefore, we set up the 228 * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here. 229 */ 230 static int avic_alloc_access_page(struct kvm *kvm) 231 { 232 void __user *ret; 233 int r = 0; 234 235 mutex_lock(&kvm->slots_lock); 236 237 if (kvm->arch.apic_access_memslot_enabled) 238 goto out; 239 240 ret = __x86_set_memory_region(kvm, 241 APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 242 APIC_DEFAULT_PHYS_BASE, 243 PAGE_SIZE); 244 if (IS_ERR(ret)) { 245 r = PTR_ERR(ret); 246 goto out; 247 } 248 249 kvm->arch.apic_access_memslot_enabled = true; 250 out: 251 mutex_unlock(&kvm->slots_lock); 252 return r; 253 } 254 255 static int avic_init_backing_page(struct kvm_vcpu *vcpu) 256 { 257 u64 *entry, new_entry; 258 int id = vcpu->vcpu_id; 259 struct vcpu_svm *svm = to_svm(vcpu); 260 261 if (id >= AVIC_MAX_PHYSICAL_ID_COUNT) 262 return -EINVAL; 263 264 if (!vcpu->arch.apic->regs) 265 return -EINVAL; 266 267 if (kvm_apicv_activated(vcpu->kvm)) { 268 int ret; 269 270 ret = avic_alloc_access_page(vcpu->kvm); 271 if (ret) 272 return ret; 273 } 274 275 svm->avic_backing_page = virt_to_page(vcpu->arch.apic->regs); 276 277 /* Setting AVIC backing page address in the phy APIC ID table */ 278 entry = avic_get_physical_id_entry(vcpu, id); 279 if (!entry) 280 return -EINVAL; 281 282 new_entry = __sme_set((page_to_phys(svm->avic_backing_page) & 283 AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) | 284 AVIC_PHYSICAL_ID_ENTRY_VALID_MASK); 285 WRITE_ONCE(*entry, new_entry); 286 287 svm->avic_physical_id_cache = entry; 288 289 return 0; 290 } 291 292 static void avic_kick_target_vcpus(struct kvm *kvm, struct kvm_lapic *source, 293 u32 icrl, u32 icrh) 294 { 295 struct kvm_vcpu *vcpu; 296 unsigned long i; 297 298 kvm_for_each_vcpu(i, vcpu, kvm) { 299 bool m = kvm_apic_match_dest(vcpu, source, 300 icrl & APIC_SHORT_MASK, 301 GET_APIC_DEST_FIELD(icrh), 302 icrl & APIC_DEST_MASK); 303 304 if (m && !avic_vcpu_is_running(vcpu)) 305 kvm_vcpu_wake_up(vcpu); 306 } 307 } 308 309 int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu) 310 { 311 struct vcpu_svm *svm = to_svm(vcpu); 312 u32 icrh = svm->vmcb->control.exit_info_1 >> 32; 313 u32 icrl = svm->vmcb->control.exit_info_1; 314 u32 id = svm->vmcb->control.exit_info_2 >> 32; 315 u32 index = svm->vmcb->control.exit_info_2 & 0xFF; 316 struct kvm_lapic *apic = vcpu->arch.apic; 317 318 trace_kvm_avic_incomplete_ipi(vcpu->vcpu_id, icrh, icrl, id, index); 319 320 switch (id) { 321 case AVIC_IPI_FAILURE_INVALID_INT_TYPE: 322 /* 323 * AVIC hardware handles the generation of 324 * IPIs when the specified Message Type is Fixed 325 * (also known as fixed delivery mode) and 326 * the Trigger Mode is edge-triggered. The hardware 327 * also supports self and broadcast delivery modes 328 * specified via the Destination Shorthand(DSH) 329 * field of the ICRL. Logical and physical APIC ID 330 * formats are supported. All other IPI types cause 331 * a #VMEXIT, which needs to emulated. 332 */ 333 kvm_lapic_reg_write(apic, APIC_ICR2, icrh); 334 kvm_lapic_reg_write(apic, APIC_ICR, icrl); 335 break; 336 case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING: 337 /* 338 * At this point, we expect that the AVIC HW has already 339 * set the appropriate IRR bits on the valid target 340 * vcpus. So, we just need to kick the appropriate vcpu. 341 */ 342 avic_kick_target_vcpus(vcpu->kvm, apic, icrl, icrh); 343 break; 344 case AVIC_IPI_FAILURE_INVALID_TARGET: 345 WARN_ONCE(1, "Invalid IPI target: index=%u, vcpu=%d, icr=%#0x:%#0x\n", 346 index, vcpu->vcpu_id, icrh, icrl); 347 break; 348 case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE: 349 WARN_ONCE(1, "Invalid backing page\n"); 350 break; 351 default: 352 pr_err("Unknown IPI interception\n"); 353 } 354 355 return 1; 356 } 357 358 static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat) 359 { 360 struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); 361 int index; 362 u32 *logical_apic_id_table; 363 int dlid = GET_APIC_LOGICAL_ID(ldr); 364 365 if (!dlid) 366 return NULL; 367 368 if (flat) { /* flat */ 369 index = ffs(dlid) - 1; 370 if (index > 7) 371 return NULL; 372 } else { /* cluster */ 373 int cluster = (dlid & 0xf0) >> 4; 374 int apic = ffs(dlid & 0x0f) - 1; 375 376 if ((apic < 0) || (apic > 7) || 377 (cluster >= 0xf)) 378 return NULL; 379 index = (cluster << 2) + apic; 380 } 381 382 logical_apic_id_table = (u32 *) page_address(kvm_svm->avic_logical_id_table_page); 383 384 return &logical_apic_id_table[index]; 385 } 386 387 static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr) 388 { 389 bool flat; 390 u32 *entry, new_entry; 391 392 flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT; 393 entry = avic_get_logical_id_entry(vcpu, ldr, flat); 394 if (!entry) 395 return -EINVAL; 396 397 new_entry = READ_ONCE(*entry); 398 new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK; 399 new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK); 400 new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK; 401 WRITE_ONCE(*entry, new_entry); 402 403 return 0; 404 } 405 406 static void avic_invalidate_logical_id_entry(struct kvm_vcpu *vcpu) 407 { 408 struct vcpu_svm *svm = to_svm(vcpu); 409 bool flat = svm->dfr_reg == APIC_DFR_FLAT; 410 u32 *entry = avic_get_logical_id_entry(vcpu, svm->ldr_reg, flat); 411 412 if (entry) 413 clear_bit(AVIC_LOGICAL_ID_ENTRY_VALID_BIT, (unsigned long *)entry); 414 } 415 416 static int avic_handle_ldr_update(struct kvm_vcpu *vcpu) 417 { 418 int ret = 0; 419 struct vcpu_svm *svm = to_svm(vcpu); 420 u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR); 421 u32 id = kvm_xapic_id(vcpu->arch.apic); 422 423 if (ldr == svm->ldr_reg) 424 return 0; 425 426 avic_invalidate_logical_id_entry(vcpu); 427 428 if (ldr) 429 ret = avic_ldr_write(vcpu, id, ldr); 430 431 if (!ret) 432 svm->ldr_reg = ldr; 433 434 return ret; 435 } 436 437 static int avic_handle_apic_id_update(struct kvm_vcpu *vcpu) 438 { 439 u64 *old, *new; 440 struct vcpu_svm *svm = to_svm(vcpu); 441 u32 id = kvm_xapic_id(vcpu->arch.apic); 442 443 if (vcpu->vcpu_id == id) 444 return 0; 445 446 old = avic_get_physical_id_entry(vcpu, vcpu->vcpu_id); 447 new = avic_get_physical_id_entry(vcpu, id); 448 if (!new || !old) 449 return 1; 450 451 /* We need to move physical_id_entry to new offset */ 452 *new = *old; 453 *old = 0ULL; 454 to_svm(vcpu)->avic_physical_id_cache = new; 455 456 /* 457 * Also update the guest physical APIC ID in the logical 458 * APIC ID table entry if already setup the LDR. 459 */ 460 if (svm->ldr_reg) 461 avic_handle_ldr_update(vcpu); 462 463 return 0; 464 } 465 466 static void avic_handle_dfr_update(struct kvm_vcpu *vcpu) 467 { 468 struct vcpu_svm *svm = to_svm(vcpu); 469 u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR); 470 471 if (svm->dfr_reg == dfr) 472 return; 473 474 avic_invalidate_logical_id_entry(vcpu); 475 svm->dfr_reg = dfr; 476 } 477 478 static int avic_unaccel_trap_write(struct vcpu_svm *svm) 479 { 480 struct kvm_lapic *apic = svm->vcpu.arch.apic; 481 u32 offset = svm->vmcb->control.exit_info_1 & 482 AVIC_UNACCEL_ACCESS_OFFSET_MASK; 483 484 switch (offset) { 485 case APIC_ID: 486 if (avic_handle_apic_id_update(&svm->vcpu)) 487 return 0; 488 break; 489 case APIC_LDR: 490 if (avic_handle_ldr_update(&svm->vcpu)) 491 return 0; 492 break; 493 case APIC_DFR: 494 avic_handle_dfr_update(&svm->vcpu); 495 break; 496 default: 497 break; 498 } 499 500 kvm_lapic_reg_write(apic, offset, kvm_lapic_get_reg(apic, offset)); 501 502 return 1; 503 } 504 505 static bool is_avic_unaccelerated_access_trap(u32 offset) 506 { 507 bool ret = false; 508 509 switch (offset) { 510 case APIC_ID: 511 case APIC_EOI: 512 case APIC_RRR: 513 case APIC_LDR: 514 case APIC_DFR: 515 case APIC_SPIV: 516 case APIC_ESR: 517 case APIC_ICR: 518 case APIC_LVTT: 519 case APIC_LVTTHMR: 520 case APIC_LVTPC: 521 case APIC_LVT0: 522 case APIC_LVT1: 523 case APIC_LVTERR: 524 case APIC_TMICT: 525 case APIC_TDCR: 526 ret = true; 527 break; 528 default: 529 break; 530 } 531 return ret; 532 } 533 534 int avic_unaccelerated_access_interception(struct kvm_vcpu *vcpu) 535 { 536 struct vcpu_svm *svm = to_svm(vcpu); 537 int ret = 0; 538 u32 offset = svm->vmcb->control.exit_info_1 & 539 AVIC_UNACCEL_ACCESS_OFFSET_MASK; 540 u32 vector = svm->vmcb->control.exit_info_2 & 541 AVIC_UNACCEL_ACCESS_VECTOR_MASK; 542 bool write = (svm->vmcb->control.exit_info_1 >> 32) & 543 AVIC_UNACCEL_ACCESS_WRITE_MASK; 544 bool trap = is_avic_unaccelerated_access_trap(offset); 545 546 trace_kvm_avic_unaccelerated_access(vcpu->vcpu_id, offset, 547 trap, write, vector); 548 if (trap) { 549 /* Handling Trap */ 550 WARN_ONCE(!write, "svm: Handling trap read.\n"); 551 ret = avic_unaccel_trap_write(svm); 552 } else { 553 /* Handling Fault */ 554 ret = kvm_emulate_instruction(vcpu, 0); 555 } 556 557 return ret; 558 } 559 560 int avic_init_vcpu(struct vcpu_svm *svm) 561 { 562 int ret; 563 struct kvm_vcpu *vcpu = &svm->vcpu; 564 565 if (!enable_apicv || !irqchip_in_kernel(vcpu->kvm)) 566 return 0; 567 568 ret = avic_init_backing_page(vcpu); 569 if (ret) 570 return ret; 571 572 INIT_LIST_HEAD(&svm->ir_list); 573 spin_lock_init(&svm->ir_list_lock); 574 svm->dfr_reg = APIC_DFR_FLAT; 575 576 return ret; 577 } 578 579 void avic_post_state_restore(struct kvm_vcpu *vcpu) 580 { 581 if (avic_handle_apic_id_update(vcpu) != 0) 582 return; 583 avic_handle_dfr_update(vcpu); 584 avic_handle_ldr_update(vcpu); 585 } 586 587 void svm_set_virtual_apic_mode(struct kvm_vcpu *vcpu) 588 { 589 return; 590 } 591 592 void svm_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr) 593 { 594 } 595 596 void svm_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr) 597 { 598 } 599 600 static int svm_set_pi_irte_mode(struct kvm_vcpu *vcpu, bool activate) 601 { 602 int ret = 0; 603 unsigned long flags; 604 struct amd_svm_iommu_ir *ir; 605 struct vcpu_svm *svm = to_svm(vcpu); 606 607 if (!kvm_arch_has_assigned_device(vcpu->kvm)) 608 return 0; 609 610 /* 611 * Here, we go through the per-vcpu ir_list to update all existing 612 * interrupt remapping table entry targeting this vcpu. 613 */ 614 spin_lock_irqsave(&svm->ir_list_lock, flags); 615 616 if (list_empty(&svm->ir_list)) 617 goto out; 618 619 list_for_each_entry(ir, &svm->ir_list, node) { 620 if (activate) 621 ret = amd_iommu_activate_guest_mode(ir->data); 622 else 623 ret = amd_iommu_deactivate_guest_mode(ir->data); 624 if (ret) 625 break; 626 } 627 out: 628 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 629 return ret; 630 } 631 632 void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) 633 { 634 struct vcpu_svm *svm = to_svm(vcpu); 635 struct vmcb *vmcb = svm->vmcb01.ptr; 636 bool activated = kvm_vcpu_apicv_active(vcpu); 637 638 if (!enable_apicv) 639 return; 640 641 if (activated) { 642 /** 643 * During AVIC temporary deactivation, guest could update 644 * APIC ID, DFR and LDR registers, which would not be trapped 645 * by avic_unaccelerated_access_interception(). In this case, 646 * we need to check and update the AVIC logical APIC ID table 647 * accordingly before re-activating. 648 */ 649 avic_post_state_restore(vcpu); 650 vmcb->control.int_ctl |= AVIC_ENABLE_MASK; 651 } else { 652 vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK; 653 } 654 vmcb_mark_dirty(vmcb, VMCB_AVIC); 655 656 if (activated) 657 avic_vcpu_load(vcpu, vcpu->cpu); 658 else 659 avic_vcpu_put(vcpu); 660 661 svm_set_pi_irte_mode(vcpu, activated); 662 } 663 664 void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) 665 { 666 return; 667 } 668 669 int svm_deliver_avic_intr(struct kvm_vcpu *vcpu, int vec) 670 { 671 if (!vcpu->arch.apicv_active) 672 return -1; 673 674 kvm_lapic_set_irr(vec, vcpu->arch.apic); 675 smp_mb__after_atomic(); 676 677 if (avic_vcpu_is_running(vcpu)) { 678 int cpu = READ_ONCE(vcpu->cpu); 679 680 /* 681 * Note, the vCPU could get migrated to a different pCPU at any 682 * point, which could result in signalling the wrong/previous 683 * pCPU. But if that happens the vCPU is guaranteed to do a 684 * VMRUN (after being migrated) and thus will process pending 685 * interrupts, i.e. a doorbell is not needed (and the spurious 686 * one is harmless). 687 */ 688 if (cpu != get_cpu()) 689 wrmsrl(SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpu)); 690 put_cpu(); 691 } else 692 kvm_vcpu_wake_up(vcpu); 693 694 return 0; 695 } 696 697 bool svm_dy_apicv_has_pending_interrupt(struct kvm_vcpu *vcpu) 698 { 699 return false; 700 } 701 702 static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi) 703 { 704 unsigned long flags; 705 struct amd_svm_iommu_ir *cur; 706 707 spin_lock_irqsave(&svm->ir_list_lock, flags); 708 list_for_each_entry(cur, &svm->ir_list, node) { 709 if (cur->data != pi->ir_data) 710 continue; 711 list_del(&cur->node); 712 kfree(cur); 713 break; 714 } 715 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 716 } 717 718 static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi) 719 { 720 int ret = 0; 721 unsigned long flags; 722 struct amd_svm_iommu_ir *ir; 723 724 /** 725 * In some cases, the existing irte is updated and re-set, 726 * so we need to check here if it's already been * added 727 * to the ir_list. 728 */ 729 if (pi->ir_data && (pi->prev_ga_tag != 0)) { 730 struct kvm *kvm = svm->vcpu.kvm; 731 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag); 732 struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id); 733 struct vcpu_svm *prev_svm; 734 735 if (!prev_vcpu) { 736 ret = -EINVAL; 737 goto out; 738 } 739 740 prev_svm = to_svm(prev_vcpu); 741 svm_ir_list_del(prev_svm, pi); 742 } 743 744 /** 745 * Allocating new amd_iommu_pi_data, which will get 746 * add to the per-vcpu ir_list. 747 */ 748 ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL_ACCOUNT); 749 if (!ir) { 750 ret = -ENOMEM; 751 goto out; 752 } 753 ir->data = pi->ir_data; 754 755 spin_lock_irqsave(&svm->ir_list_lock, flags); 756 list_add(&ir->node, &svm->ir_list); 757 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 758 out: 759 return ret; 760 } 761 762 /* 763 * Note: 764 * The HW cannot support posting multicast/broadcast 765 * interrupts to a vCPU. So, we still use legacy interrupt 766 * remapping for these kind of interrupts. 767 * 768 * For lowest-priority interrupts, we only support 769 * those with single CPU as the destination, e.g. user 770 * configures the interrupts via /proc/irq or uses 771 * irqbalance to make the interrupts single-CPU. 772 */ 773 static int 774 get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, 775 struct vcpu_data *vcpu_info, struct vcpu_svm **svm) 776 { 777 struct kvm_lapic_irq irq; 778 struct kvm_vcpu *vcpu = NULL; 779 780 kvm_set_msi_irq(kvm, e, &irq); 781 782 if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) || 783 !kvm_irq_is_postable(&irq)) { 784 pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n", 785 __func__, irq.vector); 786 return -1; 787 } 788 789 pr_debug("SVM: %s: use GA mode for irq %u\n", __func__, 790 irq.vector); 791 *svm = to_svm(vcpu); 792 vcpu_info->pi_desc_addr = __sme_set(page_to_phys((*svm)->avic_backing_page)); 793 vcpu_info->vector = irq.vector; 794 795 return 0; 796 } 797 798 /* 799 * svm_update_pi_irte - set IRTE for Posted-Interrupts 800 * 801 * @kvm: kvm 802 * @host_irq: host irq of the interrupt 803 * @guest_irq: gsi of the interrupt 804 * @set: set or unset PI 805 * returns 0 on success, < 0 on failure 806 */ 807 int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq, 808 uint32_t guest_irq, bool set) 809 { 810 struct kvm_kernel_irq_routing_entry *e; 811 struct kvm_irq_routing_table *irq_rt; 812 int idx, ret = -EINVAL; 813 814 if (!kvm_arch_has_assigned_device(kvm) || 815 !irq_remapping_cap(IRQ_POSTING_CAP)) 816 return 0; 817 818 pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n", 819 __func__, host_irq, guest_irq, set); 820 821 idx = srcu_read_lock(&kvm->irq_srcu); 822 irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu); 823 WARN_ON(guest_irq >= irq_rt->nr_rt_entries); 824 825 hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) { 826 struct vcpu_data vcpu_info; 827 struct vcpu_svm *svm = NULL; 828 829 if (e->type != KVM_IRQ_ROUTING_MSI) 830 continue; 831 832 /** 833 * Here, we setup with legacy mode in the following cases: 834 * 1. When cannot target interrupt to a specific vcpu. 835 * 2. Unsetting posted interrupt. 836 * 3. APIC virtualization is disabled for the vcpu. 837 * 4. IRQ has incompatible delivery mode (SMI, INIT, etc) 838 */ 839 if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set && 840 kvm_vcpu_apicv_active(&svm->vcpu)) { 841 struct amd_iommu_pi_data pi; 842 843 /* Try to enable guest_mode in IRTE */ 844 pi.base = __sme_set(page_to_phys(svm->avic_backing_page) & 845 AVIC_HPA_MASK); 846 pi.ga_tag = AVIC_GATAG(to_kvm_svm(kvm)->avic_vm_id, 847 svm->vcpu.vcpu_id); 848 pi.is_guest_mode = true; 849 pi.vcpu_data = &vcpu_info; 850 ret = irq_set_vcpu_affinity(host_irq, &pi); 851 852 /** 853 * Here, we successfully setting up vcpu affinity in 854 * IOMMU guest mode. Now, we need to store the posted 855 * interrupt information in a per-vcpu ir_list so that 856 * we can reference to them directly when we update vcpu 857 * scheduling information in IOMMU irte. 858 */ 859 if (!ret && pi.is_guest_mode) 860 svm_ir_list_add(svm, &pi); 861 } else { 862 /* Use legacy mode in IRTE */ 863 struct amd_iommu_pi_data pi; 864 865 /** 866 * Here, pi is used to: 867 * - Tell IOMMU to use legacy mode for this interrupt. 868 * - Retrieve ga_tag of prior interrupt remapping data. 869 */ 870 pi.prev_ga_tag = 0; 871 pi.is_guest_mode = false; 872 ret = irq_set_vcpu_affinity(host_irq, &pi); 873 874 /** 875 * Check if the posted interrupt was previously 876 * setup with the guest_mode by checking if the ga_tag 877 * was cached. If so, we need to clean up the per-vcpu 878 * ir_list. 879 */ 880 if (!ret && pi.prev_ga_tag) { 881 int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag); 882 struct kvm_vcpu *vcpu; 883 884 vcpu = kvm_get_vcpu_by_id(kvm, id); 885 if (vcpu) 886 svm_ir_list_del(to_svm(vcpu), &pi); 887 } 888 } 889 890 if (!ret && svm) { 891 trace_kvm_pi_irte_update(host_irq, svm->vcpu.vcpu_id, 892 e->gsi, vcpu_info.vector, 893 vcpu_info.pi_desc_addr, set); 894 } 895 896 if (ret < 0) { 897 pr_err("%s: failed to update PI IRTE\n", __func__); 898 goto out; 899 } 900 } 901 902 ret = 0; 903 out: 904 srcu_read_unlock(&kvm->irq_srcu, idx); 905 return ret; 906 } 907 908 bool svm_check_apicv_inhibit_reasons(ulong bit) 909 { 910 ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) | 911 BIT(APICV_INHIBIT_REASON_ABSENT) | 912 BIT(APICV_INHIBIT_REASON_HYPERV) | 913 BIT(APICV_INHIBIT_REASON_NESTED) | 914 BIT(APICV_INHIBIT_REASON_IRQWIN) | 915 BIT(APICV_INHIBIT_REASON_PIT_REINJ) | 916 BIT(APICV_INHIBIT_REASON_X2APIC) | 917 BIT(APICV_INHIBIT_REASON_BLOCKIRQ); 918 919 return supported & BIT(bit); 920 } 921 922 923 static inline int 924 avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r) 925 { 926 int ret = 0; 927 unsigned long flags; 928 struct amd_svm_iommu_ir *ir; 929 struct vcpu_svm *svm = to_svm(vcpu); 930 931 if (!kvm_arch_has_assigned_device(vcpu->kvm)) 932 return 0; 933 934 /* 935 * Here, we go through the per-vcpu ir_list to update all existing 936 * interrupt remapping table entry targeting this vcpu. 937 */ 938 spin_lock_irqsave(&svm->ir_list_lock, flags); 939 940 if (list_empty(&svm->ir_list)) 941 goto out; 942 943 list_for_each_entry(ir, &svm->ir_list, node) { 944 ret = amd_iommu_update_ga(cpu, r, ir->data); 945 if (ret) 946 break; 947 } 948 out: 949 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 950 return ret; 951 } 952 953 void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 954 { 955 u64 entry; 956 /* ID = 0xff (broadcast), ID > 0xff (reserved) */ 957 int h_physical_id = kvm_cpu_get_apicid(cpu); 958 struct vcpu_svm *svm = to_svm(vcpu); 959 960 /* 961 * Since the host physical APIC id is 8 bits, 962 * we can support host APIC ID upto 255. 963 */ 964 if (WARN_ON(h_physical_id > AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK)) 965 return; 966 967 entry = READ_ONCE(*(svm->avic_physical_id_cache)); 968 WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK); 969 970 entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK; 971 entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK); 972 973 entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; 974 if (svm->avic_is_running) 975 entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; 976 977 WRITE_ONCE(*(svm->avic_physical_id_cache), entry); 978 avic_update_iommu_vcpu_affinity(vcpu, h_physical_id, 979 svm->avic_is_running); 980 } 981 982 void avic_vcpu_put(struct kvm_vcpu *vcpu) 983 { 984 u64 entry; 985 struct vcpu_svm *svm = to_svm(vcpu); 986 987 entry = READ_ONCE(*(svm->avic_physical_id_cache)); 988 if (entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK) 989 avic_update_iommu_vcpu_affinity(vcpu, -1, 0); 990 991 entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; 992 WRITE_ONCE(*(svm->avic_physical_id_cache), entry); 993 } 994 995 /* 996 * This function is called during VCPU halt/unhalt. 997 */ 998 static void avic_set_running(struct kvm_vcpu *vcpu, bool is_run) 999 { 1000 struct vcpu_svm *svm = to_svm(vcpu); 1001 int cpu = get_cpu(); 1002 1003 WARN_ON(cpu != vcpu->cpu); 1004 svm->avic_is_running = is_run; 1005 1006 if (kvm_vcpu_apicv_active(vcpu)) { 1007 if (is_run) 1008 avic_vcpu_load(vcpu, cpu); 1009 else 1010 avic_vcpu_put(vcpu); 1011 } 1012 put_cpu(); 1013 } 1014 1015 void svm_vcpu_blocking(struct kvm_vcpu *vcpu) 1016 { 1017 avic_set_running(vcpu, false); 1018 } 1019 1020 void svm_vcpu_unblocking(struct kvm_vcpu *vcpu) 1021 { 1022 if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu)) 1023 kvm_vcpu_update_apicv(vcpu); 1024 avic_set_running(vcpu, true); 1025 } 1026