1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * AMD SVM support 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Yaniv Kamay <yaniv@qumranet.com> 12 * Avi Kivity <avi@qumranet.com> 13 */ 14 15 #define pr_fmt(fmt) "SVM: " fmt 16 17 #include <linux/kvm_types.h> 18 #include <linux/hashtable.h> 19 #include <linux/amd-iommu.h> 20 #include <linux/kvm_host.h> 21 22 #include <asm/irq_remapping.h> 23 24 #include "trace.h" 25 #include "lapic.h" 26 #include "x86.h" 27 #include "irq.h" 28 #include "svm.h" 29 30 /* enable / disable AVIC */ 31 int avic; 32 #ifdef CONFIG_X86_LOCAL_APIC 33 module_param(avic, int, S_IRUGO); 34 #endif 35 36 #define SVM_AVIC_DOORBELL 0xc001011b 37 38 #define AVIC_HPA_MASK ~((0xFFFULL << 52) | 0xFFF) 39 40 /* 41 * 0xff is broadcast, so the max index allowed for physical APIC ID 42 * table is 0xfe. APIC IDs above 0xff are reserved. 43 */ 44 #define AVIC_MAX_PHYSICAL_ID_COUNT 255 45 46 #define AVIC_UNACCEL_ACCESS_WRITE_MASK 1 47 #define AVIC_UNACCEL_ACCESS_OFFSET_MASK 0xFF0 48 #define AVIC_UNACCEL_ACCESS_VECTOR_MASK 0xFFFFFFFF 49 50 /* AVIC GATAG is encoded using VM and VCPU IDs */ 51 #define AVIC_VCPU_ID_BITS 8 52 #define AVIC_VCPU_ID_MASK ((1 << AVIC_VCPU_ID_BITS) - 1) 53 54 #define AVIC_VM_ID_BITS 24 55 #define AVIC_VM_ID_NR (1 << AVIC_VM_ID_BITS) 56 #define AVIC_VM_ID_MASK ((1 << AVIC_VM_ID_BITS) - 1) 57 58 #define AVIC_GATAG(x, y) (((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \ 59 (y & AVIC_VCPU_ID_MASK)) 60 #define AVIC_GATAG_TO_VMID(x) ((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK) 61 #define AVIC_GATAG_TO_VCPUID(x) (x & AVIC_VCPU_ID_MASK) 62 63 /* Note: 64 * This hash table is used to map VM_ID to a struct kvm_svm, 65 * when handling AMD IOMMU GALOG notification to schedule in 66 * a particular vCPU. 67 */ 68 #define SVM_VM_DATA_HASH_BITS 8 69 static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS); 70 static u32 next_vm_id = 0; 71 static bool next_vm_id_wrapped = 0; 72 static DEFINE_SPINLOCK(svm_vm_data_hash_lock); 73 74 /* 75 * This is a wrapper of struct amd_iommu_ir_data. 76 */ 77 struct amd_svm_iommu_ir { 78 struct list_head node; /* Used by SVM for per-vcpu ir_list */ 79 void *data; /* Storing pointer to struct amd_ir_data */ 80 }; 81 82 enum avic_ipi_failure_cause { 83 AVIC_IPI_FAILURE_INVALID_INT_TYPE, 84 AVIC_IPI_FAILURE_TARGET_NOT_RUNNING, 85 AVIC_IPI_FAILURE_INVALID_TARGET, 86 AVIC_IPI_FAILURE_INVALID_BACKING_PAGE, 87 }; 88 89 /* Note: 90 * This function is called from IOMMU driver to notify 91 * SVM to schedule in a particular vCPU of a particular VM. 92 */ 93 int avic_ga_log_notifier(u32 ga_tag) 94 { 95 unsigned long flags; 96 struct kvm_svm *kvm_svm; 97 struct kvm_vcpu *vcpu = NULL; 98 u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag); 99 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag); 100 101 pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id); 102 trace_kvm_avic_ga_log(vm_id, vcpu_id); 103 104 spin_lock_irqsave(&svm_vm_data_hash_lock, flags); 105 hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) { 106 if (kvm_svm->avic_vm_id != vm_id) 107 continue; 108 vcpu = kvm_get_vcpu_by_id(&kvm_svm->kvm, vcpu_id); 109 break; 110 } 111 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); 112 113 /* Note: 114 * At this point, the IOMMU should have already set the pending 115 * bit in the vAPIC backing page. So, we just need to schedule 116 * in the vcpu. 117 */ 118 if (vcpu) 119 kvm_vcpu_wake_up(vcpu); 120 121 return 0; 122 } 123 124 void avic_vm_destroy(struct kvm *kvm) 125 { 126 unsigned long flags; 127 struct kvm_svm *kvm_svm = to_kvm_svm(kvm); 128 129 if (!avic) 130 return; 131 132 if (kvm_svm->avic_logical_id_table_page) 133 __free_page(kvm_svm->avic_logical_id_table_page); 134 if (kvm_svm->avic_physical_id_table_page) 135 __free_page(kvm_svm->avic_physical_id_table_page); 136 137 spin_lock_irqsave(&svm_vm_data_hash_lock, flags); 138 hash_del(&kvm_svm->hnode); 139 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); 140 } 141 142 int avic_vm_init(struct kvm *kvm) 143 { 144 unsigned long flags; 145 int err = -ENOMEM; 146 struct kvm_svm *kvm_svm = to_kvm_svm(kvm); 147 struct kvm_svm *k2; 148 struct page *p_page; 149 struct page *l_page; 150 u32 vm_id; 151 152 if (!avic) 153 return 0; 154 155 /* Allocating physical APIC ID table (4KB) */ 156 p_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 157 if (!p_page) 158 goto free_avic; 159 160 kvm_svm->avic_physical_id_table_page = p_page; 161 162 /* Allocating logical APIC ID table (4KB) */ 163 l_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 164 if (!l_page) 165 goto free_avic; 166 167 kvm_svm->avic_logical_id_table_page = l_page; 168 169 spin_lock_irqsave(&svm_vm_data_hash_lock, flags); 170 again: 171 vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK; 172 if (vm_id == 0) { /* id is 1-based, zero is not okay */ 173 next_vm_id_wrapped = 1; 174 goto again; 175 } 176 /* Is it still in use? Only possible if wrapped at least once */ 177 if (next_vm_id_wrapped) { 178 hash_for_each_possible(svm_vm_data_hash, k2, hnode, vm_id) { 179 if (k2->avic_vm_id == vm_id) 180 goto again; 181 } 182 } 183 kvm_svm->avic_vm_id = vm_id; 184 hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id); 185 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); 186 187 return 0; 188 189 free_avic: 190 avic_vm_destroy(kvm); 191 return err; 192 } 193 194 void avic_init_vmcb(struct vcpu_svm *svm) 195 { 196 struct vmcb *vmcb = svm->vmcb; 197 struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm); 198 phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page)); 199 phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page)); 200 phys_addr_t ppa = __sme_set(page_to_phys(kvm_svm->avic_physical_id_table_page)); 201 202 vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK; 203 vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK; 204 vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK; 205 vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT; 206 if (kvm_apicv_activated(svm->vcpu.kvm)) 207 vmcb->control.int_ctl |= AVIC_ENABLE_MASK; 208 else 209 vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK; 210 } 211 212 static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu, 213 unsigned int index) 214 { 215 u64 *avic_physical_id_table; 216 struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); 217 218 if (index >= AVIC_MAX_PHYSICAL_ID_COUNT) 219 return NULL; 220 221 avic_physical_id_table = page_address(kvm_svm->avic_physical_id_table_page); 222 223 return &avic_physical_id_table[index]; 224 } 225 226 /** 227 * Note: 228 * AVIC hardware walks the nested page table to check permissions, 229 * but does not use the SPA address specified in the leaf page 230 * table entry since it uses address in the AVIC_BACKING_PAGE pointer 231 * field of the VMCB. Therefore, we set up the 232 * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here. 233 */ 234 static int avic_update_access_page(struct kvm *kvm, bool activate) 235 { 236 void __user *ret; 237 int r = 0; 238 239 mutex_lock(&kvm->slots_lock); 240 /* 241 * During kvm_destroy_vm(), kvm_pit_set_reinject() could trigger 242 * APICv mode change, which update APIC_ACCESS_PAGE_PRIVATE_MEMSLOT 243 * memory region. So, we need to ensure that kvm->mm == current->mm. 244 */ 245 if ((kvm->arch.apic_access_page_done == activate) || 246 (kvm->mm != current->mm)) 247 goto out; 248 249 ret = __x86_set_memory_region(kvm, 250 APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 251 APIC_DEFAULT_PHYS_BASE, 252 activate ? PAGE_SIZE : 0); 253 if (IS_ERR(ret)) { 254 r = PTR_ERR(ret); 255 goto out; 256 } 257 258 kvm->arch.apic_access_page_done = activate; 259 out: 260 mutex_unlock(&kvm->slots_lock); 261 return r; 262 } 263 264 static int avic_init_backing_page(struct kvm_vcpu *vcpu) 265 { 266 u64 *entry, new_entry; 267 int id = vcpu->vcpu_id; 268 struct vcpu_svm *svm = to_svm(vcpu); 269 270 if (id >= AVIC_MAX_PHYSICAL_ID_COUNT) 271 return -EINVAL; 272 273 if (!vcpu->arch.apic->regs) 274 return -EINVAL; 275 276 if (kvm_apicv_activated(vcpu->kvm)) { 277 int ret; 278 279 ret = avic_update_access_page(vcpu->kvm, true); 280 if (ret) 281 return ret; 282 } 283 284 svm->avic_backing_page = virt_to_page(vcpu->arch.apic->regs); 285 286 /* Setting AVIC backing page address in the phy APIC ID table */ 287 entry = avic_get_physical_id_entry(vcpu, id); 288 if (!entry) 289 return -EINVAL; 290 291 new_entry = __sme_set((page_to_phys(svm->avic_backing_page) & 292 AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) | 293 AVIC_PHYSICAL_ID_ENTRY_VALID_MASK); 294 WRITE_ONCE(*entry, new_entry); 295 296 svm->avic_physical_id_cache = entry; 297 298 return 0; 299 } 300 301 static void avic_kick_target_vcpus(struct kvm *kvm, struct kvm_lapic *source, 302 u32 icrl, u32 icrh) 303 { 304 struct kvm_vcpu *vcpu; 305 int i; 306 307 kvm_for_each_vcpu(i, vcpu, kvm) { 308 bool m = kvm_apic_match_dest(vcpu, source, 309 icrl & APIC_SHORT_MASK, 310 GET_APIC_DEST_FIELD(icrh), 311 icrl & APIC_DEST_MASK); 312 313 if (m && !avic_vcpu_is_running(vcpu)) 314 kvm_vcpu_wake_up(vcpu); 315 } 316 } 317 318 int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu) 319 { 320 struct vcpu_svm *svm = to_svm(vcpu); 321 u32 icrh = svm->vmcb->control.exit_info_1 >> 32; 322 u32 icrl = svm->vmcb->control.exit_info_1; 323 u32 id = svm->vmcb->control.exit_info_2 >> 32; 324 u32 index = svm->vmcb->control.exit_info_2 & 0xFF; 325 struct kvm_lapic *apic = vcpu->arch.apic; 326 327 trace_kvm_avic_incomplete_ipi(vcpu->vcpu_id, icrh, icrl, id, index); 328 329 switch (id) { 330 case AVIC_IPI_FAILURE_INVALID_INT_TYPE: 331 /* 332 * AVIC hardware handles the generation of 333 * IPIs when the specified Message Type is Fixed 334 * (also known as fixed delivery mode) and 335 * the Trigger Mode is edge-triggered. The hardware 336 * also supports self and broadcast delivery modes 337 * specified via the Destination Shorthand(DSH) 338 * field of the ICRL. Logical and physical APIC ID 339 * formats are supported. All other IPI types cause 340 * a #VMEXIT, which needs to emulated. 341 */ 342 kvm_lapic_reg_write(apic, APIC_ICR2, icrh); 343 kvm_lapic_reg_write(apic, APIC_ICR, icrl); 344 break; 345 case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING: 346 /* 347 * At this point, we expect that the AVIC HW has already 348 * set the appropriate IRR bits on the valid target 349 * vcpus. So, we just need to kick the appropriate vcpu. 350 */ 351 avic_kick_target_vcpus(vcpu->kvm, apic, icrl, icrh); 352 break; 353 case AVIC_IPI_FAILURE_INVALID_TARGET: 354 WARN_ONCE(1, "Invalid IPI target: index=%u, vcpu=%d, icr=%#0x:%#0x\n", 355 index, vcpu->vcpu_id, icrh, icrl); 356 break; 357 case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE: 358 WARN_ONCE(1, "Invalid backing page\n"); 359 break; 360 default: 361 pr_err("Unknown IPI interception\n"); 362 } 363 364 return 1; 365 } 366 367 static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat) 368 { 369 struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); 370 int index; 371 u32 *logical_apic_id_table; 372 int dlid = GET_APIC_LOGICAL_ID(ldr); 373 374 if (!dlid) 375 return NULL; 376 377 if (flat) { /* flat */ 378 index = ffs(dlid) - 1; 379 if (index > 7) 380 return NULL; 381 } else { /* cluster */ 382 int cluster = (dlid & 0xf0) >> 4; 383 int apic = ffs(dlid & 0x0f) - 1; 384 385 if ((apic < 0) || (apic > 7) || 386 (cluster >= 0xf)) 387 return NULL; 388 index = (cluster << 2) + apic; 389 } 390 391 logical_apic_id_table = (u32 *) page_address(kvm_svm->avic_logical_id_table_page); 392 393 return &logical_apic_id_table[index]; 394 } 395 396 static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr) 397 { 398 bool flat; 399 u32 *entry, new_entry; 400 401 flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT; 402 entry = avic_get_logical_id_entry(vcpu, ldr, flat); 403 if (!entry) 404 return -EINVAL; 405 406 new_entry = READ_ONCE(*entry); 407 new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK; 408 new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK); 409 new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK; 410 WRITE_ONCE(*entry, new_entry); 411 412 return 0; 413 } 414 415 static void avic_invalidate_logical_id_entry(struct kvm_vcpu *vcpu) 416 { 417 struct vcpu_svm *svm = to_svm(vcpu); 418 bool flat = svm->dfr_reg == APIC_DFR_FLAT; 419 u32 *entry = avic_get_logical_id_entry(vcpu, svm->ldr_reg, flat); 420 421 if (entry) 422 clear_bit(AVIC_LOGICAL_ID_ENTRY_VALID_BIT, (unsigned long *)entry); 423 } 424 425 static int avic_handle_ldr_update(struct kvm_vcpu *vcpu) 426 { 427 int ret = 0; 428 struct vcpu_svm *svm = to_svm(vcpu); 429 u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR); 430 u32 id = kvm_xapic_id(vcpu->arch.apic); 431 432 if (ldr == svm->ldr_reg) 433 return 0; 434 435 avic_invalidate_logical_id_entry(vcpu); 436 437 if (ldr) 438 ret = avic_ldr_write(vcpu, id, ldr); 439 440 if (!ret) 441 svm->ldr_reg = ldr; 442 443 return ret; 444 } 445 446 static int avic_handle_apic_id_update(struct kvm_vcpu *vcpu) 447 { 448 u64 *old, *new; 449 struct vcpu_svm *svm = to_svm(vcpu); 450 u32 id = kvm_xapic_id(vcpu->arch.apic); 451 452 if (vcpu->vcpu_id == id) 453 return 0; 454 455 old = avic_get_physical_id_entry(vcpu, vcpu->vcpu_id); 456 new = avic_get_physical_id_entry(vcpu, id); 457 if (!new || !old) 458 return 1; 459 460 /* We need to move physical_id_entry to new offset */ 461 *new = *old; 462 *old = 0ULL; 463 to_svm(vcpu)->avic_physical_id_cache = new; 464 465 /* 466 * Also update the guest physical APIC ID in the logical 467 * APIC ID table entry if already setup the LDR. 468 */ 469 if (svm->ldr_reg) 470 avic_handle_ldr_update(vcpu); 471 472 return 0; 473 } 474 475 static void avic_handle_dfr_update(struct kvm_vcpu *vcpu) 476 { 477 struct vcpu_svm *svm = to_svm(vcpu); 478 u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR); 479 480 if (svm->dfr_reg == dfr) 481 return; 482 483 avic_invalidate_logical_id_entry(vcpu); 484 svm->dfr_reg = dfr; 485 } 486 487 static int avic_unaccel_trap_write(struct vcpu_svm *svm) 488 { 489 struct kvm_lapic *apic = svm->vcpu.arch.apic; 490 u32 offset = svm->vmcb->control.exit_info_1 & 491 AVIC_UNACCEL_ACCESS_OFFSET_MASK; 492 493 switch (offset) { 494 case APIC_ID: 495 if (avic_handle_apic_id_update(&svm->vcpu)) 496 return 0; 497 break; 498 case APIC_LDR: 499 if (avic_handle_ldr_update(&svm->vcpu)) 500 return 0; 501 break; 502 case APIC_DFR: 503 avic_handle_dfr_update(&svm->vcpu); 504 break; 505 default: 506 break; 507 } 508 509 kvm_lapic_reg_write(apic, offset, kvm_lapic_get_reg(apic, offset)); 510 511 return 1; 512 } 513 514 static bool is_avic_unaccelerated_access_trap(u32 offset) 515 { 516 bool ret = false; 517 518 switch (offset) { 519 case APIC_ID: 520 case APIC_EOI: 521 case APIC_RRR: 522 case APIC_LDR: 523 case APIC_DFR: 524 case APIC_SPIV: 525 case APIC_ESR: 526 case APIC_ICR: 527 case APIC_LVTT: 528 case APIC_LVTTHMR: 529 case APIC_LVTPC: 530 case APIC_LVT0: 531 case APIC_LVT1: 532 case APIC_LVTERR: 533 case APIC_TMICT: 534 case APIC_TDCR: 535 ret = true; 536 break; 537 default: 538 break; 539 } 540 return ret; 541 } 542 543 int avic_unaccelerated_access_interception(struct kvm_vcpu *vcpu) 544 { 545 struct vcpu_svm *svm = to_svm(vcpu); 546 int ret = 0; 547 u32 offset = svm->vmcb->control.exit_info_1 & 548 AVIC_UNACCEL_ACCESS_OFFSET_MASK; 549 u32 vector = svm->vmcb->control.exit_info_2 & 550 AVIC_UNACCEL_ACCESS_VECTOR_MASK; 551 bool write = (svm->vmcb->control.exit_info_1 >> 32) & 552 AVIC_UNACCEL_ACCESS_WRITE_MASK; 553 bool trap = is_avic_unaccelerated_access_trap(offset); 554 555 trace_kvm_avic_unaccelerated_access(vcpu->vcpu_id, offset, 556 trap, write, vector); 557 if (trap) { 558 /* Handling Trap */ 559 WARN_ONCE(!write, "svm: Handling trap read.\n"); 560 ret = avic_unaccel_trap_write(svm); 561 } else { 562 /* Handling Fault */ 563 ret = kvm_emulate_instruction(vcpu, 0); 564 } 565 566 return ret; 567 } 568 569 int avic_init_vcpu(struct vcpu_svm *svm) 570 { 571 int ret; 572 struct kvm_vcpu *vcpu = &svm->vcpu; 573 574 if (!avic || !irqchip_in_kernel(vcpu->kvm)) 575 return 0; 576 577 ret = avic_init_backing_page(vcpu); 578 if (ret) 579 return ret; 580 581 INIT_LIST_HEAD(&svm->ir_list); 582 spin_lock_init(&svm->ir_list_lock); 583 svm->dfr_reg = APIC_DFR_FLAT; 584 585 return ret; 586 } 587 588 void avic_post_state_restore(struct kvm_vcpu *vcpu) 589 { 590 if (avic_handle_apic_id_update(vcpu) != 0) 591 return; 592 avic_handle_dfr_update(vcpu); 593 avic_handle_ldr_update(vcpu); 594 } 595 596 void svm_toggle_avic_for_irq_window(struct kvm_vcpu *vcpu, bool activate) 597 { 598 if (!avic || !lapic_in_kernel(vcpu)) 599 return; 600 601 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 602 kvm_request_apicv_update(vcpu->kvm, activate, 603 APICV_INHIBIT_REASON_IRQWIN); 604 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 605 } 606 607 void svm_set_virtual_apic_mode(struct kvm_vcpu *vcpu) 608 { 609 return; 610 } 611 612 void svm_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr) 613 { 614 } 615 616 void svm_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr) 617 { 618 } 619 620 static int svm_set_pi_irte_mode(struct kvm_vcpu *vcpu, bool activate) 621 { 622 int ret = 0; 623 unsigned long flags; 624 struct amd_svm_iommu_ir *ir; 625 struct vcpu_svm *svm = to_svm(vcpu); 626 627 if (!kvm_arch_has_assigned_device(vcpu->kvm)) 628 return 0; 629 630 /* 631 * Here, we go through the per-vcpu ir_list to update all existing 632 * interrupt remapping table entry targeting this vcpu. 633 */ 634 spin_lock_irqsave(&svm->ir_list_lock, flags); 635 636 if (list_empty(&svm->ir_list)) 637 goto out; 638 639 list_for_each_entry(ir, &svm->ir_list, node) { 640 if (activate) 641 ret = amd_iommu_activate_guest_mode(ir->data); 642 else 643 ret = amd_iommu_deactivate_guest_mode(ir->data); 644 if (ret) 645 break; 646 } 647 out: 648 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 649 return ret; 650 } 651 652 void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) 653 { 654 struct vcpu_svm *svm = to_svm(vcpu); 655 struct vmcb *vmcb = svm->vmcb; 656 bool activated = kvm_vcpu_apicv_active(vcpu); 657 658 if (!avic) 659 return; 660 661 if (activated) { 662 /** 663 * During AVIC temporary deactivation, guest could update 664 * APIC ID, DFR and LDR registers, which would not be trapped 665 * by avic_unaccelerated_access_interception(). In this case, 666 * we need to check and update the AVIC logical APIC ID table 667 * accordingly before re-activating. 668 */ 669 avic_post_state_restore(vcpu); 670 vmcb->control.int_ctl |= AVIC_ENABLE_MASK; 671 } else { 672 vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK; 673 } 674 vmcb_mark_dirty(vmcb, VMCB_AVIC); 675 676 svm_set_pi_irte_mode(vcpu, activated); 677 } 678 679 void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) 680 { 681 return; 682 } 683 684 int svm_deliver_avic_intr(struct kvm_vcpu *vcpu, int vec) 685 { 686 if (!vcpu->arch.apicv_active) 687 return -1; 688 689 kvm_lapic_set_irr(vec, vcpu->arch.apic); 690 smp_mb__after_atomic(); 691 692 if (avic_vcpu_is_running(vcpu)) { 693 int cpuid = vcpu->cpu; 694 695 if (cpuid != get_cpu()) 696 wrmsrl(SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpuid)); 697 put_cpu(); 698 } else 699 kvm_vcpu_wake_up(vcpu); 700 701 return 0; 702 } 703 704 bool svm_dy_apicv_has_pending_interrupt(struct kvm_vcpu *vcpu) 705 { 706 return false; 707 } 708 709 static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi) 710 { 711 unsigned long flags; 712 struct amd_svm_iommu_ir *cur; 713 714 spin_lock_irqsave(&svm->ir_list_lock, flags); 715 list_for_each_entry(cur, &svm->ir_list, node) { 716 if (cur->data != pi->ir_data) 717 continue; 718 list_del(&cur->node); 719 kfree(cur); 720 break; 721 } 722 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 723 } 724 725 static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi) 726 { 727 int ret = 0; 728 unsigned long flags; 729 struct amd_svm_iommu_ir *ir; 730 731 /** 732 * In some cases, the existing irte is updated and re-set, 733 * so we need to check here if it's already been * added 734 * to the ir_list. 735 */ 736 if (pi->ir_data && (pi->prev_ga_tag != 0)) { 737 struct kvm *kvm = svm->vcpu.kvm; 738 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag); 739 struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id); 740 struct vcpu_svm *prev_svm; 741 742 if (!prev_vcpu) { 743 ret = -EINVAL; 744 goto out; 745 } 746 747 prev_svm = to_svm(prev_vcpu); 748 svm_ir_list_del(prev_svm, pi); 749 } 750 751 /** 752 * Allocating new amd_iommu_pi_data, which will get 753 * add to the per-vcpu ir_list. 754 */ 755 ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL_ACCOUNT); 756 if (!ir) { 757 ret = -ENOMEM; 758 goto out; 759 } 760 ir->data = pi->ir_data; 761 762 spin_lock_irqsave(&svm->ir_list_lock, flags); 763 list_add(&ir->node, &svm->ir_list); 764 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 765 out: 766 return ret; 767 } 768 769 /** 770 * Note: 771 * The HW cannot support posting multicast/broadcast 772 * interrupts to a vCPU. So, we still use legacy interrupt 773 * remapping for these kind of interrupts. 774 * 775 * For lowest-priority interrupts, we only support 776 * those with single CPU as the destination, e.g. user 777 * configures the interrupts via /proc/irq or uses 778 * irqbalance to make the interrupts single-CPU. 779 */ 780 static int 781 get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, 782 struct vcpu_data *vcpu_info, struct vcpu_svm **svm) 783 { 784 struct kvm_lapic_irq irq; 785 struct kvm_vcpu *vcpu = NULL; 786 787 kvm_set_msi_irq(kvm, e, &irq); 788 789 if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) || 790 !kvm_irq_is_postable(&irq)) { 791 pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n", 792 __func__, irq.vector); 793 return -1; 794 } 795 796 pr_debug("SVM: %s: use GA mode for irq %u\n", __func__, 797 irq.vector); 798 *svm = to_svm(vcpu); 799 vcpu_info->pi_desc_addr = __sme_set(page_to_phys((*svm)->avic_backing_page)); 800 vcpu_info->vector = irq.vector; 801 802 return 0; 803 } 804 805 /* 806 * svm_update_pi_irte - set IRTE for Posted-Interrupts 807 * 808 * @kvm: kvm 809 * @host_irq: host irq of the interrupt 810 * @guest_irq: gsi of the interrupt 811 * @set: set or unset PI 812 * returns 0 on success, < 0 on failure 813 */ 814 int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq, 815 uint32_t guest_irq, bool set) 816 { 817 struct kvm_kernel_irq_routing_entry *e; 818 struct kvm_irq_routing_table *irq_rt; 819 int idx, ret = -EINVAL; 820 821 if (!kvm_arch_has_assigned_device(kvm) || 822 !irq_remapping_cap(IRQ_POSTING_CAP)) 823 return 0; 824 825 pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n", 826 __func__, host_irq, guest_irq, set); 827 828 idx = srcu_read_lock(&kvm->irq_srcu); 829 irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu); 830 WARN_ON(guest_irq >= irq_rt->nr_rt_entries); 831 832 hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) { 833 struct vcpu_data vcpu_info; 834 struct vcpu_svm *svm = NULL; 835 836 if (e->type != KVM_IRQ_ROUTING_MSI) 837 continue; 838 839 /** 840 * Here, we setup with legacy mode in the following cases: 841 * 1. When cannot target interrupt to a specific vcpu. 842 * 2. Unsetting posted interrupt. 843 * 3. APIC virtualization is disabled for the vcpu. 844 * 4. IRQ has incompatible delivery mode (SMI, INIT, etc) 845 */ 846 if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set && 847 kvm_vcpu_apicv_active(&svm->vcpu)) { 848 struct amd_iommu_pi_data pi; 849 850 /* Try to enable guest_mode in IRTE */ 851 pi.base = __sme_set(page_to_phys(svm->avic_backing_page) & 852 AVIC_HPA_MASK); 853 pi.ga_tag = AVIC_GATAG(to_kvm_svm(kvm)->avic_vm_id, 854 svm->vcpu.vcpu_id); 855 pi.is_guest_mode = true; 856 pi.vcpu_data = &vcpu_info; 857 ret = irq_set_vcpu_affinity(host_irq, &pi); 858 859 /** 860 * Here, we successfully setting up vcpu affinity in 861 * IOMMU guest mode. Now, we need to store the posted 862 * interrupt information in a per-vcpu ir_list so that 863 * we can reference to them directly when we update vcpu 864 * scheduling information in IOMMU irte. 865 */ 866 if (!ret && pi.is_guest_mode) 867 svm_ir_list_add(svm, &pi); 868 } else { 869 /* Use legacy mode in IRTE */ 870 struct amd_iommu_pi_data pi; 871 872 /** 873 * Here, pi is used to: 874 * - Tell IOMMU to use legacy mode for this interrupt. 875 * - Retrieve ga_tag of prior interrupt remapping data. 876 */ 877 pi.prev_ga_tag = 0; 878 pi.is_guest_mode = false; 879 ret = irq_set_vcpu_affinity(host_irq, &pi); 880 881 /** 882 * Check if the posted interrupt was previously 883 * setup with the guest_mode by checking if the ga_tag 884 * was cached. If so, we need to clean up the per-vcpu 885 * ir_list. 886 */ 887 if (!ret && pi.prev_ga_tag) { 888 int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag); 889 struct kvm_vcpu *vcpu; 890 891 vcpu = kvm_get_vcpu_by_id(kvm, id); 892 if (vcpu) 893 svm_ir_list_del(to_svm(vcpu), &pi); 894 } 895 } 896 897 if (!ret && svm) { 898 trace_kvm_pi_irte_update(host_irq, svm->vcpu.vcpu_id, 899 e->gsi, vcpu_info.vector, 900 vcpu_info.pi_desc_addr, set); 901 } 902 903 if (ret < 0) { 904 pr_err("%s: failed to update PI IRTE\n", __func__); 905 goto out; 906 } 907 } 908 909 ret = 0; 910 out: 911 srcu_read_unlock(&kvm->irq_srcu, idx); 912 return ret; 913 } 914 915 bool svm_check_apicv_inhibit_reasons(ulong bit) 916 { 917 ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) | 918 BIT(APICV_INHIBIT_REASON_HYPERV) | 919 BIT(APICV_INHIBIT_REASON_NESTED) | 920 BIT(APICV_INHIBIT_REASON_IRQWIN) | 921 BIT(APICV_INHIBIT_REASON_PIT_REINJ) | 922 BIT(APICV_INHIBIT_REASON_X2APIC); 923 924 return supported & BIT(bit); 925 } 926 927 void svm_pre_update_apicv_exec_ctrl(struct kvm *kvm, bool activate) 928 { 929 avic_update_access_page(kvm, activate); 930 } 931 932 static inline int 933 avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r) 934 { 935 int ret = 0; 936 unsigned long flags; 937 struct amd_svm_iommu_ir *ir; 938 struct vcpu_svm *svm = to_svm(vcpu); 939 940 if (!kvm_arch_has_assigned_device(vcpu->kvm)) 941 return 0; 942 943 /* 944 * Here, we go through the per-vcpu ir_list to update all existing 945 * interrupt remapping table entry targeting this vcpu. 946 */ 947 spin_lock_irqsave(&svm->ir_list_lock, flags); 948 949 if (list_empty(&svm->ir_list)) 950 goto out; 951 952 list_for_each_entry(ir, &svm->ir_list, node) { 953 ret = amd_iommu_update_ga(cpu, r, ir->data); 954 if (ret) 955 break; 956 } 957 out: 958 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 959 return ret; 960 } 961 962 void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 963 { 964 u64 entry; 965 /* ID = 0xff (broadcast), ID > 0xff (reserved) */ 966 int h_physical_id = kvm_cpu_get_apicid(cpu); 967 struct vcpu_svm *svm = to_svm(vcpu); 968 969 if (!kvm_vcpu_apicv_active(vcpu)) 970 return; 971 972 /* 973 * Since the host physical APIC id is 8 bits, 974 * we can support host APIC ID upto 255. 975 */ 976 if (WARN_ON(h_physical_id > AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK)) 977 return; 978 979 entry = READ_ONCE(*(svm->avic_physical_id_cache)); 980 WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK); 981 982 entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK; 983 entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK); 984 985 entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; 986 if (svm->avic_is_running) 987 entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; 988 989 WRITE_ONCE(*(svm->avic_physical_id_cache), entry); 990 avic_update_iommu_vcpu_affinity(vcpu, h_physical_id, 991 svm->avic_is_running); 992 } 993 994 void avic_vcpu_put(struct kvm_vcpu *vcpu) 995 { 996 u64 entry; 997 struct vcpu_svm *svm = to_svm(vcpu); 998 999 if (!kvm_vcpu_apicv_active(vcpu)) 1000 return; 1001 1002 entry = READ_ONCE(*(svm->avic_physical_id_cache)); 1003 if (entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK) 1004 avic_update_iommu_vcpu_affinity(vcpu, -1, 0); 1005 1006 entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; 1007 WRITE_ONCE(*(svm->avic_physical_id_cache), entry); 1008 } 1009 1010 /** 1011 * This function is called during VCPU halt/unhalt. 1012 */ 1013 static void avic_set_running(struct kvm_vcpu *vcpu, bool is_run) 1014 { 1015 struct vcpu_svm *svm = to_svm(vcpu); 1016 1017 svm->avic_is_running = is_run; 1018 if (is_run) 1019 avic_vcpu_load(vcpu, vcpu->cpu); 1020 else 1021 avic_vcpu_put(vcpu); 1022 } 1023 1024 void svm_vcpu_blocking(struct kvm_vcpu *vcpu) 1025 { 1026 avic_set_running(vcpu, false); 1027 } 1028 1029 void svm_vcpu_unblocking(struct kvm_vcpu *vcpu) 1030 { 1031 if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu)) 1032 kvm_vcpu_update_apicv(vcpu); 1033 avic_set_running(vcpu, true); 1034 } 1035