1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * AMD SVM support 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Yaniv Kamay <yaniv@qumranet.com> 12 * Avi Kivity <avi@qumranet.com> 13 */ 14 15 #define pr_fmt(fmt) "SVM: " fmt 16 17 #include <linux/kvm_types.h> 18 #include <linux/hashtable.h> 19 #include <linux/amd-iommu.h> 20 #include <linux/kvm_host.h> 21 22 #include <asm/irq_remapping.h> 23 24 #include "trace.h" 25 #include "lapic.h" 26 #include "x86.h" 27 #include "irq.h" 28 #include "svm.h" 29 30 /* enable / disable AVIC */ 31 bool avic; 32 module_param(avic, bool, S_IRUGO); 33 34 #define SVM_AVIC_DOORBELL 0xc001011b 35 36 #define AVIC_HPA_MASK ~((0xFFFULL << 52) | 0xFFF) 37 38 /* 39 * 0xff is broadcast, so the max index allowed for physical APIC ID 40 * table is 0xfe. APIC IDs above 0xff are reserved. 41 */ 42 #define AVIC_MAX_PHYSICAL_ID_COUNT 255 43 44 #define AVIC_UNACCEL_ACCESS_WRITE_MASK 1 45 #define AVIC_UNACCEL_ACCESS_OFFSET_MASK 0xFF0 46 #define AVIC_UNACCEL_ACCESS_VECTOR_MASK 0xFFFFFFFF 47 48 /* AVIC GATAG is encoded using VM and VCPU IDs */ 49 #define AVIC_VCPU_ID_BITS 8 50 #define AVIC_VCPU_ID_MASK ((1 << AVIC_VCPU_ID_BITS) - 1) 51 52 #define AVIC_VM_ID_BITS 24 53 #define AVIC_VM_ID_NR (1 << AVIC_VM_ID_BITS) 54 #define AVIC_VM_ID_MASK ((1 << AVIC_VM_ID_BITS) - 1) 55 56 #define AVIC_GATAG(x, y) (((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \ 57 (y & AVIC_VCPU_ID_MASK)) 58 #define AVIC_GATAG_TO_VMID(x) ((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK) 59 #define AVIC_GATAG_TO_VCPUID(x) (x & AVIC_VCPU_ID_MASK) 60 61 /* Note: 62 * This hash table is used to map VM_ID to a struct kvm_svm, 63 * when handling AMD IOMMU GALOG notification to schedule in 64 * a particular vCPU. 65 */ 66 #define SVM_VM_DATA_HASH_BITS 8 67 static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS); 68 static u32 next_vm_id = 0; 69 static bool next_vm_id_wrapped = 0; 70 static DEFINE_SPINLOCK(svm_vm_data_hash_lock); 71 72 /* 73 * This is a wrapper of struct amd_iommu_ir_data. 74 */ 75 struct amd_svm_iommu_ir { 76 struct list_head node; /* Used by SVM for per-vcpu ir_list */ 77 void *data; /* Storing pointer to struct amd_ir_data */ 78 }; 79 80 enum avic_ipi_failure_cause { 81 AVIC_IPI_FAILURE_INVALID_INT_TYPE, 82 AVIC_IPI_FAILURE_TARGET_NOT_RUNNING, 83 AVIC_IPI_FAILURE_INVALID_TARGET, 84 AVIC_IPI_FAILURE_INVALID_BACKING_PAGE, 85 }; 86 87 /* Note: 88 * This function is called from IOMMU driver to notify 89 * SVM to schedule in a particular vCPU of a particular VM. 90 */ 91 int avic_ga_log_notifier(u32 ga_tag) 92 { 93 unsigned long flags; 94 struct kvm_svm *kvm_svm; 95 struct kvm_vcpu *vcpu = NULL; 96 u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag); 97 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag); 98 99 pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id); 100 trace_kvm_avic_ga_log(vm_id, vcpu_id); 101 102 spin_lock_irqsave(&svm_vm_data_hash_lock, flags); 103 hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) { 104 if (kvm_svm->avic_vm_id != vm_id) 105 continue; 106 vcpu = kvm_get_vcpu_by_id(&kvm_svm->kvm, vcpu_id); 107 break; 108 } 109 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); 110 111 /* Note: 112 * At this point, the IOMMU should have already set the pending 113 * bit in the vAPIC backing page. So, we just need to schedule 114 * in the vcpu. 115 */ 116 if (vcpu) 117 kvm_vcpu_wake_up(vcpu); 118 119 return 0; 120 } 121 122 void avic_vm_destroy(struct kvm *kvm) 123 { 124 unsigned long flags; 125 struct kvm_svm *kvm_svm = to_kvm_svm(kvm); 126 127 if (!avic) 128 return; 129 130 if (kvm_svm->avic_logical_id_table_page) 131 __free_page(kvm_svm->avic_logical_id_table_page); 132 if (kvm_svm->avic_physical_id_table_page) 133 __free_page(kvm_svm->avic_physical_id_table_page); 134 135 spin_lock_irqsave(&svm_vm_data_hash_lock, flags); 136 hash_del(&kvm_svm->hnode); 137 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); 138 } 139 140 int avic_vm_init(struct kvm *kvm) 141 { 142 unsigned long flags; 143 int err = -ENOMEM; 144 struct kvm_svm *kvm_svm = to_kvm_svm(kvm); 145 struct kvm_svm *k2; 146 struct page *p_page; 147 struct page *l_page; 148 u32 vm_id; 149 150 if (!avic) 151 return 0; 152 153 /* Allocating physical APIC ID table (4KB) */ 154 p_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 155 if (!p_page) 156 goto free_avic; 157 158 kvm_svm->avic_physical_id_table_page = p_page; 159 160 /* Allocating logical APIC ID table (4KB) */ 161 l_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 162 if (!l_page) 163 goto free_avic; 164 165 kvm_svm->avic_logical_id_table_page = l_page; 166 167 spin_lock_irqsave(&svm_vm_data_hash_lock, flags); 168 again: 169 vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK; 170 if (vm_id == 0) { /* id is 1-based, zero is not okay */ 171 next_vm_id_wrapped = 1; 172 goto again; 173 } 174 /* Is it still in use? Only possible if wrapped at least once */ 175 if (next_vm_id_wrapped) { 176 hash_for_each_possible(svm_vm_data_hash, k2, hnode, vm_id) { 177 if (k2->avic_vm_id == vm_id) 178 goto again; 179 } 180 } 181 kvm_svm->avic_vm_id = vm_id; 182 hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id); 183 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); 184 185 return 0; 186 187 free_avic: 188 avic_vm_destroy(kvm); 189 return err; 190 } 191 192 void avic_init_vmcb(struct vcpu_svm *svm) 193 { 194 struct vmcb *vmcb = svm->vmcb; 195 struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm); 196 phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page)); 197 phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page)); 198 phys_addr_t ppa = __sme_set(page_to_phys(kvm_svm->avic_physical_id_table_page)); 199 200 vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK; 201 vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK; 202 vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK; 203 vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT; 204 if (kvm_apicv_activated(svm->vcpu.kvm)) 205 vmcb->control.int_ctl |= AVIC_ENABLE_MASK; 206 else 207 vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK; 208 } 209 210 static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu, 211 unsigned int index) 212 { 213 u64 *avic_physical_id_table; 214 struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); 215 216 if (index >= AVIC_MAX_PHYSICAL_ID_COUNT) 217 return NULL; 218 219 avic_physical_id_table = page_address(kvm_svm->avic_physical_id_table_page); 220 221 return &avic_physical_id_table[index]; 222 } 223 224 /** 225 * Note: 226 * AVIC hardware walks the nested page table to check permissions, 227 * but does not use the SPA address specified in the leaf page 228 * table entry since it uses address in the AVIC_BACKING_PAGE pointer 229 * field of the VMCB. Therefore, we set up the 230 * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here. 231 */ 232 static int avic_update_access_page(struct kvm *kvm, bool activate) 233 { 234 void __user *ret; 235 int r = 0; 236 237 mutex_lock(&kvm->slots_lock); 238 /* 239 * During kvm_destroy_vm(), kvm_pit_set_reinject() could trigger 240 * APICv mode change, which update APIC_ACCESS_PAGE_PRIVATE_MEMSLOT 241 * memory region. So, we need to ensure that kvm->mm == current->mm. 242 */ 243 if ((kvm->arch.apic_access_page_done == activate) || 244 (kvm->mm != current->mm)) 245 goto out; 246 247 ret = __x86_set_memory_region(kvm, 248 APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 249 APIC_DEFAULT_PHYS_BASE, 250 activate ? PAGE_SIZE : 0); 251 if (IS_ERR(ret)) { 252 r = PTR_ERR(ret); 253 goto out; 254 } 255 256 kvm->arch.apic_access_page_done = activate; 257 out: 258 mutex_unlock(&kvm->slots_lock); 259 return r; 260 } 261 262 static int avic_init_backing_page(struct kvm_vcpu *vcpu) 263 { 264 u64 *entry, new_entry; 265 int id = vcpu->vcpu_id; 266 struct vcpu_svm *svm = to_svm(vcpu); 267 268 if (id >= AVIC_MAX_PHYSICAL_ID_COUNT) 269 return -EINVAL; 270 271 if (!vcpu->arch.apic->regs) 272 return -EINVAL; 273 274 if (kvm_apicv_activated(vcpu->kvm)) { 275 int ret; 276 277 ret = avic_update_access_page(vcpu->kvm, true); 278 if (ret) 279 return ret; 280 } 281 282 svm->avic_backing_page = virt_to_page(vcpu->arch.apic->regs); 283 284 /* Setting AVIC backing page address in the phy APIC ID table */ 285 entry = avic_get_physical_id_entry(vcpu, id); 286 if (!entry) 287 return -EINVAL; 288 289 new_entry = __sme_set((page_to_phys(svm->avic_backing_page) & 290 AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) | 291 AVIC_PHYSICAL_ID_ENTRY_VALID_MASK); 292 WRITE_ONCE(*entry, new_entry); 293 294 svm->avic_physical_id_cache = entry; 295 296 return 0; 297 } 298 299 static void avic_kick_target_vcpus(struct kvm *kvm, struct kvm_lapic *source, 300 u32 icrl, u32 icrh) 301 { 302 struct kvm_vcpu *vcpu; 303 int i; 304 305 kvm_for_each_vcpu(i, vcpu, kvm) { 306 bool m = kvm_apic_match_dest(vcpu, source, 307 icrl & APIC_SHORT_MASK, 308 GET_APIC_DEST_FIELD(icrh), 309 icrl & APIC_DEST_MASK); 310 311 if (m && !avic_vcpu_is_running(vcpu)) 312 kvm_vcpu_wake_up(vcpu); 313 } 314 } 315 316 int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu) 317 { 318 struct vcpu_svm *svm = to_svm(vcpu); 319 u32 icrh = svm->vmcb->control.exit_info_1 >> 32; 320 u32 icrl = svm->vmcb->control.exit_info_1; 321 u32 id = svm->vmcb->control.exit_info_2 >> 32; 322 u32 index = svm->vmcb->control.exit_info_2 & 0xFF; 323 struct kvm_lapic *apic = vcpu->arch.apic; 324 325 trace_kvm_avic_incomplete_ipi(vcpu->vcpu_id, icrh, icrl, id, index); 326 327 switch (id) { 328 case AVIC_IPI_FAILURE_INVALID_INT_TYPE: 329 /* 330 * AVIC hardware handles the generation of 331 * IPIs when the specified Message Type is Fixed 332 * (also known as fixed delivery mode) and 333 * the Trigger Mode is edge-triggered. The hardware 334 * also supports self and broadcast delivery modes 335 * specified via the Destination Shorthand(DSH) 336 * field of the ICRL. Logical and physical APIC ID 337 * formats are supported. All other IPI types cause 338 * a #VMEXIT, which needs to emulated. 339 */ 340 kvm_lapic_reg_write(apic, APIC_ICR2, icrh); 341 kvm_lapic_reg_write(apic, APIC_ICR, icrl); 342 break; 343 case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING: 344 /* 345 * At this point, we expect that the AVIC HW has already 346 * set the appropriate IRR bits on the valid target 347 * vcpus. So, we just need to kick the appropriate vcpu. 348 */ 349 avic_kick_target_vcpus(vcpu->kvm, apic, icrl, icrh); 350 break; 351 case AVIC_IPI_FAILURE_INVALID_TARGET: 352 WARN_ONCE(1, "Invalid IPI target: index=%u, vcpu=%d, icr=%#0x:%#0x\n", 353 index, vcpu->vcpu_id, icrh, icrl); 354 break; 355 case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE: 356 WARN_ONCE(1, "Invalid backing page\n"); 357 break; 358 default: 359 pr_err("Unknown IPI interception\n"); 360 } 361 362 return 1; 363 } 364 365 static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat) 366 { 367 struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); 368 int index; 369 u32 *logical_apic_id_table; 370 int dlid = GET_APIC_LOGICAL_ID(ldr); 371 372 if (!dlid) 373 return NULL; 374 375 if (flat) { /* flat */ 376 index = ffs(dlid) - 1; 377 if (index > 7) 378 return NULL; 379 } else { /* cluster */ 380 int cluster = (dlid & 0xf0) >> 4; 381 int apic = ffs(dlid & 0x0f) - 1; 382 383 if ((apic < 0) || (apic > 7) || 384 (cluster >= 0xf)) 385 return NULL; 386 index = (cluster << 2) + apic; 387 } 388 389 logical_apic_id_table = (u32 *) page_address(kvm_svm->avic_logical_id_table_page); 390 391 return &logical_apic_id_table[index]; 392 } 393 394 static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr) 395 { 396 bool flat; 397 u32 *entry, new_entry; 398 399 flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT; 400 entry = avic_get_logical_id_entry(vcpu, ldr, flat); 401 if (!entry) 402 return -EINVAL; 403 404 new_entry = READ_ONCE(*entry); 405 new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK; 406 new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK); 407 new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK; 408 WRITE_ONCE(*entry, new_entry); 409 410 return 0; 411 } 412 413 static void avic_invalidate_logical_id_entry(struct kvm_vcpu *vcpu) 414 { 415 struct vcpu_svm *svm = to_svm(vcpu); 416 bool flat = svm->dfr_reg == APIC_DFR_FLAT; 417 u32 *entry = avic_get_logical_id_entry(vcpu, svm->ldr_reg, flat); 418 419 if (entry) 420 clear_bit(AVIC_LOGICAL_ID_ENTRY_VALID_BIT, (unsigned long *)entry); 421 } 422 423 static int avic_handle_ldr_update(struct kvm_vcpu *vcpu) 424 { 425 int ret = 0; 426 struct vcpu_svm *svm = to_svm(vcpu); 427 u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR); 428 u32 id = kvm_xapic_id(vcpu->arch.apic); 429 430 if (ldr == svm->ldr_reg) 431 return 0; 432 433 avic_invalidate_logical_id_entry(vcpu); 434 435 if (ldr) 436 ret = avic_ldr_write(vcpu, id, ldr); 437 438 if (!ret) 439 svm->ldr_reg = ldr; 440 441 return ret; 442 } 443 444 static int avic_handle_apic_id_update(struct kvm_vcpu *vcpu) 445 { 446 u64 *old, *new; 447 struct vcpu_svm *svm = to_svm(vcpu); 448 u32 id = kvm_xapic_id(vcpu->arch.apic); 449 450 if (vcpu->vcpu_id == id) 451 return 0; 452 453 old = avic_get_physical_id_entry(vcpu, vcpu->vcpu_id); 454 new = avic_get_physical_id_entry(vcpu, id); 455 if (!new || !old) 456 return 1; 457 458 /* We need to move physical_id_entry to new offset */ 459 *new = *old; 460 *old = 0ULL; 461 to_svm(vcpu)->avic_physical_id_cache = new; 462 463 /* 464 * Also update the guest physical APIC ID in the logical 465 * APIC ID table entry if already setup the LDR. 466 */ 467 if (svm->ldr_reg) 468 avic_handle_ldr_update(vcpu); 469 470 return 0; 471 } 472 473 static void avic_handle_dfr_update(struct kvm_vcpu *vcpu) 474 { 475 struct vcpu_svm *svm = to_svm(vcpu); 476 u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR); 477 478 if (svm->dfr_reg == dfr) 479 return; 480 481 avic_invalidate_logical_id_entry(vcpu); 482 svm->dfr_reg = dfr; 483 } 484 485 static int avic_unaccel_trap_write(struct vcpu_svm *svm) 486 { 487 struct kvm_lapic *apic = svm->vcpu.arch.apic; 488 u32 offset = svm->vmcb->control.exit_info_1 & 489 AVIC_UNACCEL_ACCESS_OFFSET_MASK; 490 491 switch (offset) { 492 case APIC_ID: 493 if (avic_handle_apic_id_update(&svm->vcpu)) 494 return 0; 495 break; 496 case APIC_LDR: 497 if (avic_handle_ldr_update(&svm->vcpu)) 498 return 0; 499 break; 500 case APIC_DFR: 501 avic_handle_dfr_update(&svm->vcpu); 502 break; 503 default: 504 break; 505 } 506 507 kvm_lapic_reg_write(apic, offset, kvm_lapic_get_reg(apic, offset)); 508 509 return 1; 510 } 511 512 static bool is_avic_unaccelerated_access_trap(u32 offset) 513 { 514 bool ret = false; 515 516 switch (offset) { 517 case APIC_ID: 518 case APIC_EOI: 519 case APIC_RRR: 520 case APIC_LDR: 521 case APIC_DFR: 522 case APIC_SPIV: 523 case APIC_ESR: 524 case APIC_ICR: 525 case APIC_LVTT: 526 case APIC_LVTTHMR: 527 case APIC_LVTPC: 528 case APIC_LVT0: 529 case APIC_LVT1: 530 case APIC_LVTERR: 531 case APIC_TMICT: 532 case APIC_TDCR: 533 ret = true; 534 break; 535 default: 536 break; 537 } 538 return ret; 539 } 540 541 int avic_unaccelerated_access_interception(struct kvm_vcpu *vcpu) 542 { 543 struct vcpu_svm *svm = to_svm(vcpu); 544 int ret = 0; 545 u32 offset = svm->vmcb->control.exit_info_1 & 546 AVIC_UNACCEL_ACCESS_OFFSET_MASK; 547 u32 vector = svm->vmcb->control.exit_info_2 & 548 AVIC_UNACCEL_ACCESS_VECTOR_MASK; 549 bool write = (svm->vmcb->control.exit_info_1 >> 32) & 550 AVIC_UNACCEL_ACCESS_WRITE_MASK; 551 bool trap = is_avic_unaccelerated_access_trap(offset); 552 553 trace_kvm_avic_unaccelerated_access(vcpu->vcpu_id, offset, 554 trap, write, vector); 555 if (trap) { 556 /* Handling Trap */ 557 WARN_ONCE(!write, "svm: Handling trap read.\n"); 558 ret = avic_unaccel_trap_write(svm); 559 } else { 560 /* Handling Fault */ 561 ret = kvm_emulate_instruction(vcpu, 0); 562 } 563 564 return ret; 565 } 566 567 int avic_init_vcpu(struct vcpu_svm *svm) 568 { 569 int ret; 570 struct kvm_vcpu *vcpu = &svm->vcpu; 571 572 if (!avic || !irqchip_in_kernel(vcpu->kvm)) 573 return 0; 574 575 ret = avic_init_backing_page(vcpu); 576 if (ret) 577 return ret; 578 579 INIT_LIST_HEAD(&svm->ir_list); 580 spin_lock_init(&svm->ir_list_lock); 581 svm->dfr_reg = APIC_DFR_FLAT; 582 583 return ret; 584 } 585 586 void avic_post_state_restore(struct kvm_vcpu *vcpu) 587 { 588 if (avic_handle_apic_id_update(vcpu) != 0) 589 return; 590 avic_handle_dfr_update(vcpu); 591 avic_handle_ldr_update(vcpu); 592 } 593 594 void svm_toggle_avic_for_irq_window(struct kvm_vcpu *vcpu, bool activate) 595 { 596 if (!avic || !lapic_in_kernel(vcpu)) 597 return; 598 599 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 600 kvm_request_apicv_update(vcpu->kvm, activate, 601 APICV_INHIBIT_REASON_IRQWIN); 602 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 603 } 604 605 void svm_set_virtual_apic_mode(struct kvm_vcpu *vcpu) 606 { 607 return; 608 } 609 610 void svm_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr) 611 { 612 } 613 614 void svm_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr) 615 { 616 } 617 618 static int svm_set_pi_irte_mode(struct kvm_vcpu *vcpu, bool activate) 619 { 620 int ret = 0; 621 unsigned long flags; 622 struct amd_svm_iommu_ir *ir; 623 struct vcpu_svm *svm = to_svm(vcpu); 624 625 if (!kvm_arch_has_assigned_device(vcpu->kvm)) 626 return 0; 627 628 /* 629 * Here, we go through the per-vcpu ir_list to update all existing 630 * interrupt remapping table entry targeting this vcpu. 631 */ 632 spin_lock_irqsave(&svm->ir_list_lock, flags); 633 634 if (list_empty(&svm->ir_list)) 635 goto out; 636 637 list_for_each_entry(ir, &svm->ir_list, node) { 638 if (activate) 639 ret = amd_iommu_activate_guest_mode(ir->data); 640 else 641 ret = amd_iommu_deactivate_guest_mode(ir->data); 642 if (ret) 643 break; 644 } 645 out: 646 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 647 return ret; 648 } 649 650 void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) 651 { 652 struct vcpu_svm *svm = to_svm(vcpu); 653 struct vmcb *vmcb = svm->vmcb; 654 bool activated = kvm_vcpu_apicv_active(vcpu); 655 656 if (!avic) 657 return; 658 659 if (activated) { 660 /** 661 * During AVIC temporary deactivation, guest could update 662 * APIC ID, DFR and LDR registers, which would not be trapped 663 * by avic_unaccelerated_access_interception(). In this case, 664 * we need to check and update the AVIC logical APIC ID table 665 * accordingly before re-activating. 666 */ 667 avic_post_state_restore(vcpu); 668 vmcb->control.int_ctl |= AVIC_ENABLE_MASK; 669 } else { 670 vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK; 671 } 672 vmcb_mark_dirty(vmcb, VMCB_AVIC); 673 674 svm_set_pi_irte_mode(vcpu, activated); 675 } 676 677 void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) 678 { 679 return; 680 } 681 682 int svm_deliver_avic_intr(struct kvm_vcpu *vcpu, int vec) 683 { 684 if (!vcpu->arch.apicv_active) 685 return -1; 686 687 kvm_lapic_set_irr(vec, vcpu->arch.apic); 688 smp_mb__after_atomic(); 689 690 if (avic_vcpu_is_running(vcpu)) { 691 int cpuid = vcpu->cpu; 692 693 if (cpuid != get_cpu()) 694 wrmsrl(SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpuid)); 695 put_cpu(); 696 } else 697 kvm_vcpu_wake_up(vcpu); 698 699 return 0; 700 } 701 702 bool svm_dy_apicv_has_pending_interrupt(struct kvm_vcpu *vcpu) 703 { 704 return false; 705 } 706 707 static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi) 708 { 709 unsigned long flags; 710 struct amd_svm_iommu_ir *cur; 711 712 spin_lock_irqsave(&svm->ir_list_lock, flags); 713 list_for_each_entry(cur, &svm->ir_list, node) { 714 if (cur->data != pi->ir_data) 715 continue; 716 list_del(&cur->node); 717 kfree(cur); 718 break; 719 } 720 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 721 } 722 723 static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi) 724 { 725 int ret = 0; 726 unsigned long flags; 727 struct amd_svm_iommu_ir *ir; 728 729 /** 730 * In some cases, the existing irte is updated and re-set, 731 * so we need to check here if it's already been * added 732 * to the ir_list. 733 */ 734 if (pi->ir_data && (pi->prev_ga_tag != 0)) { 735 struct kvm *kvm = svm->vcpu.kvm; 736 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag); 737 struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id); 738 struct vcpu_svm *prev_svm; 739 740 if (!prev_vcpu) { 741 ret = -EINVAL; 742 goto out; 743 } 744 745 prev_svm = to_svm(prev_vcpu); 746 svm_ir_list_del(prev_svm, pi); 747 } 748 749 /** 750 * Allocating new amd_iommu_pi_data, which will get 751 * add to the per-vcpu ir_list. 752 */ 753 ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL_ACCOUNT); 754 if (!ir) { 755 ret = -ENOMEM; 756 goto out; 757 } 758 ir->data = pi->ir_data; 759 760 spin_lock_irqsave(&svm->ir_list_lock, flags); 761 list_add(&ir->node, &svm->ir_list); 762 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 763 out: 764 return ret; 765 } 766 767 /** 768 * Note: 769 * The HW cannot support posting multicast/broadcast 770 * interrupts to a vCPU. So, we still use legacy interrupt 771 * remapping for these kind of interrupts. 772 * 773 * For lowest-priority interrupts, we only support 774 * those with single CPU as the destination, e.g. user 775 * configures the interrupts via /proc/irq or uses 776 * irqbalance to make the interrupts single-CPU. 777 */ 778 static int 779 get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, 780 struct vcpu_data *vcpu_info, struct vcpu_svm **svm) 781 { 782 struct kvm_lapic_irq irq; 783 struct kvm_vcpu *vcpu = NULL; 784 785 kvm_set_msi_irq(kvm, e, &irq); 786 787 if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) || 788 !kvm_irq_is_postable(&irq)) { 789 pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n", 790 __func__, irq.vector); 791 return -1; 792 } 793 794 pr_debug("SVM: %s: use GA mode for irq %u\n", __func__, 795 irq.vector); 796 *svm = to_svm(vcpu); 797 vcpu_info->pi_desc_addr = __sme_set(page_to_phys((*svm)->avic_backing_page)); 798 vcpu_info->vector = irq.vector; 799 800 return 0; 801 } 802 803 /* 804 * svm_update_pi_irte - set IRTE for Posted-Interrupts 805 * 806 * @kvm: kvm 807 * @host_irq: host irq of the interrupt 808 * @guest_irq: gsi of the interrupt 809 * @set: set or unset PI 810 * returns 0 on success, < 0 on failure 811 */ 812 int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq, 813 uint32_t guest_irq, bool set) 814 { 815 struct kvm_kernel_irq_routing_entry *e; 816 struct kvm_irq_routing_table *irq_rt; 817 int idx, ret = -EINVAL; 818 819 if (!kvm_arch_has_assigned_device(kvm) || 820 !irq_remapping_cap(IRQ_POSTING_CAP)) 821 return 0; 822 823 pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n", 824 __func__, host_irq, guest_irq, set); 825 826 idx = srcu_read_lock(&kvm->irq_srcu); 827 irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu); 828 WARN_ON(guest_irq >= irq_rt->nr_rt_entries); 829 830 hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) { 831 struct vcpu_data vcpu_info; 832 struct vcpu_svm *svm = NULL; 833 834 if (e->type != KVM_IRQ_ROUTING_MSI) 835 continue; 836 837 /** 838 * Here, we setup with legacy mode in the following cases: 839 * 1. When cannot target interrupt to a specific vcpu. 840 * 2. Unsetting posted interrupt. 841 * 3. APIC virtualization is disabled for the vcpu. 842 * 4. IRQ has incompatible delivery mode (SMI, INIT, etc) 843 */ 844 if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set && 845 kvm_vcpu_apicv_active(&svm->vcpu)) { 846 struct amd_iommu_pi_data pi; 847 848 /* Try to enable guest_mode in IRTE */ 849 pi.base = __sme_set(page_to_phys(svm->avic_backing_page) & 850 AVIC_HPA_MASK); 851 pi.ga_tag = AVIC_GATAG(to_kvm_svm(kvm)->avic_vm_id, 852 svm->vcpu.vcpu_id); 853 pi.is_guest_mode = true; 854 pi.vcpu_data = &vcpu_info; 855 ret = irq_set_vcpu_affinity(host_irq, &pi); 856 857 /** 858 * Here, we successfully setting up vcpu affinity in 859 * IOMMU guest mode. Now, we need to store the posted 860 * interrupt information in a per-vcpu ir_list so that 861 * we can reference to them directly when we update vcpu 862 * scheduling information in IOMMU irte. 863 */ 864 if (!ret && pi.is_guest_mode) 865 svm_ir_list_add(svm, &pi); 866 } else { 867 /* Use legacy mode in IRTE */ 868 struct amd_iommu_pi_data pi; 869 870 /** 871 * Here, pi is used to: 872 * - Tell IOMMU to use legacy mode for this interrupt. 873 * - Retrieve ga_tag of prior interrupt remapping data. 874 */ 875 pi.prev_ga_tag = 0; 876 pi.is_guest_mode = false; 877 ret = irq_set_vcpu_affinity(host_irq, &pi); 878 879 /** 880 * Check if the posted interrupt was previously 881 * setup with the guest_mode by checking if the ga_tag 882 * was cached. If so, we need to clean up the per-vcpu 883 * ir_list. 884 */ 885 if (!ret && pi.prev_ga_tag) { 886 int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag); 887 struct kvm_vcpu *vcpu; 888 889 vcpu = kvm_get_vcpu_by_id(kvm, id); 890 if (vcpu) 891 svm_ir_list_del(to_svm(vcpu), &pi); 892 } 893 } 894 895 if (!ret && svm) { 896 trace_kvm_pi_irte_update(host_irq, svm->vcpu.vcpu_id, 897 e->gsi, vcpu_info.vector, 898 vcpu_info.pi_desc_addr, set); 899 } 900 901 if (ret < 0) { 902 pr_err("%s: failed to update PI IRTE\n", __func__); 903 goto out; 904 } 905 } 906 907 ret = 0; 908 out: 909 srcu_read_unlock(&kvm->irq_srcu, idx); 910 return ret; 911 } 912 913 bool svm_check_apicv_inhibit_reasons(ulong bit) 914 { 915 ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) | 916 BIT(APICV_INHIBIT_REASON_HYPERV) | 917 BIT(APICV_INHIBIT_REASON_NESTED) | 918 BIT(APICV_INHIBIT_REASON_IRQWIN) | 919 BIT(APICV_INHIBIT_REASON_PIT_REINJ) | 920 BIT(APICV_INHIBIT_REASON_X2APIC); 921 922 return supported & BIT(bit); 923 } 924 925 void svm_pre_update_apicv_exec_ctrl(struct kvm *kvm, bool activate) 926 { 927 avic_update_access_page(kvm, activate); 928 } 929 930 static inline int 931 avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r) 932 { 933 int ret = 0; 934 unsigned long flags; 935 struct amd_svm_iommu_ir *ir; 936 struct vcpu_svm *svm = to_svm(vcpu); 937 938 if (!kvm_arch_has_assigned_device(vcpu->kvm)) 939 return 0; 940 941 /* 942 * Here, we go through the per-vcpu ir_list to update all existing 943 * interrupt remapping table entry targeting this vcpu. 944 */ 945 spin_lock_irqsave(&svm->ir_list_lock, flags); 946 947 if (list_empty(&svm->ir_list)) 948 goto out; 949 950 list_for_each_entry(ir, &svm->ir_list, node) { 951 ret = amd_iommu_update_ga(cpu, r, ir->data); 952 if (ret) 953 break; 954 } 955 out: 956 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 957 return ret; 958 } 959 960 void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 961 { 962 u64 entry; 963 /* ID = 0xff (broadcast), ID > 0xff (reserved) */ 964 int h_physical_id = kvm_cpu_get_apicid(cpu); 965 struct vcpu_svm *svm = to_svm(vcpu); 966 967 if (!kvm_vcpu_apicv_active(vcpu)) 968 return; 969 970 /* 971 * Since the host physical APIC id is 8 bits, 972 * we can support host APIC ID upto 255. 973 */ 974 if (WARN_ON(h_physical_id > AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK)) 975 return; 976 977 entry = READ_ONCE(*(svm->avic_physical_id_cache)); 978 WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK); 979 980 entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK; 981 entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK); 982 983 entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; 984 if (svm->avic_is_running) 985 entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; 986 987 WRITE_ONCE(*(svm->avic_physical_id_cache), entry); 988 avic_update_iommu_vcpu_affinity(vcpu, h_physical_id, 989 svm->avic_is_running); 990 } 991 992 void avic_vcpu_put(struct kvm_vcpu *vcpu) 993 { 994 u64 entry; 995 struct vcpu_svm *svm = to_svm(vcpu); 996 997 if (!kvm_vcpu_apicv_active(vcpu)) 998 return; 999 1000 entry = READ_ONCE(*(svm->avic_physical_id_cache)); 1001 if (entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK) 1002 avic_update_iommu_vcpu_affinity(vcpu, -1, 0); 1003 1004 entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; 1005 WRITE_ONCE(*(svm->avic_physical_id_cache), entry); 1006 } 1007 1008 /** 1009 * This function is called during VCPU halt/unhalt. 1010 */ 1011 static void avic_set_running(struct kvm_vcpu *vcpu, bool is_run) 1012 { 1013 struct vcpu_svm *svm = to_svm(vcpu); 1014 1015 svm->avic_is_running = is_run; 1016 if (is_run) 1017 avic_vcpu_load(vcpu, vcpu->cpu); 1018 else 1019 avic_vcpu_put(vcpu); 1020 } 1021 1022 void svm_vcpu_blocking(struct kvm_vcpu *vcpu) 1023 { 1024 avic_set_running(vcpu, false); 1025 } 1026 1027 void svm_vcpu_unblocking(struct kvm_vcpu *vcpu) 1028 { 1029 if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu)) 1030 kvm_vcpu_update_apicv(vcpu); 1031 avic_set_running(vcpu, true); 1032 } 1033