xref: /openbmc/linux/arch/x86/kvm/svm/avic.c (revision 9a22717b)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Kernel-based Virtual Machine driver for Linux
4  *
5  * AMD SVM support
6  *
7  * Copyright (C) 2006 Qumranet, Inc.
8  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
9  *
10  * Authors:
11  *   Yaniv Kamay  <yaniv@qumranet.com>
12  *   Avi Kivity   <avi@qumranet.com>
13  */
14 
15 #define pr_fmt(fmt) "SVM: " fmt
16 
17 #include <linux/kvm_types.h>
18 #include <linux/hashtable.h>
19 #include <linux/amd-iommu.h>
20 #include <linux/kvm_host.h>
21 
22 #include <asm/irq_remapping.h>
23 
24 #include "trace.h"
25 #include "lapic.h"
26 #include "x86.h"
27 #include "irq.h"
28 #include "svm.h"
29 
30 /* AVIC GATAG is encoded using VM and VCPU IDs */
31 #define AVIC_VCPU_ID_BITS		8
32 #define AVIC_VCPU_ID_MASK		((1 << AVIC_VCPU_ID_BITS) - 1)
33 
34 #define AVIC_VM_ID_BITS			24
35 #define AVIC_VM_ID_NR			(1 << AVIC_VM_ID_BITS)
36 #define AVIC_VM_ID_MASK			((1 << AVIC_VM_ID_BITS) - 1)
37 
38 #define AVIC_GATAG(x, y)		(((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \
39 						(y & AVIC_VCPU_ID_MASK))
40 #define AVIC_GATAG_TO_VMID(x)		((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK)
41 #define AVIC_GATAG_TO_VCPUID(x)		(x & AVIC_VCPU_ID_MASK)
42 
43 /* Note:
44  * This hash table is used to map VM_ID to a struct kvm_svm,
45  * when handling AMD IOMMU GALOG notification to schedule in
46  * a particular vCPU.
47  */
48 #define SVM_VM_DATA_HASH_BITS	8
49 static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS);
50 static u32 next_vm_id = 0;
51 static bool next_vm_id_wrapped = 0;
52 static DEFINE_SPINLOCK(svm_vm_data_hash_lock);
53 
54 /*
55  * This is a wrapper of struct amd_iommu_ir_data.
56  */
57 struct amd_svm_iommu_ir {
58 	struct list_head node;	/* Used by SVM for per-vcpu ir_list */
59 	void *data;		/* Storing pointer to struct amd_ir_data */
60 };
61 
62 
63 /* Note:
64  * This function is called from IOMMU driver to notify
65  * SVM to schedule in a particular vCPU of a particular VM.
66  */
67 int avic_ga_log_notifier(u32 ga_tag)
68 {
69 	unsigned long flags;
70 	struct kvm_svm *kvm_svm;
71 	struct kvm_vcpu *vcpu = NULL;
72 	u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag);
73 	u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag);
74 
75 	pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id);
76 	trace_kvm_avic_ga_log(vm_id, vcpu_id);
77 
78 	spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
79 	hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) {
80 		if (kvm_svm->avic_vm_id != vm_id)
81 			continue;
82 		vcpu = kvm_get_vcpu_by_id(&kvm_svm->kvm, vcpu_id);
83 		break;
84 	}
85 	spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
86 
87 	/* Note:
88 	 * At this point, the IOMMU should have already set the pending
89 	 * bit in the vAPIC backing page. So, we just need to schedule
90 	 * in the vcpu.
91 	 */
92 	if (vcpu)
93 		kvm_vcpu_wake_up(vcpu);
94 
95 	return 0;
96 }
97 
98 void avic_vm_destroy(struct kvm *kvm)
99 {
100 	unsigned long flags;
101 	struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
102 
103 	if (!enable_apicv)
104 		return;
105 
106 	if (kvm_svm->avic_logical_id_table_page)
107 		__free_page(kvm_svm->avic_logical_id_table_page);
108 	if (kvm_svm->avic_physical_id_table_page)
109 		__free_page(kvm_svm->avic_physical_id_table_page);
110 
111 	spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
112 	hash_del(&kvm_svm->hnode);
113 	spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
114 }
115 
116 int avic_vm_init(struct kvm *kvm)
117 {
118 	unsigned long flags;
119 	int err = -ENOMEM;
120 	struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
121 	struct kvm_svm *k2;
122 	struct page *p_page;
123 	struct page *l_page;
124 	u32 vm_id;
125 
126 	if (!enable_apicv)
127 		return 0;
128 
129 	/* Allocating physical APIC ID table (4KB) */
130 	p_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
131 	if (!p_page)
132 		goto free_avic;
133 
134 	kvm_svm->avic_physical_id_table_page = p_page;
135 
136 	/* Allocating logical APIC ID table (4KB) */
137 	l_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
138 	if (!l_page)
139 		goto free_avic;
140 
141 	kvm_svm->avic_logical_id_table_page = l_page;
142 
143 	spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
144  again:
145 	vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK;
146 	if (vm_id == 0) { /* id is 1-based, zero is not okay */
147 		next_vm_id_wrapped = 1;
148 		goto again;
149 	}
150 	/* Is it still in use? Only possible if wrapped at least once */
151 	if (next_vm_id_wrapped) {
152 		hash_for_each_possible(svm_vm_data_hash, k2, hnode, vm_id) {
153 			if (k2->avic_vm_id == vm_id)
154 				goto again;
155 		}
156 	}
157 	kvm_svm->avic_vm_id = vm_id;
158 	hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id);
159 	spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
160 
161 	return 0;
162 
163 free_avic:
164 	avic_vm_destroy(kvm);
165 	return err;
166 }
167 
168 void avic_init_vmcb(struct vcpu_svm *svm)
169 {
170 	struct vmcb *vmcb = svm->vmcb;
171 	struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm);
172 	phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page));
173 	phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page));
174 	phys_addr_t ppa = __sme_set(page_to_phys(kvm_svm->avic_physical_id_table_page));
175 
176 	vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK;
177 	vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK;
178 	vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK;
179 	vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT;
180 	vmcb->control.avic_vapic_bar = APIC_DEFAULT_PHYS_BASE & VMCB_AVIC_APIC_BAR_MASK;
181 
182 	if (kvm_apicv_activated(svm->vcpu.kvm))
183 		vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
184 	else
185 		vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK;
186 }
187 
188 static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu,
189 				       unsigned int index)
190 {
191 	u64 *avic_physical_id_table;
192 	struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
193 
194 	if (index >= AVIC_MAX_PHYSICAL_ID_COUNT)
195 		return NULL;
196 
197 	avic_physical_id_table = page_address(kvm_svm->avic_physical_id_table_page);
198 
199 	return &avic_physical_id_table[index];
200 }
201 
202 /*
203  * Note:
204  * AVIC hardware walks the nested page table to check permissions,
205  * but does not use the SPA address specified in the leaf page
206  * table entry since it uses  address in the AVIC_BACKING_PAGE pointer
207  * field of the VMCB. Therefore, we set up the
208  * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here.
209  */
210 static int avic_alloc_access_page(struct kvm *kvm)
211 {
212 	void __user *ret;
213 	int r = 0;
214 
215 	mutex_lock(&kvm->slots_lock);
216 
217 	if (kvm->arch.apic_access_memslot_enabled)
218 		goto out;
219 
220 	ret = __x86_set_memory_region(kvm,
221 				      APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
222 				      APIC_DEFAULT_PHYS_BASE,
223 				      PAGE_SIZE);
224 	if (IS_ERR(ret)) {
225 		r = PTR_ERR(ret);
226 		goto out;
227 	}
228 
229 	kvm->arch.apic_access_memslot_enabled = true;
230 out:
231 	mutex_unlock(&kvm->slots_lock);
232 	return r;
233 }
234 
235 static int avic_init_backing_page(struct kvm_vcpu *vcpu)
236 {
237 	u64 *entry, new_entry;
238 	int id = vcpu->vcpu_id;
239 	struct vcpu_svm *svm = to_svm(vcpu);
240 
241 	if (id >= AVIC_MAX_PHYSICAL_ID_COUNT)
242 		return -EINVAL;
243 
244 	if (!vcpu->arch.apic->regs)
245 		return -EINVAL;
246 
247 	if (kvm_apicv_activated(vcpu->kvm)) {
248 		int ret;
249 
250 		ret = avic_alloc_access_page(vcpu->kvm);
251 		if (ret)
252 			return ret;
253 	}
254 
255 	svm->avic_backing_page = virt_to_page(vcpu->arch.apic->regs);
256 
257 	/* Setting AVIC backing page address in the phy APIC ID table */
258 	entry = avic_get_physical_id_entry(vcpu, id);
259 	if (!entry)
260 		return -EINVAL;
261 
262 	new_entry = __sme_set((page_to_phys(svm->avic_backing_page) &
263 			      AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) |
264 			      AVIC_PHYSICAL_ID_ENTRY_VALID_MASK);
265 	WRITE_ONCE(*entry, new_entry);
266 
267 	svm->avic_physical_id_cache = entry;
268 
269 	return 0;
270 }
271 
272 void avic_ring_doorbell(struct kvm_vcpu *vcpu)
273 {
274 	/*
275 	 * Note, the vCPU could get migrated to a different pCPU at any point,
276 	 * which could result in signalling the wrong/previous pCPU.  But if
277 	 * that happens the vCPU is guaranteed to do a VMRUN (after being
278 	 * migrated) and thus will process pending interrupts, i.e. a doorbell
279 	 * is not needed (and the spurious one is harmless).
280 	 */
281 	int cpu = READ_ONCE(vcpu->cpu);
282 
283 	if (cpu != get_cpu())
284 		wrmsrl(MSR_AMD64_SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpu));
285 	put_cpu();
286 }
287 
288 static void avic_kick_target_vcpus(struct kvm *kvm, struct kvm_lapic *source,
289 				   u32 icrl, u32 icrh)
290 {
291 	struct kvm_vcpu *vcpu;
292 	unsigned long i;
293 
294 	/*
295 	 * Wake any target vCPUs that are blocking, i.e. waiting for a wake
296 	 * event.  There's no need to signal doorbells, as hardware has handled
297 	 * vCPUs that were in guest at the time of the IPI, and vCPUs that have
298 	 * since entered the guest will have processed pending IRQs at VMRUN.
299 	 */
300 	kvm_for_each_vcpu(i, vcpu, kvm) {
301 		if (kvm_apic_match_dest(vcpu, source, icrl & APIC_SHORT_MASK,
302 					GET_APIC_DEST_FIELD(icrh),
303 					icrl & APIC_DEST_MASK)) {
304 			vcpu->arch.apic->irr_pending = true;
305 			svm_complete_interrupt_delivery(vcpu,
306 							icrl & APIC_MODE_MASK,
307 							icrl & APIC_INT_LEVELTRIG,
308 							icrl & APIC_VECTOR_MASK);
309 		}
310 	}
311 }
312 
313 int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu)
314 {
315 	struct vcpu_svm *svm = to_svm(vcpu);
316 	u32 icrh = svm->vmcb->control.exit_info_1 >> 32;
317 	u32 icrl = svm->vmcb->control.exit_info_1;
318 	u32 id = svm->vmcb->control.exit_info_2 >> 32;
319 	u32 index = svm->vmcb->control.exit_info_2 & 0xFF;
320 	struct kvm_lapic *apic = vcpu->arch.apic;
321 
322 	trace_kvm_avic_incomplete_ipi(vcpu->vcpu_id, icrh, icrl, id, index);
323 
324 	switch (id) {
325 	case AVIC_IPI_FAILURE_INVALID_INT_TYPE:
326 		/*
327 		 * Emulate IPIs that are not handled by AVIC hardware, which
328 		 * only virtualizes Fixed, Edge-Triggered INTRs.  The exit is
329 		 * a trap, e.g. ICR holds the correct value and RIP has been
330 		 * advanced, KVM is responsible only for emulating the IPI.
331 		 * Sadly, hardware may sometimes leave the BUSY flag set, in
332 		 * which case KVM needs to emulate the ICR write as well in
333 		 * order to clear the BUSY flag.
334 		 */
335 		if (icrl & APIC_ICR_BUSY)
336 			kvm_apic_write_nodecode(vcpu, APIC_ICR);
337 		else
338 			kvm_apic_send_ipi(apic, icrl, icrh);
339 		break;
340 	case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING:
341 		/*
342 		 * At this point, we expect that the AVIC HW has already
343 		 * set the appropriate IRR bits on the valid target
344 		 * vcpus. So, we just need to kick the appropriate vcpu.
345 		 */
346 		avic_kick_target_vcpus(vcpu->kvm, apic, icrl, icrh);
347 		break;
348 	case AVIC_IPI_FAILURE_INVALID_TARGET:
349 		break;
350 	case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE:
351 		WARN_ONCE(1, "Invalid backing page\n");
352 		break;
353 	default:
354 		pr_err("Unknown IPI interception\n");
355 	}
356 
357 	return 1;
358 }
359 
360 static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat)
361 {
362 	struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
363 	int index;
364 	u32 *logical_apic_id_table;
365 	int dlid = GET_APIC_LOGICAL_ID(ldr);
366 
367 	if (!dlid)
368 		return NULL;
369 
370 	if (flat) { /* flat */
371 		index = ffs(dlid) - 1;
372 		if (index > 7)
373 			return NULL;
374 	} else { /* cluster */
375 		int cluster = (dlid & 0xf0) >> 4;
376 		int apic = ffs(dlid & 0x0f) - 1;
377 
378 		if ((apic < 0) || (apic > 7) ||
379 		    (cluster >= 0xf))
380 			return NULL;
381 		index = (cluster << 2) + apic;
382 	}
383 
384 	logical_apic_id_table = (u32 *) page_address(kvm_svm->avic_logical_id_table_page);
385 
386 	return &logical_apic_id_table[index];
387 }
388 
389 static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr)
390 {
391 	bool flat;
392 	u32 *entry, new_entry;
393 
394 	flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT;
395 	entry = avic_get_logical_id_entry(vcpu, ldr, flat);
396 	if (!entry)
397 		return -EINVAL;
398 
399 	new_entry = READ_ONCE(*entry);
400 	new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK;
401 	new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK);
402 	new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK;
403 	WRITE_ONCE(*entry, new_entry);
404 
405 	return 0;
406 }
407 
408 static void avic_invalidate_logical_id_entry(struct kvm_vcpu *vcpu)
409 {
410 	struct vcpu_svm *svm = to_svm(vcpu);
411 	bool flat = svm->dfr_reg == APIC_DFR_FLAT;
412 	u32 *entry = avic_get_logical_id_entry(vcpu, svm->ldr_reg, flat);
413 
414 	if (entry)
415 		clear_bit(AVIC_LOGICAL_ID_ENTRY_VALID_BIT, (unsigned long *)entry);
416 }
417 
418 static int avic_handle_ldr_update(struct kvm_vcpu *vcpu)
419 {
420 	int ret = 0;
421 	struct vcpu_svm *svm = to_svm(vcpu);
422 	u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR);
423 	u32 id = kvm_xapic_id(vcpu->arch.apic);
424 
425 	if (ldr == svm->ldr_reg)
426 		return 0;
427 
428 	avic_invalidate_logical_id_entry(vcpu);
429 
430 	if (ldr)
431 		ret = avic_ldr_write(vcpu, id, ldr);
432 
433 	if (!ret)
434 		svm->ldr_reg = ldr;
435 
436 	return ret;
437 }
438 
439 static int avic_handle_apic_id_update(struct kvm_vcpu *vcpu)
440 {
441 	u64 *old, *new;
442 	struct vcpu_svm *svm = to_svm(vcpu);
443 	u32 id = kvm_xapic_id(vcpu->arch.apic);
444 
445 	if (vcpu->vcpu_id == id)
446 		return 0;
447 
448 	old = avic_get_physical_id_entry(vcpu, vcpu->vcpu_id);
449 	new = avic_get_physical_id_entry(vcpu, id);
450 	if (!new || !old)
451 		return 1;
452 
453 	/* We need to move physical_id_entry to new offset */
454 	*new = *old;
455 	*old = 0ULL;
456 	to_svm(vcpu)->avic_physical_id_cache = new;
457 
458 	/*
459 	 * Also update the guest physical APIC ID in the logical
460 	 * APIC ID table entry if already setup the LDR.
461 	 */
462 	if (svm->ldr_reg)
463 		avic_handle_ldr_update(vcpu);
464 
465 	return 0;
466 }
467 
468 static void avic_handle_dfr_update(struct kvm_vcpu *vcpu)
469 {
470 	struct vcpu_svm *svm = to_svm(vcpu);
471 	u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR);
472 
473 	if (svm->dfr_reg == dfr)
474 		return;
475 
476 	avic_invalidate_logical_id_entry(vcpu);
477 	svm->dfr_reg = dfr;
478 }
479 
480 static int avic_unaccel_trap_write(struct kvm_vcpu *vcpu)
481 {
482 	u32 offset = to_svm(vcpu)->vmcb->control.exit_info_1 &
483 				AVIC_UNACCEL_ACCESS_OFFSET_MASK;
484 
485 	switch (offset) {
486 	case APIC_ID:
487 		if (avic_handle_apic_id_update(vcpu))
488 			return 0;
489 		break;
490 	case APIC_LDR:
491 		if (avic_handle_ldr_update(vcpu))
492 			return 0;
493 		break;
494 	case APIC_DFR:
495 		avic_handle_dfr_update(vcpu);
496 		break;
497 	default:
498 		break;
499 	}
500 
501 	kvm_apic_write_nodecode(vcpu, offset);
502 	return 1;
503 }
504 
505 static bool is_avic_unaccelerated_access_trap(u32 offset)
506 {
507 	bool ret = false;
508 
509 	switch (offset) {
510 	case APIC_ID:
511 	case APIC_EOI:
512 	case APIC_RRR:
513 	case APIC_LDR:
514 	case APIC_DFR:
515 	case APIC_SPIV:
516 	case APIC_ESR:
517 	case APIC_ICR:
518 	case APIC_LVTT:
519 	case APIC_LVTTHMR:
520 	case APIC_LVTPC:
521 	case APIC_LVT0:
522 	case APIC_LVT1:
523 	case APIC_LVTERR:
524 	case APIC_TMICT:
525 	case APIC_TDCR:
526 		ret = true;
527 		break;
528 	default:
529 		break;
530 	}
531 	return ret;
532 }
533 
534 int avic_unaccelerated_access_interception(struct kvm_vcpu *vcpu)
535 {
536 	struct vcpu_svm *svm = to_svm(vcpu);
537 	int ret = 0;
538 	u32 offset = svm->vmcb->control.exit_info_1 &
539 		     AVIC_UNACCEL_ACCESS_OFFSET_MASK;
540 	u32 vector = svm->vmcb->control.exit_info_2 &
541 		     AVIC_UNACCEL_ACCESS_VECTOR_MASK;
542 	bool write = (svm->vmcb->control.exit_info_1 >> 32) &
543 		     AVIC_UNACCEL_ACCESS_WRITE_MASK;
544 	bool trap = is_avic_unaccelerated_access_trap(offset);
545 
546 	trace_kvm_avic_unaccelerated_access(vcpu->vcpu_id, offset,
547 					    trap, write, vector);
548 	if (trap) {
549 		/* Handling Trap */
550 		WARN_ONCE(!write, "svm: Handling trap read.\n");
551 		ret = avic_unaccel_trap_write(vcpu);
552 	} else {
553 		/* Handling Fault */
554 		ret = kvm_emulate_instruction(vcpu, 0);
555 	}
556 
557 	return ret;
558 }
559 
560 int avic_init_vcpu(struct vcpu_svm *svm)
561 {
562 	int ret;
563 	struct kvm_vcpu *vcpu = &svm->vcpu;
564 
565 	if (!enable_apicv || !irqchip_in_kernel(vcpu->kvm))
566 		return 0;
567 
568 	ret = avic_init_backing_page(vcpu);
569 	if (ret)
570 		return ret;
571 
572 	INIT_LIST_HEAD(&svm->ir_list);
573 	spin_lock_init(&svm->ir_list_lock);
574 	svm->dfr_reg = APIC_DFR_FLAT;
575 
576 	return ret;
577 }
578 
579 void avic_apicv_post_state_restore(struct kvm_vcpu *vcpu)
580 {
581 	if (avic_handle_apic_id_update(vcpu) != 0)
582 		return;
583 	avic_handle_dfr_update(vcpu);
584 	avic_handle_ldr_update(vcpu);
585 }
586 
587 static int avic_set_pi_irte_mode(struct kvm_vcpu *vcpu, bool activate)
588 {
589 	int ret = 0;
590 	unsigned long flags;
591 	struct amd_svm_iommu_ir *ir;
592 	struct vcpu_svm *svm = to_svm(vcpu);
593 
594 	if (!kvm_arch_has_assigned_device(vcpu->kvm))
595 		return 0;
596 
597 	/*
598 	 * Here, we go through the per-vcpu ir_list to update all existing
599 	 * interrupt remapping table entry targeting this vcpu.
600 	 */
601 	spin_lock_irqsave(&svm->ir_list_lock, flags);
602 
603 	if (list_empty(&svm->ir_list))
604 		goto out;
605 
606 	list_for_each_entry(ir, &svm->ir_list, node) {
607 		if (activate)
608 			ret = amd_iommu_activate_guest_mode(ir->data);
609 		else
610 			ret = amd_iommu_deactivate_guest_mode(ir->data);
611 		if (ret)
612 			break;
613 	}
614 out:
615 	spin_unlock_irqrestore(&svm->ir_list_lock, flags);
616 	return ret;
617 }
618 
619 static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
620 {
621 	unsigned long flags;
622 	struct amd_svm_iommu_ir *cur;
623 
624 	spin_lock_irqsave(&svm->ir_list_lock, flags);
625 	list_for_each_entry(cur, &svm->ir_list, node) {
626 		if (cur->data != pi->ir_data)
627 			continue;
628 		list_del(&cur->node);
629 		kfree(cur);
630 		break;
631 	}
632 	spin_unlock_irqrestore(&svm->ir_list_lock, flags);
633 }
634 
635 static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
636 {
637 	int ret = 0;
638 	unsigned long flags;
639 	struct amd_svm_iommu_ir *ir;
640 
641 	/**
642 	 * In some cases, the existing irte is updated and re-set,
643 	 * so we need to check here if it's already been * added
644 	 * to the ir_list.
645 	 */
646 	if (pi->ir_data && (pi->prev_ga_tag != 0)) {
647 		struct kvm *kvm = svm->vcpu.kvm;
648 		u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag);
649 		struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
650 		struct vcpu_svm *prev_svm;
651 
652 		if (!prev_vcpu) {
653 			ret = -EINVAL;
654 			goto out;
655 		}
656 
657 		prev_svm = to_svm(prev_vcpu);
658 		svm_ir_list_del(prev_svm, pi);
659 	}
660 
661 	/**
662 	 * Allocating new amd_iommu_pi_data, which will get
663 	 * add to the per-vcpu ir_list.
664 	 */
665 	ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL_ACCOUNT);
666 	if (!ir) {
667 		ret = -ENOMEM;
668 		goto out;
669 	}
670 	ir->data = pi->ir_data;
671 
672 	spin_lock_irqsave(&svm->ir_list_lock, flags);
673 	list_add(&ir->node, &svm->ir_list);
674 	spin_unlock_irqrestore(&svm->ir_list_lock, flags);
675 out:
676 	return ret;
677 }
678 
679 /*
680  * Note:
681  * The HW cannot support posting multicast/broadcast
682  * interrupts to a vCPU. So, we still use legacy interrupt
683  * remapping for these kind of interrupts.
684  *
685  * For lowest-priority interrupts, we only support
686  * those with single CPU as the destination, e.g. user
687  * configures the interrupts via /proc/irq or uses
688  * irqbalance to make the interrupts single-CPU.
689  */
690 static int
691 get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e,
692 		 struct vcpu_data *vcpu_info, struct vcpu_svm **svm)
693 {
694 	struct kvm_lapic_irq irq;
695 	struct kvm_vcpu *vcpu = NULL;
696 
697 	kvm_set_msi_irq(kvm, e, &irq);
698 
699 	if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) ||
700 	    !kvm_irq_is_postable(&irq)) {
701 		pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n",
702 			 __func__, irq.vector);
703 		return -1;
704 	}
705 
706 	pr_debug("SVM: %s: use GA mode for irq %u\n", __func__,
707 		 irq.vector);
708 	*svm = to_svm(vcpu);
709 	vcpu_info->pi_desc_addr = __sme_set(page_to_phys((*svm)->avic_backing_page));
710 	vcpu_info->vector = irq.vector;
711 
712 	return 0;
713 }
714 
715 /*
716  * avic_pi_update_irte - set IRTE for Posted-Interrupts
717  *
718  * @kvm: kvm
719  * @host_irq: host irq of the interrupt
720  * @guest_irq: gsi of the interrupt
721  * @set: set or unset PI
722  * returns 0 on success, < 0 on failure
723  */
724 int avic_pi_update_irte(struct kvm *kvm, unsigned int host_irq,
725 			uint32_t guest_irq, bool set)
726 {
727 	struct kvm_kernel_irq_routing_entry *e;
728 	struct kvm_irq_routing_table *irq_rt;
729 	int idx, ret = 0;
730 
731 	if (!kvm_arch_has_assigned_device(kvm) ||
732 	    !irq_remapping_cap(IRQ_POSTING_CAP))
733 		return 0;
734 
735 	pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n",
736 		 __func__, host_irq, guest_irq, set);
737 
738 	idx = srcu_read_lock(&kvm->irq_srcu);
739 	irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
740 
741 	if (guest_irq >= irq_rt->nr_rt_entries ||
742 		hlist_empty(&irq_rt->map[guest_irq])) {
743 		pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n",
744 			     guest_irq, irq_rt->nr_rt_entries);
745 		goto out;
746 	}
747 
748 	hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
749 		struct vcpu_data vcpu_info;
750 		struct vcpu_svm *svm = NULL;
751 
752 		if (e->type != KVM_IRQ_ROUTING_MSI)
753 			continue;
754 
755 		/**
756 		 * Here, we setup with legacy mode in the following cases:
757 		 * 1. When cannot target interrupt to a specific vcpu.
758 		 * 2. Unsetting posted interrupt.
759 		 * 3. APIC virtualization is disabled for the vcpu.
760 		 * 4. IRQ has incompatible delivery mode (SMI, INIT, etc)
761 		 */
762 		if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set &&
763 		    kvm_vcpu_apicv_active(&svm->vcpu)) {
764 			struct amd_iommu_pi_data pi;
765 
766 			/* Try to enable guest_mode in IRTE */
767 			pi.base = __sme_set(page_to_phys(svm->avic_backing_page) &
768 					    AVIC_HPA_MASK);
769 			pi.ga_tag = AVIC_GATAG(to_kvm_svm(kvm)->avic_vm_id,
770 						     svm->vcpu.vcpu_id);
771 			pi.is_guest_mode = true;
772 			pi.vcpu_data = &vcpu_info;
773 			ret = irq_set_vcpu_affinity(host_irq, &pi);
774 
775 			/**
776 			 * Here, we successfully setting up vcpu affinity in
777 			 * IOMMU guest mode. Now, we need to store the posted
778 			 * interrupt information in a per-vcpu ir_list so that
779 			 * we can reference to them directly when we update vcpu
780 			 * scheduling information in IOMMU irte.
781 			 */
782 			if (!ret && pi.is_guest_mode)
783 				svm_ir_list_add(svm, &pi);
784 		} else {
785 			/* Use legacy mode in IRTE */
786 			struct amd_iommu_pi_data pi;
787 
788 			/**
789 			 * Here, pi is used to:
790 			 * - Tell IOMMU to use legacy mode for this interrupt.
791 			 * - Retrieve ga_tag of prior interrupt remapping data.
792 			 */
793 			pi.prev_ga_tag = 0;
794 			pi.is_guest_mode = false;
795 			ret = irq_set_vcpu_affinity(host_irq, &pi);
796 
797 			/**
798 			 * Check if the posted interrupt was previously
799 			 * setup with the guest_mode by checking if the ga_tag
800 			 * was cached. If so, we need to clean up the per-vcpu
801 			 * ir_list.
802 			 */
803 			if (!ret && pi.prev_ga_tag) {
804 				int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag);
805 				struct kvm_vcpu *vcpu;
806 
807 				vcpu = kvm_get_vcpu_by_id(kvm, id);
808 				if (vcpu)
809 					svm_ir_list_del(to_svm(vcpu), &pi);
810 			}
811 		}
812 
813 		if (!ret && svm) {
814 			trace_kvm_pi_irte_update(host_irq, svm->vcpu.vcpu_id,
815 						 e->gsi, vcpu_info.vector,
816 						 vcpu_info.pi_desc_addr, set);
817 		}
818 
819 		if (ret < 0) {
820 			pr_err("%s: failed to update PI IRTE\n", __func__);
821 			goto out;
822 		}
823 	}
824 
825 	ret = 0;
826 out:
827 	srcu_read_unlock(&kvm->irq_srcu, idx);
828 	return ret;
829 }
830 
831 bool avic_check_apicv_inhibit_reasons(enum kvm_apicv_inhibit reason)
832 {
833 	ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) |
834 			  BIT(APICV_INHIBIT_REASON_ABSENT) |
835 			  BIT(APICV_INHIBIT_REASON_HYPERV) |
836 			  BIT(APICV_INHIBIT_REASON_NESTED) |
837 			  BIT(APICV_INHIBIT_REASON_IRQWIN) |
838 			  BIT(APICV_INHIBIT_REASON_PIT_REINJ) |
839 			  BIT(APICV_INHIBIT_REASON_X2APIC) |
840 			  BIT(APICV_INHIBIT_REASON_BLOCKIRQ);
841 
842 	return supported & BIT(reason);
843 }
844 
845 
846 static inline int
847 avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r)
848 {
849 	int ret = 0;
850 	unsigned long flags;
851 	struct amd_svm_iommu_ir *ir;
852 	struct vcpu_svm *svm = to_svm(vcpu);
853 
854 	if (!kvm_arch_has_assigned_device(vcpu->kvm))
855 		return 0;
856 
857 	/*
858 	 * Here, we go through the per-vcpu ir_list to update all existing
859 	 * interrupt remapping table entry targeting this vcpu.
860 	 */
861 	spin_lock_irqsave(&svm->ir_list_lock, flags);
862 
863 	if (list_empty(&svm->ir_list))
864 		goto out;
865 
866 	list_for_each_entry(ir, &svm->ir_list, node) {
867 		ret = amd_iommu_update_ga(cpu, r, ir->data);
868 		if (ret)
869 			break;
870 	}
871 out:
872 	spin_unlock_irqrestore(&svm->ir_list_lock, flags);
873 	return ret;
874 }
875 
876 void __avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
877 {
878 	u64 entry;
879 	int h_physical_id = kvm_cpu_get_apicid(cpu);
880 	struct vcpu_svm *svm = to_svm(vcpu);
881 
882 	lockdep_assert_preemption_disabled();
883 
884 	if (WARN_ON(h_physical_id & ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK))
885 		return;
886 
887 	/*
888 	 * No need to update anything if the vCPU is blocking, i.e. if the vCPU
889 	 * is being scheduled in after being preempted.  The CPU entries in the
890 	 * Physical APIC table and IRTE are consumed iff IsRun{ning} is '1'.
891 	 * If the vCPU was migrated, its new CPU value will be stuffed when the
892 	 * vCPU unblocks.
893 	 */
894 	if (kvm_vcpu_is_blocking(vcpu))
895 		return;
896 
897 	entry = READ_ONCE(*(svm->avic_physical_id_cache));
898 	WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
899 
900 	entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK;
901 	entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK);
902 	entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
903 
904 	WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
905 	avic_update_iommu_vcpu_affinity(vcpu, h_physical_id, true);
906 }
907 
908 void __avic_vcpu_put(struct kvm_vcpu *vcpu)
909 {
910 	u64 entry;
911 	struct vcpu_svm *svm = to_svm(vcpu);
912 
913 	lockdep_assert_preemption_disabled();
914 
915 	entry = READ_ONCE(*(svm->avic_physical_id_cache));
916 
917 	/* Nothing to do if IsRunning == '0' due to vCPU blocking. */
918 	if (!(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK))
919 		return;
920 
921 	avic_update_iommu_vcpu_affinity(vcpu, -1, 0);
922 
923 	entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
924 	WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
925 }
926 
927 static void avic_vcpu_load(struct kvm_vcpu *vcpu)
928 {
929 	int cpu = get_cpu();
930 
931 	WARN_ON(cpu != vcpu->cpu);
932 
933 	__avic_vcpu_load(vcpu, cpu);
934 
935 	put_cpu();
936 }
937 
938 static void avic_vcpu_put(struct kvm_vcpu *vcpu)
939 {
940 	preempt_disable();
941 
942 	__avic_vcpu_put(vcpu);
943 
944 	preempt_enable();
945 }
946 
947 void avic_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
948 {
949 	struct vcpu_svm *svm = to_svm(vcpu);
950 	struct vmcb *vmcb = svm->vmcb01.ptr;
951 	bool activated = kvm_vcpu_apicv_active(vcpu);
952 
953 	if (!enable_apicv)
954 		return;
955 
956 	if (activated) {
957 		/**
958 		 * During AVIC temporary deactivation, guest could update
959 		 * APIC ID, DFR and LDR registers, which would not be trapped
960 		 * by avic_unaccelerated_access_interception(). In this case,
961 		 * we need to check and update the AVIC logical APIC ID table
962 		 * accordingly before re-activating.
963 		 */
964 		avic_apicv_post_state_restore(vcpu);
965 		vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
966 	} else {
967 		vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK;
968 	}
969 	vmcb_mark_dirty(vmcb, VMCB_AVIC);
970 
971 	if (activated)
972 		avic_vcpu_load(vcpu);
973 	else
974 		avic_vcpu_put(vcpu);
975 
976 	avic_set_pi_irte_mode(vcpu, activated);
977 }
978 
979 void avic_vcpu_blocking(struct kvm_vcpu *vcpu)
980 {
981 	if (!kvm_vcpu_apicv_active(vcpu))
982 		return;
983 
984        /*
985         * Unload the AVIC when the vCPU is about to block, _before_
986         * the vCPU actually blocks.
987         *
988         * Any IRQs that arrive before IsRunning=0 will not cause an
989         * incomplete IPI vmexit on the source, therefore vIRR will also
990         * be checked by kvm_vcpu_check_block() before blocking.  The
991         * memory barrier implicit in set_current_state orders writing
992         * IsRunning=0 before reading the vIRR.  The processor needs a
993         * matching memory barrier on interrupt delivery between writing
994         * IRR and reading IsRunning; the lack of this barrier might be
995         * the cause of errata #1235).
996         */
997 	avic_vcpu_put(vcpu);
998 }
999 
1000 void avic_vcpu_unblocking(struct kvm_vcpu *vcpu)
1001 {
1002 	if (!kvm_vcpu_apicv_active(vcpu))
1003 		return;
1004 
1005 	avic_vcpu_load(vcpu);
1006 }
1007