xref: /openbmc/linux/arch/arm64/kvm/vgic/vgic.c (revision 81dc9504)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2015, 2016 ARM Ltd.
4  */
5 
6 #include <linux/interrupt.h>
7 #include <linux/irq.h>
8 #include <linux/kvm.h>
9 #include <linux/kvm_host.h>
10 #include <linux/list_sort.h>
11 #include <linux/nospec.h>
12 
13 #include <asm/kvm_hyp.h>
14 
15 #include "vgic.h"
16 
17 #define CREATE_TRACE_POINTS
18 #include "trace.h"
19 
20 struct vgic_global kvm_vgic_global_state __ro_after_init = {
21 	.gicv3_cpuif = STATIC_KEY_FALSE_INIT,
22 };
23 
24 /*
25  * Locking order is always:
26  * kvm->lock (mutex)
27  *   vcpu->mutex (mutex)
28  *     kvm->arch.config_lock (mutex)
29  *       its->cmd_lock (mutex)
30  *         its->its_lock (mutex)
31  *           vgic_cpu->ap_list_lock		must be taken with IRQs disabled
32  *             kvm->lpi_list_lock		must be taken with IRQs disabled
33  *               vgic_irq->irq_lock		must be taken with IRQs disabled
34  *
35  * As the ap_list_lock might be taken from the timer interrupt handler,
36  * we have to disable IRQs before taking this lock and everything lower
37  * than it.
38  *
39  * If you need to take multiple locks, always take the upper lock first,
40  * then the lower ones, e.g. first take the its_lock, then the irq_lock.
41  * If you are already holding a lock and need to take a higher one, you
42  * have to drop the lower ranking lock first and re-acquire it after having
43  * taken the upper one.
44  *
45  * When taking more than one ap_list_lock at the same time, always take the
46  * lowest numbered VCPU's ap_list_lock first, so:
47  *   vcpuX->vcpu_id < vcpuY->vcpu_id:
48  *     raw_spin_lock(vcpuX->arch.vgic_cpu.ap_list_lock);
49  *     raw_spin_lock(vcpuY->arch.vgic_cpu.ap_list_lock);
50  *
51  * Since the VGIC must support injecting virtual interrupts from ISRs, we have
52  * to use the raw_spin_lock_irqsave/raw_spin_unlock_irqrestore versions of outer
53  * spinlocks for any lock that may be taken while injecting an interrupt.
54  */
55 
56 /*
57  * Iterate over the VM's list of mapped LPIs to find the one with a
58  * matching interrupt ID and return a reference to the IRQ structure.
59  */
vgic_get_lpi(struct kvm * kvm,u32 intid)60 static struct vgic_irq *vgic_get_lpi(struct kvm *kvm, u32 intid)
61 {
62 	struct vgic_dist *dist = &kvm->arch.vgic;
63 	struct vgic_irq *irq = NULL;
64 	unsigned long flags;
65 
66 	raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
67 
68 	list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) {
69 		if (irq->intid != intid)
70 			continue;
71 
72 		/*
73 		 * This increases the refcount, the caller is expected to
74 		 * call vgic_put_irq() later once it's finished with the IRQ.
75 		 */
76 		vgic_get_irq_kref(irq);
77 		goto out_unlock;
78 	}
79 	irq = NULL;
80 
81 out_unlock:
82 	raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
83 
84 	return irq;
85 }
86 
87 /*
88  * This looks up the virtual interrupt ID to get the corresponding
89  * struct vgic_irq. It also increases the refcount, so any caller is expected
90  * to call vgic_put_irq() once it's finished with this IRQ.
91  */
vgic_get_irq(struct kvm * kvm,struct kvm_vcpu * vcpu,u32 intid)92 struct vgic_irq *vgic_get_irq(struct kvm *kvm, struct kvm_vcpu *vcpu,
93 			      u32 intid)
94 {
95 	/* SGIs and PPIs */
96 	if (intid <= VGIC_MAX_PRIVATE) {
97 		intid = array_index_nospec(intid, VGIC_MAX_PRIVATE + 1);
98 		return &vcpu->arch.vgic_cpu.private_irqs[intid];
99 	}
100 
101 	/* SPIs */
102 	if (intid < (kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS)) {
103 		intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS);
104 		return &kvm->arch.vgic.spis[intid - VGIC_NR_PRIVATE_IRQS];
105 	}
106 
107 	/* LPIs */
108 	if (intid >= VGIC_MIN_LPI)
109 		return vgic_get_lpi(kvm, intid);
110 
111 	return NULL;
112 }
113 
114 /*
115  * We can't do anything in here, because we lack the kvm pointer to
116  * lock and remove the item from the lpi_list. So we keep this function
117  * empty and use the return value of kref_put() to trigger the freeing.
118  */
vgic_irq_release(struct kref * ref)119 static void vgic_irq_release(struct kref *ref)
120 {
121 }
122 
123 /*
124  * Drop the refcount on the LPI. Must be called with lpi_list_lock held.
125  */
__vgic_put_lpi_locked(struct kvm * kvm,struct vgic_irq * irq)126 void __vgic_put_lpi_locked(struct kvm *kvm, struct vgic_irq *irq)
127 {
128 	struct vgic_dist *dist = &kvm->arch.vgic;
129 
130 	if (!kref_put(&irq->refcount, vgic_irq_release))
131 		return;
132 
133 	list_del(&irq->lpi_list);
134 	dist->lpi_list_count--;
135 
136 	kfree(irq);
137 }
138 
vgic_put_irq(struct kvm * kvm,struct vgic_irq * irq)139 void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq)
140 {
141 	struct vgic_dist *dist = &kvm->arch.vgic;
142 	unsigned long flags;
143 
144 	if (irq->intid < VGIC_MIN_LPI)
145 		return;
146 
147 	raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
148 	__vgic_put_lpi_locked(kvm, irq);
149 	raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
150 }
151 
vgic_flush_pending_lpis(struct kvm_vcpu * vcpu)152 void vgic_flush_pending_lpis(struct kvm_vcpu *vcpu)
153 {
154 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
155 	struct vgic_irq *irq, *tmp;
156 	unsigned long flags;
157 
158 	raw_spin_lock_irqsave(&vgic_cpu->ap_list_lock, flags);
159 
160 	list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) {
161 		if (irq->intid >= VGIC_MIN_LPI) {
162 			raw_spin_lock(&irq->irq_lock);
163 			list_del(&irq->ap_list);
164 			irq->vcpu = NULL;
165 			raw_spin_unlock(&irq->irq_lock);
166 			vgic_put_irq(vcpu->kvm, irq);
167 		}
168 	}
169 
170 	raw_spin_unlock_irqrestore(&vgic_cpu->ap_list_lock, flags);
171 }
172 
vgic_irq_set_phys_pending(struct vgic_irq * irq,bool pending)173 void vgic_irq_set_phys_pending(struct vgic_irq *irq, bool pending)
174 {
175 	WARN_ON(irq_set_irqchip_state(irq->host_irq,
176 				      IRQCHIP_STATE_PENDING,
177 				      pending));
178 }
179 
vgic_get_phys_line_level(struct vgic_irq * irq)180 bool vgic_get_phys_line_level(struct vgic_irq *irq)
181 {
182 	bool line_level;
183 
184 	BUG_ON(!irq->hw);
185 
186 	if (irq->ops && irq->ops->get_input_level)
187 		return irq->ops->get_input_level(irq->intid);
188 
189 	WARN_ON(irq_get_irqchip_state(irq->host_irq,
190 				      IRQCHIP_STATE_PENDING,
191 				      &line_level));
192 	return line_level;
193 }
194 
195 /* Set/Clear the physical active state */
vgic_irq_set_phys_active(struct vgic_irq * irq,bool active)196 void vgic_irq_set_phys_active(struct vgic_irq *irq, bool active)
197 {
198 
199 	BUG_ON(!irq->hw);
200 	WARN_ON(irq_set_irqchip_state(irq->host_irq,
201 				      IRQCHIP_STATE_ACTIVE,
202 				      active));
203 }
204 
205 /**
206  * kvm_vgic_target_oracle - compute the target vcpu for an irq
207  *
208  * @irq:	The irq to route. Must be already locked.
209  *
210  * Based on the current state of the interrupt (enabled, pending,
211  * active, vcpu and target_vcpu), compute the next vcpu this should be
212  * given to. Return NULL if this shouldn't be injected at all.
213  *
214  * Requires the IRQ lock to be held.
215  */
vgic_target_oracle(struct vgic_irq * irq)216 static struct kvm_vcpu *vgic_target_oracle(struct vgic_irq *irq)
217 {
218 	lockdep_assert_held(&irq->irq_lock);
219 
220 	/* If the interrupt is active, it must stay on the current vcpu */
221 	if (irq->active)
222 		return irq->vcpu ? : irq->target_vcpu;
223 
224 	/*
225 	 * If the IRQ is not active but enabled and pending, we should direct
226 	 * it to its configured target VCPU.
227 	 * If the distributor is disabled, pending interrupts shouldn't be
228 	 * forwarded.
229 	 */
230 	if (irq->enabled && irq_is_pending(irq)) {
231 		if (unlikely(irq->target_vcpu &&
232 			     !irq->target_vcpu->kvm->arch.vgic.enabled))
233 			return NULL;
234 
235 		return irq->target_vcpu;
236 	}
237 
238 	/* If neither active nor pending and enabled, then this IRQ should not
239 	 * be queued to any VCPU.
240 	 */
241 	return NULL;
242 }
243 
244 /*
245  * The order of items in the ap_lists defines how we'll pack things in LRs as
246  * well, the first items in the list being the first things populated in the
247  * LRs.
248  *
249  * A hard rule is that active interrupts can never be pushed out of the LRs
250  * (and therefore take priority) since we cannot reliably trap on deactivation
251  * of IRQs and therefore they have to be present in the LRs.
252  *
253  * Otherwise things should be sorted by the priority field and the GIC
254  * hardware support will take care of preemption of priority groups etc.
255  *
256  * Return negative if "a" sorts before "b", 0 to preserve order, and positive
257  * to sort "b" before "a".
258  */
vgic_irq_cmp(void * priv,const struct list_head * a,const struct list_head * b)259 static int vgic_irq_cmp(void *priv, const struct list_head *a,
260 			const struct list_head *b)
261 {
262 	struct vgic_irq *irqa = container_of(a, struct vgic_irq, ap_list);
263 	struct vgic_irq *irqb = container_of(b, struct vgic_irq, ap_list);
264 	bool penda, pendb;
265 	int ret;
266 
267 	/*
268 	 * list_sort may call this function with the same element when
269 	 * the list is fairly long.
270 	 */
271 	if (unlikely(irqa == irqb))
272 		return 0;
273 
274 	raw_spin_lock(&irqa->irq_lock);
275 	raw_spin_lock_nested(&irqb->irq_lock, SINGLE_DEPTH_NESTING);
276 
277 	if (irqa->active || irqb->active) {
278 		ret = (int)irqb->active - (int)irqa->active;
279 		goto out;
280 	}
281 
282 	penda = irqa->enabled && irq_is_pending(irqa);
283 	pendb = irqb->enabled && irq_is_pending(irqb);
284 
285 	if (!penda || !pendb) {
286 		ret = (int)pendb - (int)penda;
287 		goto out;
288 	}
289 
290 	/* Both pending and enabled, sort by priority */
291 	ret = irqa->priority - irqb->priority;
292 out:
293 	raw_spin_unlock(&irqb->irq_lock);
294 	raw_spin_unlock(&irqa->irq_lock);
295 	return ret;
296 }
297 
298 /* Must be called with the ap_list_lock held */
vgic_sort_ap_list(struct kvm_vcpu * vcpu)299 static void vgic_sort_ap_list(struct kvm_vcpu *vcpu)
300 {
301 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
302 
303 	lockdep_assert_held(&vgic_cpu->ap_list_lock);
304 
305 	list_sort(NULL, &vgic_cpu->ap_list_head, vgic_irq_cmp);
306 }
307 
308 /*
309  * Only valid injection if changing level for level-triggered IRQs or for a
310  * rising edge, and in-kernel connected IRQ lines can only be controlled by
311  * their owner.
312  */
vgic_validate_injection(struct vgic_irq * irq,bool level,void * owner)313 static bool vgic_validate_injection(struct vgic_irq *irq, bool level, void *owner)
314 {
315 	if (irq->owner != owner)
316 		return false;
317 
318 	switch (irq->config) {
319 	case VGIC_CONFIG_LEVEL:
320 		return irq->line_level != level;
321 	case VGIC_CONFIG_EDGE:
322 		return level;
323 	}
324 
325 	return false;
326 }
327 
328 /*
329  * Check whether an IRQ needs to (and can) be queued to a VCPU's ap list.
330  * Do the queuing if necessary, taking the right locks in the right order.
331  * Returns true when the IRQ was queued, false otherwise.
332  *
333  * Needs to be entered with the IRQ lock already held, but will return
334  * with all locks dropped.
335  */
vgic_queue_irq_unlock(struct kvm * kvm,struct vgic_irq * irq,unsigned long flags)336 bool vgic_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq,
337 			   unsigned long flags)
338 {
339 	struct kvm_vcpu *vcpu;
340 
341 	lockdep_assert_held(&irq->irq_lock);
342 
343 retry:
344 	vcpu = vgic_target_oracle(irq);
345 	if (irq->vcpu || !vcpu) {
346 		/*
347 		 * If this IRQ is already on a VCPU's ap_list, then it
348 		 * cannot be moved or modified and there is no more work for
349 		 * us to do.
350 		 *
351 		 * Otherwise, if the irq is not pending and enabled, it does
352 		 * not need to be inserted into an ap_list and there is also
353 		 * no more work for us to do.
354 		 */
355 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
356 
357 		/*
358 		 * We have to kick the VCPU here, because we could be
359 		 * queueing an edge-triggered interrupt for which we
360 		 * get no EOI maintenance interrupt. In that case,
361 		 * while the IRQ is already on the VCPU's AP list, the
362 		 * VCPU could have EOI'ed the original interrupt and
363 		 * won't see this one until it exits for some other
364 		 * reason.
365 		 */
366 		if (vcpu) {
367 			kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
368 			kvm_vcpu_kick(vcpu);
369 		}
370 		return false;
371 	}
372 
373 	/*
374 	 * We must unlock the irq lock to take the ap_list_lock where
375 	 * we are going to insert this new pending interrupt.
376 	 */
377 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
378 
379 	/* someone can do stuff here, which we re-check below */
380 
381 	raw_spin_lock_irqsave(&vcpu->arch.vgic_cpu.ap_list_lock, flags);
382 	raw_spin_lock(&irq->irq_lock);
383 
384 	/*
385 	 * Did something change behind our backs?
386 	 *
387 	 * There are two cases:
388 	 * 1) The irq lost its pending state or was disabled behind our
389 	 *    backs and/or it was queued to another VCPU's ap_list.
390 	 * 2) Someone changed the affinity on this irq behind our
391 	 *    backs and we are now holding the wrong ap_list_lock.
392 	 *
393 	 * In both cases, drop the locks and retry.
394 	 */
395 
396 	if (unlikely(irq->vcpu || vcpu != vgic_target_oracle(irq))) {
397 		raw_spin_unlock(&irq->irq_lock);
398 		raw_spin_unlock_irqrestore(&vcpu->arch.vgic_cpu.ap_list_lock,
399 					   flags);
400 
401 		raw_spin_lock_irqsave(&irq->irq_lock, flags);
402 		goto retry;
403 	}
404 
405 	/*
406 	 * Grab a reference to the irq to reflect the fact that it is
407 	 * now in the ap_list.
408 	 */
409 	vgic_get_irq_kref(irq);
410 	list_add_tail(&irq->ap_list, &vcpu->arch.vgic_cpu.ap_list_head);
411 	irq->vcpu = vcpu;
412 
413 	raw_spin_unlock(&irq->irq_lock);
414 	raw_spin_unlock_irqrestore(&vcpu->arch.vgic_cpu.ap_list_lock, flags);
415 
416 	kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
417 	kvm_vcpu_kick(vcpu);
418 
419 	return true;
420 }
421 
422 /**
423  * kvm_vgic_inject_irq - Inject an IRQ from a device to the vgic
424  * @kvm:     The VM structure pointer
425  * @cpuid:   The CPU for PPIs
426  * @intid:   The INTID to inject a new state to.
427  * @level:   Edge-triggered:  true:  to trigger the interrupt
428  *			      false: to ignore the call
429  *	     Level-sensitive  true:  raise the input signal
430  *			      false: lower the input signal
431  * @owner:   The opaque pointer to the owner of the IRQ being raised to verify
432  *           that the caller is allowed to inject this IRQ.  Userspace
433  *           injections will have owner == NULL.
434  *
435  * The VGIC is not concerned with devices being active-LOW or active-HIGH for
436  * level-sensitive interrupts.  You can think of the level parameter as 1
437  * being HIGH and 0 being LOW and all devices being active-HIGH.
438  */
kvm_vgic_inject_irq(struct kvm * kvm,int cpuid,unsigned int intid,bool level,void * owner)439 int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int intid,
440 			bool level, void *owner)
441 {
442 	struct kvm_vcpu *vcpu;
443 	struct vgic_irq *irq;
444 	unsigned long flags;
445 	int ret;
446 
447 	trace_vgic_update_irq_pending(cpuid, intid, level);
448 
449 	ret = vgic_lazy_init(kvm);
450 	if (ret)
451 		return ret;
452 
453 	vcpu = kvm_get_vcpu(kvm, cpuid);
454 	if (!vcpu && intid < VGIC_NR_PRIVATE_IRQS)
455 		return -EINVAL;
456 
457 	irq = vgic_get_irq(kvm, vcpu, intid);
458 	if (!irq)
459 		return -EINVAL;
460 
461 	raw_spin_lock_irqsave(&irq->irq_lock, flags);
462 
463 	if (!vgic_validate_injection(irq, level, owner)) {
464 		/* Nothing to see here, move along... */
465 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
466 		vgic_put_irq(kvm, irq);
467 		return 0;
468 	}
469 
470 	if (irq->config == VGIC_CONFIG_LEVEL)
471 		irq->line_level = level;
472 	else
473 		irq->pending_latch = true;
474 
475 	vgic_queue_irq_unlock(kvm, irq, flags);
476 	vgic_put_irq(kvm, irq);
477 
478 	return 0;
479 }
480 
481 /* @irq->irq_lock must be held */
kvm_vgic_map_irq(struct kvm_vcpu * vcpu,struct vgic_irq * irq,unsigned int host_irq,struct irq_ops * ops)482 static int kvm_vgic_map_irq(struct kvm_vcpu *vcpu, struct vgic_irq *irq,
483 			    unsigned int host_irq,
484 			    struct irq_ops *ops)
485 {
486 	struct irq_desc *desc;
487 	struct irq_data *data;
488 
489 	/*
490 	 * Find the physical IRQ number corresponding to @host_irq
491 	 */
492 	desc = irq_to_desc(host_irq);
493 	if (!desc) {
494 		kvm_err("%s: no interrupt descriptor\n", __func__);
495 		return -EINVAL;
496 	}
497 	data = irq_desc_get_irq_data(desc);
498 	while (data->parent_data)
499 		data = data->parent_data;
500 
501 	irq->hw = true;
502 	irq->host_irq = host_irq;
503 	irq->hwintid = data->hwirq;
504 	irq->ops = ops;
505 	return 0;
506 }
507 
508 /* @irq->irq_lock must be held */
kvm_vgic_unmap_irq(struct vgic_irq * irq)509 static inline void kvm_vgic_unmap_irq(struct vgic_irq *irq)
510 {
511 	irq->hw = false;
512 	irq->hwintid = 0;
513 	irq->ops = NULL;
514 }
515 
kvm_vgic_map_phys_irq(struct kvm_vcpu * vcpu,unsigned int host_irq,u32 vintid,struct irq_ops * ops)516 int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, unsigned int host_irq,
517 			  u32 vintid, struct irq_ops *ops)
518 {
519 	struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, vintid);
520 	unsigned long flags;
521 	int ret;
522 
523 	BUG_ON(!irq);
524 
525 	raw_spin_lock_irqsave(&irq->irq_lock, flags);
526 	ret = kvm_vgic_map_irq(vcpu, irq, host_irq, ops);
527 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
528 	vgic_put_irq(vcpu->kvm, irq);
529 
530 	return ret;
531 }
532 
533 /**
534  * kvm_vgic_reset_mapped_irq - Reset a mapped IRQ
535  * @vcpu: The VCPU pointer
536  * @vintid: The INTID of the interrupt
537  *
538  * Reset the active and pending states of a mapped interrupt.  Kernel
539  * subsystems injecting mapped interrupts should reset their interrupt lines
540  * when we are doing a reset of the VM.
541  */
kvm_vgic_reset_mapped_irq(struct kvm_vcpu * vcpu,u32 vintid)542 void kvm_vgic_reset_mapped_irq(struct kvm_vcpu *vcpu, u32 vintid)
543 {
544 	struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, vintid);
545 	unsigned long flags;
546 
547 	if (!irq->hw)
548 		goto out;
549 
550 	raw_spin_lock_irqsave(&irq->irq_lock, flags);
551 	irq->active = false;
552 	irq->pending_latch = false;
553 	irq->line_level = false;
554 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
555 out:
556 	vgic_put_irq(vcpu->kvm, irq);
557 }
558 
kvm_vgic_unmap_phys_irq(struct kvm_vcpu * vcpu,unsigned int vintid)559 int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, unsigned int vintid)
560 {
561 	struct vgic_irq *irq;
562 	unsigned long flags;
563 
564 	if (!vgic_initialized(vcpu->kvm))
565 		return -EAGAIN;
566 
567 	irq = vgic_get_irq(vcpu->kvm, vcpu, vintid);
568 	BUG_ON(!irq);
569 
570 	raw_spin_lock_irqsave(&irq->irq_lock, flags);
571 	kvm_vgic_unmap_irq(irq);
572 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
573 	vgic_put_irq(vcpu->kvm, irq);
574 
575 	return 0;
576 }
577 
kvm_vgic_get_map(struct kvm_vcpu * vcpu,unsigned int vintid)578 int kvm_vgic_get_map(struct kvm_vcpu *vcpu, unsigned int vintid)
579 {
580 	struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, vintid);
581 	unsigned long flags;
582 	int ret = -1;
583 
584 	raw_spin_lock_irqsave(&irq->irq_lock, flags);
585 	if (irq->hw)
586 		ret = irq->hwintid;
587 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
588 
589 	vgic_put_irq(vcpu->kvm, irq);
590 	return ret;
591 }
592 
593 /**
594  * kvm_vgic_set_owner - Set the owner of an interrupt for a VM
595  *
596  * @vcpu:   Pointer to the VCPU (used for PPIs)
597  * @intid:  The virtual INTID identifying the interrupt (PPI or SPI)
598  * @owner:  Opaque pointer to the owner
599  *
600  * Returns 0 if intid is not already used by another in-kernel device and the
601  * owner is set, otherwise returns an error code.
602  */
kvm_vgic_set_owner(struct kvm_vcpu * vcpu,unsigned int intid,void * owner)603 int kvm_vgic_set_owner(struct kvm_vcpu *vcpu, unsigned int intid, void *owner)
604 {
605 	struct vgic_irq *irq;
606 	unsigned long flags;
607 	int ret = 0;
608 
609 	if (!vgic_initialized(vcpu->kvm))
610 		return -EAGAIN;
611 
612 	/* SGIs and LPIs cannot be wired up to any device */
613 	if (!irq_is_ppi(intid) && !vgic_valid_spi(vcpu->kvm, intid))
614 		return -EINVAL;
615 
616 	irq = vgic_get_irq(vcpu->kvm, vcpu, intid);
617 	raw_spin_lock_irqsave(&irq->irq_lock, flags);
618 	if (irq->owner && irq->owner != owner)
619 		ret = -EEXIST;
620 	else
621 		irq->owner = owner;
622 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
623 
624 	return ret;
625 }
626 
627 /**
628  * vgic_prune_ap_list - Remove non-relevant interrupts from the list
629  *
630  * @vcpu: The VCPU pointer
631  *
632  * Go over the list of "interesting" interrupts, and prune those that we
633  * won't have to consider in the near future.
634  */
vgic_prune_ap_list(struct kvm_vcpu * vcpu)635 static void vgic_prune_ap_list(struct kvm_vcpu *vcpu)
636 {
637 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
638 	struct vgic_irq *irq, *tmp;
639 
640 	DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());
641 
642 retry:
643 	raw_spin_lock(&vgic_cpu->ap_list_lock);
644 
645 	list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) {
646 		struct kvm_vcpu *target_vcpu, *vcpuA, *vcpuB;
647 		bool target_vcpu_needs_kick = false;
648 
649 		raw_spin_lock(&irq->irq_lock);
650 
651 		BUG_ON(vcpu != irq->vcpu);
652 
653 		target_vcpu = vgic_target_oracle(irq);
654 
655 		if (!target_vcpu) {
656 			/*
657 			 * We don't need to process this interrupt any
658 			 * further, move it off the list.
659 			 */
660 			list_del(&irq->ap_list);
661 			irq->vcpu = NULL;
662 			raw_spin_unlock(&irq->irq_lock);
663 
664 			/*
665 			 * This vgic_put_irq call matches the
666 			 * vgic_get_irq_kref in vgic_queue_irq_unlock,
667 			 * where we added the LPI to the ap_list. As
668 			 * we remove the irq from the list, we drop
669 			 * also drop the refcount.
670 			 */
671 			vgic_put_irq(vcpu->kvm, irq);
672 			continue;
673 		}
674 
675 		if (target_vcpu == vcpu) {
676 			/* We're on the right CPU */
677 			raw_spin_unlock(&irq->irq_lock);
678 			continue;
679 		}
680 
681 		/* This interrupt looks like it has to be migrated. */
682 
683 		raw_spin_unlock(&irq->irq_lock);
684 		raw_spin_unlock(&vgic_cpu->ap_list_lock);
685 
686 		/*
687 		 * Ensure locking order by always locking the smallest
688 		 * ID first.
689 		 */
690 		if (vcpu->vcpu_id < target_vcpu->vcpu_id) {
691 			vcpuA = vcpu;
692 			vcpuB = target_vcpu;
693 		} else {
694 			vcpuA = target_vcpu;
695 			vcpuB = vcpu;
696 		}
697 
698 		raw_spin_lock(&vcpuA->arch.vgic_cpu.ap_list_lock);
699 		raw_spin_lock_nested(&vcpuB->arch.vgic_cpu.ap_list_lock,
700 				      SINGLE_DEPTH_NESTING);
701 		raw_spin_lock(&irq->irq_lock);
702 
703 		/*
704 		 * If the affinity has been preserved, move the
705 		 * interrupt around. Otherwise, it means things have
706 		 * changed while the interrupt was unlocked, and we
707 		 * need to replay this.
708 		 *
709 		 * In all cases, we cannot trust the list not to have
710 		 * changed, so we restart from the beginning.
711 		 */
712 		if (target_vcpu == vgic_target_oracle(irq)) {
713 			struct vgic_cpu *new_cpu = &target_vcpu->arch.vgic_cpu;
714 
715 			list_del(&irq->ap_list);
716 			irq->vcpu = target_vcpu;
717 			list_add_tail(&irq->ap_list, &new_cpu->ap_list_head);
718 			target_vcpu_needs_kick = true;
719 		}
720 
721 		raw_spin_unlock(&irq->irq_lock);
722 		raw_spin_unlock(&vcpuB->arch.vgic_cpu.ap_list_lock);
723 		raw_spin_unlock(&vcpuA->arch.vgic_cpu.ap_list_lock);
724 
725 		if (target_vcpu_needs_kick) {
726 			kvm_make_request(KVM_REQ_IRQ_PENDING, target_vcpu);
727 			kvm_vcpu_kick(target_vcpu);
728 		}
729 
730 		goto retry;
731 	}
732 
733 	raw_spin_unlock(&vgic_cpu->ap_list_lock);
734 }
735 
vgic_fold_lr_state(struct kvm_vcpu * vcpu)736 static inline void vgic_fold_lr_state(struct kvm_vcpu *vcpu)
737 {
738 	if (kvm_vgic_global_state.type == VGIC_V2)
739 		vgic_v2_fold_lr_state(vcpu);
740 	else
741 		vgic_v3_fold_lr_state(vcpu);
742 }
743 
744 /* Requires the irq_lock to be held. */
vgic_populate_lr(struct kvm_vcpu * vcpu,struct vgic_irq * irq,int lr)745 static inline void vgic_populate_lr(struct kvm_vcpu *vcpu,
746 				    struct vgic_irq *irq, int lr)
747 {
748 	lockdep_assert_held(&irq->irq_lock);
749 
750 	if (kvm_vgic_global_state.type == VGIC_V2)
751 		vgic_v2_populate_lr(vcpu, irq, lr);
752 	else
753 		vgic_v3_populate_lr(vcpu, irq, lr);
754 }
755 
vgic_clear_lr(struct kvm_vcpu * vcpu,int lr)756 static inline void vgic_clear_lr(struct kvm_vcpu *vcpu, int lr)
757 {
758 	if (kvm_vgic_global_state.type == VGIC_V2)
759 		vgic_v2_clear_lr(vcpu, lr);
760 	else
761 		vgic_v3_clear_lr(vcpu, lr);
762 }
763 
vgic_set_underflow(struct kvm_vcpu * vcpu)764 static inline void vgic_set_underflow(struct kvm_vcpu *vcpu)
765 {
766 	if (kvm_vgic_global_state.type == VGIC_V2)
767 		vgic_v2_set_underflow(vcpu);
768 	else
769 		vgic_v3_set_underflow(vcpu);
770 }
771 
772 /* Requires the ap_list_lock to be held. */
compute_ap_list_depth(struct kvm_vcpu * vcpu,bool * multi_sgi)773 static int compute_ap_list_depth(struct kvm_vcpu *vcpu,
774 				 bool *multi_sgi)
775 {
776 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
777 	struct vgic_irq *irq;
778 	int count = 0;
779 
780 	*multi_sgi = false;
781 
782 	lockdep_assert_held(&vgic_cpu->ap_list_lock);
783 
784 	list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
785 		int w;
786 
787 		raw_spin_lock(&irq->irq_lock);
788 		/* GICv2 SGIs can count for more than one... */
789 		w = vgic_irq_get_lr_count(irq);
790 		raw_spin_unlock(&irq->irq_lock);
791 
792 		count += w;
793 		*multi_sgi |= (w > 1);
794 	}
795 	return count;
796 }
797 
798 /* Requires the VCPU's ap_list_lock to be held. */
vgic_flush_lr_state(struct kvm_vcpu * vcpu)799 static void vgic_flush_lr_state(struct kvm_vcpu *vcpu)
800 {
801 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
802 	struct vgic_irq *irq;
803 	int count;
804 	bool multi_sgi;
805 	u8 prio = 0xff;
806 	int i = 0;
807 
808 	lockdep_assert_held(&vgic_cpu->ap_list_lock);
809 
810 	count = compute_ap_list_depth(vcpu, &multi_sgi);
811 	if (count > kvm_vgic_global_state.nr_lr || multi_sgi)
812 		vgic_sort_ap_list(vcpu);
813 
814 	count = 0;
815 
816 	list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
817 		raw_spin_lock(&irq->irq_lock);
818 
819 		/*
820 		 * If we have multi-SGIs in the pipeline, we need to
821 		 * guarantee that they are all seen before any IRQ of
822 		 * lower priority. In that case, we need to filter out
823 		 * these interrupts by exiting early. This is easy as
824 		 * the AP list has been sorted already.
825 		 */
826 		if (multi_sgi && irq->priority > prio) {
827 			_raw_spin_unlock(&irq->irq_lock);
828 			break;
829 		}
830 
831 		if (likely(vgic_target_oracle(irq) == vcpu)) {
832 			vgic_populate_lr(vcpu, irq, count++);
833 
834 			if (irq->source)
835 				prio = irq->priority;
836 		}
837 
838 		raw_spin_unlock(&irq->irq_lock);
839 
840 		if (count == kvm_vgic_global_state.nr_lr) {
841 			if (!list_is_last(&irq->ap_list,
842 					  &vgic_cpu->ap_list_head))
843 				vgic_set_underflow(vcpu);
844 			break;
845 		}
846 	}
847 
848 	/* Nuke remaining LRs */
849 	for (i = count ; i < kvm_vgic_global_state.nr_lr; i++)
850 		vgic_clear_lr(vcpu, i);
851 
852 	if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
853 		vcpu->arch.vgic_cpu.vgic_v2.used_lrs = count;
854 	else
855 		vcpu->arch.vgic_cpu.vgic_v3.used_lrs = count;
856 }
857 
can_access_vgic_from_kernel(void)858 static inline bool can_access_vgic_from_kernel(void)
859 {
860 	/*
861 	 * GICv2 can always be accessed from the kernel because it is
862 	 * memory-mapped, and VHE systems can access GICv3 EL2 system
863 	 * registers.
864 	 */
865 	return !static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif) || has_vhe();
866 }
867 
vgic_save_state(struct kvm_vcpu * vcpu)868 static inline void vgic_save_state(struct kvm_vcpu *vcpu)
869 {
870 	if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
871 		vgic_v2_save_state(vcpu);
872 	else
873 		__vgic_v3_save_state(&vcpu->arch.vgic_cpu.vgic_v3);
874 }
875 
876 /* Sync back the hardware VGIC state into our emulation after a guest's run. */
kvm_vgic_sync_hwstate(struct kvm_vcpu * vcpu)877 void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
878 {
879 	int used_lrs;
880 
881 	/* An empty ap_list_head implies used_lrs == 0 */
882 	if (list_empty(&vcpu->arch.vgic_cpu.ap_list_head))
883 		return;
884 
885 	if (can_access_vgic_from_kernel())
886 		vgic_save_state(vcpu);
887 
888 	if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
889 		used_lrs = vcpu->arch.vgic_cpu.vgic_v2.used_lrs;
890 	else
891 		used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
892 
893 	if (used_lrs)
894 		vgic_fold_lr_state(vcpu);
895 	vgic_prune_ap_list(vcpu);
896 }
897 
vgic_restore_state(struct kvm_vcpu * vcpu)898 static inline void vgic_restore_state(struct kvm_vcpu *vcpu)
899 {
900 	if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
901 		vgic_v2_restore_state(vcpu);
902 	else
903 		__vgic_v3_restore_state(&vcpu->arch.vgic_cpu.vgic_v3);
904 }
905 
906 /* Flush our emulation state into the GIC hardware before entering the guest. */
kvm_vgic_flush_hwstate(struct kvm_vcpu * vcpu)907 void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
908 {
909 	/*
910 	 * If there are no virtual interrupts active or pending for this
911 	 * VCPU, then there is no work to do and we can bail out without
912 	 * taking any lock.  There is a potential race with someone injecting
913 	 * interrupts to the VCPU, but it is a benign race as the VCPU will
914 	 * either observe the new interrupt before or after doing this check,
915 	 * and introducing additional synchronization mechanism doesn't change
916 	 * this.
917 	 *
918 	 * Note that we still need to go through the whole thing if anything
919 	 * can be directly injected (GICv4).
920 	 */
921 	if (list_empty(&vcpu->arch.vgic_cpu.ap_list_head) &&
922 	    !vgic_supports_direct_msis(vcpu->kvm))
923 		return;
924 
925 	DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());
926 
927 	if (!list_empty(&vcpu->arch.vgic_cpu.ap_list_head)) {
928 		raw_spin_lock(&vcpu->arch.vgic_cpu.ap_list_lock);
929 		vgic_flush_lr_state(vcpu);
930 		raw_spin_unlock(&vcpu->arch.vgic_cpu.ap_list_lock);
931 	}
932 
933 	if (can_access_vgic_from_kernel())
934 		vgic_restore_state(vcpu);
935 
936 	if (vgic_supports_direct_msis(vcpu->kvm))
937 		vgic_v4_commit(vcpu);
938 }
939 
kvm_vgic_load(struct kvm_vcpu * vcpu)940 void kvm_vgic_load(struct kvm_vcpu *vcpu)
941 {
942 	if (unlikely(!vgic_initialized(vcpu->kvm)))
943 		return;
944 
945 	if (kvm_vgic_global_state.type == VGIC_V2)
946 		vgic_v2_load(vcpu);
947 	else
948 		vgic_v3_load(vcpu);
949 }
950 
kvm_vgic_put(struct kvm_vcpu * vcpu)951 void kvm_vgic_put(struct kvm_vcpu *vcpu)
952 {
953 	if (unlikely(!vgic_initialized(vcpu->kvm)))
954 		return;
955 
956 	if (kvm_vgic_global_state.type == VGIC_V2)
957 		vgic_v2_put(vcpu);
958 	else
959 		vgic_v3_put(vcpu);
960 }
961 
kvm_vgic_vmcr_sync(struct kvm_vcpu * vcpu)962 void kvm_vgic_vmcr_sync(struct kvm_vcpu *vcpu)
963 {
964 	if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
965 		return;
966 
967 	if (kvm_vgic_global_state.type == VGIC_V2)
968 		vgic_v2_vmcr_sync(vcpu);
969 	else
970 		vgic_v3_vmcr_sync(vcpu);
971 }
972 
kvm_vgic_vcpu_pending_irq(struct kvm_vcpu * vcpu)973 int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
974 {
975 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
976 	struct vgic_irq *irq;
977 	bool pending = false;
978 	unsigned long flags;
979 	struct vgic_vmcr vmcr;
980 
981 	if (!vcpu->kvm->arch.vgic.enabled)
982 		return false;
983 
984 	if (vcpu->arch.vgic_cpu.vgic_v3.its_vpe.pending_last)
985 		return true;
986 
987 	vgic_get_vmcr(vcpu, &vmcr);
988 
989 	raw_spin_lock_irqsave(&vgic_cpu->ap_list_lock, flags);
990 
991 	list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
992 		raw_spin_lock(&irq->irq_lock);
993 		pending = irq_is_pending(irq) && irq->enabled &&
994 			  !irq->active &&
995 			  irq->priority < vmcr.pmr;
996 		raw_spin_unlock(&irq->irq_lock);
997 
998 		if (pending)
999 			break;
1000 	}
1001 
1002 	raw_spin_unlock_irqrestore(&vgic_cpu->ap_list_lock, flags);
1003 
1004 	return pending;
1005 }
1006 
vgic_kick_vcpus(struct kvm * kvm)1007 void vgic_kick_vcpus(struct kvm *kvm)
1008 {
1009 	struct kvm_vcpu *vcpu;
1010 	unsigned long c;
1011 
1012 	/*
1013 	 * We've injected an interrupt, time to find out who deserves
1014 	 * a good kick...
1015 	 */
1016 	kvm_for_each_vcpu(c, vcpu, kvm) {
1017 		if (kvm_vgic_vcpu_pending_irq(vcpu)) {
1018 			kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
1019 			kvm_vcpu_kick(vcpu);
1020 		}
1021 	}
1022 }
1023 
kvm_vgic_map_is_active(struct kvm_vcpu * vcpu,unsigned int vintid)1024 bool kvm_vgic_map_is_active(struct kvm_vcpu *vcpu, unsigned int vintid)
1025 {
1026 	struct vgic_irq *irq;
1027 	bool map_is_active;
1028 	unsigned long flags;
1029 
1030 	if (!vgic_initialized(vcpu->kvm))
1031 		return false;
1032 
1033 	irq = vgic_get_irq(vcpu->kvm, vcpu, vintid);
1034 	raw_spin_lock_irqsave(&irq->irq_lock, flags);
1035 	map_is_active = irq->hw && irq->active;
1036 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
1037 	vgic_put_irq(vcpu->kvm, irq);
1038 
1039 	return map_is_active;
1040 }
1041 
1042 /*
1043  * Level-triggered mapped IRQs are special because we only observe rising
1044  * edges as input to the VGIC.
1045  *
1046  * If the guest never acked the interrupt we have to sample the physical
1047  * line and set the line level, because the device state could have changed
1048  * or we simply need to process the still pending interrupt later.
1049  *
1050  * We could also have entered the guest with the interrupt active+pending.
1051  * On the next exit, we need to re-evaluate the pending state, as it could
1052  * otherwise result in a spurious interrupt by injecting a now potentially
1053  * stale pending state.
1054  *
1055  * If this causes us to lower the level, we have to also clear the physical
1056  * active state, since we will otherwise never be told when the interrupt
1057  * becomes asserted again.
1058  *
1059  * Another case is when the interrupt requires a helping hand on
1060  * deactivation (no HW deactivation, for example).
1061  */
vgic_irq_handle_resampling(struct vgic_irq * irq,bool lr_deactivated,bool lr_pending)1062 void vgic_irq_handle_resampling(struct vgic_irq *irq,
1063 				bool lr_deactivated, bool lr_pending)
1064 {
1065 	if (vgic_irq_is_mapped_level(irq)) {
1066 		bool resample = false;
1067 
1068 		if (unlikely(vgic_irq_needs_resampling(irq))) {
1069 			resample = !(irq->active || irq->pending_latch);
1070 		} else if (lr_pending || (lr_deactivated && irq->line_level)) {
1071 			irq->line_level = vgic_get_phys_line_level(irq);
1072 			resample = !irq->line_level;
1073 		}
1074 
1075 		if (resample)
1076 			vgic_irq_set_phys_active(irq, false);
1077 	}
1078 }
1079