xref: /openbmc/linux/arch/powerpc/kvm/book3s_xive.c (revision 932c6dea)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright 2017 Benjamin Herrenschmidt, IBM Corporation.
4  */
5 
6 #define pr_fmt(fmt) "xive-kvm: " fmt
7 
8 #include <linux/kernel.h>
9 #include <linux/kvm_host.h>
10 #include <linux/err.h>
11 #include <linux/gfp.h>
12 #include <linux/spinlock.h>
13 #include <linux/delay.h>
14 #include <linux/percpu.h>
15 #include <linux/cpumask.h>
16 #include <linux/uaccess.h>
17 #include <linux/irqdomain.h>
18 #include <asm/kvm_book3s.h>
19 #include <asm/kvm_ppc.h>
20 #include <asm/hvcall.h>
21 #include <asm/xics.h>
22 #include <asm/xive.h>
23 #include <asm/xive-regs.h>
24 #include <asm/debug.h>
25 #include <asm/time.h>
26 #include <asm/opal.h>
27 
28 #include <linux/debugfs.h>
29 #include <linux/seq_file.h>
30 
31 #include "book3s_xive.h"
32 
33 #define __x_eoi_page(xd)	((void __iomem *)((xd)->eoi_mmio))
34 #define __x_trig_page(xd)	((void __iomem *)((xd)->trig_mmio))
35 
36 /* Dummy interrupt used when taking interrupts out of a queue in H_CPPR */
37 #define XICS_DUMMY	1
38 
xive_vm_ack_pending(struct kvmppc_xive_vcpu * xc)39 static void xive_vm_ack_pending(struct kvmppc_xive_vcpu *xc)
40 {
41 	u8 cppr;
42 	u16 ack;
43 
44 	/*
45 	 * Ensure any previous store to CPPR is ordered vs.
46 	 * the subsequent loads from PIPR or ACK.
47 	 */
48 	eieio();
49 
50 	/* Perform the acknowledge OS to register cycle. */
51 	ack = be16_to_cpu(__raw_readw(xive_tima + TM_SPC_ACK_OS_REG));
52 
53 	/* Synchronize subsequent queue accesses */
54 	mb();
55 
56 	/* XXX Check grouping level */
57 
58 	/* Anything ? */
59 	if (!((ack >> 8) & TM_QW1_NSR_EO))
60 		return;
61 
62 	/* Grab CPPR of the most favored pending interrupt */
63 	cppr = ack & 0xff;
64 	if (cppr < 8)
65 		xc->pending |= 1 << cppr;
66 
67 	/* Check consistency */
68 	if (cppr >= xc->hw_cppr)
69 		pr_warn("KVM-XIVE: CPU %d odd ack CPPR, got %d at %d\n",
70 			smp_processor_id(), cppr, xc->hw_cppr);
71 
72 	/*
73 	 * Update our image of the HW CPPR. We don't yet modify
74 	 * xc->cppr, this will be done as we scan for interrupts
75 	 * in the queues.
76 	 */
77 	xc->hw_cppr = cppr;
78 }
79 
xive_vm_esb_load(struct xive_irq_data * xd,u32 offset)80 static u8 xive_vm_esb_load(struct xive_irq_data *xd, u32 offset)
81 {
82 	u64 val;
83 
84 	if (offset == XIVE_ESB_SET_PQ_10 && xd->flags & XIVE_IRQ_FLAG_STORE_EOI)
85 		offset |= XIVE_ESB_LD_ST_MO;
86 
87 	val = __raw_readq(__x_eoi_page(xd) + offset);
88 #ifdef __LITTLE_ENDIAN__
89 	val >>= 64-8;
90 #endif
91 	return (u8)val;
92 }
93 
94 
xive_vm_source_eoi(u32 hw_irq,struct xive_irq_data * xd)95 static void xive_vm_source_eoi(u32 hw_irq, struct xive_irq_data *xd)
96 {
97 	/* If the XIVE supports the new "store EOI facility, use it */
98 	if (xd->flags & XIVE_IRQ_FLAG_STORE_EOI)
99 		__raw_writeq(0, __x_eoi_page(xd) + XIVE_ESB_STORE_EOI);
100 	else if (xd->flags & XIVE_IRQ_FLAG_LSI) {
101 		/*
102 		 * For LSIs the HW EOI cycle is used rather than PQ bits,
103 		 * as they are automatically re-triggred in HW when still
104 		 * pending.
105 		 */
106 		__raw_readq(__x_eoi_page(xd) + XIVE_ESB_LOAD_EOI);
107 	} else {
108 		uint64_t eoi_val;
109 
110 		/*
111 		 * Otherwise for EOI, we use the special MMIO that does
112 		 * a clear of both P and Q and returns the old Q,
113 		 * except for LSIs where we use the "EOI cycle" special
114 		 * load.
115 		 *
116 		 * This allows us to then do a re-trigger if Q was set
117 		 * rather than synthetizing an interrupt in software
118 		 */
119 		eoi_val = xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_00);
120 
121 		/* Re-trigger if needed */
122 		if ((eoi_val & 1) && __x_trig_page(xd))
123 			__raw_writeq(0, __x_trig_page(xd));
124 	}
125 }
126 
127 enum {
128 	scan_fetch,
129 	scan_poll,
130 	scan_eoi,
131 };
132 
xive_vm_scan_interrupts(struct kvmppc_xive_vcpu * xc,u8 pending,int scan_type)133 static u32 xive_vm_scan_interrupts(struct kvmppc_xive_vcpu *xc,
134 				       u8 pending, int scan_type)
135 {
136 	u32 hirq = 0;
137 	u8 prio = 0xff;
138 
139 	/* Find highest pending priority */
140 	while ((xc->mfrr != 0xff || pending != 0) && hirq == 0) {
141 		struct xive_q *q;
142 		u32 idx, toggle;
143 		__be32 *qpage;
144 
145 		/*
146 		 * If pending is 0 this will return 0xff which is what
147 		 * we want
148 		 */
149 		prio = ffs(pending) - 1;
150 
151 		/* Don't scan past the guest cppr */
152 		if (prio >= xc->cppr || prio > 7) {
153 			if (xc->mfrr < xc->cppr) {
154 				prio = xc->mfrr;
155 				hirq = XICS_IPI;
156 			}
157 			break;
158 		}
159 
160 		/* Grab queue and pointers */
161 		q = &xc->queues[prio];
162 		idx = q->idx;
163 		toggle = q->toggle;
164 
165 		/*
166 		 * Snapshot the queue page. The test further down for EOI
167 		 * must use the same "copy" that was used by __xive_read_eq
168 		 * since qpage can be set concurrently and we don't want
169 		 * to miss an EOI.
170 		 */
171 		qpage = READ_ONCE(q->qpage);
172 
173 skip_ipi:
174 		/*
175 		 * Try to fetch from the queue. Will return 0 for a
176 		 * non-queueing priority (ie, qpage = 0).
177 		 */
178 		hirq = __xive_read_eq(qpage, q->msk, &idx, &toggle);
179 
180 		/*
181 		 * If this was a signal for an MFFR change done by
182 		 * H_IPI we skip it. Additionally, if we were fetching
183 		 * we EOI it now, thus re-enabling reception of a new
184 		 * such signal.
185 		 *
186 		 * We also need to do that if prio is 0 and we had no
187 		 * page for the queue. In this case, we have non-queued
188 		 * IPI that needs to be EOId.
189 		 *
190 		 * This is safe because if we have another pending MFRR
191 		 * change that wasn't observed above, the Q bit will have
192 		 * been set and another occurrence of the IPI will trigger.
193 		 */
194 		if (hirq == XICS_IPI || (prio == 0 && !qpage)) {
195 			if (scan_type == scan_fetch) {
196 				xive_vm_source_eoi(xc->vp_ipi,
197 						       &xc->vp_ipi_data);
198 				q->idx = idx;
199 				q->toggle = toggle;
200 			}
201 			/* Loop back on same queue with updated idx/toggle */
202 			WARN_ON(hirq && hirq != XICS_IPI);
203 			if (hirq)
204 				goto skip_ipi;
205 		}
206 
207 		/* If it's the dummy interrupt, continue searching */
208 		if (hirq == XICS_DUMMY)
209 			goto skip_ipi;
210 
211 		/* Clear the pending bit if the queue is now empty */
212 		if (!hirq) {
213 			pending &= ~(1 << prio);
214 
215 			/*
216 			 * Check if the queue count needs adjusting due to
217 			 * interrupts being moved away.
218 			 */
219 			if (atomic_read(&q->pending_count)) {
220 				int p = atomic_xchg(&q->pending_count, 0);
221 
222 				if (p) {
223 					WARN_ON(p > atomic_read(&q->count));
224 					atomic_sub(p, &q->count);
225 				}
226 			}
227 		}
228 
229 		/*
230 		 * If the most favoured prio we found pending is less
231 		 * favored (or equal) than a pending IPI, we return
232 		 * the IPI instead.
233 		 */
234 		if (prio >= xc->mfrr && xc->mfrr < xc->cppr) {
235 			prio = xc->mfrr;
236 			hirq = XICS_IPI;
237 			break;
238 		}
239 
240 		/* If fetching, update queue pointers */
241 		if (scan_type == scan_fetch) {
242 			q->idx = idx;
243 			q->toggle = toggle;
244 		}
245 	}
246 
247 	/* If we are just taking a "peek", do nothing else */
248 	if (scan_type == scan_poll)
249 		return hirq;
250 
251 	/* Update the pending bits */
252 	xc->pending = pending;
253 
254 	/*
255 	 * If this is an EOI that's it, no CPPR adjustment done here,
256 	 * all we needed was cleanup the stale pending bits and check
257 	 * if there's anything left.
258 	 */
259 	if (scan_type == scan_eoi)
260 		return hirq;
261 
262 	/*
263 	 * If we found an interrupt, adjust what the guest CPPR should
264 	 * be as if we had just fetched that interrupt from HW.
265 	 *
266 	 * Note: This can only make xc->cppr smaller as the previous
267 	 * loop will only exit with hirq != 0 if prio is lower than
268 	 * the current xc->cppr. Thus we don't need to re-check xc->mfrr
269 	 * for pending IPIs.
270 	 */
271 	if (hirq)
272 		xc->cppr = prio;
273 	/*
274 	 * If it was an IPI the HW CPPR might have been lowered too much
275 	 * as the HW interrupt we use for IPIs is routed to priority 0.
276 	 *
277 	 * We re-sync it here.
278 	 */
279 	if (xc->cppr != xc->hw_cppr) {
280 		xc->hw_cppr = xc->cppr;
281 		__raw_writeb(xc->cppr, xive_tima + TM_QW1_OS + TM_CPPR);
282 	}
283 
284 	return hirq;
285 }
286 
xive_vm_h_xirr(struct kvm_vcpu * vcpu)287 static unsigned long xive_vm_h_xirr(struct kvm_vcpu *vcpu)
288 {
289 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
290 	u8 old_cppr;
291 	u32 hirq;
292 
293 	pr_devel("H_XIRR\n");
294 
295 	xc->stat_vm_h_xirr++;
296 
297 	/* First collect pending bits from HW */
298 	xive_vm_ack_pending(xc);
299 
300 	pr_devel(" new pending=0x%02x hw_cppr=%d cppr=%d\n",
301 		 xc->pending, xc->hw_cppr, xc->cppr);
302 
303 	/* Grab previous CPPR and reverse map it */
304 	old_cppr = xive_prio_to_guest(xc->cppr);
305 
306 	/* Scan for actual interrupts */
307 	hirq = xive_vm_scan_interrupts(xc, xc->pending, scan_fetch);
308 
309 	pr_devel(" got hirq=0x%x hw_cppr=%d cppr=%d\n",
310 		 hirq, xc->hw_cppr, xc->cppr);
311 
312 	/* That should never hit */
313 	if (hirq & 0xff000000)
314 		pr_warn("XIVE: Weird guest interrupt number 0x%08x\n", hirq);
315 
316 	/*
317 	 * XXX We could check if the interrupt is masked here and
318 	 * filter it. If we chose to do so, we would need to do:
319 	 *
320 	 *    if (masked) {
321 	 *        lock();
322 	 *        if (masked) {
323 	 *            old_Q = true;
324 	 *            hirq = 0;
325 	 *        }
326 	 *        unlock();
327 	 *    }
328 	 */
329 
330 	/* Return interrupt and old CPPR in GPR4 */
331 	vcpu->arch.regs.gpr[4] = hirq | (old_cppr << 24);
332 
333 	return H_SUCCESS;
334 }
335 
xive_vm_h_ipoll(struct kvm_vcpu * vcpu,unsigned long server)336 static unsigned long xive_vm_h_ipoll(struct kvm_vcpu *vcpu, unsigned long server)
337 {
338 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
339 	u8 pending = xc->pending;
340 	u32 hirq;
341 
342 	pr_devel("H_IPOLL(server=%ld)\n", server);
343 
344 	xc->stat_vm_h_ipoll++;
345 
346 	/* Grab the target VCPU if not the current one */
347 	if (xc->server_num != server) {
348 		vcpu = kvmppc_xive_find_server(vcpu->kvm, server);
349 		if (!vcpu)
350 			return H_PARAMETER;
351 		xc = vcpu->arch.xive_vcpu;
352 
353 		/* Scan all priorities */
354 		pending = 0xff;
355 	} else {
356 		/* Grab pending interrupt if any */
357 		__be64 qw1 = __raw_readq(xive_tima + TM_QW1_OS);
358 		u8 pipr = be64_to_cpu(qw1) & 0xff;
359 
360 		if (pipr < 8)
361 			pending |= 1 << pipr;
362 	}
363 
364 	hirq = xive_vm_scan_interrupts(xc, pending, scan_poll);
365 
366 	/* Return interrupt and old CPPR in GPR4 */
367 	vcpu->arch.regs.gpr[4] = hirq | (xc->cppr << 24);
368 
369 	return H_SUCCESS;
370 }
371 
xive_vm_push_pending_to_hw(struct kvmppc_xive_vcpu * xc)372 static void xive_vm_push_pending_to_hw(struct kvmppc_xive_vcpu *xc)
373 {
374 	u8 pending, prio;
375 
376 	pending = xc->pending;
377 	if (xc->mfrr != 0xff) {
378 		if (xc->mfrr < 8)
379 			pending |= 1 << xc->mfrr;
380 		else
381 			pending |= 0x80;
382 	}
383 	if (!pending)
384 		return;
385 	prio = ffs(pending) - 1;
386 
387 	__raw_writeb(prio, xive_tima + TM_SPC_SET_OS_PENDING);
388 }
389 
xive_vm_scan_for_rerouted_irqs(struct kvmppc_xive * xive,struct kvmppc_xive_vcpu * xc)390 static void xive_vm_scan_for_rerouted_irqs(struct kvmppc_xive *xive,
391 					       struct kvmppc_xive_vcpu *xc)
392 {
393 	unsigned int prio;
394 
395 	/* For each priority that is now masked */
396 	for (prio = xc->cppr; prio < KVMPPC_XIVE_Q_COUNT; prio++) {
397 		struct xive_q *q = &xc->queues[prio];
398 		struct kvmppc_xive_irq_state *state;
399 		struct kvmppc_xive_src_block *sb;
400 		u32 idx, toggle, entry, irq, hw_num;
401 		struct xive_irq_data *xd;
402 		__be32 *qpage;
403 		u16 src;
404 
405 		idx = q->idx;
406 		toggle = q->toggle;
407 		qpage = READ_ONCE(q->qpage);
408 		if (!qpage)
409 			continue;
410 
411 		/* For each interrupt in the queue */
412 		for (;;) {
413 			entry = be32_to_cpup(qpage + idx);
414 
415 			/* No more ? */
416 			if ((entry >> 31) == toggle)
417 				break;
418 			irq = entry & 0x7fffffff;
419 
420 			/* Skip dummies and IPIs */
421 			if (irq == XICS_DUMMY || irq == XICS_IPI)
422 				goto next;
423 			sb = kvmppc_xive_find_source(xive, irq, &src);
424 			if (!sb)
425 				goto next;
426 			state = &sb->irq_state[src];
427 
428 			/* Has it been rerouted ? */
429 			if (xc->server_num == state->act_server)
430 				goto next;
431 
432 			/*
433 			 * Allright, it *has* been re-routed, kill it from
434 			 * the queue.
435 			 */
436 			qpage[idx] = cpu_to_be32((entry & 0x80000000) | XICS_DUMMY);
437 
438 			/* Find the HW interrupt */
439 			kvmppc_xive_select_irq(state, &hw_num, &xd);
440 
441 			/* If it's not an LSI, set PQ to 11 the EOI will force a resend */
442 			if (!(xd->flags & XIVE_IRQ_FLAG_LSI))
443 				xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_11);
444 
445 			/* EOI the source */
446 			xive_vm_source_eoi(hw_num, xd);
447 
448 next:
449 			idx = (idx + 1) & q->msk;
450 			if (idx == 0)
451 				toggle ^= 1;
452 		}
453 	}
454 }
455 
xive_vm_h_cppr(struct kvm_vcpu * vcpu,unsigned long cppr)456 static int xive_vm_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr)
457 {
458 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
459 	struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
460 	u8 old_cppr;
461 
462 	pr_devel("H_CPPR(cppr=%ld)\n", cppr);
463 
464 	xc->stat_vm_h_cppr++;
465 
466 	/* Map CPPR */
467 	cppr = xive_prio_from_guest(cppr);
468 
469 	/* Remember old and update SW state */
470 	old_cppr = xc->cppr;
471 	xc->cppr = cppr;
472 
473 	/*
474 	 * Order the above update of xc->cppr with the subsequent
475 	 * read of xc->mfrr inside push_pending_to_hw()
476 	 */
477 	smp_mb();
478 
479 	if (cppr > old_cppr) {
480 		/*
481 		 * We are masking less, we need to look for pending things
482 		 * to deliver and set VP pending bits accordingly to trigger
483 		 * a new interrupt otherwise we might miss MFRR changes for
484 		 * which we have optimized out sending an IPI signal.
485 		 */
486 		xive_vm_push_pending_to_hw(xc);
487 	} else {
488 		/*
489 		 * We are masking more, we need to check the queue for any
490 		 * interrupt that has been routed to another CPU, take
491 		 * it out (replace it with the dummy) and retrigger it.
492 		 *
493 		 * This is necessary since those interrupts may otherwise
494 		 * never be processed, at least not until this CPU restores
495 		 * its CPPR.
496 		 *
497 		 * This is in theory racy vs. HW adding new interrupts to
498 		 * the queue. In practice this works because the interesting
499 		 * cases are when the guest has done a set_xive() to move the
500 		 * interrupt away, which flushes the xive, followed by the
501 		 * target CPU doing a H_CPPR. So any new interrupt coming into
502 		 * the queue must still be routed to us and isn't a source
503 		 * of concern.
504 		 */
505 		xive_vm_scan_for_rerouted_irqs(xive, xc);
506 	}
507 
508 	/* Apply new CPPR */
509 	xc->hw_cppr = cppr;
510 	__raw_writeb(cppr, xive_tima + TM_QW1_OS + TM_CPPR);
511 
512 	return H_SUCCESS;
513 }
514 
xive_vm_h_eoi(struct kvm_vcpu * vcpu,unsigned long xirr)515 static int xive_vm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr)
516 {
517 	struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
518 	struct kvmppc_xive_src_block *sb;
519 	struct kvmppc_xive_irq_state *state;
520 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
521 	struct xive_irq_data *xd;
522 	u8 new_cppr = xirr >> 24;
523 	u32 irq = xirr & 0x00ffffff, hw_num;
524 	u16 src;
525 	int rc = 0;
526 
527 	pr_devel("H_EOI(xirr=%08lx)\n", xirr);
528 
529 	xc->stat_vm_h_eoi++;
530 
531 	xc->cppr = xive_prio_from_guest(new_cppr);
532 
533 	/*
534 	 * IPIs are synthetized from MFRR and thus don't need
535 	 * any special EOI handling. The underlying interrupt
536 	 * used to signal MFRR changes is EOId when fetched from
537 	 * the queue.
538 	 */
539 	if (irq == XICS_IPI || irq == 0) {
540 		/*
541 		 * This barrier orders the setting of xc->cppr vs.
542 		 * subsequent test of xc->mfrr done inside
543 		 * scan_interrupts and push_pending_to_hw
544 		 */
545 		smp_mb();
546 		goto bail;
547 	}
548 
549 	/* Find interrupt source */
550 	sb = kvmppc_xive_find_source(xive, irq, &src);
551 	if (!sb) {
552 		pr_devel(" source not found !\n");
553 		rc = H_PARAMETER;
554 		/* Same as above */
555 		smp_mb();
556 		goto bail;
557 	}
558 	state = &sb->irq_state[src];
559 	kvmppc_xive_select_irq(state, &hw_num, &xd);
560 
561 	state->in_eoi = true;
562 
563 	/*
564 	 * This barrier orders both setting of in_eoi above vs,
565 	 * subsequent test of guest_priority, and the setting
566 	 * of xc->cppr vs. subsequent test of xc->mfrr done inside
567 	 * scan_interrupts and push_pending_to_hw
568 	 */
569 	smp_mb();
570 
571 again:
572 	if (state->guest_priority == MASKED) {
573 		arch_spin_lock(&sb->lock);
574 		if (state->guest_priority != MASKED) {
575 			arch_spin_unlock(&sb->lock);
576 			goto again;
577 		}
578 		pr_devel(" EOI on saved P...\n");
579 
580 		/* Clear old_p, that will cause unmask to perform an EOI */
581 		state->old_p = false;
582 
583 		arch_spin_unlock(&sb->lock);
584 	} else {
585 		pr_devel(" EOI on source...\n");
586 
587 		/* Perform EOI on the source */
588 		xive_vm_source_eoi(hw_num, xd);
589 
590 		/* If it's an emulated LSI, check level and resend */
591 		if (state->lsi && state->asserted)
592 			__raw_writeq(0, __x_trig_page(xd));
593 
594 	}
595 
596 	/*
597 	 * This barrier orders the above guest_priority check
598 	 * and spin_lock/unlock with clearing in_eoi below.
599 	 *
600 	 * It also has to be a full mb() as it must ensure
601 	 * the MMIOs done in source_eoi() are completed before
602 	 * state->in_eoi is visible.
603 	 */
604 	mb();
605 	state->in_eoi = false;
606 bail:
607 
608 	/* Re-evaluate pending IRQs and update HW */
609 	xive_vm_scan_interrupts(xc, xc->pending, scan_eoi);
610 	xive_vm_push_pending_to_hw(xc);
611 	pr_devel(" after scan pending=%02x\n", xc->pending);
612 
613 	/* Apply new CPPR */
614 	xc->hw_cppr = xc->cppr;
615 	__raw_writeb(xc->cppr, xive_tima + TM_QW1_OS + TM_CPPR);
616 
617 	return rc;
618 }
619 
xive_vm_h_ipi(struct kvm_vcpu * vcpu,unsigned long server,unsigned long mfrr)620 static int xive_vm_h_ipi(struct kvm_vcpu *vcpu, unsigned long server,
621 			       unsigned long mfrr)
622 {
623 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
624 
625 	pr_devel("H_IPI(server=%08lx,mfrr=%ld)\n", server, mfrr);
626 
627 	xc->stat_vm_h_ipi++;
628 
629 	/* Find target */
630 	vcpu = kvmppc_xive_find_server(vcpu->kvm, server);
631 	if (!vcpu)
632 		return H_PARAMETER;
633 	xc = vcpu->arch.xive_vcpu;
634 
635 	/* Locklessly write over MFRR */
636 	xc->mfrr = mfrr;
637 
638 	/*
639 	 * The load of xc->cppr below and the subsequent MMIO store
640 	 * to the IPI must happen after the above mfrr update is
641 	 * globally visible so that:
642 	 *
643 	 * - Synchronize with another CPU doing an H_EOI or a H_CPPR
644 	 *   updating xc->cppr then reading xc->mfrr.
645 	 *
646 	 * - The target of the IPI sees the xc->mfrr update
647 	 */
648 	mb();
649 
650 	/* Shoot the IPI if most favored than target cppr */
651 	if (mfrr < xc->cppr)
652 		__raw_writeq(0, __x_trig_page(&xc->vp_ipi_data));
653 
654 	return H_SUCCESS;
655 }
656 
657 /*
658  * We leave a gap of a couple of interrupts in the queue to
659  * account for the IPI and additional safety guard.
660  */
661 #define XIVE_Q_GAP	2
662 
kvmppc_xive_vcpu_has_save_restore(struct kvm_vcpu * vcpu)663 static bool kvmppc_xive_vcpu_has_save_restore(struct kvm_vcpu *vcpu)
664 {
665 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
666 
667 	/* Check enablement at VP level */
668 	return xc->vp_cam & TM_QW1W2_HO;
669 }
670 
kvmppc_xive_check_save_restore(struct kvm_vcpu * vcpu)671 bool kvmppc_xive_check_save_restore(struct kvm_vcpu *vcpu)
672 {
673 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
674 	struct kvmppc_xive *xive = xc->xive;
675 
676 	if (xive->flags & KVMPPC_XIVE_FLAG_SAVE_RESTORE)
677 		return kvmppc_xive_vcpu_has_save_restore(vcpu);
678 
679 	return true;
680 }
681 
682 /*
683  * Push a vcpu's context to the XIVE on guest entry.
684  * This assumes we are in virtual mode (MMU on)
685  */
kvmppc_xive_push_vcpu(struct kvm_vcpu * vcpu)686 void kvmppc_xive_push_vcpu(struct kvm_vcpu *vcpu)
687 {
688 	void __iomem *tima = local_paca->kvm_hstate.xive_tima_virt;
689 	u64 pq;
690 
691 	/*
692 	 * Nothing to do if the platform doesn't have a XIVE
693 	 * or this vCPU doesn't have its own XIVE context
694 	 * (e.g. because it's not using an in-kernel interrupt controller).
695 	 */
696 	if (!tima || !vcpu->arch.xive_cam_word)
697 		return;
698 
699 	eieio();
700 	if (!kvmppc_xive_vcpu_has_save_restore(vcpu))
701 		__raw_writeq(vcpu->arch.xive_saved_state.w01, tima + TM_QW1_OS);
702 	__raw_writel(vcpu->arch.xive_cam_word, tima + TM_QW1_OS + TM_WORD2);
703 	vcpu->arch.xive_pushed = 1;
704 	eieio();
705 
706 	/*
707 	 * We clear the irq_pending flag. There is a small chance of a
708 	 * race vs. the escalation interrupt happening on another
709 	 * processor setting it again, but the only consequence is to
710 	 * cause a spurious wakeup on the next H_CEDE, which is not an
711 	 * issue.
712 	 */
713 	vcpu->arch.irq_pending = 0;
714 
715 	/*
716 	 * In single escalation mode, if the escalation interrupt is
717 	 * on, we mask it.
718 	 */
719 	if (vcpu->arch.xive_esc_on) {
720 		pq = __raw_readq((void __iomem *)(vcpu->arch.xive_esc_vaddr +
721 						  XIVE_ESB_SET_PQ_01));
722 		mb();
723 
724 		/*
725 		 * We have a possible subtle race here: The escalation
726 		 * interrupt might have fired and be on its way to the
727 		 * host queue while we mask it, and if we unmask it
728 		 * early enough (re-cede right away), there is a
729 		 * theoretical possibility that it fires again, thus
730 		 * landing in the target queue more than once which is
731 		 * a big no-no.
732 		 *
733 		 * Fortunately, solving this is rather easy. If the
734 		 * above load setting PQ to 01 returns a previous
735 		 * value where P is set, then we know the escalation
736 		 * interrupt is somewhere on its way to the host. In
737 		 * that case we simply don't clear the xive_esc_on
738 		 * flag below. It will be eventually cleared by the
739 		 * handler for the escalation interrupt.
740 		 *
741 		 * Then, when doing a cede, we check that flag again
742 		 * before re-enabling the escalation interrupt, and if
743 		 * set, we abort the cede.
744 		 */
745 		if (!(pq & XIVE_ESB_VAL_P))
746 			/* Now P is 0, we can clear the flag */
747 			vcpu->arch.xive_esc_on = 0;
748 	}
749 }
750 EXPORT_SYMBOL_GPL(kvmppc_xive_push_vcpu);
751 
752 /*
753  * Pull a vcpu's context from the XIVE on guest exit.
754  * This assumes we are in virtual mode (MMU on)
755  */
kvmppc_xive_pull_vcpu(struct kvm_vcpu * vcpu)756 void kvmppc_xive_pull_vcpu(struct kvm_vcpu *vcpu)
757 {
758 	void __iomem *tima = local_paca->kvm_hstate.xive_tima_virt;
759 
760 	if (!vcpu->arch.xive_pushed)
761 		return;
762 
763 	/*
764 	 * Should not have been pushed if there is no tima
765 	 */
766 	if (WARN_ON(!tima))
767 		return;
768 
769 	eieio();
770 	/* First load to pull the context, we ignore the value */
771 	__raw_readl(tima + TM_SPC_PULL_OS_CTX);
772 	/* Second load to recover the context state (Words 0 and 1) */
773 	if (!kvmppc_xive_vcpu_has_save_restore(vcpu))
774 		vcpu->arch.xive_saved_state.w01 = __raw_readq(tima + TM_QW1_OS);
775 
776 	/* Fixup some of the state for the next load */
777 	vcpu->arch.xive_saved_state.lsmfb = 0;
778 	vcpu->arch.xive_saved_state.ack = 0xff;
779 	vcpu->arch.xive_pushed = 0;
780 	eieio();
781 }
782 EXPORT_SYMBOL_GPL(kvmppc_xive_pull_vcpu);
783 
kvmppc_xive_rearm_escalation(struct kvm_vcpu * vcpu)784 bool kvmppc_xive_rearm_escalation(struct kvm_vcpu *vcpu)
785 {
786 	void __iomem *esc_vaddr = (void __iomem *)vcpu->arch.xive_esc_vaddr;
787 	bool ret = true;
788 
789 	if (!esc_vaddr)
790 		return ret;
791 
792 	/* we are using XIVE with single escalation */
793 
794 	if (vcpu->arch.xive_esc_on) {
795 		/*
796 		 * If we still have a pending escalation, abort the cede,
797 		 * and we must set PQ to 10 rather than 00 so that we don't
798 		 * potentially end up with two entries for the escalation
799 		 * interrupt in the XIVE interrupt queue.  In that case
800 		 * we also don't want to set xive_esc_on to 1 here in
801 		 * case we race with xive_esc_irq().
802 		 */
803 		ret = false;
804 		/*
805 		 * The escalation interrupts are special as we don't EOI them.
806 		 * There is no need to use the load-after-store ordering offset
807 		 * to set PQ to 10 as we won't use StoreEOI.
808 		 */
809 		__raw_readq(esc_vaddr + XIVE_ESB_SET_PQ_10);
810 	} else {
811 		vcpu->arch.xive_esc_on = true;
812 		mb();
813 		__raw_readq(esc_vaddr + XIVE_ESB_SET_PQ_00);
814 	}
815 	mb();
816 
817 	return ret;
818 }
819 EXPORT_SYMBOL_GPL(kvmppc_xive_rearm_escalation);
820 
821 /*
822  * This is a simple trigger for a generic XIVE IRQ. This must
823  * only be called for interrupts that support a trigger page
824  */
xive_irq_trigger(struct xive_irq_data * xd)825 static bool xive_irq_trigger(struct xive_irq_data *xd)
826 {
827 	/* This should be only for MSIs */
828 	if (WARN_ON(xd->flags & XIVE_IRQ_FLAG_LSI))
829 		return false;
830 
831 	/* Those interrupts should always have a trigger page */
832 	if (WARN_ON(!xd->trig_mmio))
833 		return false;
834 
835 	out_be64(xd->trig_mmio, 0);
836 
837 	return true;
838 }
839 
xive_esc_irq(int irq,void * data)840 static irqreturn_t xive_esc_irq(int irq, void *data)
841 {
842 	struct kvm_vcpu *vcpu = data;
843 
844 	vcpu->arch.irq_pending = 1;
845 	smp_mb();
846 	if (vcpu->arch.ceded || vcpu->arch.nested)
847 		kvmppc_fast_vcpu_kick(vcpu);
848 
849 	/* Since we have the no-EOI flag, the interrupt is effectively
850 	 * disabled now. Clearing xive_esc_on means we won't bother
851 	 * doing so on the next entry.
852 	 *
853 	 * This also allows the entry code to know that if a PQ combination
854 	 * of 10 is observed while xive_esc_on is true, it means the queue
855 	 * contains an unprocessed escalation interrupt. We don't make use of
856 	 * that knowledge today but might (see comment in book3s_hv_rmhandler.S)
857 	 */
858 	vcpu->arch.xive_esc_on = false;
859 
860 	/* This orders xive_esc_on = false vs. subsequent stale_p = true */
861 	smp_wmb();	/* goes with smp_mb() in cleanup_single_escalation */
862 
863 	return IRQ_HANDLED;
864 }
865 
kvmppc_xive_attach_escalation(struct kvm_vcpu * vcpu,u8 prio,bool single_escalation)866 int kvmppc_xive_attach_escalation(struct kvm_vcpu *vcpu, u8 prio,
867 				  bool single_escalation)
868 {
869 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
870 	struct xive_q *q = &xc->queues[prio];
871 	char *name = NULL;
872 	int rc;
873 
874 	/* Already there ? */
875 	if (xc->esc_virq[prio])
876 		return 0;
877 
878 	/* Hook up the escalation interrupt */
879 	xc->esc_virq[prio] = irq_create_mapping(NULL, q->esc_irq);
880 	if (!xc->esc_virq[prio]) {
881 		pr_err("Failed to map escalation interrupt for queue %d of VCPU %d\n",
882 		       prio, xc->server_num);
883 		return -EIO;
884 	}
885 
886 	if (single_escalation)
887 		name = kasprintf(GFP_KERNEL, "kvm-%d-%d",
888 				 vcpu->kvm->arch.lpid, xc->server_num);
889 	else
890 		name = kasprintf(GFP_KERNEL, "kvm-%d-%d-%d",
891 				 vcpu->kvm->arch.lpid, xc->server_num, prio);
892 	if (!name) {
893 		pr_err("Failed to allocate escalation irq name for queue %d of VCPU %d\n",
894 		       prio, xc->server_num);
895 		rc = -ENOMEM;
896 		goto error;
897 	}
898 
899 	pr_devel("Escalation %s irq %d (prio %d)\n", name, xc->esc_virq[prio], prio);
900 
901 	rc = request_irq(xc->esc_virq[prio], xive_esc_irq,
902 			 IRQF_NO_THREAD, name, vcpu);
903 	if (rc) {
904 		pr_err("Failed to request escalation interrupt for queue %d of VCPU %d\n",
905 		       prio, xc->server_num);
906 		goto error;
907 	}
908 	xc->esc_virq_names[prio] = name;
909 
910 	/* In single escalation mode, we grab the ESB MMIO of the
911 	 * interrupt and mask it. Also populate the VCPU v/raddr
912 	 * of the ESB page for use by asm entry/exit code. Finally
913 	 * set the XIVE_IRQ_FLAG_NO_EOI flag which will prevent the
914 	 * core code from performing an EOI on the escalation
915 	 * interrupt, thus leaving it effectively masked after
916 	 * it fires once.
917 	 */
918 	if (single_escalation) {
919 		struct irq_data *d = irq_get_irq_data(xc->esc_virq[prio]);
920 		struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
921 
922 		xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01);
923 		vcpu->arch.xive_esc_raddr = xd->eoi_page;
924 		vcpu->arch.xive_esc_vaddr = (__force u64)xd->eoi_mmio;
925 		xd->flags |= XIVE_IRQ_FLAG_NO_EOI;
926 	}
927 
928 	return 0;
929 error:
930 	irq_dispose_mapping(xc->esc_virq[prio]);
931 	xc->esc_virq[prio] = 0;
932 	kfree(name);
933 	return rc;
934 }
935 
xive_provision_queue(struct kvm_vcpu * vcpu,u8 prio)936 static int xive_provision_queue(struct kvm_vcpu *vcpu, u8 prio)
937 {
938 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
939 	struct kvmppc_xive *xive = xc->xive;
940 	struct xive_q *q =  &xc->queues[prio];
941 	void *qpage;
942 	int rc;
943 
944 	if (WARN_ON(q->qpage))
945 		return 0;
946 
947 	/* Allocate the queue and retrieve infos on current node for now */
948 	qpage = (__be32 *)__get_free_pages(GFP_KERNEL, xive->q_page_order);
949 	if (!qpage) {
950 		pr_err("Failed to allocate queue %d for VCPU %d\n",
951 		       prio, xc->server_num);
952 		return -ENOMEM;
953 	}
954 	memset(qpage, 0, 1 << xive->q_order);
955 
956 	/*
957 	 * Reconfigure the queue. This will set q->qpage only once the
958 	 * queue is fully configured. This is a requirement for prio 0
959 	 * as we will stop doing EOIs for every IPI as soon as we observe
960 	 * qpage being non-NULL, and instead will only EOI when we receive
961 	 * corresponding queue 0 entries
962 	 */
963 	rc = xive_native_configure_queue(xc->vp_id, q, prio, qpage,
964 					 xive->q_order, true);
965 	if (rc)
966 		pr_err("Failed to configure queue %d for VCPU %d\n",
967 		       prio, xc->server_num);
968 	return rc;
969 }
970 
971 /* Called with xive->lock held */
xive_check_provisioning(struct kvm * kvm,u8 prio)972 static int xive_check_provisioning(struct kvm *kvm, u8 prio)
973 {
974 	struct kvmppc_xive *xive = kvm->arch.xive;
975 	struct kvm_vcpu *vcpu;
976 	unsigned long i;
977 	int rc;
978 
979 	lockdep_assert_held(&xive->lock);
980 
981 	/* Already provisioned ? */
982 	if (xive->qmap & (1 << prio))
983 		return 0;
984 
985 	pr_devel("Provisioning prio... %d\n", prio);
986 
987 	/* Provision each VCPU and enable escalations if needed */
988 	kvm_for_each_vcpu(i, vcpu, kvm) {
989 		if (!vcpu->arch.xive_vcpu)
990 			continue;
991 		rc = xive_provision_queue(vcpu, prio);
992 		if (rc == 0 && !kvmppc_xive_has_single_escalation(xive))
993 			kvmppc_xive_attach_escalation(vcpu, prio,
994 						      kvmppc_xive_has_single_escalation(xive));
995 		if (rc)
996 			return rc;
997 	}
998 
999 	/* Order previous stores and mark it as provisioned */
1000 	mb();
1001 	xive->qmap |= (1 << prio);
1002 	return 0;
1003 }
1004 
xive_inc_q_pending(struct kvm * kvm,u32 server,u8 prio)1005 static void xive_inc_q_pending(struct kvm *kvm, u32 server, u8 prio)
1006 {
1007 	struct kvm_vcpu *vcpu;
1008 	struct kvmppc_xive_vcpu *xc;
1009 	struct xive_q *q;
1010 
1011 	/* Locate target server */
1012 	vcpu = kvmppc_xive_find_server(kvm, server);
1013 	if (!vcpu) {
1014 		pr_warn("%s: Can't find server %d\n", __func__, server);
1015 		return;
1016 	}
1017 	xc = vcpu->arch.xive_vcpu;
1018 	if (WARN_ON(!xc))
1019 		return;
1020 
1021 	q = &xc->queues[prio];
1022 	atomic_inc(&q->pending_count);
1023 }
1024 
xive_try_pick_queue(struct kvm_vcpu * vcpu,u8 prio)1025 static int xive_try_pick_queue(struct kvm_vcpu *vcpu, u8 prio)
1026 {
1027 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1028 	struct xive_q *q;
1029 	u32 max;
1030 
1031 	if (WARN_ON(!xc))
1032 		return -ENXIO;
1033 	if (!xc->valid)
1034 		return -ENXIO;
1035 
1036 	q = &xc->queues[prio];
1037 	if (WARN_ON(!q->qpage))
1038 		return -ENXIO;
1039 
1040 	/* Calculate max number of interrupts in that queue. */
1041 	max = (q->msk + 1) - XIVE_Q_GAP;
1042 	return atomic_add_unless(&q->count, 1, max) ? 0 : -EBUSY;
1043 }
1044 
kvmppc_xive_select_target(struct kvm * kvm,u32 * server,u8 prio)1045 int kvmppc_xive_select_target(struct kvm *kvm, u32 *server, u8 prio)
1046 {
1047 	struct kvm_vcpu *vcpu;
1048 	unsigned long i;
1049 	int rc;
1050 
1051 	/* Locate target server */
1052 	vcpu = kvmppc_xive_find_server(kvm, *server);
1053 	if (!vcpu) {
1054 		pr_devel("Can't find server %d\n", *server);
1055 		return -EINVAL;
1056 	}
1057 
1058 	pr_devel("Finding irq target on 0x%x/%d...\n", *server, prio);
1059 
1060 	/* Try pick it */
1061 	rc = xive_try_pick_queue(vcpu, prio);
1062 	if (rc == 0)
1063 		return rc;
1064 
1065 	pr_devel(" .. failed, looking up candidate...\n");
1066 
1067 	/* Failed, pick another VCPU */
1068 	kvm_for_each_vcpu(i, vcpu, kvm) {
1069 		if (!vcpu->arch.xive_vcpu)
1070 			continue;
1071 		rc = xive_try_pick_queue(vcpu, prio);
1072 		if (rc == 0) {
1073 			*server = vcpu->arch.xive_vcpu->server_num;
1074 			pr_devel("  found on 0x%x/%d\n", *server, prio);
1075 			return rc;
1076 		}
1077 	}
1078 	pr_devel("  no available target !\n");
1079 
1080 	/* No available target ! */
1081 	return -EBUSY;
1082 }
1083 
xive_lock_and_mask(struct kvmppc_xive * xive,struct kvmppc_xive_src_block * sb,struct kvmppc_xive_irq_state * state)1084 static u8 xive_lock_and_mask(struct kvmppc_xive *xive,
1085 			     struct kvmppc_xive_src_block *sb,
1086 			     struct kvmppc_xive_irq_state *state)
1087 {
1088 	struct xive_irq_data *xd;
1089 	u32 hw_num;
1090 	u8 old_prio;
1091 	u64 val;
1092 
1093 	/*
1094 	 * Take the lock, set masked, try again if racing
1095 	 * with H_EOI
1096 	 */
1097 	for (;;) {
1098 		arch_spin_lock(&sb->lock);
1099 		old_prio = state->guest_priority;
1100 		state->guest_priority = MASKED;
1101 		mb();
1102 		if (!state->in_eoi)
1103 			break;
1104 		state->guest_priority = old_prio;
1105 		arch_spin_unlock(&sb->lock);
1106 	}
1107 
1108 	/* No change ? Bail */
1109 	if (old_prio == MASKED)
1110 		return old_prio;
1111 
1112 	/* Get the right irq */
1113 	kvmppc_xive_select_irq(state, &hw_num, &xd);
1114 
1115 	/* Set PQ to 10, return old P and old Q and remember them */
1116 	val = xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_10);
1117 	state->old_p = !!(val & 2);
1118 	state->old_q = !!(val & 1);
1119 
1120 	/*
1121 	 * Synchronize hardware to sensure the queues are updated when
1122 	 * masking
1123 	 */
1124 	xive_native_sync_source(hw_num);
1125 
1126 	return old_prio;
1127 }
1128 
xive_lock_for_unmask(struct kvmppc_xive_src_block * sb,struct kvmppc_xive_irq_state * state)1129 static void xive_lock_for_unmask(struct kvmppc_xive_src_block *sb,
1130 				 struct kvmppc_xive_irq_state *state)
1131 {
1132 	/*
1133 	 * Take the lock try again if racing with H_EOI
1134 	 */
1135 	for (;;) {
1136 		arch_spin_lock(&sb->lock);
1137 		if (!state->in_eoi)
1138 			break;
1139 		arch_spin_unlock(&sb->lock);
1140 	}
1141 }
1142 
xive_finish_unmask(struct kvmppc_xive * xive,struct kvmppc_xive_src_block * sb,struct kvmppc_xive_irq_state * state,u8 prio)1143 static void xive_finish_unmask(struct kvmppc_xive *xive,
1144 			       struct kvmppc_xive_src_block *sb,
1145 			       struct kvmppc_xive_irq_state *state,
1146 			       u8 prio)
1147 {
1148 	struct xive_irq_data *xd;
1149 	u32 hw_num;
1150 
1151 	/* If we aren't changing a thing, move on */
1152 	if (state->guest_priority != MASKED)
1153 		goto bail;
1154 
1155 	/* Get the right irq */
1156 	kvmppc_xive_select_irq(state, &hw_num, &xd);
1157 
1158 	/* Old Q set, set PQ to 11 */
1159 	if (state->old_q)
1160 		xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_11);
1161 
1162 	/*
1163 	 * If not old P, then perform an "effective" EOI,
1164 	 * on the source. This will handle the cases where
1165 	 * FW EOI is needed.
1166 	 */
1167 	if (!state->old_p)
1168 		xive_vm_source_eoi(hw_num, xd);
1169 
1170 	/* Synchronize ordering and mark unmasked */
1171 	mb();
1172 bail:
1173 	state->guest_priority = prio;
1174 }
1175 
1176 /*
1177  * Target an interrupt to a given server/prio, this will fallback
1178  * to another server if necessary and perform the HW targetting
1179  * updates as needed
1180  *
1181  * NOTE: Must be called with the state lock held
1182  */
xive_target_interrupt(struct kvm * kvm,struct kvmppc_xive_irq_state * state,u32 server,u8 prio)1183 static int xive_target_interrupt(struct kvm *kvm,
1184 				 struct kvmppc_xive_irq_state *state,
1185 				 u32 server, u8 prio)
1186 {
1187 	struct kvmppc_xive *xive = kvm->arch.xive;
1188 	u32 hw_num;
1189 	int rc;
1190 
1191 	/*
1192 	 * This will return a tentative server and actual
1193 	 * priority. The count for that new target will have
1194 	 * already been incremented.
1195 	 */
1196 	rc = kvmppc_xive_select_target(kvm, &server, prio);
1197 
1198 	/*
1199 	 * We failed to find a target ? Not much we can do
1200 	 * at least until we support the GIQ.
1201 	 */
1202 	if (rc)
1203 		return rc;
1204 
1205 	/*
1206 	 * Increment the old queue pending count if there
1207 	 * was one so that the old queue count gets adjusted later
1208 	 * when observed to be empty.
1209 	 */
1210 	if (state->act_priority != MASKED)
1211 		xive_inc_q_pending(kvm,
1212 				   state->act_server,
1213 				   state->act_priority);
1214 	/*
1215 	 * Update state and HW
1216 	 */
1217 	state->act_priority = prio;
1218 	state->act_server = server;
1219 
1220 	/* Get the right irq */
1221 	kvmppc_xive_select_irq(state, &hw_num, NULL);
1222 
1223 	return xive_native_configure_irq(hw_num,
1224 					 kvmppc_xive_vp(xive, server),
1225 					 prio, state->number);
1226 }
1227 
1228 /*
1229  * Targetting rules: In order to avoid losing track of
1230  * pending interrupts across mask and unmask, which would
1231  * allow queue overflows, we implement the following rules:
1232  *
1233  *  - Unless it was never enabled (or we run out of capacity)
1234  *    an interrupt is always targetted at a valid server/queue
1235  *    pair even when "masked" by the guest. This pair tends to
1236  *    be the last one used but it can be changed under some
1237  *    circumstances. That allows us to separate targetting
1238  *    from masking, we only handle accounting during (re)targetting,
1239  *    this also allows us to let an interrupt drain into its target
1240  *    queue after masking, avoiding complex schemes to remove
1241  *    interrupts out of remote processor queues.
1242  *
1243  *  - When masking, we set PQ to 10 and save the previous value
1244  *    of P and Q.
1245  *
1246  *  - When unmasking, if saved Q was set, we set PQ to 11
1247  *    otherwise we leave PQ to the HW state which will be either
1248  *    10 if nothing happened or 11 if the interrupt fired while
1249  *    masked. Effectively we are OR'ing the previous Q into the
1250  *    HW Q.
1251  *
1252  *    Then if saved P is clear, we do an effective EOI (Q->P->Trigger)
1253  *    which will unmask the interrupt and shoot a new one if Q was
1254  *    set.
1255  *
1256  *    Otherwise (saved P is set) we leave PQ unchanged (so 10 or 11,
1257  *    effectively meaning an H_EOI from the guest is still expected
1258  *    for that interrupt).
1259  *
1260  *  - If H_EOI occurs while masked, we clear the saved P.
1261  *
1262  *  - When changing target, we account on the new target and
1263  *    increment a separate "pending" counter on the old one.
1264  *    This pending counter will be used to decrement the old
1265  *    target's count when its queue has been observed empty.
1266  */
1267 
kvmppc_xive_set_xive(struct kvm * kvm,u32 irq,u32 server,u32 priority)1268 int kvmppc_xive_set_xive(struct kvm *kvm, u32 irq, u32 server,
1269 			 u32 priority)
1270 {
1271 	struct kvmppc_xive *xive = kvm->arch.xive;
1272 	struct kvmppc_xive_src_block *sb;
1273 	struct kvmppc_xive_irq_state *state;
1274 	u8 new_act_prio;
1275 	int rc = 0;
1276 	u16 idx;
1277 
1278 	if (!xive)
1279 		return -ENODEV;
1280 
1281 	pr_devel("set_xive ! irq 0x%x server 0x%x prio %d\n",
1282 		 irq, server, priority);
1283 
1284 	/* First, check provisioning of queues */
1285 	if (priority != MASKED) {
1286 		mutex_lock(&xive->lock);
1287 		rc = xive_check_provisioning(xive->kvm,
1288 			      xive_prio_from_guest(priority));
1289 		mutex_unlock(&xive->lock);
1290 	}
1291 	if (rc) {
1292 		pr_devel("  provisioning failure %d !\n", rc);
1293 		return rc;
1294 	}
1295 
1296 	sb = kvmppc_xive_find_source(xive, irq, &idx);
1297 	if (!sb)
1298 		return -EINVAL;
1299 	state = &sb->irq_state[idx];
1300 
1301 	/*
1302 	 * We first handle masking/unmasking since the locking
1303 	 * might need to be retried due to EOIs, we'll handle
1304 	 * targetting changes later. These functions will return
1305 	 * with the SB lock held.
1306 	 *
1307 	 * xive_lock_and_mask() will also set state->guest_priority
1308 	 * but won't otherwise change other fields of the state.
1309 	 *
1310 	 * xive_lock_for_unmask will not actually unmask, this will
1311 	 * be done later by xive_finish_unmask() once the targetting
1312 	 * has been done, so we don't try to unmask an interrupt
1313 	 * that hasn't yet been targetted.
1314 	 */
1315 	if (priority == MASKED)
1316 		xive_lock_and_mask(xive, sb, state);
1317 	else
1318 		xive_lock_for_unmask(sb, state);
1319 
1320 
1321 	/*
1322 	 * Then we handle targetting.
1323 	 *
1324 	 * First calculate a new "actual priority"
1325 	 */
1326 	new_act_prio = state->act_priority;
1327 	if (priority != MASKED)
1328 		new_act_prio = xive_prio_from_guest(priority);
1329 
1330 	pr_devel(" new_act_prio=%x act_server=%x act_prio=%x\n",
1331 		 new_act_prio, state->act_server, state->act_priority);
1332 
1333 	/*
1334 	 * Then check if we actually need to change anything,
1335 	 *
1336 	 * The condition for re-targetting the interrupt is that
1337 	 * we have a valid new priority (new_act_prio is not 0xff)
1338 	 * and either the server or the priority changed.
1339 	 *
1340 	 * Note: If act_priority was ff and the new priority is
1341 	 *       also ff, we don't do anything and leave the interrupt
1342 	 *       untargetted. An attempt of doing an int_on on an
1343 	 *       untargetted interrupt will fail. If that is a problem
1344 	 *       we could initialize interrupts with valid default
1345 	 */
1346 
1347 	if (new_act_prio != MASKED &&
1348 	    (state->act_server != server ||
1349 	     state->act_priority != new_act_prio))
1350 		rc = xive_target_interrupt(kvm, state, server, new_act_prio);
1351 
1352 	/*
1353 	 * Perform the final unmasking of the interrupt source
1354 	 * if necessary
1355 	 */
1356 	if (priority != MASKED)
1357 		xive_finish_unmask(xive, sb, state, priority);
1358 
1359 	/*
1360 	 * Finally Update saved_priority to match. Only int_on/off
1361 	 * set this field to a different value.
1362 	 */
1363 	state->saved_priority = priority;
1364 
1365 	arch_spin_unlock(&sb->lock);
1366 	return rc;
1367 }
1368 
kvmppc_xive_get_xive(struct kvm * kvm,u32 irq,u32 * server,u32 * priority)1369 int kvmppc_xive_get_xive(struct kvm *kvm, u32 irq, u32 *server,
1370 			 u32 *priority)
1371 {
1372 	struct kvmppc_xive *xive = kvm->arch.xive;
1373 	struct kvmppc_xive_src_block *sb;
1374 	struct kvmppc_xive_irq_state *state;
1375 	u16 idx;
1376 
1377 	if (!xive)
1378 		return -ENODEV;
1379 
1380 	sb = kvmppc_xive_find_source(xive, irq, &idx);
1381 	if (!sb)
1382 		return -EINVAL;
1383 	state = &sb->irq_state[idx];
1384 	arch_spin_lock(&sb->lock);
1385 	*server = state->act_server;
1386 	*priority = state->guest_priority;
1387 	arch_spin_unlock(&sb->lock);
1388 
1389 	return 0;
1390 }
1391 
kvmppc_xive_int_on(struct kvm * kvm,u32 irq)1392 int kvmppc_xive_int_on(struct kvm *kvm, u32 irq)
1393 {
1394 	struct kvmppc_xive *xive = kvm->arch.xive;
1395 	struct kvmppc_xive_src_block *sb;
1396 	struct kvmppc_xive_irq_state *state;
1397 	u16 idx;
1398 
1399 	if (!xive)
1400 		return -ENODEV;
1401 
1402 	sb = kvmppc_xive_find_source(xive, irq, &idx);
1403 	if (!sb)
1404 		return -EINVAL;
1405 	state = &sb->irq_state[idx];
1406 
1407 	pr_devel("int_on(irq=0x%x)\n", irq);
1408 
1409 	/*
1410 	 * Check if interrupt was not targetted
1411 	 */
1412 	if (state->act_priority == MASKED) {
1413 		pr_devel("int_on on untargetted interrupt\n");
1414 		return -EINVAL;
1415 	}
1416 
1417 	/* If saved_priority is 0xff, do nothing */
1418 	if (state->saved_priority == MASKED)
1419 		return 0;
1420 
1421 	/*
1422 	 * Lock and unmask it.
1423 	 */
1424 	xive_lock_for_unmask(sb, state);
1425 	xive_finish_unmask(xive, sb, state, state->saved_priority);
1426 	arch_spin_unlock(&sb->lock);
1427 
1428 	return 0;
1429 }
1430 
kvmppc_xive_int_off(struct kvm * kvm,u32 irq)1431 int kvmppc_xive_int_off(struct kvm *kvm, u32 irq)
1432 {
1433 	struct kvmppc_xive *xive = kvm->arch.xive;
1434 	struct kvmppc_xive_src_block *sb;
1435 	struct kvmppc_xive_irq_state *state;
1436 	u16 idx;
1437 
1438 	if (!xive)
1439 		return -ENODEV;
1440 
1441 	sb = kvmppc_xive_find_source(xive, irq, &idx);
1442 	if (!sb)
1443 		return -EINVAL;
1444 	state = &sb->irq_state[idx];
1445 
1446 	pr_devel("int_off(irq=0x%x)\n", irq);
1447 
1448 	/*
1449 	 * Lock and mask
1450 	 */
1451 	state->saved_priority = xive_lock_and_mask(xive, sb, state);
1452 	arch_spin_unlock(&sb->lock);
1453 
1454 	return 0;
1455 }
1456 
xive_restore_pending_irq(struct kvmppc_xive * xive,u32 irq)1457 static bool xive_restore_pending_irq(struct kvmppc_xive *xive, u32 irq)
1458 {
1459 	struct kvmppc_xive_src_block *sb;
1460 	struct kvmppc_xive_irq_state *state;
1461 	u16 idx;
1462 
1463 	sb = kvmppc_xive_find_source(xive, irq, &idx);
1464 	if (!sb)
1465 		return false;
1466 	state = &sb->irq_state[idx];
1467 	if (!state->valid)
1468 		return false;
1469 
1470 	/*
1471 	 * Trigger the IPI. This assumes we never restore a pass-through
1472 	 * interrupt which should be safe enough
1473 	 */
1474 	xive_irq_trigger(&state->ipi_data);
1475 
1476 	return true;
1477 }
1478 
kvmppc_xive_get_icp(struct kvm_vcpu * vcpu)1479 u64 kvmppc_xive_get_icp(struct kvm_vcpu *vcpu)
1480 {
1481 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1482 
1483 	if (!xc)
1484 		return 0;
1485 
1486 	/* Return the per-cpu state for state saving/migration */
1487 	return (u64)xc->cppr << KVM_REG_PPC_ICP_CPPR_SHIFT |
1488 	       (u64)xc->mfrr << KVM_REG_PPC_ICP_MFRR_SHIFT |
1489 	       (u64)0xff << KVM_REG_PPC_ICP_PPRI_SHIFT;
1490 }
1491 
kvmppc_xive_set_icp(struct kvm_vcpu * vcpu,u64 icpval)1492 int kvmppc_xive_set_icp(struct kvm_vcpu *vcpu, u64 icpval)
1493 {
1494 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1495 	struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
1496 	u8 cppr, mfrr;
1497 	u32 xisr;
1498 
1499 	if (!xc || !xive)
1500 		return -ENOENT;
1501 
1502 	/* Grab individual state fields. We don't use pending_pri */
1503 	cppr = icpval >> KVM_REG_PPC_ICP_CPPR_SHIFT;
1504 	xisr = (icpval >> KVM_REG_PPC_ICP_XISR_SHIFT) &
1505 		KVM_REG_PPC_ICP_XISR_MASK;
1506 	mfrr = icpval >> KVM_REG_PPC_ICP_MFRR_SHIFT;
1507 
1508 	pr_devel("set_icp vcpu %d cppr=0x%x mfrr=0x%x xisr=0x%x\n",
1509 		 xc->server_num, cppr, mfrr, xisr);
1510 
1511 	/*
1512 	 * We can't update the state of a "pushed" VCPU, but that
1513 	 * shouldn't happen because the vcpu->mutex makes running a
1514 	 * vcpu mutually exclusive with doing one_reg get/set on it.
1515 	 */
1516 	if (WARN_ON(vcpu->arch.xive_pushed))
1517 		return -EIO;
1518 
1519 	/* Update VCPU HW saved state */
1520 	vcpu->arch.xive_saved_state.cppr = cppr;
1521 	xc->hw_cppr = xc->cppr = cppr;
1522 
1523 	/*
1524 	 * Update MFRR state. If it's not 0xff, we mark the VCPU as
1525 	 * having a pending MFRR change, which will re-evaluate the
1526 	 * target. The VCPU will thus potentially get a spurious
1527 	 * interrupt but that's not a big deal.
1528 	 */
1529 	xc->mfrr = mfrr;
1530 	if (mfrr < cppr)
1531 		xive_irq_trigger(&xc->vp_ipi_data);
1532 
1533 	/*
1534 	 * Now saved XIRR is "interesting". It means there's something in
1535 	 * the legacy "1 element" queue... for an IPI we simply ignore it,
1536 	 * as the MFRR restore will handle that. For anything else we need
1537 	 * to force a resend of the source.
1538 	 * However the source may not have been setup yet. If that's the
1539 	 * case, we keep that info and increment a counter in the xive to
1540 	 * tell subsequent xive_set_source() to go look.
1541 	 */
1542 	if (xisr > XICS_IPI && !xive_restore_pending_irq(xive, xisr)) {
1543 		xc->delayed_irq = xisr;
1544 		xive->delayed_irqs++;
1545 		pr_devel("  xisr restore delayed\n");
1546 	}
1547 
1548 	return 0;
1549 }
1550 
kvmppc_xive_set_mapped(struct kvm * kvm,unsigned long guest_irq,unsigned long host_irq)1551 int kvmppc_xive_set_mapped(struct kvm *kvm, unsigned long guest_irq,
1552 			   unsigned long host_irq)
1553 {
1554 	struct kvmppc_xive *xive = kvm->arch.xive;
1555 	struct kvmppc_xive_src_block *sb;
1556 	struct kvmppc_xive_irq_state *state;
1557 	struct irq_data *host_data =
1558 		irq_domain_get_irq_data(irq_get_default_host(), host_irq);
1559 	unsigned int hw_irq = (unsigned int)irqd_to_hwirq(host_data);
1560 	u16 idx;
1561 	u8 prio;
1562 	int rc;
1563 
1564 	if (!xive)
1565 		return -ENODEV;
1566 
1567 	pr_debug("%s: GIRQ 0x%lx host IRQ %ld XIVE HW IRQ 0x%x\n",
1568 		 __func__, guest_irq, host_irq, hw_irq);
1569 
1570 	sb = kvmppc_xive_find_source(xive, guest_irq, &idx);
1571 	if (!sb)
1572 		return -EINVAL;
1573 	state = &sb->irq_state[idx];
1574 
1575 	/*
1576 	 * Mark the passed-through interrupt as going to a VCPU,
1577 	 * this will prevent further EOIs and similar operations
1578 	 * from the XIVE code. It will also mask the interrupt
1579 	 * to either PQ=10 or 11 state, the latter if the interrupt
1580 	 * is pending. This will allow us to unmask or retrigger it
1581 	 * after routing it to the guest with a simple EOI.
1582 	 *
1583 	 * The "state" argument is a "token", all it needs is to be
1584 	 * non-NULL to switch to passed-through or NULL for the
1585 	 * other way around. We may not yet have an actual VCPU
1586 	 * target here and we don't really care.
1587 	 */
1588 	rc = irq_set_vcpu_affinity(host_irq, state);
1589 	if (rc) {
1590 		pr_err("Failed to set VCPU affinity for host IRQ %ld\n", host_irq);
1591 		return rc;
1592 	}
1593 
1594 	/*
1595 	 * Mask and read state of IPI. We need to know if its P bit
1596 	 * is set as that means it's potentially already using a
1597 	 * queue entry in the target
1598 	 */
1599 	prio = xive_lock_and_mask(xive, sb, state);
1600 	pr_devel(" old IPI prio %02x P:%d Q:%d\n", prio,
1601 		 state->old_p, state->old_q);
1602 
1603 	/* Turn the IPI hard off */
1604 	xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
1605 
1606 	/*
1607 	 * Reset ESB guest mapping. Needed when ESB pages are exposed
1608 	 * to the guest in XIVE native mode
1609 	 */
1610 	if (xive->ops && xive->ops->reset_mapped)
1611 		xive->ops->reset_mapped(kvm, guest_irq);
1612 
1613 	/* Grab info about irq */
1614 	state->pt_number = hw_irq;
1615 	state->pt_data = irq_data_get_irq_handler_data(host_data);
1616 
1617 	/*
1618 	 * Configure the IRQ to match the existing configuration of
1619 	 * the IPI if it was already targetted. Otherwise this will
1620 	 * mask the interrupt in a lossy way (act_priority is 0xff)
1621 	 * which is fine for a never started interrupt.
1622 	 */
1623 	xive_native_configure_irq(hw_irq,
1624 				  kvmppc_xive_vp(xive, state->act_server),
1625 				  state->act_priority, state->number);
1626 
1627 	/*
1628 	 * We do an EOI to enable the interrupt (and retrigger if needed)
1629 	 * if the guest has the interrupt unmasked and the P bit was *not*
1630 	 * set in the IPI. If it was set, we know a slot may still be in
1631 	 * use in the target queue thus we have to wait for a guest
1632 	 * originated EOI
1633 	 */
1634 	if (prio != MASKED && !state->old_p)
1635 		xive_vm_source_eoi(hw_irq, state->pt_data);
1636 
1637 	/* Clear old_p/old_q as they are no longer relevant */
1638 	state->old_p = state->old_q = false;
1639 
1640 	/* Restore guest prio (unlocks EOI) */
1641 	mb();
1642 	state->guest_priority = prio;
1643 	arch_spin_unlock(&sb->lock);
1644 
1645 	return 0;
1646 }
1647 EXPORT_SYMBOL_GPL(kvmppc_xive_set_mapped);
1648 
kvmppc_xive_clr_mapped(struct kvm * kvm,unsigned long guest_irq,unsigned long host_irq)1649 int kvmppc_xive_clr_mapped(struct kvm *kvm, unsigned long guest_irq,
1650 			   unsigned long host_irq)
1651 {
1652 	struct kvmppc_xive *xive = kvm->arch.xive;
1653 	struct kvmppc_xive_src_block *sb;
1654 	struct kvmppc_xive_irq_state *state;
1655 	u16 idx;
1656 	u8 prio;
1657 	int rc;
1658 
1659 	if (!xive)
1660 		return -ENODEV;
1661 
1662 	pr_debug("%s: GIRQ 0x%lx host IRQ %ld\n", __func__, guest_irq, host_irq);
1663 
1664 	sb = kvmppc_xive_find_source(xive, guest_irq, &idx);
1665 	if (!sb)
1666 		return -EINVAL;
1667 	state = &sb->irq_state[idx];
1668 
1669 	/*
1670 	 * Mask and read state of IRQ. We need to know if its P bit
1671 	 * is set as that means it's potentially already using a
1672 	 * queue entry in the target
1673 	 */
1674 	prio = xive_lock_and_mask(xive, sb, state);
1675 	pr_devel(" old IRQ prio %02x P:%d Q:%d\n", prio,
1676 		 state->old_p, state->old_q);
1677 
1678 	/*
1679 	 * If old_p is set, the interrupt is pending, we switch it to
1680 	 * PQ=11. This will force a resend in the host so the interrupt
1681 	 * isn't lost to whatever host driver may pick it up
1682 	 */
1683 	if (state->old_p)
1684 		xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_11);
1685 
1686 	/* Release the passed-through interrupt to the host */
1687 	rc = irq_set_vcpu_affinity(host_irq, NULL);
1688 	if (rc) {
1689 		pr_err("Failed to clr VCPU affinity for host IRQ %ld\n", host_irq);
1690 		return rc;
1691 	}
1692 
1693 	/* Forget about the IRQ */
1694 	state->pt_number = 0;
1695 	state->pt_data = NULL;
1696 
1697 	/*
1698 	 * Reset ESB guest mapping. Needed when ESB pages are exposed
1699 	 * to the guest in XIVE native mode
1700 	 */
1701 	if (xive->ops && xive->ops->reset_mapped) {
1702 		xive->ops->reset_mapped(kvm, guest_irq);
1703 	}
1704 
1705 	/* Reconfigure the IPI */
1706 	xive_native_configure_irq(state->ipi_number,
1707 				  kvmppc_xive_vp(xive, state->act_server),
1708 				  state->act_priority, state->number);
1709 
1710 	/*
1711 	 * If old_p is set (we have a queue entry potentially
1712 	 * occupied) or the interrupt is masked, we set the IPI
1713 	 * to PQ=10 state. Otherwise we just re-enable it (PQ=00).
1714 	 */
1715 	if (prio == MASKED || state->old_p)
1716 		xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_10);
1717 	else
1718 		xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_00);
1719 
1720 	/* Restore guest prio (unlocks EOI) */
1721 	mb();
1722 	state->guest_priority = prio;
1723 	arch_spin_unlock(&sb->lock);
1724 
1725 	return 0;
1726 }
1727 EXPORT_SYMBOL_GPL(kvmppc_xive_clr_mapped);
1728 
kvmppc_xive_disable_vcpu_interrupts(struct kvm_vcpu * vcpu)1729 void kvmppc_xive_disable_vcpu_interrupts(struct kvm_vcpu *vcpu)
1730 {
1731 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1732 	struct kvm *kvm = vcpu->kvm;
1733 	struct kvmppc_xive *xive = kvm->arch.xive;
1734 	int i, j;
1735 
1736 	for (i = 0; i <= xive->max_sbid; i++) {
1737 		struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
1738 
1739 		if (!sb)
1740 			continue;
1741 		for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) {
1742 			struct kvmppc_xive_irq_state *state = &sb->irq_state[j];
1743 
1744 			if (!state->valid)
1745 				continue;
1746 			if (state->act_priority == MASKED)
1747 				continue;
1748 			if (state->act_server != xc->server_num)
1749 				continue;
1750 
1751 			/* Clean it up */
1752 			arch_spin_lock(&sb->lock);
1753 			state->act_priority = MASKED;
1754 			xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
1755 			xive_native_configure_irq(state->ipi_number, 0, MASKED, 0);
1756 			if (state->pt_number) {
1757 				xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_01);
1758 				xive_native_configure_irq(state->pt_number, 0, MASKED, 0);
1759 			}
1760 			arch_spin_unlock(&sb->lock);
1761 		}
1762 	}
1763 
1764 	/* Disable vcpu's escalation interrupt */
1765 	if (vcpu->arch.xive_esc_on) {
1766 		__raw_readq((void __iomem *)(vcpu->arch.xive_esc_vaddr +
1767 					     XIVE_ESB_SET_PQ_01));
1768 		vcpu->arch.xive_esc_on = false;
1769 	}
1770 
1771 	/*
1772 	 * Clear pointers to escalation interrupt ESB.
1773 	 * This is safe because the vcpu->mutex is held, preventing
1774 	 * any other CPU from concurrently executing a KVM_RUN ioctl.
1775 	 */
1776 	vcpu->arch.xive_esc_vaddr = 0;
1777 	vcpu->arch.xive_esc_raddr = 0;
1778 }
1779 
1780 /*
1781  * In single escalation mode, the escalation interrupt is marked so
1782  * that EOI doesn't re-enable it, but just sets the stale_p flag to
1783  * indicate that the P bit has already been dealt with.  However, the
1784  * assembly code that enters the guest sets PQ to 00 without clearing
1785  * stale_p (because it has no easy way to address it).  Hence we have
1786  * to adjust stale_p before shutting down the interrupt.
1787  */
xive_cleanup_single_escalation(struct kvm_vcpu * vcpu,int irq)1788 void xive_cleanup_single_escalation(struct kvm_vcpu *vcpu, int irq)
1789 {
1790 	struct irq_data *d = irq_get_irq_data(irq);
1791 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
1792 
1793 	/*
1794 	 * This slightly odd sequence gives the right result
1795 	 * (i.e. stale_p set if xive_esc_on is false) even if
1796 	 * we race with xive_esc_irq() and xive_irq_eoi().
1797 	 */
1798 	xd->stale_p = false;
1799 	smp_mb();		/* paired with smb_wmb in xive_esc_irq */
1800 	if (!vcpu->arch.xive_esc_on)
1801 		xd->stale_p = true;
1802 }
1803 
kvmppc_xive_cleanup_vcpu(struct kvm_vcpu * vcpu)1804 void kvmppc_xive_cleanup_vcpu(struct kvm_vcpu *vcpu)
1805 {
1806 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1807 	struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
1808 	int i;
1809 
1810 	if (!kvmppc_xics_enabled(vcpu))
1811 		return;
1812 
1813 	if (!xc)
1814 		return;
1815 
1816 	pr_devel("cleanup_vcpu(cpu=%d)\n", xc->server_num);
1817 
1818 	/* Ensure no interrupt is still routed to that VP */
1819 	xc->valid = false;
1820 	kvmppc_xive_disable_vcpu_interrupts(vcpu);
1821 
1822 	/* Mask the VP IPI */
1823 	xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_01);
1824 
1825 	/* Free escalations */
1826 	for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
1827 		if (xc->esc_virq[i]) {
1828 			if (kvmppc_xive_has_single_escalation(xc->xive))
1829 				xive_cleanup_single_escalation(vcpu, xc->esc_virq[i]);
1830 			free_irq(xc->esc_virq[i], vcpu);
1831 			irq_dispose_mapping(xc->esc_virq[i]);
1832 			kfree(xc->esc_virq_names[i]);
1833 		}
1834 	}
1835 
1836 	/* Disable the VP */
1837 	xive_native_disable_vp(xc->vp_id);
1838 
1839 	/* Clear the cam word so guest entry won't try to push context */
1840 	vcpu->arch.xive_cam_word = 0;
1841 
1842 	/* Free the queues */
1843 	for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
1844 		struct xive_q *q = &xc->queues[i];
1845 
1846 		xive_native_disable_queue(xc->vp_id, q, i);
1847 		if (q->qpage) {
1848 			free_pages((unsigned long)q->qpage,
1849 				   xive->q_page_order);
1850 			q->qpage = NULL;
1851 		}
1852 	}
1853 
1854 	/* Free the IPI */
1855 	if (xc->vp_ipi) {
1856 		xive_cleanup_irq_data(&xc->vp_ipi_data);
1857 		xive_native_free_irq(xc->vp_ipi);
1858 	}
1859 	/* Free the VP */
1860 	kfree(xc);
1861 
1862 	/* Cleanup the vcpu */
1863 	vcpu->arch.irq_type = KVMPPC_IRQ_DEFAULT;
1864 	vcpu->arch.xive_vcpu = NULL;
1865 }
1866 
kvmppc_xive_vcpu_id_valid(struct kvmppc_xive * xive,u32 cpu)1867 static bool kvmppc_xive_vcpu_id_valid(struct kvmppc_xive *xive, u32 cpu)
1868 {
1869 	/* We have a block of xive->nr_servers VPs. We just need to check
1870 	 * packed vCPU ids are below that.
1871 	 */
1872 	return kvmppc_pack_vcpu_id(xive->kvm, cpu) < xive->nr_servers;
1873 }
1874 
kvmppc_xive_compute_vp_id(struct kvmppc_xive * xive,u32 cpu,u32 * vp)1875 int kvmppc_xive_compute_vp_id(struct kvmppc_xive *xive, u32 cpu, u32 *vp)
1876 {
1877 	u32 vp_id;
1878 
1879 	if (!kvmppc_xive_vcpu_id_valid(xive, cpu)) {
1880 		pr_devel("Out of bounds !\n");
1881 		return -EINVAL;
1882 	}
1883 
1884 	if (xive->vp_base == XIVE_INVALID_VP) {
1885 		xive->vp_base = xive_native_alloc_vp_block(xive->nr_servers);
1886 		pr_devel("VP_Base=%x nr_servers=%d\n", xive->vp_base, xive->nr_servers);
1887 
1888 		if (xive->vp_base == XIVE_INVALID_VP)
1889 			return -ENOSPC;
1890 	}
1891 
1892 	vp_id = kvmppc_xive_vp(xive, cpu);
1893 	if (kvmppc_xive_vp_in_use(xive->kvm, vp_id)) {
1894 		pr_devel("Duplicate !\n");
1895 		return -EEXIST;
1896 	}
1897 
1898 	*vp = vp_id;
1899 
1900 	return 0;
1901 }
1902 
kvmppc_xive_connect_vcpu(struct kvm_device * dev,struct kvm_vcpu * vcpu,u32 cpu)1903 int kvmppc_xive_connect_vcpu(struct kvm_device *dev,
1904 			     struct kvm_vcpu *vcpu, u32 cpu)
1905 {
1906 	struct kvmppc_xive *xive = dev->private;
1907 	struct kvmppc_xive_vcpu *xc;
1908 	int i, r = -EBUSY;
1909 	u32 vp_id;
1910 
1911 	pr_devel("connect_vcpu(cpu=%d)\n", cpu);
1912 
1913 	if (dev->ops != &kvm_xive_ops) {
1914 		pr_devel("Wrong ops !\n");
1915 		return -EPERM;
1916 	}
1917 	if (xive->kvm != vcpu->kvm)
1918 		return -EPERM;
1919 	if (vcpu->arch.irq_type != KVMPPC_IRQ_DEFAULT)
1920 		return -EBUSY;
1921 
1922 	/* We need to synchronize with queue provisioning */
1923 	mutex_lock(&xive->lock);
1924 
1925 	r = kvmppc_xive_compute_vp_id(xive, cpu, &vp_id);
1926 	if (r)
1927 		goto bail;
1928 
1929 	xc = kzalloc(sizeof(*xc), GFP_KERNEL);
1930 	if (!xc) {
1931 		r = -ENOMEM;
1932 		goto bail;
1933 	}
1934 
1935 	vcpu->arch.xive_vcpu = xc;
1936 	xc->xive = xive;
1937 	xc->vcpu = vcpu;
1938 	xc->server_num = cpu;
1939 	xc->vp_id = vp_id;
1940 	xc->mfrr = 0xff;
1941 	xc->valid = true;
1942 
1943 	r = xive_native_get_vp_info(xc->vp_id, &xc->vp_cam, &xc->vp_chip_id);
1944 	if (r)
1945 		goto bail;
1946 
1947 	if (!kvmppc_xive_check_save_restore(vcpu)) {
1948 		pr_err("inconsistent save-restore setup for VCPU %d\n", cpu);
1949 		r = -EIO;
1950 		goto bail;
1951 	}
1952 
1953 	/* Configure VCPU fields for use by assembly push/pull */
1954 	vcpu->arch.xive_saved_state.w01 = cpu_to_be64(0xff000000);
1955 	vcpu->arch.xive_cam_word = cpu_to_be32(xc->vp_cam | TM_QW1W2_VO);
1956 
1957 	/* Allocate IPI */
1958 	xc->vp_ipi = xive_native_alloc_irq();
1959 	if (!xc->vp_ipi) {
1960 		pr_err("Failed to allocate xive irq for VCPU IPI\n");
1961 		r = -EIO;
1962 		goto bail;
1963 	}
1964 	pr_devel(" IPI=0x%x\n", xc->vp_ipi);
1965 
1966 	r = xive_native_populate_irq_data(xc->vp_ipi, &xc->vp_ipi_data);
1967 	if (r)
1968 		goto bail;
1969 
1970 	/*
1971 	 * Enable the VP first as the single escalation mode will
1972 	 * affect escalation interrupts numbering
1973 	 */
1974 	r = xive_native_enable_vp(xc->vp_id, kvmppc_xive_has_single_escalation(xive));
1975 	if (r) {
1976 		pr_err("Failed to enable VP in OPAL, err %d\n", r);
1977 		goto bail;
1978 	}
1979 
1980 	/*
1981 	 * Initialize queues. Initially we set them all for no queueing
1982 	 * and we enable escalation for queue 0 only which we'll use for
1983 	 * our mfrr change notifications. If the VCPU is hot-plugged, we
1984 	 * do handle provisioning however based on the existing "map"
1985 	 * of enabled queues.
1986 	 */
1987 	for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
1988 		struct xive_q *q = &xc->queues[i];
1989 
1990 		/* Single escalation, no queue 7 */
1991 		if (i == 7 && kvmppc_xive_has_single_escalation(xive))
1992 			break;
1993 
1994 		/* Is queue already enabled ? Provision it */
1995 		if (xive->qmap & (1 << i)) {
1996 			r = xive_provision_queue(vcpu, i);
1997 			if (r == 0 && !kvmppc_xive_has_single_escalation(xive))
1998 				kvmppc_xive_attach_escalation(
1999 					vcpu, i, kvmppc_xive_has_single_escalation(xive));
2000 			if (r)
2001 				goto bail;
2002 		} else {
2003 			r = xive_native_configure_queue(xc->vp_id,
2004 							q, i, NULL, 0, true);
2005 			if (r) {
2006 				pr_err("Failed to configure queue %d for VCPU %d\n",
2007 				       i, cpu);
2008 				goto bail;
2009 			}
2010 		}
2011 	}
2012 
2013 	/* If not done above, attach priority 0 escalation */
2014 	r = kvmppc_xive_attach_escalation(vcpu, 0, kvmppc_xive_has_single_escalation(xive));
2015 	if (r)
2016 		goto bail;
2017 
2018 	/* Route the IPI */
2019 	r = xive_native_configure_irq(xc->vp_ipi, xc->vp_id, 0, XICS_IPI);
2020 	if (!r)
2021 		xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_00);
2022 
2023 bail:
2024 	mutex_unlock(&xive->lock);
2025 	if (r) {
2026 		kvmppc_xive_cleanup_vcpu(vcpu);
2027 		return r;
2028 	}
2029 
2030 	vcpu->arch.irq_type = KVMPPC_IRQ_XICS;
2031 	return 0;
2032 }
2033 
2034 /*
2035  * Scanning of queues before/after migration save
2036  */
xive_pre_save_set_queued(struct kvmppc_xive * xive,u32 irq)2037 static void xive_pre_save_set_queued(struct kvmppc_xive *xive, u32 irq)
2038 {
2039 	struct kvmppc_xive_src_block *sb;
2040 	struct kvmppc_xive_irq_state *state;
2041 	u16 idx;
2042 
2043 	sb = kvmppc_xive_find_source(xive, irq, &idx);
2044 	if (!sb)
2045 		return;
2046 
2047 	state = &sb->irq_state[idx];
2048 
2049 	/* Some sanity checking */
2050 	if (!state->valid) {
2051 		pr_err("invalid irq 0x%x in cpu queue!\n", irq);
2052 		return;
2053 	}
2054 
2055 	/*
2056 	 * If the interrupt is in a queue it should have P set.
2057 	 * We warn so that gets reported. A backtrace isn't useful
2058 	 * so no need to use a WARN_ON.
2059 	 */
2060 	if (!state->saved_p)
2061 		pr_err("Interrupt 0x%x is marked in a queue but P not set !\n", irq);
2062 
2063 	/* Set flag */
2064 	state->in_queue = true;
2065 }
2066 
xive_pre_save_mask_irq(struct kvmppc_xive * xive,struct kvmppc_xive_src_block * sb,u32 irq)2067 static void xive_pre_save_mask_irq(struct kvmppc_xive *xive,
2068 				   struct kvmppc_xive_src_block *sb,
2069 				   u32 irq)
2070 {
2071 	struct kvmppc_xive_irq_state *state = &sb->irq_state[irq];
2072 
2073 	if (!state->valid)
2074 		return;
2075 
2076 	/* Mask and save state, this will also sync HW queues */
2077 	state->saved_scan_prio = xive_lock_and_mask(xive, sb, state);
2078 
2079 	/* Transfer P and Q */
2080 	state->saved_p = state->old_p;
2081 	state->saved_q = state->old_q;
2082 
2083 	/* Unlock */
2084 	arch_spin_unlock(&sb->lock);
2085 }
2086 
xive_pre_save_unmask_irq(struct kvmppc_xive * xive,struct kvmppc_xive_src_block * sb,u32 irq)2087 static void xive_pre_save_unmask_irq(struct kvmppc_xive *xive,
2088 				     struct kvmppc_xive_src_block *sb,
2089 				     u32 irq)
2090 {
2091 	struct kvmppc_xive_irq_state *state = &sb->irq_state[irq];
2092 
2093 	if (!state->valid)
2094 		return;
2095 
2096 	/*
2097 	 * Lock / exclude EOI (not technically necessary if the
2098 	 * guest isn't running concurrently. If this becomes a
2099 	 * performance issue we can probably remove the lock.
2100 	 */
2101 	xive_lock_for_unmask(sb, state);
2102 
2103 	/* Restore mask/prio if it wasn't masked */
2104 	if (state->saved_scan_prio != MASKED)
2105 		xive_finish_unmask(xive, sb, state, state->saved_scan_prio);
2106 
2107 	/* Unlock */
2108 	arch_spin_unlock(&sb->lock);
2109 }
2110 
xive_pre_save_queue(struct kvmppc_xive * xive,struct xive_q * q)2111 static void xive_pre_save_queue(struct kvmppc_xive *xive, struct xive_q *q)
2112 {
2113 	u32 idx = q->idx;
2114 	u32 toggle = q->toggle;
2115 	u32 irq;
2116 
2117 	do {
2118 		irq = __xive_read_eq(q->qpage, q->msk, &idx, &toggle);
2119 		if (irq > XICS_IPI)
2120 			xive_pre_save_set_queued(xive, irq);
2121 	} while(irq);
2122 }
2123 
xive_pre_save_scan(struct kvmppc_xive * xive)2124 static void xive_pre_save_scan(struct kvmppc_xive *xive)
2125 {
2126 	struct kvm_vcpu *vcpu = NULL;
2127 	unsigned long i;
2128 	int j;
2129 
2130 	/*
2131 	 * See comment in xive_get_source() about how this
2132 	 * work. Collect a stable state for all interrupts
2133 	 */
2134 	for (i = 0; i <= xive->max_sbid; i++) {
2135 		struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
2136 		if (!sb)
2137 			continue;
2138 		for (j = 0;  j < KVMPPC_XICS_IRQ_PER_ICS; j++)
2139 			xive_pre_save_mask_irq(xive, sb, j);
2140 	}
2141 
2142 	/* Then scan the queues and update the "in_queue" flag */
2143 	kvm_for_each_vcpu(i, vcpu, xive->kvm) {
2144 		struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
2145 		if (!xc)
2146 			continue;
2147 		for (j = 0; j < KVMPPC_XIVE_Q_COUNT; j++) {
2148 			if (xc->queues[j].qpage)
2149 				xive_pre_save_queue(xive, &xc->queues[j]);
2150 		}
2151 	}
2152 
2153 	/* Finally restore interrupt states */
2154 	for (i = 0; i <= xive->max_sbid; i++) {
2155 		struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
2156 		if (!sb)
2157 			continue;
2158 		for (j = 0;  j < KVMPPC_XICS_IRQ_PER_ICS; j++)
2159 			xive_pre_save_unmask_irq(xive, sb, j);
2160 	}
2161 }
2162 
xive_post_save_scan(struct kvmppc_xive * xive)2163 static void xive_post_save_scan(struct kvmppc_xive *xive)
2164 {
2165 	u32 i, j;
2166 
2167 	/* Clear all the in_queue flags */
2168 	for (i = 0; i <= xive->max_sbid; i++) {
2169 		struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
2170 		if (!sb)
2171 			continue;
2172 		for (j = 0;  j < KVMPPC_XICS_IRQ_PER_ICS; j++)
2173 			sb->irq_state[j].in_queue = false;
2174 	}
2175 
2176 	/* Next get_source() will do a new scan */
2177 	xive->saved_src_count = 0;
2178 }
2179 
2180 /*
2181  * This returns the source configuration and state to user space.
2182  */
xive_get_source(struct kvmppc_xive * xive,long irq,u64 addr)2183 static int xive_get_source(struct kvmppc_xive *xive, long irq, u64 addr)
2184 {
2185 	struct kvmppc_xive_src_block *sb;
2186 	struct kvmppc_xive_irq_state *state;
2187 	u64 __user *ubufp = (u64 __user *) addr;
2188 	u64 val, prio;
2189 	u16 idx;
2190 
2191 	sb = kvmppc_xive_find_source(xive, irq, &idx);
2192 	if (!sb)
2193 		return -ENOENT;
2194 
2195 	state = &sb->irq_state[idx];
2196 
2197 	if (!state->valid)
2198 		return -ENOENT;
2199 
2200 	pr_devel("get_source(%ld)...\n", irq);
2201 
2202 	/*
2203 	 * So to properly save the state into something that looks like a
2204 	 * XICS migration stream we cannot treat interrupts individually.
2205 	 *
2206 	 * We need, instead, mask them all (& save their previous PQ state)
2207 	 * to get a stable state in the HW, then sync them to ensure that
2208 	 * any interrupt that had already fired hits its queue, and finally
2209 	 * scan all the queues to collect which interrupts are still present
2210 	 * in the queues, so we can set the "pending" flag on them and
2211 	 * they can be resent on restore.
2212 	 *
2213 	 * So we do it all when the "first" interrupt gets saved, all the
2214 	 * state is collected at that point, the rest of xive_get_source()
2215 	 * will merely collect and convert that state to the expected
2216 	 * userspace bit mask.
2217 	 */
2218 	if (xive->saved_src_count == 0)
2219 		xive_pre_save_scan(xive);
2220 	xive->saved_src_count++;
2221 
2222 	/* Convert saved state into something compatible with xics */
2223 	val = state->act_server;
2224 	prio = state->saved_scan_prio;
2225 
2226 	if (prio == MASKED) {
2227 		val |= KVM_XICS_MASKED;
2228 		prio = state->saved_priority;
2229 	}
2230 	val |= prio << KVM_XICS_PRIORITY_SHIFT;
2231 	if (state->lsi) {
2232 		val |= KVM_XICS_LEVEL_SENSITIVE;
2233 		if (state->saved_p)
2234 			val |= KVM_XICS_PENDING;
2235 	} else {
2236 		if (state->saved_p)
2237 			val |= KVM_XICS_PRESENTED;
2238 
2239 		if (state->saved_q)
2240 			val |= KVM_XICS_QUEUED;
2241 
2242 		/*
2243 		 * We mark it pending (which will attempt a re-delivery)
2244 		 * if we are in a queue *or* we were masked and had
2245 		 * Q set which is equivalent to the XICS "masked pending"
2246 		 * state
2247 		 */
2248 		if (state->in_queue || (prio == MASKED && state->saved_q))
2249 			val |= KVM_XICS_PENDING;
2250 	}
2251 
2252 	/*
2253 	 * If that was the last interrupt saved, reset the
2254 	 * in_queue flags
2255 	 */
2256 	if (xive->saved_src_count == xive->src_count)
2257 		xive_post_save_scan(xive);
2258 
2259 	/* Copy the result to userspace */
2260 	if (put_user(val, ubufp))
2261 		return -EFAULT;
2262 
2263 	return 0;
2264 }
2265 
kvmppc_xive_create_src_block(struct kvmppc_xive * xive,int irq)2266 struct kvmppc_xive_src_block *kvmppc_xive_create_src_block(
2267 	struct kvmppc_xive *xive, int irq)
2268 {
2269 	struct kvmppc_xive_src_block *sb;
2270 	int i, bid;
2271 
2272 	bid = irq >> KVMPPC_XICS_ICS_SHIFT;
2273 
2274 	mutex_lock(&xive->lock);
2275 
2276 	/* block already exists - somebody else got here first */
2277 	if (xive->src_blocks[bid])
2278 		goto out;
2279 
2280 	/* Create the ICS */
2281 	sb = kzalloc(sizeof(*sb), GFP_KERNEL);
2282 	if (!sb)
2283 		goto out;
2284 
2285 	sb->id = bid;
2286 
2287 	for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
2288 		sb->irq_state[i].number = (bid << KVMPPC_XICS_ICS_SHIFT) | i;
2289 		sb->irq_state[i].eisn = 0;
2290 		sb->irq_state[i].guest_priority = MASKED;
2291 		sb->irq_state[i].saved_priority = MASKED;
2292 		sb->irq_state[i].act_priority = MASKED;
2293 	}
2294 	smp_wmb();
2295 	xive->src_blocks[bid] = sb;
2296 
2297 	if (bid > xive->max_sbid)
2298 		xive->max_sbid = bid;
2299 
2300 out:
2301 	mutex_unlock(&xive->lock);
2302 	return xive->src_blocks[bid];
2303 }
2304 
xive_check_delayed_irq(struct kvmppc_xive * xive,u32 irq)2305 static bool xive_check_delayed_irq(struct kvmppc_xive *xive, u32 irq)
2306 {
2307 	struct kvm *kvm = xive->kvm;
2308 	struct kvm_vcpu *vcpu = NULL;
2309 	unsigned long i;
2310 
2311 	kvm_for_each_vcpu(i, vcpu, kvm) {
2312 		struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
2313 
2314 		if (!xc)
2315 			continue;
2316 
2317 		if (xc->delayed_irq == irq) {
2318 			xc->delayed_irq = 0;
2319 			xive->delayed_irqs--;
2320 			return true;
2321 		}
2322 	}
2323 	return false;
2324 }
2325 
xive_set_source(struct kvmppc_xive * xive,long irq,u64 addr)2326 static int xive_set_source(struct kvmppc_xive *xive, long irq, u64 addr)
2327 {
2328 	struct kvmppc_xive_src_block *sb;
2329 	struct kvmppc_xive_irq_state *state;
2330 	u64 __user *ubufp = (u64 __user *) addr;
2331 	u16 idx;
2332 	u64 val;
2333 	u8 act_prio, guest_prio;
2334 	u32 server;
2335 	int rc = 0;
2336 
2337 	if (irq < KVMPPC_XICS_FIRST_IRQ || irq >= KVMPPC_XICS_NR_IRQS)
2338 		return -ENOENT;
2339 
2340 	pr_devel("set_source(irq=0x%lx)\n", irq);
2341 
2342 	/* Find the source */
2343 	sb = kvmppc_xive_find_source(xive, irq, &idx);
2344 	if (!sb) {
2345 		pr_devel("No source, creating source block...\n");
2346 		sb = kvmppc_xive_create_src_block(xive, irq);
2347 		if (!sb) {
2348 			pr_devel("Failed to create block...\n");
2349 			return -ENOMEM;
2350 		}
2351 	}
2352 	state = &sb->irq_state[idx];
2353 
2354 	/* Read user passed data */
2355 	if (get_user(val, ubufp)) {
2356 		pr_devel("fault getting user info !\n");
2357 		return -EFAULT;
2358 	}
2359 
2360 	server = val & KVM_XICS_DESTINATION_MASK;
2361 	guest_prio = val >> KVM_XICS_PRIORITY_SHIFT;
2362 
2363 	pr_devel("  val=0x016%llx (server=0x%x, guest_prio=%d)\n",
2364 		 val, server, guest_prio);
2365 
2366 	/*
2367 	 * If the source doesn't already have an IPI, allocate
2368 	 * one and get the corresponding data
2369 	 */
2370 	if (!state->ipi_number) {
2371 		state->ipi_number = xive_native_alloc_irq();
2372 		if (state->ipi_number == 0) {
2373 			pr_devel("Failed to allocate IPI !\n");
2374 			return -ENOMEM;
2375 		}
2376 		xive_native_populate_irq_data(state->ipi_number, &state->ipi_data);
2377 		pr_devel(" src_ipi=0x%x\n", state->ipi_number);
2378 	}
2379 
2380 	/*
2381 	 * We use lock_and_mask() to set us in the right masked
2382 	 * state. We will override that state from the saved state
2383 	 * further down, but this will handle the cases of interrupts
2384 	 * that need FW masking. We set the initial guest_priority to
2385 	 * 0 before calling it to ensure it actually performs the masking.
2386 	 */
2387 	state->guest_priority = 0;
2388 	xive_lock_and_mask(xive, sb, state);
2389 
2390 	/*
2391 	 * Now, we select a target if we have one. If we don't we
2392 	 * leave the interrupt untargetted. It means that an interrupt
2393 	 * can become "untargetted" across migration if it was masked
2394 	 * by set_xive() but there is little we can do about it.
2395 	 */
2396 
2397 	/* First convert prio and mark interrupt as untargetted */
2398 	act_prio = xive_prio_from_guest(guest_prio);
2399 	state->act_priority = MASKED;
2400 
2401 	/*
2402 	 * We need to drop the lock due to the mutex below. Hopefully
2403 	 * nothing is touching that interrupt yet since it hasn't been
2404 	 * advertized to a running guest yet
2405 	 */
2406 	arch_spin_unlock(&sb->lock);
2407 
2408 	/* If we have a priority target the interrupt */
2409 	if (act_prio != MASKED) {
2410 		/* First, check provisioning of queues */
2411 		mutex_lock(&xive->lock);
2412 		rc = xive_check_provisioning(xive->kvm, act_prio);
2413 		mutex_unlock(&xive->lock);
2414 
2415 		/* Target interrupt */
2416 		if (rc == 0)
2417 			rc = xive_target_interrupt(xive->kvm, state,
2418 						   server, act_prio);
2419 		/*
2420 		 * If provisioning or targetting failed, leave it
2421 		 * alone and masked. It will remain disabled until
2422 		 * the guest re-targets it.
2423 		 */
2424 	}
2425 
2426 	/*
2427 	 * Find out if this was a delayed irq stashed in an ICP,
2428 	 * in which case, treat it as pending
2429 	 */
2430 	if (xive->delayed_irqs && xive_check_delayed_irq(xive, irq)) {
2431 		val |= KVM_XICS_PENDING;
2432 		pr_devel("  Found delayed ! forcing PENDING !\n");
2433 	}
2434 
2435 	/* Cleanup the SW state */
2436 	state->old_p = false;
2437 	state->old_q = false;
2438 	state->lsi = false;
2439 	state->asserted = false;
2440 
2441 	/* Restore LSI state */
2442 	if (val & KVM_XICS_LEVEL_SENSITIVE) {
2443 		state->lsi = true;
2444 		if (val & KVM_XICS_PENDING)
2445 			state->asserted = true;
2446 		pr_devel("  LSI ! Asserted=%d\n", state->asserted);
2447 	}
2448 
2449 	/*
2450 	 * Restore P and Q. If the interrupt was pending, we
2451 	 * force Q and !P, which will trigger a resend.
2452 	 *
2453 	 * That means that a guest that had both an interrupt
2454 	 * pending (queued) and Q set will restore with only
2455 	 * one instance of that interrupt instead of 2, but that
2456 	 * is perfectly fine as coalescing interrupts that haven't
2457 	 * been presented yet is always allowed.
2458 	 */
2459 	if (val & KVM_XICS_PRESENTED && !(val & KVM_XICS_PENDING))
2460 		state->old_p = true;
2461 	if (val & KVM_XICS_QUEUED || val & KVM_XICS_PENDING)
2462 		state->old_q = true;
2463 
2464 	pr_devel("  P=%d, Q=%d\n", state->old_p, state->old_q);
2465 
2466 	/*
2467 	 * If the interrupt was unmasked, update guest priority and
2468 	 * perform the appropriate state transition and do a
2469 	 * re-trigger if necessary.
2470 	 */
2471 	if (val & KVM_XICS_MASKED) {
2472 		pr_devel("  masked, saving prio\n");
2473 		state->guest_priority = MASKED;
2474 		state->saved_priority = guest_prio;
2475 	} else {
2476 		pr_devel("  unmasked, restoring to prio %d\n", guest_prio);
2477 		xive_finish_unmask(xive, sb, state, guest_prio);
2478 		state->saved_priority = guest_prio;
2479 	}
2480 
2481 	/* Increment the number of valid sources and mark this one valid */
2482 	if (!state->valid)
2483 		xive->src_count++;
2484 	state->valid = true;
2485 
2486 	return 0;
2487 }
2488 
kvmppc_xive_set_irq(struct kvm * kvm,int irq_source_id,u32 irq,int level,bool line_status)2489 int kvmppc_xive_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
2490 			bool line_status)
2491 {
2492 	struct kvmppc_xive *xive = kvm->arch.xive;
2493 	struct kvmppc_xive_src_block *sb;
2494 	struct kvmppc_xive_irq_state *state;
2495 	u16 idx;
2496 
2497 	if (!xive)
2498 		return -ENODEV;
2499 
2500 	sb = kvmppc_xive_find_source(xive, irq, &idx);
2501 	if (!sb)
2502 		return -EINVAL;
2503 
2504 	/* Perform locklessly .... (we need to do some RCUisms here...) */
2505 	state = &sb->irq_state[idx];
2506 	if (!state->valid)
2507 		return -EINVAL;
2508 
2509 	/* We don't allow a trigger on a passed-through interrupt */
2510 	if (state->pt_number)
2511 		return -EINVAL;
2512 
2513 	if ((level == 1 && state->lsi) || level == KVM_INTERRUPT_SET_LEVEL)
2514 		state->asserted = true;
2515 	else if (level == 0 || level == KVM_INTERRUPT_UNSET) {
2516 		state->asserted = false;
2517 		return 0;
2518 	}
2519 
2520 	/* Trigger the IPI */
2521 	xive_irq_trigger(&state->ipi_data);
2522 
2523 	return 0;
2524 }
2525 
kvmppc_xive_set_nr_servers(struct kvmppc_xive * xive,u64 addr)2526 int kvmppc_xive_set_nr_servers(struct kvmppc_xive *xive, u64 addr)
2527 {
2528 	u32 __user *ubufp = (u32 __user *) addr;
2529 	u32 nr_servers;
2530 	int rc = 0;
2531 
2532 	if (get_user(nr_servers, ubufp))
2533 		return -EFAULT;
2534 
2535 	pr_devel("%s nr_servers=%u\n", __func__, nr_servers);
2536 
2537 	if (!nr_servers || nr_servers > KVM_MAX_VCPU_IDS)
2538 		return -EINVAL;
2539 
2540 	mutex_lock(&xive->lock);
2541 	if (xive->vp_base != XIVE_INVALID_VP)
2542 		/* The VP block is allocated once and freed when the device
2543 		 * is released. Better not allow to change its size since its
2544 		 * used by connect_vcpu to validate vCPU ids are valid (eg,
2545 		 * setting it back to a higher value could allow connect_vcpu
2546 		 * to come up with a VP id that goes beyond the VP block, which
2547 		 * is likely to cause a crash in OPAL).
2548 		 */
2549 		rc = -EBUSY;
2550 	else if (nr_servers > KVM_MAX_VCPUS)
2551 		/* We don't need more servers. Higher vCPU ids get packed
2552 		 * down below KVM_MAX_VCPUS by kvmppc_pack_vcpu_id().
2553 		 */
2554 		xive->nr_servers = KVM_MAX_VCPUS;
2555 	else
2556 		xive->nr_servers = nr_servers;
2557 
2558 	mutex_unlock(&xive->lock);
2559 
2560 	return rc;
2561 }
2562 
xive_set_attr(struct kvm_device * dev,struct kvm_device_attr * attr)2563 static int xive_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2564 {
2565 	struct kvmppc_xive *xive = dev->private;
2566 
2567 	/* We honor the existing XICS ioctl */
2568 	switch (attr->group) {
2569 	case KVM_DEV_XICS_GRP_SOURCES:
2570 		return xive_set_source(xive, attr->attr, attr->addr);
2571 	case KVM_DEV_XICS_GRP_CTRL:
2572 		switch (attr->attr) {
2573 		case KVM_DEV_XICS_NR_SERVERS:
2574 			return kvmppc_xive_set_nr_servers(xive, attr->addr);
2575 		}
2576 	}
2577 	return -ENXIO;
2578 }
2579 
xive_get_attr(struct kvm_device * dev,struct kvm_device_attr * attr)2580 static int xive_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2581 {
2582 	struct kvmppc_xive *xive = dev->private;
2583 
2584 	/* We honor the existing XICS ioctl */
2585 	switch (attr->group) {
2586 	case KVM_DEV_XICS_GRP_SOURCES:
2587 		return xive_get_source(xive, attr->attr, attr->addr);
2588 	}
2589 	return -ENXIO;
2590 }
2591 
xive_has_attr(struct kvm_device * dev,struct kvm_device_attr * attr)2592 static int xive_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2593 {
2594 	/* We honor the same limits as XICS, at least for now */
2595 	switch (attr->group) {
2596 	case KVM_DEV_XICS_GRP_SOURCES:
2597 		if (attr->attr >= KVMPPC_XICS_FIRST_IRQ &&
2598 		    attr->attr < KVMPPC_XICS_NR_IRQS)
2599 			return 0;
2600 		break;
2601 	case KVM_DEV_XICS_GRP_CTRL:
2602 		switch (attr->attr) {
2603 		case KVM_DEV_XICS_NR_SERVERS:
2604 			return 0;
2605 		}
2606 	}
2607 	return -ENXIO;
2608 }
2609 
kvmppc_xive_cleanup_irq(u32 hw_num,struct xive_irq_data * xd)2610 static void kvmppc_xive_cleanup_irq(u32 hw_num, struct xive_irq_data *xd)
2611 {
2612 	xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01);
2613 	xive_native_configure_irq(hw_num, 0, MASKED, 0);
2614 }
2615 
kvmppc_xive_free_sources(struct kvmppc_xive_src_block * sb)2616 void kvmppc_xive_free_sources(struct kvmppc_xive_src_block *sb)
2617 {
2618 	int i;
2619 
2620 	for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
2621 		struct kvmppc_xive_irq_state *state = &sb->irq_state[i];
2622 
2623 		if (!state->valid)
2624 			continue;
2625 
2626 		kvmppc_xive_cleanup_irq(state->ipi_number, &state->ipi_data);
2627 		xive_cleanup_irq_data(&state->ipi_data);
2628 		xive_native_free_irq(state->ipi_number);
2629 
2630 		/* Pass-through, cleanup too but keep IRQ hw data */
2631 		if (state->pt_number)
2632 			kvmppc_xive_cleanup_irq(state->pt_number, state->pt_data);
2633 
2634 		state->valid = false;
2635 	}
2636 }
2637 
2638 /*
2639  * Called when device fd is closed.  kvm->lock is held.
2640  */
kvmppc_xive_release(struct kvm_device * dev)2641 static void kvmppc_xive_release(struct kvm_device *dev)
2642 {
2643 	struct kvmppc_xive *xive = dev->private;
2644 	struct kvm *kvm = xive->kvm;
2645 	struct kvm_vcpu *vcpu;
2646 	unsigned long i;
2647 
2648 	pr_devel("Releasing xive device\n");
2649 
2650 	/*
2651 	 * Since this is the device release function, we know that
2652 	 * userspace does not have any open fd referring to the
2653 	 * device.  Therefore there can not be any of the device
2654 	 * attribute set/get functions being executed concurrently,
2655 	 * and similarly, the connect_vcpu and set/clr_mapped
2656 	 * functions also cannot be being executed.
2657 	 */
2658 
2659 	debugfs_remove(xive->dentry);
2660 
2661 	/*
2662 	 * We should clean up the vCPU interrupt presenters first.
2663 	 */
2664 	kvm_for_each_vcpu(i, vcpu, kvm) {
2665 		/*
2666 		 * Take vcpu->mutex to ensure that no one_reg get/set ioctl
2667 		 * (i.e. kvmppc_xive_[gs]et_icp) can be done concurrently.
2668 		 * Holding the vcpu->mutex also means that the vcpu cannot
2669 		 * be executing the KVM_RUN ioctl, and therefore it cannot
2670 		 * be executing the XIVE push or pull code or accessing
2671 		 * the XIVE MMIO regions.
2672 		 */
2673 		mutex_lock(&vcpu->mutex);
2674 		kvmppc_xive_cleanup_vcpu(vcpu);
2675 		mutex_unlock(&vcpu->mutex);
2676 	}
2677 
2678 	/*
2679 	 * Now that we have cleared vcpu->arch.xive_vcpu, vcpu->arch.irq_type
2680 	 * and vcpu->arch.xive_esc_[vr]addr on each vcpu, we are safe
2681 	 * against xive code getting called during vcpu execution or
2682 	 * set/get one_reg operations.
2683 	 */
2684 	kvm->arch.xive = NULL;
2685 
2686 	/* Mask and free interrupts */
2687 	for (i = 0; i <= xive->max_sbid; i++) {
2688 		if (xive->src_blocks[i])
2689 			kvmppc_xive_free_sources(xive->src_blocks[i]);
2690 		kfree(xive->src_blocks[i]);
2691 		xive->src_blocks[i] = NULL;
2692 	}
2693 
2694 	if (xive->vp_base != XIVE_INVALID_VP)
2695 		xive_native_free_vp_block(xive->vp_base);
2696 
2697 	/*
2698 	 * A reference of the kvmppc_xive pointer is now kept under
2699 	 * the xive_devices struct of the machine for reuse. It is
2700 	 * freed when the VM is destroyed for now until we fix all the
2701 	 * execution paths.
2702 	 */
2703 
2704 	kfree(dev);
2705 }
2706 
2707 /*
2708  * When the guest chooses the interrupt mode (XICS legacy or XIVE
2709  * native), the VM will switch of KVM device. The previous device will
2710  * be "released" before the new one is created.
2711  *
2712  * Until we are sure all execution paths are well protected, provide a
2713  * fail safe (transitional) method for device destruction, in which
2714  * the XIVE device pointer is recycled and not directly freed.
2715  */
kvmppc_xive_get_device(struct kvm * kvm,u32 type)2716 struct kvmppc_xive *kvmppc_xive_get_device(struct kvm *kvm, u32 type)
2717 {
2718 	struct kvmppc_xive **kvm_xive_device = type == KVM_DEV_TYPE_XIVE ?
2719 		&kvm->arch.xive_devices.native :
2720 		&kvm->arch.xive_devices.xics_on_xive;
2721 	struct kvmppc_xive *xive = *kvm_xive_device;
2722 
2723 	if (!xive) {
2724 		xive = kzalloc(sizeof(*xive), GFP_KERNEL);
2725 		*kvm_xive_device = xive;
2726 	} else {
2727 		memset(xive, 0, sizeof(*xive));
2728 	}
2729 
2730 	return xive;
2731 }
2732 
2733 /*
2734  * Create a XICS device with XIVE backend.  kvm->lock is held.
2735  */
kvmppc_xive_create(struct kvm_device * dev,u32 type)2736 static int kvmppc_xive_create(struct kvm_device *dev, u32 type)
2737 {
2738 	struct kvmppc_xive *xive;
2739 	struct kvm *kvm = dev->kvm;
2740 
2741 	pr_devel("Creating xive for partition\n");
2742 
2743 	/* Already there ? */
2744 	if (kvm->arch.xive)
2745 		return -EEXIST;
2746 
2747 	xive = kvmppc_xive_get_device(kvm, type);
2748 	if (!xive)
2749 		return -ENOMEM;
2750 
2751 	dev->private = xive;
2752 	xive->dev = dev;
2753 	xive->kvm = kvm;
2754 	mutex_init(&xive->lock);
2755 
2756 	/* We use the default queue size set by the host */
2757 	xive->q_order = xive_native_default_eq_shift();
2758 	if (xive->q_order < PAGE_SHIFT)
2759 		xive->q_page_order = 0;
2760 	else
2761 		xive->q_page_order = xive->q_order - PAGE_SHIFT;
2762 
2763 	/* VP allocation is delayed to the first call to connect_vcpu */
2764 	xive->vp_base = XIVE_INVALID_VP;
2765 	/* KVM_MAX_VCPUS limits the number of VMs to roughly 64 per sockets
2766 	 * on a POWER9 system.
2767 	 */
2768 	xive->nr_servers = KVM_MAX_VCPUS;
2769 
2770 	if (xive_native_has_single_escalation())
2771 		xive->flags |= KVMPPC_XIVE_FLAG_SINGLE_ESCALATION;
2772 
2773 	if (xive_native_has_save_restore())
2774 		xive->flags |= KVMPPC_XIVE_FLAG_SAVE_RESTORE;
2775 
2776 	kvm->arch.xive = xive;
2777 	return 0;
2778 }
2779 
kvmppc_xive_xics_hcall(struct kvm_vcpu * vcpu,u32 req)2780 int kvmppc_xive_xics_hcall(struct kvm_vcpu *vcpu, u32 req)
2781 {
2782 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
2783 
2784 	/* The VM should have configured XICS mode before doing XICS hcalls. */
2785 	if (!kvmppc_xics_enabled(vcpu))
2786 		return H_TOO_HARD;
2787 
2788 	switch (req) {
2789 	case H_XIRR:
2790 		return xive_vm_h_xirr(vcpu);
2791 	case H_CPPR:
2792 		return xive_vm_h_cppr(vcpu, kvmppc_get_gpr(vcpu, 4));
2793 	case H_EOI:
2794 		return xive_vm_h_eoi(vcpu, kvmppc_get_gpr(vcpu, 4));
2795 	case H_IPI:
2796 		return xive_vm_h_ipi(vcpu, kvmppc_get_gpr(vcpu, 4),
2797 					  kvmppc_get_gpr(vcpu, 5));
2798 	case H_IPOLL:
2799 		return xive_vm_h_ipoll(vcpu, kvmppc_get_gpr(vcpu, 4));
2800 	case H_XIRR_X:
2801 		xive_vm_h_xirr(vcpu);
2802 		kvmppc_set_gpr(vcpu, 5, get_tb() + vc->tb_offset);
2803 		return H_SUCCESS;
2804 	}
2805 
2806 	return H_UNSUPPORTED;
2807 }
2808 EXPORT_SYMBOL_GPL(kvmppc_xive_xics_hcall);
2809 
kvmppc_xive_debug_show_queues(struct seq_file * m,struct kvm_vcpu * vcpu)2810 int kvmppc_xive_debug_show_queues(struct seq_file *m, struct kvm_vcpu *vcpu)
2811 {
2812 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
2813 	unsigned int i;
2814 
2815 	for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
2816 		struct xive_q *q = &xc->queues[i];
2817 		u32 i0, i1, idx;
2818 
2819 		if (!q->qpage && !xc->esc_virq[i])
2820 			continue;
2821 
2822 		if (q->qpage) {
2823 			seq_printf(m, "    q[%d]: ", i);
2824 			idx = q->idx;
2825 			i0 = be32_to_cpup(q->qpage + idx);
2826 			idx = (idx + 1) & q->msk;
2827 			i1 = be32_to_cpup(q->qpage + idx);
2828 			seq_printf(m, "T=%d %08x %08x...\n", q->toggle,
2829 				   i0, i1);
2830 		}
2831 		if (xc->esc_virq[i]) {
2832 			struct irq_data *d = irq_get_irq_data(xc->esc_virq[i]);
2833 			struct xive_irq_data *xd =
2834 				irq_data_get_irq_handler_data(d);
2835 			u64 pq = xive_vm_esb_load(xd, XIVE_ESB_GET);
2836 
2837 			seq_printf(m, "    ESC %d %c%c EOI @%llx",
2838 				   xc->esc_virq[i],
2839 				   (pq & XIVE_ESB_VAL_P) ? 'P' : '-',
2840 				   (pq & XIVE_ESB_VAL_Q) ? 'Q' : '-',
2841 				   xd->eoi_page);
2842 			seq_puts(m, "\n");
2843 		}
2844 	}
2845 	return 0;
2846 }
2847 
kvmppc_xive_debug_show_sources(struct seq_file * m,struct kvmppc_xive_src_block * sb)2848 void kvmppc_xive_debug_show_sources(struct seq_file *m,
2849 				    struct kvmppc_xive_src_block *sb)
2850 {
2851 	int i;
2852 
2853 	seq_puts(m, "    LISN      HW/CHIP   TYPE    PQ      EISN    CPU/PRIO\n");
2854 	for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
2855 		struct kvmppc_xive_irq_state *state = &sb->irq_state[i];
2856 		struct xive_irq_data *xd;
2857 		u64 pq;
2858 		u32 hw_num;
2859 
2860 		if (!state->valid)
2861 			continue;
2862 
2863 		kvmppc_xive_select_irq(state, &hw_num, &xd);
2864 
2865 		pq = xive_vm_esb_load(xd, XIVE_ESB_GET);
2866 
2867 		seq_printf(m, "%08x  %08x/%02x", state->number, hw_num,
2868 			   xd->src_chip);
2869 		if (state->lsi)
2870 			seq_printf(m, " %cLSI", state->asserted ? '^' : ' ');
2871 		else
2872 			seq_puts(m, "  MSI");
2873 
2874 		seq_printf(m, " %s  %c%c  %08x   % 4d/%d",
2875 			   state->ipi_number == hw_num ? "IPI" : " PT",
2876 			   pq & XIVE_ESB_VAL_P ? 'P' : '-',
2877 			   pq & XIVE_ESB_VAL_Q ? 'Q' : '-',
2878 			   state->eisn, state->act_server,
2879 			   state->act_priority);
2880 
2881 		seq_puts(m, "\n");
2882 	}
2883 }
2884 
xive_debug_show(struct seq_file * m,void * private)2885 static int xive_debug_show(struct seq_file *m, void *private)
2886 {
2887 	struct kvmppc_xive *xive = m->private;
2888 	struct kvm *kvm = xive->kvm;
2889 	struct kvm_vcpu *vcpu;
2890 	u64 t_rm_h_xirr = 0;
2891 	u64 t_rm_h_ipoll = 0;
2892 	u64 t_rm_h_cppr = 0;
2893 	u64 t_rm_h_eoi = 0;
2894 	u64 t_rm_h_ipi = 0;
2895 	u64 t_vm_h_xirr = 0;
2896 	u64 t_vm_h_ipoll = 0;
2897 	u64 t_vm_h_cppr = 0;
2898 	u64 t_vm_h_eoi = 0;
2899 	u64 t_vm_h_ipi = 0;
2900 	unsigned long i;
2901 
2902 	if (!kvm)
2903 		return 0;
2904 
2905 	seq_puts(m, "=========\nVCPU state\n=========\n");
2906 
2907 	kvm_for_each_vcpu(i, vcpu, kvm) {
2908 		struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
2909 
2910 		if (!xc)
2911 			continue;
2912 
2913 		seq_printf(m, "VCPU %d: VP:%#x/%02x\n"
2914 			 "    CPPR:%#x HWCPPR:%#x MFRR:%#x PEND:%#x h_xirr: R=%lld V=%lld\n",
2915 			 xc->server_num, xc->vp_id, xc->vp_chip_id,
2916 			 xc->cppr, xc->hw_cppr,
2917 			 xc->mfrr, xc->pending,
2918 			 xc->stat_rm_h_xirr, xc->stat_vm_h_xirr);
2919 
2920 		kvmppc_xive_debug_show_queues(m, vcpu);
2921 
2922 		t_rm_h_xirr += xc->stat_rm_h_xirr;
2923 		t_rm_h_ipoll += xc->stat_rm_h_ipoll;
2924 		t_rm_h_cppr += xc->stat_rm_h_cppr;
2925 		t_rm_h_eoi += xc->stat_rm_h_eoi;
2926 		t_rm_h_ipi += xc->stat_rm_h_ipi;
2927 		t_vm_h_xirr += xc->stat_vm_h_xirr;
2928 		t_vm_h_ipoll += xc->stat_vm_h_ipoll;
2929 		t_vm_h_cppr += xc->stat_vm_h_cppr;
2930 		t_vm_h_eoi += xc->stat_vm_h_eoi;
2931 		t_vm_h_ipi += xc->stat_vm_h_ipi;
2932 	}
2933 
2934 	seq_puts(m, "Hcalls totals\n");
2935 	seq_printf(m, " H_XIRR  R=%10lld V=%10lld\n", t_rm_h_xirr, t_vm_h_xirr);
2936 	seq_printf(m, " H_IPOLL R=%10lld V=%10lld\n", t_rm_h_ipoll, t_vm_h_ipoll);
2937 	seq_printf(m, " H_CPPR  R=%10lld V=%10lld\n", t_rm_h_cppr, t_vm_h_cppr);
2938 	seq_printf(m, " H_EOI   R=%10lld V=%10lld\n", t_rm_h_eoi, t_vm_h_eoi);
2939 	seq_printf(m, " H_IPI   R=%10lld V=%10lld\n", t_rm_h_ipi, t_vm_h_ipi);
2940 
2941 	seq_puts(m, "=========\nSources\n=========\n");
2942 
2943 	for (i = 0; i <= xive->max_sbid; i++) {
2944 		struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
2945 
2946 		if (sb) {
2947 			arch_spin_lock(&sb->lock);
2948 			kvmppc_xive_debug_show_sources(m, sb);
2949 			arch_spin_unlock(&sb->lock);
2950 		}
2951 	}
2952 
2953 	return 0;
2954 }
2955 
2956 DEFINE_SHOW_ATTRIBUTE(xive_debug);
2957 
xive_debugfs_init(struct kvmppc_xive * xive)2958 static void xive_debugfs_init(struct kvmppc_xive *xive)
2959 {
2960 	xive->dentry = debugfs_create_file("xive", S_IRUGO, xive->kvm->debugfs_dentry,
2961 					   xive, &xive_debug_fops);
2962 
2963 	pr_debug("%s: created\n", __func__);
2964 }
2965 
kvmppc_xive_init(struct kvm_device * dev)2966 static void kvmppc_xive_init(struct kvm_device *dev)
2967 {
2968 	struct kvmppc_xive *xive = dev->private;
2969 
2970 	/* Register some debug interfaces */
2971 	xive_debugfs_init(xive);
2972 }
2973 
2974 struct kvm_device_ops kvm_xive_ops = {
2975 	.name = "kvm-xive",
2976 	.create = kvmppc_xive_create,
2977 	.init = kvmppc_xive_init,
2978 	.release = kvmppc_xive_release,
2979 	.set_attr = xive_set_attr,
2980 	.get_attr = xive_get_attr,
2981 	.has_attr = xive_has_attr,
2982 };
2983