1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
25af50993SBenjamin Herrenschmidt /*
35af50993SBenjamin Herrenschmidt * Copyright 2017 Benjamin Herrenschmidt, IBM Corporation.
45af50993SBenjamin Herrenschmidt */
55af50993SBenjamin Herrenschmidt
65af50993SBenjamin Herrenschmidt #define pr_fmt(fmt) "xive-kvm: " fmt
75af50993SBenjamin Herrenschmidt
85af50993SBenjamin Herrenschmidt #include <linux/kernel.h>
95af50993SBenjamin Herrenschmidt #include <linux/kvm_host.h>
105af50993SBenjamin Herrenschmidt #include <linux/err.h>
115af50993SBenjamin Herrenschmidt #include <linux/gfp.h>
125af50993SBenjamin Herrenschmidt #include <linux/spinlock.h>
135af50993SBenjamin Herrenschmidt #include <linux/delay.h>
145af50993SBenjamin Herrenschmidt #include <linux/percpu.h>
155af50993SBenjamin Herrenschmidt #include <linux/cpumask.h>
165bb866deSAl Viro #include <linux/uaccess.h>
1713a9a5d1SMarc Zyngier #include <linux/irqdomain.h>
185af50993SBenjamin Herrenschmidt #include <asm/kvm_book3s.h>
195af50993SBenjamin Herrenschmidt #include <asm/kvm_ppc.h>
205af50993SBenjamin Herrenschmidt #include <asm/hvcall.h>
215af50993SBenjamin Herrenschmidt #include <asm/xics.h>
225af50993SBenjamin Herrenschmidt #include <asm/xive.h>
235af50993SBenjamin Herrenschmidt #include <asm/xive-regs.h>
245af50993SBenjamin Herrenschmidt #include <asm/debug.h>
255af50993SBenjamin Herrenschmidt #include <asm/time.h>
265af50993SBenjamin Herrenschmidt #include <asm/opal.h>
275af50993SBenjamin Herrenschmidt
285af50993SBenjamin Herrenschmidt #include <linux/debugfs.h>
295af50993SBenjamin Herrenschmidt #include <linux/seq_file.h>
305af50993SBenjamin Herrenschmidt
315af50993SBenjamin Herrenschmidt #include "book3s_xive.h"
325af50993SBenjamin Herrenschmidt
335af50993SBenjamin Herrenschmidt #define __x_eoi_page(xd) ((void __iomem *)((xd)->eoi_mmio))
345af50993SBenjamin Herrenschmidt #define __x_trig_page(xd) ((void __iomem *)((xd)->trig_mmio))
355af50993SBenjamin Herrenschmidt
36b22af904SAlexey Kardashevskiy /* Dummy interrupt used when taking interrupts out of a queue in H_CPPR */
37b22af904SAlexey Kardashevskiy #define XICS_DUMMY 1
38b22af904SAlexey Kardashevskiy
xive_vm_ack_pending(struct kvmppc_xive_vcpu * xc)39b22af904SAlexey Kardashevskiy static void xive_vm_ack_pending(struct kvmppc_xive_vcpu *xc)
40b22af904SAlexey Kardashevskiy {
41b22af904SAlexey Kardashevskiy u8 cppr;
42b22af904SAlexey Kardashevskiy u16 ack;
43b22af904SAlexey Kardashevskiy
44b22af904SAlexey Kardashevskiy /*
45b22af904SAlexey Kardashevskiy * Ensure any previous store to CPPR is ordered vs.
46b22af904SAlexey Kardashevskiy * the subsequent loads from PIPR or ACK.
47b22af904SAlexey Kardashevskiy */
48b22af904SAlexey Kardashevskiy eieio();
49b22af904SAlexey Kardashevskiy
50b22af904SAlexey Kardashevskiy /* Perform the acknowledge OS to register cycle. */
51b22af904SAlexey Kardashevskiy ack = be16_to_cpu(__raw_readw(xive_tima + TM_SPC_ACK_OS_REG));
52b22af904SAlexey Kardashevskiy
53b22af904SAlexey Kardashevskiy /* Synchronize subsequent queue accesses */
54b22af904SAlexey Kardashevskiy mb();
55b22af904SAlexey Kardashevskiy
56b22af904SAlexey Kardashevskiy /* XXX Check grouping level */
57b22af904SAlexey Kardashevskiy
58b22af904SAlexey Kardashevskiy /* Anything ? */
59b22af904SAlexey Kardashevskiy if (!((ack >> 8) & TM_QW1_NSR_EO))
60b22af904SAlexey Kardashevskiy return;
61b22af904SAlexey Kardashevskiy
62b22af904SAlexey Kardashevskiy /* Grab CPPR of the most favored pending interrupt */
63b22af904SAlexey Kardashevskiy cppr = ack & 0xff;
64b22af904SAlexey Kardashevskiy if (cppr < 8)
65b22af904SAlexey Kardashevskiy xc->pending |= 1 << cppr;
66b22af904SAlexey Kardashevskiy
67b22af904SAlexey Kardashevskiy /* Check consistency */
68b22af904SAlexey Kardashevskiy if (cppr >= xc->hw_cppr)
69b22af904SAlexey Kardashevskiy pr_warn("KVM-XIVE: CPU %d odd ack CPPR, got %d at %d\n",
70b22af904SAlexey Kardashevskiy smp_processor_id(), cppr, xc->hw_cppr);
71b22af904SAlexey Kardashevskiy
72b22af904SAlexey Kardashevskiy /*
73b22af904SAlexey Kardashevskiy * Update our image of the HW CPPR. We don't yet modify
74b22af904SAlexey Kardashevskiy * xc->cppr, this will be done as we scan for interrupts
75b22af904SAlexey Kardashevskiy * in the queues.
76b22af904SAlexey Kardashevskiy */
77b22af904SAlexey Kardashevskiy xc->hw_cppr = cppr;
78b22af904SAlexey Kardashevskiy }
79b22af904SAlexey Kardashevskiy
xive_vm_esb_load(struct xive_irq_data * xd,u32 offset)80b22af904SAlexey Kardashevskiy static u8 xive_vm_esb_load(struct xive_irq_data *xd, u32 offset)
81b22af904SAlexey Kardashevskiy {
82b22af904SAlexey Kardashevskiy u64 val;
83b22af904SAlexey Kardashevskiy
84b22af904SAlexey Kardashevskiy if (offset == XIVE_ESB_SET_PQ_10 && xd->flags & XIVE_IRQ_FLAG_STORE_EOI)
85b22af904SAlexey Kardashevskiy offset |= XIVE_ESB_LD_ST_MO;
86b22af904SAlexey Kardashevskiy
87b22af904SAlexey Kardashevskiy val = __raw_readq(__x_eoi_page(xd) + offset);
88b22af904SAlexey Kardashevskiy #ifdef __LITTLE_ENDIAN__
89b22af904SAlexey Kardashevskiy val >>= 64-8;
90b22af904SAlexey Kardashevskiy #endif
91b22af904SAlexey Kardashevskiy return (u8)val;
92b22af904SAlexey Kardashevskiy }
93b22af904SAlexey Kardashevskiy
94b22af904SAlexey Kardashevskiy
xive_vm_source_eoi(u32 hw_irq,struct xive_irq_data * xd)95b22af904SAlexey Kardashevskiy static void xive_vm_source_eoi(u32 hw_irq, struct xive_irq_data *xd)
96b22af904SAlexey Kardashevskiy {
97b22af904SAlexey Kardashevskiy /* If the XIVE supports the new "store EOI facility, use it */
98b22af904SAlexey Kardashevskiy if (xd->flags & XIVE_IRQ_FLAG_STORE_EOI)
99b22af904SAlexey Kardashevskiy __raw_writeq(0, __x_eoi_page(xd) + XIVE_ESB_STORE_EOI);
100b22af904SAlexey Kardashevskiy else if (xd->flags & XIVE_IRQ_FLAG_LSI) {
101b22af904SAlexey Kardashevskiy /*
102b22af904SAlexey Kardashevskiy * For LSIs the HW EOI cycle is used rather than PQ bits,
103b22af904SAlexey Kardashevskiy * as they are automatically re-triggred in HW when still
104b22af904SAlexey Kardashevskiy * pending.
105b22af904SAlexey Kardashevskiy */
106b22af904SAlexey Kardashevskiy __raw_readq(__x_eoi_page(xd) + XIVE_ESB_LOAD_EOI);
107b22af904SAlexey Kardashevskiy } else {
108b22af904SAlexey Kardashevskiy uint64_t eoi_val;
109b22af904SAlexey Kardashevskiy
110b22af904SAlexey Kardashevskiy /*
111b22af904SAlexey Kardashevskiy * Otherwise for EOI, we use the special MMIO that does
112b22af904SAlexey Kardashevskiy * a clear of both P and Q and returns the old Q,
113b22af904SAlexey Kardashevskiy * except for LSIs where we use the "EOI cycle" special
114b22af904SAlexey Kardashevskiy * load.
115b22af904SAlexey Kardashevskiy *
116b22af904SAlexey Kardashevskiy * This allows us to then do a re-trigger if Q was set
117b22af904SAlexey Kardashevskiy * rather than synthetizing an interrupt in software
118b22af904SAlexey Kardashevskiy */
119b22af904SAlexey Kardashevskiy eoi_val = xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_00);
120b22af904SAlexey Kardashevskiy
121b22af904SAlexey Kardashevskiy /* Re-trigger if needed */
122b22af904SAlexey Kardashevskiy if ((eoi_val & 1) && __x_trig_page(xd))
123b22af904SAlexey Kardashevskiy __raw_writeq(0, __x_trig_page(xd));
124b22af904SAlexey Kardashevskiy }
125b22af904SAlexey Kardashevskiy }
126b22af904SAlexey Kardashevskiy
127b22af904SAlexey Kardashevskiy enum {
128b22af904SAlexey Kardashevskiy scan_fetch,
129b22af904SAlexey Kardashevskiy scan_poll,
130b22af904SAlexey Kardashevskiy scan_eoi,
131b22af904SAlexey Kardashevskiy };
132b22af904SAlexey Kardashevskiy
xive_vm_scan_interrupts(struct kvmppc_xive_vcpu * xc,u8 pending,int scan_type)133b22af904SAlexey Kardashevskiy static u32 xive_vm_scan_interrupts(struct kvmppc_xive_vcpu *xc,
134b22af904SAlexey Kardashevskiy u8 pending, int scan_type)
135b22af904SAlexey Kardashevskiy {
136b22af904SAlexey Kardashevskiy u32 hirq = 0;
137b22af904SAlexey Kardashevskiy u8 prio = 0xff;
138b22af904SAlexey Kardashevskiy
139b22af904SAlexey Kardashevskiy /* Find highest pending priority */
140b22af904SAlexey Kardashevskiy while ((xc->mfrr != 0xff || pending != 0) && hirq == 0) {
141b22af904SAlexey Kardashevskiy struct xive_q *q;
142b22af904SAlexey Kardashevskiy u32 idx, toggle;
143b22af904SAlexey Kardashevskiy __be32 *qpage;
144b22af904SAlexey Kardashevskiy
145b22af904SAlexey Kardashevskiy /*
146b22af904SAlexey Kardashevskiy * If pending is 0 this will return 0xff which is what
147b22af904SAlexey Kardashevskiy * we want
148b22af904SAlexey Kardashevskiy */
149b22af904SAlexey Kardashevskiy prio = ffs(pending) - 1;
150b22af904SAlexey Kardashevskiy
151b22af904SAlexey Kardashevskiy /* Don't scan past the guest cppr */
152b22af904SAlexey Kardashevskiy if (prio >= xc->cppr || prio > 7) {
153b22af904SAlexey Kardashevskiy if (xc->mfrr < xc->cppr) {
154b22af904SAlexey Kardashevskiy prio = xc->mfrr;
155b22af904SAlexey Kardashevskiy hirq = XICS_IPI;
156b22af904SAlexey Kardashevskiy }
157b22af904SAlexey Kardashevskiy break;
158b22af904SAlexey Kardashevskiy }
159b22af904SAlexey Kardashevskiy
160b22af904SAlexey Kardashevskiy /* Grab queue and pointers */
161b22af904SAlexey Kardashevskiy q = &xc->queues[prio];
162b22af904SAlexey Kardashevskiy idx = q->idx;
163b22af904SAlexey Kardashevskiy toggle = q->toggle;
164b22af904SAlexey Kardashevskiy
165b22af904SAlexey Kardashevskiy /*
166b22af904SAlexey Kardashevskiy * Snapshot the queue page. The test further down for EOI
167b22af904SAlexey Kardashevskiy * must use the same "copy" that was used by __xive_read_eq
168b22af904SAlexey Kardashevskiy * since qpage can be set concurrently and we don't want
169b22af904SAlexey Kardashevskiy * to miss an EOI.
170b22af904SAlexey Kardashevskiy */
171b22af904SAlexey Kardashevskiy qpage = READ_ONCE(q->qpage);
172b22af904SAlexey Kardashevskiy
173b22af904SAlexey Kardashevskiy skip_ipi:
174b22af904SAlexey Kardashevskiy /*
175b22af904SAlexey Kardashevskiy * Try to fetch from the queue. Will return 0 for a
176b22af904SAlexey Kardashevskiy * non-queueing priority (ie, qpage = 0).
177b22af904SAlexey Kardashevskiy */
178b22af904SAlexey Kardashevskiy hirq = __xive_read_eq(qpage, q->msk, &idx, &toggle);
179b22af904SAlexey Kardashevskiy
180b22af904SAlexey Kardashevskiy /*
181b22af904SAlexey Kardashevskiy * If this was a signal for an MFFR change done by
182b22af904SAlexey Kardashevskiy * H_IPI we skip it. Additionally, if we were fetching
183b22af904SAlexey Kardashevskiy * we EOI it now, thus re-enabling reception of a new
184b22af904SAlexey Kardashevskiy * such signal.
185b22af904SAlexey Kardashevskiy *
186b22af904SAlexey Kardashevskiy * We also need to do that if prio is 0 and we had no
187b22af904SAlexey Kardashevskiy * page for the queue. In this case, we have non-queued
188b22af904SAlexey Kardashevskiy * IPI that needs to be EOId.
189b22af904SAlexey Kardashevskiy *
190b22af904SAlexey Kardashevskiy * This is safe because if we have another pending MFRR
191b22af904SAlexey Kardashevskiy * change that wasn't observed above, the Q bit will have
192b22af904SAlexey Kardashevskiy * been set and another occurrence of the IPI will trigger.
193b22af904SAlexey Kardashevskiy */
194b22af904SAlexey Kardashevskiy if (hirq == XICS_IPI || (prio == 0 && !qpage)) {
195b22af904SAlexey Kardashevskiy if (scan_type == scan_fetch) {
196b22af904SAlexey Kardashevskiy xive_vm_source_eoi(xc->vp_ipi,
197b22af904SAlexey Kardashevskiy &xc->vp_ipi_data);
198b22af904SAlexey Kardashevskiy q->idx = idx;
199b22af904SAlexey Kardashevskiy q->toggle = toggle;
200b22af904SAlexey Kardashevskiy }
201b22af904SAlexey Kardashevskiy /* Loop back on same queue with updated idx/toggle */
202b22af904SAlexey Kardashevskiy WARN_ON(hirq && hirq != XICS_IPI);
203b22af904SAlexey Kardashevskiy if (hirq)
204b22af904SAlexey Kardashevskiy goto skip_ipi;
205b22af904SAlexey Kardashevskiy }
206b22af904SAlexey Kardashevskiy
207b22af904SAlexey Kardashevskiy /* If it's the dummy interrupt, continue searching */
208b22af904SAlexey Kardashevskiy if (hirq == XICS_DUMMY)
209b22af904SAlexey Kardashevskiy goto skip_ipi;
210b22af904SAlexey Kardashevskiy
211b22af904SAlexey Kardashevskiy /* Clear the pending bit if the queue is now empty */
212b22af904SAlexey Kardashevskiy if (!hirq) {
213b22af904SAlexey Kardashevskiy pending &= ~(1 << prio);
214b22af904SAlexey Kardashevskiy
215b22af904SAlexey Kardashevskiy /*
216b22af904SAlexey Kardashevskiy * Check if the queue count needs adjusting due to
217b22af904SAlexey Kardashevskiy * interrupts being moved away.
218b22af904SAlexey Kardashevskiy */
219b22af904SAlexey Kardashevskiy if (atomic_read(&q->pending_count)) {
220b22af904SAlexey Kardashevskiy int p = atomic_xchg(&q->pending_count, 0);
221b22af904SAlexey Kardashevskiy
222b22af904SAlexey Kardashevskiy if (p) {
223b22af904SAlexey Kardashevskiy WARN_ON(p > atomic_read(&q->count));
224b22af904SAlexey Kardashevskiy atomic_sub(p, &q->count);
225b22af904SAlexey Kardashevskiy }
226b22af904SAlexey Kardashevskiy }
227b22af904SAlexey Kardashevskiy }
228b22af904SAlexey Kardashevskiy
229b22af904SAlexey Kardashevskiy /*
230b22af904SAlexey Kardashevskiy * If the most favoured prio we found pending is less
231b22af904SAlexey Kardashevskiy * favored (or equal) than a pending IPI, we return
232b22af904SAlexey Kardashevskiy * the IPI instead.
233b22af904SAlexey Kardashevskiy */
234b22af904SAlexey Kardashevskiy if (prio >= xc->mfrr && xc->mfrr < xc->cppr) {
235b22af904SAlexey Kardashevskiy prio = xc->mfrr;
236b22af904SAlexey Kardashevskiy hirq = XICS_IPI;
237b22af904SAlexey Kardashevskiy break;
238b22af904SAlexey Kardashevskiy }
239b22af904SAlexey Kardashevskiy
240b22af904SAlexey Kardashevskiy /* If fetching, update queue pointers */
241b22af904SAlexey Kardashevskiy if (scan_type == scan_fetch) {
242b22af904SAlexey Kardashevskiy q->idx = idx;
243b22af904SAlexey Kardashevskiy q->toggle = toggle;
244b22af904SAlexey Kardashevskiy }
245b22af904SAlexey Kardashevskiy }
246b22af904SAlexey Kardashevskiy
247b22af904SAlexey Kardashevskiy /* If we are just taking a "peek", do nothing else */
248b22af904SAlexey Kardashevskiy if (scan_type == scan_poll)
249b22af904SAlexey Kardashevskiy return hirq;
250b22af904SAlexey Kardashevskiy
251b22af904SAlexey Kardashevskiy /* Update the pending bits */
252b22af904SAlexey Kardashevskiy xc->pending = pending;
253b22af904SAlexey Kardashevskiy
254b22af904SAlexey Kardashevskiy /*
255b22af904SAlexey Kardashevskiy * If this is an EOI that's it, no CPPR adjustment done here,
256b22af904SAlexey Kardashevskiy * all we needed was cleanup the stale pending bits and check
257b22af904SAlexey Kardashevskiy * if there's anything left.
258b22af904SAlexey Kardashevskiy */
259b22af904SAlexey Kardashevskiy if (scan_type == scan_eoi)
260b22af904SAlexey Kardashevskiy return hirq;
261b22af904SAlexey Kardashevskiy
262b22af904SAlexey Kardashevskiy /*
263b22af904SAlexey Kardashevskiy * If we found an interrupt, adjust what the guest CPPR should
264b22af904SAlexey Kardashevskiy * be as if we had just fetched that interrupt from HW.
265b22af904SAlexey Kardashevskiy *
266b22af904SAlexey Kardashevskiy * Note: This can only make xc->cppr smaller as the previous
267b22af904SAlexey Kardashevskiy * loop will only exit with hirq != 0 if prio is lower than
268b22af904SAlexey Kardashevskiy * the current xc->cppr. Thus we don't need to re-check xc->mfrr
269b22af904SAlexey Kardashevskiy * for pending IPIs.
270b22af904SAlexey Kardashevskiy */
271b22af904SAlexey Kardashevskiy if (hirq)
272b22af904SAlexey Kardashevskiy xc->cppr = prio;
273b22af904SAlexey Kardashevskiy /*
274b22af904SAlexey Kardashevskiy * If it was an IPI the HW CPPR might have been lowered too much
275b22af904SAlexey Kardashevskiy * as the HW interrupt we use for IPIs is routed to priority 0.
276b22af904SAlexey Kardashevskiy *
277b22af904SAlexey Kardashevskiy * We re-sync it here.
278b22af904SAlexey Kardashevskiy */
279b22af904SAlexey Kardashevskiy if (xc->cppr != xc->hw_cppr) {
280b22af904SAlexey Kardashevskiy xc->hw_cppr = xc->cppr;
281b22af904SAlexey Kardashevskiy __raw_writeb(xc->cppr, xive_tima + TM_QW1_OS + TM_CPPR);
282b22af904SAlexey Kardashevskiy }
283b22af904SAlexey Kardashevskiy
284b22af904SAlexey Kardashevskiy return hirq;
285b22af904SAlexey Kardashevskiy }
286b22af904SAlexey Kardashevskiy
xive_vm_h_xirr(struct kvm_vcpu * vcpu)287b22af904SAlexey Kardashevskiy static unsigned long xive_vm_h_xirr(struct kvm_vcpu *vcpu)
288b22af904SAlexey Kardashevskiy {
289b22af904SAlexey Kardashevskiy struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
290b22af904SAlexey Kardashevskiy u8 old_cppr;
291b22af904SAlexey Kardashevskiy u32 hirq;
292b22af904SAlexey Kardashevskiy
293b22af904SAlexey Kardashevskiy pr_devel("H_XIRR\n");
294b22af904SAlexey Kardashevskiy
295b22af904SAlexey Kardashevskiy xc->stat_vm_h_xirr++;
296b22af904SAlexey Kardashevskiy
297b22af904SAlexey Kardashevskiy /* First collect pending bits from HW */
298b22af904SAlexey Kardashevskiy xive_vm_ack_pending(xc);
299b22af904SAlexey Kardashevskiy
300b22af904SAlexey Kardashevskiy pr_devel(" new pending=0x%02x hw_cppr=%d cppr=%d\n",
301b22af904SAlexey Kardashevskiy xc->pending, xc->hw_cppr, xc->cppr);
302b22af904SAlexey Kardashevskiy
303b22af904SAlexey Kardashevskiy /* Grab previous CPPR and reverse map it */
304b22af904SAlexey Kardashevskiy old_cppr = xive_prio_to_guest(xc->cppr);
305b22af904SAlexey Kardashevskiy
306b22af904SAlexey Kardashevskiy /* Scan for actual interrupts */
307b22af904SAlexey Kardashevskiy hirq = xive_vm_scan_interrupts(xc, xc->pending, scan_fetch);
308b22af904SAlexey Kardashevskiy
309b22af904SAlexey Kardashevskiy pr_devel(" got hirq=0x%x hw_cppr=%d cppr=%d\n",
310b22af904SAlexey Kardashevskiy hirq, xc->hw_cppr, xc->cppr);
311b22af904SAlexey Kardashevskiy
312b22af904SAlexey Kardashevskiy /* That should never hit */
313b22af904SAlexey Kardashevskiy if (hirq & 0xff000000)
314b22af904SAlexey Kardashevskiy pr_warn("XIVE: Weird guest interrupt number 0x%08x\n", hirq);
315b22af904SAlexey Kardashevskiy
316b22af904SAlexey Kardashevskiy /*
317b22af904SAlexey Kardashevskiy * XXX We could check if the interrupt is masked here and
318b22af904SAlexey Kardashevskiy * filter it. If we chose to do so, we would need to do:
319b22af904SAlexey Kardashevskiy *
320b22af904SAlexey Kardashevskiy * if (masked) {
321b22af904SAlexey Kardashevskiy * lock();
322b22af904SAlexey Kardashevskiy * if (masked) {
323b22af904SAlexey Kardashevskiy * old_Q = true;
324b22af904SAlexey Kardashevskiy * hirq = 0;
325b22af904SAlexey Kardashevskiy * }
326b22af904SAlexey Kardashevskiy * unlock();
327b22af904SAlexey Kardashevskiy * }
328b22af904SAlexey Kardashevskiy */
329b22af904SAlexey Kardashevskiy
330b22af904SAlexey Kardashevskiy /* Return interrupt and old CPPR in GPR4 */
331b22af904SAlexey Kardashevskiy vcpu->arch.regs.gpr[4] = hirq | (old_cppr << 24);
332b22af904SAlexey Kardashevskiy
333b22af904SAlexey Kardashevskiy return H_SUCCESS;
334b22af904SAlexey Kardashevskiy }
335b22af904SAlexey Kardashevskiy
xive_vm_h_ipoll(struct kvm_vcpu * vcpu,unsigned long server)336b22af904SAlexey Kardashevskiy static unsigned long xive_vm_h_ipoll(struct kvm_vcpu *vcpu, unsigned long server)
337b22af904SAlexey Kardashevskiy {
338b22af904SAlexey Kardashevskiy struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
339b22af904SAlexey Kardashevskiy u8 pending = xc->pending;
340b22af904SAlexey Kardashevskiy u32 hirq;
341b22af904SAlexey Kardashevskiy
342b22af904SAlexey Kardashevskiy pr_devel("H_IPOLL(server=%ld)\n", server);
343b22af904SAlexey Kardashevskiy
344b22af904SAlexey Kardashevskiy xc->stat_vm_h_ipoll++;
345b22af904SAlexey Kardashevskiy
346b22af904SAlexey Kardashevskiy /* Grab the target VCPU if not the current one */
347b22af904SAlexey Kardashevskiy if (xc->server_num != server) {
348b22af904SAlexey Kardashevskiy vcpu = kvmppc_xive_find_server(vcpu->kvm, server);
349b22af904SAlexey Kardashevskiy if (!vcpu)
350b22af904SAlexey Kardashevskiy return H_PARAMETER;
351b22af904SAlexey Kardashevskiy xc = vcpu->arch.xive_vcpu;
352b22af904SAlexey Kardashevskiy
353b22af904SAlexey Kardashevskiy /* Scan all priorities */
354b22af904SAlexey Kardashevskiy pending = 0xff;
355b22af904SAlexey Kardashevskiy } else {
356b22af904SAlexey Kardashevskiy /* Grab pending interrupt if any */
357b22af904SAlexey Kardashevskiy __be64 qw1 = __raw_readq(xive_tima + TM_QW1_OS);
358b22af904SAlexey Kardashevskiy u8 pipr = be64_to_cpu(qw1) & 0xff;
359b22af904SAlexey Kardashevskiy
360b22af904SAlexey Kardashevskiy if (pipr < 8)
361b22af904SAlexey Kardashevskiy pending |= 1 << pipr;
362b22af904SAlexey Kardashevskiy }
363b22af904SAlexey Kardashevskiy
364b22af904SAlexey Kardashevskiy hirq = xive_vm_scan_interrupts(xc, pending, scan_poll);
365b22af904SAlexey Kardashevskiy
366b22af904SAlexey Kardashevskiy /* Return interrupt and old CPPR in GPR4 */
367b22af904SAlexey Kardashevskiy vcpu->arch.regs.gpr[4] = hirq | (xc->cppr << 24);
368b22af904SAlexey Kardashevskiy
369b22af904SAlexey Kardashevskiy return H_SUCCESS;
370b22af904SAlexey Kardashevskiy }
371b22af904SAlexey Kardashevskiy
xive_vm_push_pending_to_hw(struct kvmppc_xive_vcpu * xc)372b22af904SAlexey Kardashevskiy static void xive_vm_push_pending_to_hw(struct kvmppc_xive_vcpu *xc)
373b22af904SAlexey Kardashevskiy {
374b22af904SAlexey Kardashevskiy u8 pending, prio;
375b22af904SAlexey Kardashevskiy
376b22af904SAlexey Kardashevskiy pending = xc->pending;
377b22af904SAlexey Kardashevskiy if (xc->mfrr != 0xff) {
378b22af904SAlexey Kardashevskiy if (xc->mfrr < 8)
379b22af904SAlexey Kardashevskiy pending |= 1 << xc->mfrr;
380b22af904SAlexey Kardashevskiy else
381b22af904SAlexey Kardashevskiy pending |= 0x80;
382b22af904SAlexey Kardashevskiy }
383b22af904SAlexey Kardashevskiy if (!pending)
384b22af904SAlexey Kardashevskiy return;
385b22af904SAlexey Kardashevskiy prio = ffs(pending) - 1;
386b22af904SAlexey Kardashevskiy
387b22af904SAlexey Kardashevskiy __raw_writeb(prio, xive_tima + TM_SPC_SET_OS_PENDING);
388b22af904SAlexey Kardashevskiy }
389b22af904SAlexey Kardashevskiy
xive_vm_scan_for_rerouted_irqs(struct kvmppc_xive * xive,struct kvmppc_xive_vcpu * xc)390b22af904SAlexey Kardashevskiy static void xive_vm_scan_for_rerouted_irqs(struct kvmppc_xive *xive,
391b22af904SAlexey Kardashevskiy struct kvmppc_xive_vcpu *xc)
392b22af904SAlexey Kardashevskiy {
393b22af904SAlexey Kardashevskiy unsigned int prio;
394b22af904SAlexey Kardashevskiy
395b22af904SAlexey Kardashevskiy /* For each priority that is now masked */
396b22af904SAlexey Kardashevskiy for (prio = xc->cppr; prio < KVMPPC_XIVE_Q_COUNT; prio++) {
397b22af904SAlexey Kardashevskiy struct xive_q *q = &xc->queues[prio];
398b22af904SAlexey Kardashevskiy struct kvmppc_xive_irq_state *state;
399b22af904SAlexey Kardashevskiy struct kvmppc_xive_src_block *sb;
400b22af904SAlexey Kardashevskiy u32 idx, toggle, entry, irq, hw_num;
401b22af904SAlexey Kardashevskiy struct xive_irq_data *xd;
402b22af904SAlexey Kardashevskiy __be32 *qpage;
403b22af904SAlexey Kardashevskiy u16 src;
404b22af904SAlexey Kardashevskiy
405b22af904SAlexey Kardashevskiy idx = q->idx;
406b22af904SAlexey Kardashevskiy toggle = q->toggle;
407b22af904SAlexey Kardashevskiy qpage = READ_ONCE(q->qpage);
408b22af904SAlexey Kardashevskiy if (!qpage)
409b22af904SAlexey Kardashevskiy continue;
410b22af904SAlexey Kardashevskiy
411b22af904SAlexey Kardashevskiy /* For each interrupt in the queue */
412b22af904SAlexey Kardashevskiy for (;;) {
413b22af904SAlexey Kardashevskiy entry = be32_to_cpup(qpage + idx);
414b22af904SAlexey Kardashevskiy
415b22af904SAlexey Kardashevskiy /* No more ? */
416b22af904SAlexey Kardashevskiy if ((entry >> 31) == toggle)
417b22af904SAlexey Kardashevskiy break;
418b22af904SAlexey Kardashevskiy irq = entry & 0x7fffffff;
419b22af904SAlexey Kardashevskiy
420b22af904SAlexey Kardashevskiy /* Skip dummies and IPIs */
421b22af904SAlexey Kardashevskiy if (irq == XICS_DUMMY || irq == XICS_IPI)
422b22af904SAlexey Kardashevskiy goto next;
423b22af904SAlexey Kardashevskiy sb = kvmppc_xive_find_source(xive, irq, &src);
424b22af904SAlexey Kardashevskiy if (!sb)
425b22af904SAlexey Kardashevskiy goto next;
426b22af904SAlexey Kardashevskiy state = &sb->irq_state[src];
427b22af904SAlexey Kardashevskiy
428b22af904SAlexey Kardashevskiy /* Has it been rerouted ? */
429b22af904SAlexey Kardashevskiy if (xc->server_num == state->act_server)
430b22af904SAlexey Kardashevskiy goto next;
431b22af904SAlexey Kardashevskiy
432b22af904SAlexey Kardashevskiy /*
433b22af904SAlexey Kardashevskiy * Allright, it *has* been re-routed, kill it from
434b22af904SAlexey Kardashevskiy * the queue.
435b22af904SAlexey Kardashevskiy */
436b22af904SAlexey Kardashevskiy qpage[idx] = cpu_to_be32((entry & 0x80000000) | XICS_DUMMY);
437b22af904SAlexey Kardashevskiy
438b22af904SAlexey Kardashevskiy /* Find the HW interrupt */
439b22af904SAlexey Kardashevskiy kvmppc_xive_select_irq(state, &hw_num, &xd);
440b22af904SAlexey Kardashevskiy
441b22af904SAlexey Kardashevskiy /* If it's not an LSI, set PQ to 11 the EOI will force a resend */
442b22af904SAlexey Kardashevskiy if (!(xd->flags & XIVE_IRQ_FLAG_LSI))
443b22af904SAlexey Kardashevskiy xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_11);
444b22af904SAlexey Kardashevskiy
445b22af904SAlexey Kardashevskiy /* EOI the source */
446b22af904SAlexey Kardashevskiy xive_vm_source_eoi(hw_num, xd);
447b22af904SAlexey Kardashevskiy
448b22af904SAlexey Kardashevskiy next:
449b22af904SAlexey Kardashevskiy idx = (idx + 1) & q->msk;
450b22af904SAlexey Kardashevskiy if (idx == 0)
451b22af904SAlexey Kardashevskiy toggle ^= 1;
452b22af904SAlexey Kardashevskiy }
453b22af904SAlexey Kardashevskiy }
454b22af904SAlexey Kardashevskiy }
455b22af904SAlexey Kardashevskiy
xive_vm_h_cppr(struct kvm_vcpu * vcpu,unsigned long cppr)456b22af904SAlexey Kardashevskiy static int xive_vm_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr)
457b22af904SAlexey Kardashevskiy {
458b22af904SAlexey Kardashevskiy struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
459b22af904SAlexey Kardashevskiy struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
460b22af904SAlexey Kardashevskiy u8 old_cppr;
461b22af904SAlexey Kardashevskiy
462b22af904SAlexey Kardashevskiy pr_devel("H_CPPR(cppr=%ld)\n", cppr);
463b22af904SAlexey Kardashevskiy
464b22af904SAlexey Kardashevskiy xc->stat_vm_h_cppr++;
465b22af904SAlexey Kardashevskiy
466b22af904SAlexey Kardashevskiy /* Map CPPR */
467b22af904SAlexey Kardashevskiy cppr = xive_prio_from_guest(cppr);
468b22af904SAlexey Kardashevskiy
469b22af904SAlexey Kardashevskiy /* Remember old and update SW state */
470b22af904SAlexey Kardashevskiy old_cppr = xc->cppr;
471b22af904SAlexey Kardashevskiy xc->cppr = cppr;
472b22af904SAlexey Kardashevskiy
473b22af904SAlexey Kardashevskiy /*
474b22af904SAlexey Kardashevskiy * Order the above update of xc->cppr with the subsequent
475b22af904SAlexey Kardashevskiy * read of xc->mfrr inside push_pending_to_hw()
476b22af904SAlexey Kardashevskiy */
477b22af904SAlexey Kardashevskiy smp_mb();
478b22af904SAlexey Kardashevskiy
479b22af904SAlexey Kardashevskiy if (cppr > old_cppr) {
480b22af904SAlexey Kardashevskiy /*
481b22af904SAlexey Kardashevskiy * We are masking less, we need to look for pending things
482b22af904SAlexey Kardashevskiy * to deliver and set VP pending bits accordingly to trigger
483b22af904SAlexey Kardashevskiy * a new interrupt otherwise we might miss MFRR changes for
484b22af904SAlexey Kardashevskiy * which we have optimized out sending an IPI signal.
485b22af904SAlexey Kardashevskiy */
486b22af904SAlexey Kardashevskiy xive_vm_push_pending_to_hw(xc);
487b22af904SAlexey Kardashevskiy } else {
488b22af904SAlexey Kardashevskiy /*
489b22af904SAlexey Kardashevskiy * We are masking more, we need to check the queue for any
490b22af904SAlexey Kardashevskiy * interrupt that has been routed to another CPU, take
491b22af904SAlexey Kardashevskiy * it out (replace it with the dummy) and retrigger it.
492b22af904SAlexey Kardashevskiy *
493b22af904SAlexey Kardashevskiy * This is necessary since those interrupts may otherwise
494b22af904SAlexey Kardashevskiy * never be processed, at least not until this CPU restores
495b22af904SAlexey Kardashevskiy * its CPPR.
496b22af904SAlexey Kardashevskiy *
497b22af904SAlexey Kardashevskiy * This is in theory racy vs. HW adding new interrupts to
498b22af904SAlexey Kardashevskiy * the queue. In practice this works because the interesting
499b22af904SAlexey Kardashevskiy * cases are when the guest has done a set_xive() to move the
500b22af904SAlexey Kardashevskiy * interrupt away, which flushes the xive, followed by the
501b22af904SAlexey Kardashevskiy * target CPU doing a H_CPPR. So any new interrupt coming into
502b22af904SAlexey Kardashevskiy * the queue must still be routed to us and isn't a source
503b22af904SAlexey Kardashevskiy * of concern.
504b22af904SAlexey Kardashevskiy */
505b22af904SAlexey Kardashevskiy xive_vm_scan_for_rerouted_irqs(xive, xc);
506b22af904SAlexey Kardashevskiy }
507b22af904SAlexey Kardashevskiy
508b22af904SAlexey Kardashevskiy /* Apply new CPPR */
509b22af904SAlexey Kardashevskiy xc->hw_cppr = cppr;
510b22af904SAlexey Kardashevskiy __raw_writeb(cppr, xive_tima + TM_QW1_OS + TM_CPPR);
511b22af904SAlexey Kardashevskiy
512b22af904SAlexey Kardashevskiy return H_SUCCESS;
513b22af904SAlexey Kardashevskiy }
514b22af904SAlexey Kardashevskiy
xive_vm_h_eoi(struct kvm_vcpu * vcpu,unsigned long xirr)515b22af904SAlexey Kardashevskiy static int xive_vm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr)
516b22af904SAlexey Kardashevskiy {
517b22af904SAlexey Kardashevskiy struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
518b22af904SAlexey Kardashevskiy struct kvmppc_xive_src_block *sb;
519b22af904SAlexey Kardashevskiy struct kvmppc_xive_irq_state *state;
520b22af904SAlexey Kardashevskiy struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
521b22af904SAlexey Kardashevskiy struct xive_irq_data *xd;
522b22af904SAlexey Kardashevskiy u8 new_cppr = xirr >> 24;
523b22af904SAlexey Kardashevskiy u32 irq = xirr & 0x00ffffff, hw_num;
524b22af904SAlexey Kardashevskiy u16 src;
525b22af904SAlexey Kardashevskiy int rc = 0;
526b22af904SAlexey Kardashevskiy
527b22af904SAlexey Kardashevskiy pr_devel("H_EOI(xirr=%08lx)\n", xirr);
528b22af904SAlexey Kardashevskiy
529b22af904SAlexey Kardashevskiy xc->stat_vm_h_eoi++;
530b22af904SAlexey Kardashevskiy
531b22af904SAlexey Kardashevskiy xc->cppr = xive_prio_from_guest(new_cppr);
532b22af904SAlexey Kardashevskiy
533b22af904SAlexey Kardashevskiy /*
534b22af904SAlexey Kardashevskiy * IPIs are synthetized from MFRR and thus don't need
535b22af904SAlexey Kardashevskiy * any special EOI handling. The underlying interrupt
536b22af904SAlexey Kardashevskiy * used to signal MFRR changes is EOId when fetched from
537b22af904SAlexey Kardashevskiy * the queue.
538b22af904SAlexey Kardashevskiy */
539b22af904SAlexey Kardashevskiy if (irq == XICS_IPI || irq == 0) {
540b22af904SAlexey Kardashevskiy /*
541b22af904SAlexey Kardashevskiy * This barrier orders the setting of xc->cppr vs.
542b22af904SAlexey Kardashevskiy * subsequent test of xc->mfrr done inside
543b22af904SAlexey Kardashevskiy * scan_interrupts and push_pending_to_hw
544b22af904SAlexey Kardashevskiy */
545b22af904SAlexey Kardashevskiy smp_mb();
546b22af904SAlexey Kardashevskiy goto bail;
547b22af904SAlexey Kardashevskiy }
548b22af904SAlexey Kardashevskiy
549b22af904SAlexey Kardashevskiy /* Find interrupt source */
550b22af904SAlexey Kardashevskiy sb = kvmppc_xive_find_source(xive, irq, &src);
551b22af904SAlexey Kardashevskiy if (!sb) {
552b22af904SAlexey Kardashevskiy pr_devel(" source not found !\n");
553b22af904SAlexey Kardashevskiy rc = H_PARAMETER;
554b22af904SAlexey Kardashevskiy /* Same as above */
555b22af904SAlexey Kardashevskiy smp_mb();
556b22af904SAlexey Kardashevskiy goto bail;
557b22af904SAlexey Kardashevskiy }
558b22af904SAlexey Kardashevskiy state = &sb->irq_state[src];
559b22af904SAlexey Kardashevskiy kvmppc_xive_select_irq(state, &hw_num, &xd);
560b22af904SAlexey Kardashevskiy
561b22af904SAlexey Kardashevskiy state->in_eoi = true;
562b22af904SAlexey Kardashevskiy
563b22af904SAlexey Kardashevskiy /*
564b22af904SAlexey Kardashevskiy * This barrier orders both setting of in_eoi above vs,
565b22af904SAlexey Kardashevskiy * subsequent test of guest_priority, and the setting
566b22af904SAlexey Kardashevskiy * of xc->cppr vs. subsequent test of xc->mfrr done inside
567b22af904SAlexey Kardashevskiy * scan_interrupts and push_pending_to_hw
568b22af904SAlexey Kardashevskiy */
569b22af904SAlexey Kardashevskiy smp_mb();
570b22af904SAlexey Kardashevskiy
571b22af904SAlexey Kardashevskiy again:
572b22af904SAlexey Kardashevskiy if (state->guest_priority == MASKED) {
573b22af904SAlexey Kardashevskiy arch_spin_lock(&sb->lock);
574b22af904SAlexey Kardashevskiy if (state->guest_priority != MASKED) {
575b22af904SAlexey Kardashevskiy arch_spin_unlock(&sb->lock);
576b22af904SAlexey Kardashevskiy goto again;
577b22af904SAlexey Kardashevskiy }
578b22af904SAlexey Kardashevskiy pr_devel(" EOI on saved P...\n");
579b22af904SAlexey Kardashevskiy
580b22af904SAlexey Kardashevskiy /* Clear old_p, that will cause unmask to perform an EOI */
581b22af904SAlexey Kardashevskiy state->old_p = false;
582b22af904SAlexey Kardashevskiy
583b22af904SAlexey Kardashevskiy arch_spin_unlock(&sb->lock);
584b22af904SAlexey Kardashevskiy } else {
585b22af904SAlexey Kardashevskiy pr_devel(" EOI on source...\n");
586b22af904SAlexey Kardashevskiy
587b22af904SAlexey Kardashevskiy /* Perform EOI on the source */
588b22af904SAlexey Kardashevskiy xive_vm_source_eoi(hw_num, xd);
589b22af904SAlexey Kardashevskiy
590b22af904SAlexey Kardashevskiy /* If it's an emulated LSI, check level and resend */
591b22af904SAlexey Kardashevskiy if (state->lsi && state->asserted)
592b22af904SAlexey Kardashevskiy __raw_writeq(0, __x_trig_page(xd));
593b22af904SAlexey Kardashevskiy
594b22af904SAlexey Kardashevskiy }
595b22af904SAlexey Kardashevskiy
596b22af904SAlexey Kardashevskiy /*
597b22af904SAlexey Kardashevskiy * This barrier orders the above guest_priority check
598b22af904SAlexey Kardashevskiy * and spin_lock/unlock with clearing in_eoi below.
599b22af904SAlexey Kardashevskiy *
600b22af904SAlexey Kardashevskiy * It also has to be a full mb() as it must ensure
601b22af904SAlexey Kardashevskiy * the MMIOs done in source_eoi() are completed before
602b22af904SAlexey Kardashevskiy * state->in_eoi is visible.
603b22af904SAlexey Kardashevskiy */
604b22af904SAlexey Kardashevskiy mb();
605b22af904SAlexey Kardashevskiy state->in_eoi = false;
606b22af904SAlexey Kardashevskiy bail:
607b22af904SAlexey Kardashevskiy
608b22af904SAlexey Kardashevskiy /* Re-evaluate pending IRQs and update HW */
609b22af904SAlexey Kardashevskiy xive_vm_scan_interrupts(xc, xc->pending, scan_eoi);
610b22af904SAlexey Kardashevskiy xive_vm_push_pending_to_hw(xc);
611b22af904SAlexey Kardashevskiy pr_devel(" after scan pending=%02x\n", xc->pending);
612b22af904SAlexey Kardashevskiy
613b22af904SAlexey Kardashevskiy /* Apply new CPPR */
614b22af904SAlexey Kardashevskiy xc->hw_cppr = xc->cppr;
615b22af904SAlexey Kardashevskiy __raw_writeb(xc->cppr, xive_tima + TM_QW1_OS + TM_CPPR);
616b22af904SAlexey Kardashevskiy
617b22af904SAlexey Kardashevskiy return rc;
618b22af904SAlexey Kardashevskiy }
619b22af904SAlexey Kardashevskiy
xive_vm_h_ipi(struct kvm_vcpu * vcpu,unsigned long server,unsigned long mfrr)620b22af904SAlexey Kardashevskiy static int xive_vm_h_ipi(struct kvm_vcpu *vcpu, unsigned long server,
621b22af904SAlexey Kardashevskiy unsigned long mfrr)
622b22af904SAlexey Kardashevskiy {
623b22af904SAlexey Kardashevskiy struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
624b22af904SAlexey Kardashevskiy
625b22af904SAlexey Kardashevskiy pr_devel("H_IPI(server=%08lx,mfrr=%ld)\n", server, mfrr);
626b22af904SAlexey Kardashevskiy
627b22af904SAlexey Kardashevskiy xc->stat_vm_h_ipi++;
628b22af904SAlexey Kardashevskiy
629b22af904SAlexey Kardashevskiy /* Find target */
630b22af904SAlexey Kardashevskiy vcpu = kvmppc_xive_find_server(vcpu->kvm, server);
631b22af904SAlexey Kardashevskiy if (!vcpu)
632b22af904SAlexey Kardashevskiy return H_PARAMETER;
633b22af904SAlexey Kardashevskiy xc = vcpu->arch.xive_vcpu;
634b22af904SAlexey Kardashevskiy
635b22af904SAlexey Kardashevskiy /* Locklessly write over MFRR */
636b22af904SAlexey Kardashevskiy xc->mfrr = mfrr;
637b22af904SAlexey Kardashevskiy
638b22af904SAlexey Kardashevskiy /*
639b22af904SAlexey Kardashevskiy * The load of xc->cppr below and the subsequent MMIO store
640b22af904SAlexey Kardashevskiy * to the IPI must happen after the above mfrr update is
641b22af904SAlexey Kardashevskiy * globally visible so that:
642b22af904SAlexey Kardashevskiy *
643b22af904SAlexey Kardashevskiy * - Synchronize with another CPU doing an H_EOI or a H_CPPR
644b22af904SAlexey Kardashevskiy * updating xc->cppr then reading xc->mfrr.
645b22af904SAlexey Kardashevskiy *
646b22af904SAlexey Kardashevskiy * - The target of the IPI sees the xc->mfrr update
647b22af904SAlexey Kardashevskiy */
648b22af904SAlexey Kardashevskiy mb();
649b22af904SAlexey Kardashevskiy
650b22af904SAlexey Kardashevskiy /* Shoot the IPI if most favored than target cppr */
651b22af904SAlexey Kardashevskiy if (mfrr < xc->cppr)
652b22af904SAlexey Kardashevskiy __raw_writeq(0, __x_trig_page(&xc->vp_ipi_data));
653b22af904SAlexey Kardashevskiy
654b22af904SAlexey Kardashevskiy return H_SUCCESS;
655b22af904SAlexey Kardashevskiy }
6565af50993SBenjamin Herrenschmidt
6575af50993SBenjamin Herrenschmidt /*
6585af50993SBenjamin Herrenschmidt * We leave a gap of a couple of interrupts in the queue to
6595af50993SBenjamin Herrenschmidt * account for the IPI and additional safety guard.
6605af50993SBenjamin Herrenschmidt */
6615af50993SBenjamin Herrenschmidt #define XIVE_Q_GAP 2
6625af50993SBenjamin Herrenschmidt
kvmppc_xive_vcpu_has_save_restore(struct kvm_vcpu * vcpu)663f5af0a97SCédric Le Goater static bool kvmppc_xive_vcpu_has_save_restore(struct kvm_vcpu *vcpu)
664f5af0a97SCédric Le Goater {
665f5af0a97SCédric Le Goater struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
666f5af0a97SCédric Le Goater
667f5af0a97SCédric Le Goater /* Check enablement at VP level */
668f5af0a97SCédric Le Goater return xc->vp_cam & TM_QW1W2_HO;
669f5af0a97SCédric Le Goater }
670f5af0a97SCédric Le Goater
kvmppc_xive_check_save_restore(struct kvm_vcpu * vcpu)671f5af0a97SCédric Le Goater bool kvmppc_xive_check_save_restore(struct kvm_vcpu *vcpu)
672f5af0a97SCédric Le Goater {
673f5af0a97SCédric Le Goater struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
674f5af0a97SCédric Le Goater struct kvmppc_xive *xive = xc->xive;
675f5af0a97SCédric Le Goater
676f5af0a97SCédric Le Goater if (xive->flags & KVMPPC_XIVE_FLAG_SAVE_RESTORE)
677f5af0a97SCédric Le Goater return kvmppc_xive_vcpu_has_save_restore(vcpu);
678f5af0a97SCédric Le Goater
679f5af0a97SCédric Le Goater return true;
680f5af0a97SCédric Le Goater }
681f5af0a97SCédric Le Goater
6825af50993SBenjamin Herrenschmidt /*
68395a6432cSPaul Mackerras * Push a vcpu's context to the XIVE on guest entry.
68495a6432cSPaul Mackerras * This assumes we are in virtual mode (MMU on)
68595a6432cSPaul Mackerras */
kvmppc_xive_push_vcpu(struct kvm_vcpu * vcpu)68695a6432cSPaul Mackerras void kvmppc_xive_push_vcpu(struct kvm_vcpu *vcpu)
68795a6432cSPaul Mackerras {
68895a6432cSPaul Mackerras void __iomem *tima = local_paca->kvm_hstate.xive_tima_virt;
68995a6432cSPaul Mackerras u64 pq;
69095a6432cSPaul Mackerras
6918d4ba9c9SPaul Mackerras /*
6928d4ba9c9SPaul Mackerras * Nothing to do if the platform doesn't have a XIVE
6938d4ba9c9SPaul Mackerras * or this vCPU doesn't have its own XIVE context
6948d4ba9c9SPaul Mackerras * (e.g. because it's not using an in-kernel interrupt controller).
6958d4ba9c9SPaul Mackerras */
6968d4ba9c9SPaul Mackerras if (!tima || !vcpu->arch.xive_cam_word)
69795a6432cSPaul Mackerras return;
6988d4ba9c9SPaul Mackerras
69995a6432cSPaul Mackerras eieio();
700f5af0a97SCédric Le Goater if (!kvmppc_xive_vcpu_has_save_restore(vcpu))
70195a6432cSPaul Mackerras __raw_writeq(vcpu->arch.xive_saved_state.w01, tima + TM_QW1_OS);
70295a6432cSPaul Mackerras __raw_writel(vcpu->arch.xive_cam_word, tima + TM_QW1_OS + TM_WORD2);
70395a6432cSPaul Mackerras vcpu->arch.xive_pushed = 1;
70495a6432cSPaul Mackerras eieio();
70595a6432cSPaul Mackerras
70695a6432cSPaul Mackerras /*
70795a6432cSPaul Mackerras * We clear the irq_pending flag. There is a small chance of a
70895a6432cSPaul Mackerras * race vs. the escalation interrupt happening on another
70995a6432cSPaul Mackerras * processor setting it again, but the only consequence is to
71095a6432cSPaul Mackerras * cause a spurious wakeup on the next H_CEDE, which is not an
71195a6432cSPaul Mackerras * issue.
71295a6432cSPaul Mackerras */
71395a6432cSPaul Mackerras vcpu->arch.irq_pending = 0;
71495a6432cSPaul Mackerras
71595a6432cSPaul Mackerras /*
71695a6432cSPaul Mackerras * In single escalation mode, if the escalation interrupt is
71795a6432cSPaul Mackerras * on, we mask it.
71895a6432cSPaul Mackerras */
71995a6432cSPaul Mackerras if (vcpu->arch.xive_esc_on) {
72095a6432cSPaul Mackerras pq = __raw_readq((void __iomem *)(vcpu->arch.xive_esc_vaddr +
72195a6432cSPaul Mackerras XIVE_ESB_SET_PQ_01));
72295a6432cSPaul Mackerras mb();
72395a6432cSPaul Mackerras
72495a6432cSPaul Mackerras /*
72595a6432cSPaul Mackerras * We have a possible subtle race here: The escalation
72695a6432cSPaul Mackerras * interrupt might have fired and be on its way to the
72795a6432cSPaul Mackerras * host queue while we mask it, and if we unmask it
72895a6432cSPaul Mackerras * early enough (re-cede right away), there is a
7291fd02f66SJulia Lawall * theoretical possibility that it fires again, thus
73095a6432cSPaul Mackerras * landing in the target queue more than once which is
73195a6432cSPaul Mackerras * a big no-no.
73295a6432cSPaul Mackerras *
73395a6432cSPaul Mackerras * Fortunately, solving this is rather easy. If the
73495a6432cSPaul Mackerras * above load setting PQ to 01 returns a previous
73595a6432cSPaul Mackerras * value where P is set, then we know the escalation
73695a6432cSPaul Mackerras * interrupt is somewhere on its way to the host. In
73795a6432cSPaul Mackerras * that case we simply don't clear the xive_esc_on
73895a6432cSPaul Mackerras * flag below. It will be eventually cleared by the
73995a6432cSPaul Mackerras * handler for the escalation interrupt.
74095a6432cSPaul Mackerras *
74195a6432cSPaul Mackerras * Then, when doing a cede, we check that flag again
74295a6432cSPaul Mackerras * before re-enabling the escalation interrupt, and if
74395a6432cSPaul Mackerras * set, we abort the cede.
74495a6432cSPaul Mackerras */
74595a6432cSPaul Mackerras if (!(pq & XIVE_ESB_VAL_P))
74695a6432cSPaul Mackerras /* Now P is 0, we can clear the flag */
74795a6432cSPaul Mackerras vcpu->arch.xive_esc_on = 0;
74895a6432cSPaul Mackerras }
74995a6432cSPaul Mackerras }
75095a6432cSPaul Mackerras EXPORT_SYMBOL_GPL(kvmppc_xive_push_vcpu);
75195a6432cSPaul Mackerras
75295a6432cSPaul Mackerras /*
753023c3c96SNicholas Piggin * Pull a vcpu's context from the XIVE on guest exit.
754023c3c96SNicholas Piggin * This assumes we are in virtual mode (MMU on)
755023c3c96SNicholas Piggin */
kvmppc_xive_pull_vcpu(struct kvm_vcpu * vcpu)756023c3c96SNicholas Piggin void kvmppc_xive_pull_vcpu(struct kvm_vcpu *vcpu)
757023c3c96SNicholas Piggin {
758023c3c96SNicholas Piggin void __iomem *tima = local_paca->kvm_hstate.xive_tima_virt;
759023c3c96SNicholas Piggin
760023c3c96SNicholas Piggin if (!vcpu->arch.xive_pushed)
761023c3c96SNicholas Piggin return;
762023c3c96SNicholas Piggin
763023c3c96SNicholas Piggin /*
764023c3c96SNicholas Piggin * Should not have been pushed if there is no tima
765023c3c96SNicholas Piggin */
766023c3c96SNicholas Piggin if (WARN_ON(!tima))
767023c3c96SNicholas Piggin return;
768023c3c96SNicholas Piggin
769023c3c96SNicholas Piggin eieio();
770023c3c96SNicholas Piggin /* First load to pull the context, we ignore the value */
771023c3c96SNicholas Piggin __raw_readl(tima + TM_SPC_PULL_OS_CTX);
772023c3c96SNicholas Piggin /* Second load to recover the context state (Words 0 and 1) */
773f5af0a97SCédric Le Goater if (!kvmppc_xive_vcpu_has_save_restore(vcpu))
774023c3c96SNicholas Piggin vcpu->arch.xive_saved_state.w01 = __raw_readq(tima + TM_QW1_OS);
775023c3c96SNicholas Piggin
776023c3c96SNicholas Piggin /* Fixup some of the state for the next load */
777023c3c96SNicholas Piggin vcpu->arch.xive_saved_state.lsmfb = 0;
778023c3c96SNicholas Piggin vcpu->arch.xive_saved_state.ack = 0xff;
779023c3c96SNicholas Piggin vcpu->arch.xive_pushed = 0;
780023c3c96SNicholas Piggin eieio();
781023c3c96SNicholas Piggin }
782023c3c96SNicholas Piggin EXPORT_SYMBOL_GPL(kvmppc_xive_pull_vcpu);
783023c3c96SNicholas Piggin
kvmppc_xive_rearm_escalation(struct kvm_vcpu * vcpu)784ad5ace91SNicholas Piggin bool kvmppc_xive_rearm_escalation(struct kvm_vcpu *vcpu)
7859dc2babcSNicholas Piggin {
7869dc2babcSNicholas Piggin void __iomem *esc_vaddr = (void __iomem *)vcpu->arch.xive_esc_vaddr;
787ad5ace91SNicholas Piggin bool ret = true;
7889dc2babcSNicholas Piggin
7899dc2babcSNicholas Piggin if (!esc_vaddr)
790ad5ace91SNicholas Piggin return ret;
7919dc2babcSNicholas Piggin
7929dc2babcSNicholas Piggin /* we are using XIVE with single escalation */
7939dc2babcSNicholas Piggin
7949dc2babcSNicholas Piggin if (vcpu->arch.xive_esc_on) {
7959dc2babcSNicholas Piggin /*
7969dc2babcSNicholas Piggin * If we still have a pending escalation, abort the cede,
7979dc2babcSNicholas Piggin * and we must set PQ to 10 rather than 00 so that we don't
7989dc2babcSNicholas Piggin * potentially end up with two entries for the escalation
7999dc2babcSNicholas Piggin * interrupt in the XIVE interrupt queue. In that case
8009dc2babcSNicholas Piggin * we also don't want to set xive_esc_on to 1 here in
8019dc2babcSNicholas Piggin * case we race with xive_esc_irq().
8029dc2babcSNicholas Piggin */
803ad5ace91SNicholas Piggin ret = false;
8049dc2babcSNicholas Piggin /*
8059dc2babcSNicholas Piggin * The escalation interrupts are special as we don't EOI them.
8069dc2babcSNicholas Piggin * There is no need to use the load-after-store ordering offset
8079dc2babcSNicholas Piggin * to set PQ to 10 as we won't use StoreEOI.
8089dc2babcSNicholas Piggin */
8099dc2babcSNicholas Piggin __raw_readq(esc_vaddr + XIVE_ESB_SET_PQ_10);
8109dc2babcSNicholas Piggin } else {
8119dc2babcSNicholas Piggin vcpu->arch.xive_esc_on = true;
8129dc2babcSNicholas Piggin mb();
8139dc2babcSNicholas Piggin __raw_readq(esc_vaddr + XIVE_ESB_SET_PQ_00);
8149dc2babcSNicholas Piggin }
8159dc2babcSNicholas Piggin mb();
816ad5ace91SNicholas Piggin
817ad5ace91SNicholas Piggin return ret;
8189dc2babcSNicholas Piggin }
8199dc2babcSNicholas Piggin EXPORT_SYMBOL_GPL(kvmppc_xive_rearm_escalation);
8209dc2babcSNicholas Piggin
821023c3c96SNicholas Piggin /*
8225af50993SBenjamin Herrenschmidt * This is a simple trigger for a generic XIVE IRQ. This must
8235af50993SBenjamin Herrenschmidt * only be called for interrupts that support a trigger page
8245af50993SBenjamin Herrenschmidt */
xive_irq_trigger(struct xive_irq_data * xd)8255af50993SBenjamin Herrenschmidt static bool xive_irq_trigger(struct xive_irq_data *xd)
8265af50993SBenjamin Herrenschmidt {
8275af50993SBenjamin Herrenschmidt /* This should be only for MSIs */
8285af50993SBenjamin Herrenschmidt if (WARN_ON(xd->flags & XIVE_IRQ_FLAG_LSI))
8295af50993SBenjamin Herrenschmidt return false;
8305af50993SBenjamin Herrenschmidt
8315af50993SBenjamin Herrenschmidt /* Those interrupts should always have a trigger page */
8325af50993SBenjamin Herrenschmidt if (WARN_ON(!xd->trig_mmio))
8335af50993SBenjamin Herrenschmidt return false;
8345af50993SBenjamin Herrenschmidt
8355af50993SBenjamin Herrenschmidt out_be64(xd->trig_mmio, 0);
8365af50993SBenjamin Herrenschmidt
8375af50993SBenjamin Herrenschmidt return true;
8385af50993SBenjamin Herrenschmidt }
8395af50993SBenjamin Herrenschmidt
xive_esc_irq(int irq,void * data)8405af50993SBenjamin Herrenschmidt static irqreturn_t xive_esc_irq(int irq, void *data)
8415af50993SBenjamin Herrenschmidt {
8425af50993SBenjamin Herrenschmidt struct kvm_vcpu *vcpu = data;
8435af50993SBenjamin Herrenschmidt
8442267ea76SBenjamin Herrenschmidt vcpu->arch.irq_pending = 1;
8455af50993SBenjamin Herrenschmidt smp_mb();
84611681b79SNicholas Piggin if (vcpu->arch.ceded || vcpu->arch.nested)
8475af50993SBenjamin Herrenschmidt kvmppc_fast_vcpu_kick(vcpu);
8485af50993SBenjamin Herrenschmidt
8499b9b13a6SBenjamin Herrenschmidt /* Since we have the no-EOI flag, the interrupt is effectively
8509b9b13a6SBenjamin Herrenschmidt * disabled now. Clearing xive_esc_on means we won't bother
8519b9b13a6SBenjamin Herrenschmidt * doing so on the next entry.
8529b9b13a6SBenjamin Herrenschmidt *
8539b9b13a6SBenjamin Herrenschmidt * This also allows the entry code to know that if a PQ combination
8549b9b13a6SBenjamin Herrenschmidt * of 10 is observed while xive_esc_on is true, it means the queue
8559b9b13a6SBenjamin Herrenschmidt * contains an unprocessed escalation interrupt. We don't make use of
8569b9b13a6SBenjamin Herrenschmidt * that knowledge today but might (see comment in book3s_hv_rmhandler.S)
8579b9b13a6SBenjamin Herrenschmidt */
8589b9b13a6SBenjamin Herrenschmidt vcpu->arch.xive_esc_on = false;
8599b9b13a6SBenjamin Herrenschmidt
860da15c03bSPaul Mackerras /* This orders xive_esc_on = false vs. subsequent stale_p = true */
861da15c03bSPaul Mackerras smp_wmb(); /* goes with smp_mb() in cleanup_single_escalation */
862da15c03bSPaul Mackerras
8635af50993SBenjamin Herrenschmidt return IRQ_HANDLED;
8645af50993SBenjamin Herrenschmidt }
8655af50993SBenjamin Herrenschmidt
kvmppc_xive_attach_escalation(struct kvm_vcpu * vcpu,u8 prio,bool single_escalation)86613ce3297SCédric Le Goater int kvmppc_xive_attach_escalation(struct kvm_vcpu *vcpu, u8 prio,
86713ce3297SCédric Le Goater bool single_escalation)
8685af50993SBenjamin Herrenschmidt {
8695af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
8705af50993SBenjamin Herrenschmidt struct xive_q *q = &xc->queues[prio];
8715af50993SBenjamin Herrenschmidt char *name = NULL;
8725af50993SBenjamin Herrenschmidt int rc;
8735af50993SBenjamin Herrenschmidt
8745af50993SBenjamin Herrenschmidt /* Already there ? */
8755af50993SBenjamin Herrenschmidt if (xc->esc_virq[prio])
8765af50993SBenjamin Herrenschmidt return 0;
8775af50993SBenjamin Herrenschmidt
8785af50993SBenjamin Herrenschmidt /* Hook up the escalation interrupt */
8795af50993SBenjamin Herrenschmidt xc->esc_virq[prio] = irq_create_mapping(NULL, q->esc_irq);
8805af50993SBenjamin Herrenschmidt if (!xc->esc_virq[prio]) {
8815af50993SBenjamin Herrenschmidt pr_err("Failed to map escalation interrupt for queue %d of VCPU %d\n",
8825af50993SBenjamin Herrenschmidt prio, xc->server_num);
8835af50993SBenjamin Herrenschmidt return -EIO;
8845af50993SBenjamin Herrenschmidt }
8855af50993SBenjamin Herrenschmidt
88613ce3297SCédric Le Goater if (single_escalation)
887bf4159daSBenjamin Herrenschmidt name = kasprintf(GFP_KERNEL, "kvm-%d-%d",
888bf4159daSBenjamin Herrenschmidt vcpu->kvm->arch.lpid, xc->server_num);
889bf4159daSBenjamin Herrenschmidt else
8905af50993SBenjamin Herrenschmidt name = kasprintf(GFP_KERNEL, "kvm-%d-%d-%d",
8915af50993SBenjamin Herrenschmidt vcpu->kvm->arch.lpid, xc->server_num, prio);
8925af50993SBenjamin Herrenschmidt if (!name) {
8935af50993SBenjamin Herrenschmidt pr_err("Failed to allocate escalation irq name for queue %d of VCPU %d\n",
8945af50993SBenjamin Herrenschmidt prio, xc->server_num);
8955af50993SBenjamin Herrenschmidt rc = -ENOMEM;
8965af50993SBenjamin Herrenschmidt goto error;
8975af50993SBenjamin Herrenschmidt }
898bf4159daSBenjamin Herrenschmidt
899bf4159daSBenjamin Herrenschmidt pr_devel("Escalation %s irq %d (prio %d)\n", name, xc->esc_virq[prio], prio);
900bf4159daSBenjamin Herrenschmidt
9015af50993SBenjamin Herrenschmidt rc = request_irq(xc->esc_virq[prio], xive_esc_irq,
9025af50993SBenjamin Herrenschmidt IRQF_NO_THREAD, name, vcpu);
9035af50993SBenjamin Herrenschmidt if (rc) {
9045af50993SBenjamin Herrenschmidt pr_err("Failed to request escalation interrupt for queue %d of VCPU %d\n",
9055af50993SBenjamin Herrenschmidt prio, xc->server_num);
9065af50993SBenjamin Herrenschmidt goto error;
9075af50993SBenjamin Herrenschmidt }
9085af50993SBenjamin Herrenschmidt xc->esc_virq_names[prio] = name;
9099b9b13a6SBenjamin Herrenschmidt
9109b9b13a6SBenjamin Herrenschmidt /* In single escalation mode, we grab the ESB MMIO of the
9119b9b13a6SBenjamin Herrenschmidt * interrupt and mask it. Also populate the VCPU v/raddr
9129b9b13a6SBenjamin Herrenschmidt * of the ESB page for use by asm entry/exit code. Finally
9134f1c3f7bSCédric Le Goater * set the XIVE_IRQ_FLAG_NO_EOI flag which will prevent the
9149b9b13a6SBenjamin Herrenschmidt * core code from performing an EOI on the escalation
9159b9b13a6SBenjamin Herrenschmidt * interrupt, thus leaving it effectively masked after
9169b9b13a6SBenjamin Herrenschmidt * it fires once.
9179b9b13a6SBenjamin Herrenschmidt */
91813ce3297SCédric Le Goater if (single_escalation) {
9199b9b13a6SBenjamin Herrenschmidt struct irq_data *d = irq_get_irq_data(xc->esc_virq[prio]);
9209b9b13a6SBenjamin Herrenschmidt struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
9219b9b13a6SBenjamin Herrenschmidt
9229b9b13a6SBenjamin Herrenschmidt xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01);
9239b9b13a6SBenjamin Herrenschmidt vcpu->arch.xive_esc_raddr = xd->eoi_page;
9249b9b13a6SBenjamin Herrenschmidt vcpu->arch.xive_esc_vaddr = (__force u64)xd->eoi_mmio;
9254f1c3f7bSCédric Le Goater xd->flags |= XIVE_IRQ_FLAG_NO_EOI;
9269b9b13a6SBenjamin Herrenschmidt }
9279b9b13a6SBenjamin Herrenschmidt
9285af50993SBenjamin Herrenschmidt return 0;
9295af50993SBenjamin Herrenschmidt error:
9305af50993SBenjamin Herrenschmidt irq_dispose_mapping(xc->esc_virq[prio]);
9315af50993SBenjamin Herrenschmidt xc->esc_virq[prio] = 0;
9325af50993SBenjamin Herrenschmidt kfree(name);
9335af50993SBenjamin Herrenschmidt return rc;
9345af50993SBenjamin Herrenschmidt }
9355af50993SBenjamin Herrenschmidt
xive_provision_queue(struct kvm_vcpu * vcpu,u8 prio)9365af50993SBenjamin Herrenschmidt static int xive_provision_queue(struct kvm_vcpu *vcpu, u8 prio)
9375af50993SBenjamin Herrenschmidt {
9385af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
9395af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = xc->xive;
9405af50993SBenjamin Herrenschmidt struct xive_q *q = &xc->queues[prio];
9415af50993SBenjamin Herrenschmidt void *qpage;
9425af50993SBenjamin Herrenschmidt int rc;
9435af50993SBenjamin Herrenschmidt
9445af50993SBenjamin Herrenschmidt if (WARN_ON(q->qpage))
9455af50993SBenjamin Herrenschmidt return 0;
9465af50993SBenjamin Herrenschmidt
9475af50993SBenjamin Herrenschmidt /* Allocate the queue and retrieve infos on current node for now */
9485af50993SBenjamin Herrenschmidt qpage = (__be32 *)__get_free_pages(GFP_KERNEL, xive->q_page_order);
9495af50993SBenjamin Herrenschmidt if (!qpage) {
9505af50993SBenjamin Herrenschmidt pr_err("Failed to allocate queue %d for VCPU %d\n",
9515af50993SBenjamin Herrenschmidt prio, xc->server_num);
952ed7158baSIngo Molnar return -ENOMEM;
9535af50993SBenjamin Herrenschmidt }
9545af50993SBenjamin Herrenschmidt memset(qpage, 0, 1 << xive->q_order);
9555af50993SBenjamin Herrenschmidt
9565af50993SBenjamin Herrenschmidt /*
9575af50993SBenjamin Herrenschmidt * Reconfigure the queue. This will set q->qpage only once the
9585af50993SBenjamin Herrenschmidt * queue is fully configured. This is a requirement for prio 0
9595af50993SBenjamin Herrenschmidt * as we will stop doing EOIs for every IPI as soon as we observe
9605af50993SBenjamin Herrenschmidt * qpage being non-NULL, and instead will only EOI when we receive
9615af50993SBenjamin Herrenschmidt * corresponding queue 0 entries
9625af50993SBenjamin Herrenschmidt */
9635af50993SBenjamin Herrenschmidt rc = xive_native_configure_queue(xc->vp_id, q, prio, qpage,
9645af50993SBenjamin Herrenschmidt xive->q_order, true);
9655af50993SBenjamin Herrenschmidt if (rc)
9665af50993SBenjamin Herrenschmidt pr_err("Failed to configure queue %d for VCPU %d\n",
9675af50993SBenjamin Herrenschmidt prio, xc->server_num);
9685af50993SBenjamin Herrenschmidt return rc;
9695af50993SBenjamin Herrenschmidt }
9705af50993SBenjamin Herrenschmidt
9717e10b9a6SCédric Le Goater /* Called with xive->lock held */
xive_check_provisioning(struct kvm * kvm,u8 prio)9725af50993SBenjamin Herrenschmidt static int xive_check_provisioning(struct kvm *kvm, u8 prio)
9735af50993SBenjamin Herrenschmidt {
9745af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = kvm->arch.xive;
9755af50993SBenjamin Herrenschmidt struct kvm_vcpu *vcpu;
97646808a4cSMarc Zyngier unsigned long i;
97746808a4cSMarc Zyngier int rc;
9785af50993SBenjamin Herrenschmidt
9797e10b9a6SCédric Le Goater lockdep_assert_held(&xive->lock);
9805af50993SBenjamin Herrenschmidt
9815af50993SBenjamin Herrenschmidt /* Already provisioned ? */
9825af50993SBenjamin Herrenschmidt if (xive->qmap & (1 << prio))
9835af50993SBenjamin Herrenschmidt return 0;
9845af50993SBenjamin Herrenschmidt
9855af50993SBenjamin Herrenschmidt pr_devel("Provisioning prio... %d\n", prio);
9865af50993SBenjamin Herrenschmidt
987bf4159daSBenjamin Herrenschmidt /* Provision each VCPU and enable escalations if needed */
9885af50993SBenjamin Herrenschmidt kvm_for_each_vcpu(i, vcpu, kvm) {
9895af50993SBenjamin Herrenschmidt if (!vcpu->arch.xive_vcpu)
9905af50993SBenjamin Herrenschmidt continue;
9915af50993SBenjamin Herrenschmidt rc = xive_provision_queue(vcpu, prio);
992b68c6646SCédric Le Goater if (rc == 0 && !kvmppc_xive_has_single_escalation(xive))
99313ce3297SCédric Le Goater kvmppc_xive_attach_escalation(vcpu, prio,
994b68c6646SCédric Le Goater kvmppc_xive_has_single_escalation(xive));
9955af50993SBenjamin Herrenschmidt if (rc)
9965af50993SBenjamin Herrenschmidt return rc;
9975af50993SBenjamin Herrenschmidt }
9985af50993SBenjamin Herrenschmidt
9995af50993SBenjamin Herrenschmidt /* Order previous stores and mark it as provisioned */
10005af50993SBenjamin Herrenschmidt mb();
10015af50993SBenjamin Herrenschmidt xive->qmap |= (1 << prio);
10025af50993SBenjamin Herrenschmidt return 0;
10035af50993SBenjamin Herrenschmidt }
10045af50993SBenjamin Herrenschmidt
xive_inc_q_pending(struct kvm * kvm,u32 server,u8 prio)10055af50993SBenjamin Herrenschmidt static void xive_inc_q_pending(struct kvm *kvm, u32 server, u8 prio)
10065af50993SBenjamin Herrenschmidt {
10075af50993SBenjamin Herrenschmidt struct kvm_vcpu *vcpu;
10085af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc;
10095af50993SBenjamin Herrenschmidt struct xive_q *q;
10105af50993SBenjamin Herrenschmidt
10115af50993SBenjamin Herrenschmidt /* Locate target server */
10125af50993SBenjamin Herrenschmidt vcpu = kvmppc_xive_find_server(kvm, server);
10135af50993SBenjamin Herrenschmidt if (!vcpu) {
10145af50993SBenjamin Herrenschmidt pr_warn("%s: Can't find server %d\n", __func__, server);
10155af50993SBenjamin Herrenschmidt return;
10165af50993SBenjamin Herrenschmidt }
10175af50993SBenjamin Herrenschmidt xc = vcpu->arch.xive_vcpu;
10185af50993SBenjamin Herrenschmidt if (WARN_ON(!xc))
10195af50993SBenjamin Herrenschmidt return;
10205af50993SBenjamin Herrenschmidt
10215af50993SBenjamin Herrenschmidt q = &xc->queues[prio];
10225af50993SBenjamin Herrenschmidt atomic_inc(&q->pending_count);
10235af50993SBenjamin Herrenschmidt }
10245af50993SBenjamin Herrenschmidt
xive_try_pick_queue(struct kvm_vcpu * vcpu,u8 prio)10255af50993SBenjamin Herrenschmidt static int xive_try_pick_queue(struct kvm_vcpu *vcpu, u8 prio)
10265af50993SBenjamin Herrenschmidt {
10275af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
10285af50993SBenjamin Herrenschmidt struct xive_q *q;
10295af50993SBenjamin Herrenschmidt u32 max;
10305af50993SBenjamin Herrenschmidt
10315af50993SBenjamin Herrenschmidt if (WARN_ON(!xc))
10325af50993SBenjamin Herrenschmidt return -ENXIO;
10335af50993SBenjamin Herrenschmidt if (!xc->valid)
10345af50993SBenjamin Herrenschmidt return -ENXIO;
10355af50993SBenjamin Herrenschmidt
10365af50993SBenjamin Herrenschmidt q = &xc->queues[prio];
10375af50993SBenjamin Herrenschmidt if (WARN_ON(!q->qpage))
10385af50993SBenjamin Herrenschmidt return -ENXIO;
10395af50993SBenjamin Herrenschmidt
10405af50993SBenjamin Herrenschmidt /* Calculate max number of interrupts in that queue. */
10415af50993SBenjamin Herrenschmidt max = (q->msk + 1) - XIVE_Q_GAP;
10425af50993SBenjamin Herrenschmidt return atomic_add_unless(&q->count, 1, max) ? 0 : -EBUSY;
10435af50993SBenjamin Herrenschmidt }
10445af50993SBenjamin Herrenschmidt
kvmppc_xive_select_target(struct kvm * kvm,u32 * server,u8 prio)1045e8676ce5SCédric Le Goater int kvmppc_xive_select_target(struct kvm *kvm, u32 *server, u8 prio)
10465af50993SBenjamin Herrenschmidt {
10475af50993SBenjamin Herrenschmidt struct kvm_vcpu *vcpu;
104846808a4cSMarc Zyngier unsigned long i;
104946808a4cSMarc Zyngier int rc;
10505af50993SBenjamin Herrenschmidt
10515af50993SBenjamin Herrenschmidt /* Locate target server */
10525af50993SBenjamin Herrenschmidt vcpu = kvmppc_xive_find_server(kvm, *server);
10535af50993SBenjamin Herrenschmidt if (!vcpu) {
10545af50993SBenjamin Herrenschmidt pr_devel("Can't find server %d\n", *server);
10555af50993SBenjamin Herrenschmidt return -EINVAL;
10565af50993SBenjamin Herrenschmidt }
10575af50993SBenjamin Herrenschmidt
10585af50993SBenjamin Herrenschmidt pr_devel("Finding irq target on 0x%x/%d...\n", *server, prio);
10595af50993SBenjamin Herrenschmidt
10605af50993SBenjamin Herrenschmidt /* Try pick it */
10615af50993SBenjamin Herrenschmidt rc = xive_try_pick_queue(vcpu, prio);
10625af50993SBenjamin Herrenschmidt if (rc == 0)
10635af50993SBenjamin Herrenschmidt return rc;
10645af50993SBenjamin Herrenschmidt
10655af50993SBenjamin Herrenschmidt pr_devel(" .. failed, looking up candidate...\n");
10665af50993SBenjamin Herrenschmidt
10675af50993SBenjamin Herrenschmidt /* Failed, pick another VCPU */
10685af50993SBenjamin Herrenschmidt kvm_for_each_vcpu(i, vcpu, kvm) {
10695af50993SBenjamin Herrenschmidt if (!vcpu->arch.xive_vcpu)
10705af50993SBenjamin Herrenschmidt continue;
10715af50993SBenjamin Herrenschmidt rc = xive_try_pick_queue(vcpu, prio);
10725af50993SBenjamin Herrenschmidt if (rc == 0) {
10735af50993SBenjamin Herrenschmidt *server = vcpu->arch.xive_vcpu->server_num;
10745af50993SBenjamin Herrenschmidt pr_devel(" found on 0x%x/%d\n", *server, prio);
10755af50993SBenjamin Herrenschmidt return rc;
10765af50993SBenjamin Herrenschmidt }
10775af50993SBenjamin Herrenschmidt }
10785af50993SBenjamin Herrenschmidt pr_devel(" no available target !\n");
10795af50993SBenjamin Herrenschmidt
10805af50993SBenjamin Herrenschmidt /* No available target ! */
10815af50993SBenjamin Herrenschmidt return -EBUSY;
10825af50993SBenjamin Herrenschmidt }
10835af50993SBenjamin Herrenschmidt
xive_lock_and_mask(struct kvmppc_xive * xive,struct kvmppc_xive_src_block * sb,struct kvmppc_xive_irq_state * state)10845af50993SBenjamin Herrenschmidt static u8 xive_lock_and_mask(struct kvmppc_xive *xive,
10855af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb,
10865af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state)
10875af50993SBenjamin Herrenschmidt {
10885af50993SBenjamin Herrenschmidt struct xive_irq_data *xd;
10895af50993SBenjamin Herrenschmidt u32 hw_num;
10905af50993SBenjamin Herrenschmidt u8 old_prio;
10915af50993SBenjamin Herrenschmidt u64 val;
10925af50993SBenjamin Herrenschmidt
10935af50993SBenjamin Herrenschmidt /*
10945af50993SBenjamin Herrenschmidt * Take the lock, set masked, try again if racing
10955af50993SBenjamin Herrenschmidt * with H_EOI
10965af50993SBenjamin Herrenschmidt */
10975af50993SBenjamin Herrenschmidt for (;;) {
10985af50993SBenjamin Herrenschmidt arch_spin_lock(&sb->lock);
10995af50993SBenjamin Herrenschmidt old_prio = state->guest_priority;
11005af50993SBenjamin Herrenschmidt state->guest_priority = MASKED;
11015af50993SBenjamin Herrenschmidt mb();
11025af50993SBenjamin Herrenschmidt if (!state->in_eoi)
11035af50993SBenjamin Herrenschmidt break;
11045af50993SBenjamin Herrenschmidt state->guest_priority = old_prio;
11055af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock);
11065af50993SBenjamin Herrenschmidt }
11075af50993SBenjamin Herrenschmidt
11085af50993SBenjamin Herrenschmidt /* No change ? Bail */
11095af50993SBenjamin Herrenschmidt if (old_prio == MASKED)
11105af50993SBenjamin Herrenschmidt return old_prio;
11115af50993SBenjamin Herrenschmidt
11125af50993SBenjamin Herrenschmidt /* Get the right irq */
11135af50993SBenjamin Herrenschmidt kvmppc_xive_select_irq(state, &hw_num, &xd);
11145af50993SBenjamin Herrenschmidt
11155af50993SBenjamin Herrenschmidt /* Set PQ to 10, return old P and old Q and remember them */
11165af50993SBenjamin Herrenschmidt val = xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_10);
11175af50993SBenjamin Herrenschmidt state->old_p = !!(val & 2);
11185af50993SBenjamin Herrenschmidt state->old_q = !!(val & 1);
11195af50993SBenjamin Herrenschmidt
11205af50993SBenjamin Herrenschmidt /*
1121b5277d18SCédric Le Goater * Synchronize hardware to sensure the queues are updated when
1122b5277d18SCédric Le Goater * masking
11235af50993SBenjamin Herrenschmidt */
11245af50993SBenjamin Herrenschmidt xive_native_sync_source(hw_num);
11255af50993SBenjamin Herrenschmidt
11265af50993SBenjamin Herrenschmidt return old_prio;
11275af50993SBenjamin Herrenschmidt }
11285af50993SBenjamin Herrenschmidt
xive_lock_for_unmask(struct kvmppc_xive_src_block * sb,struct kvmppc_xive_irq_state * state)11295af50993SBenjamin Herrenschmidt static void xive_lock_for_unmask(struct kvmppc_xive_src_block *sb,
11305af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state)
11315af50993SBenjamin Herrenschmidt {
11325af50993SBenjamin Herrenschmidt /*
11335af50993SBenjamin Herrenschmidt * Take the lock try again if racing with H_EOI
11345af50993SBenjamin Herrenschmidt */
11355af50993SBenjamin Herrenschmidt for (;;) {
11365af50993SBenjamin Herrenschmidt arch_spin_lock(&sb->lock);
11375af50993SBenjamin Herrenschmidt if (!state->in_eoi)
11385af50993SBenjamin Herrenschmidt break;
11395af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock);
11405af50993SBenjamin Herrenschmidt }
11415af50993SBenjamin Herrenschmidt }
11425af50993SBenjamin Herrenschmidt
xive_finish_unmask(struct kvmppc_xive * xive,struct kvmppc_xive_src_block * sb,struct kvmppc_xive_irq_state * state,u8 prio)11435af50993SBenjamin Herrenschmidt static void xive_finish_unmask(struct kvmppc_xive *xive,
11445af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb,
11455af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state,
11465af50993SBenjamin Herrenschmidt u8 prio)
11475af50993SBenjamin Herrenschmidt {
11485af50993SBenjamin Herrenschmidt struct xive_irq_data *xd;
11495af50993SBenjamin Herrenschmidt u32 hw_num;
11505af50993SBenjamin Herrenschmidt
11515af50993SBenjamin Herrenschmidt /* If we aren't changing a thing, move on */
11525af50993SBenjamin Herrenschmidt if (state->guest_priority != MASKED)
11535af50993SBenjamin Herrenschmidt goto bail;
11545af50993SBenjamin Herrenschmidt
11555af50993SBenjamin Herrenschmidt /* Get the right irq */
11565af50993SBenjamin Herrenschmidt kvmppc_xive_select_irq(state, &hw_num, &xd);
11575af50993SBenjamin Herrenschmidt
11585af50993SBenjamin Herrenschmidt /* Old Q set, set PQ to 11 */
11595af50993SBenjamin Herrenschmidt if (state->old_q)
11605af50993SBenjamin Herrenschmidt xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_11);
11615af50993SBenjamin Herrenschmidt
11625af50993SBenjamin Herrenschmidt /*
11635af50993SBenjamin Herrenschmidt * If not old P, then perform an "effective" EOI,
11645af50993SBenjamin Herrenschmidt * on the source. This will handle the cases where
11655af50993SBenjamin Herrenschmidt * FW EOI is needed.
11665af50993SBenjamin Herrenschmidt */
11675af50993SBenjamin Herrenschmidt if (!state->old_p)
11685af50993SBenjamin Herrenschmidt xive_vm_source_eoi(hw_num, xd);
11695af50993SBenjamin Herrenschmidt
11705af50993SBenjamin Herrenschmidt /* Synchronize ordering and mark unmasked */
11715af50993SBenjamin Herrenschmidt mb();
11725af50993SBenjamin Herrenschmidt bail:
11735af50993SBenjamin Herrenschmidt state->guest_priority = prio;
11745af50993SBenjamin Herrenschmidt }
11755af50993SBenjamin Herrenschmidt
11765af50993SBenjamin Herrenschmidt /*
11775af50993SBenjamin Herrenschmidt * Target an interrupt to a given server/prio, this will fallback
11785af50993SBenjamin Herrenschmidt * to another server if necessary and perform the HW targetting
11795af50993SBenjamin Herrenschmidt * updates as needed
11805af50993SBenjamin Herrenschmidt *
11815af50993SBenjamin Herrenschmidt * NOTE: Must be called with the state lock held
11825af50993SBenjamin Herrenschmidt */
xive_target_interrupt(struct kvm * kvm,struct kvmppc_xive_irq_state * state,u32 server,u8 prio)11835af50993SBenjamin Herrenschmidt static int xive_target_interrupt(struct kvm *kvm,
11845af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state,
11855af50993SBenjamin Herrenschmidt u32 server, u8 prio)
11865af50993SBenjamin Herrenschmidt {
11875af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = kvm->arch.xive;
11885af50993SBenjamin Herrenschmidt u32 hw_num;
11895af50993SBenjamin Herrenschmidt int rc;
11905af50993SBenjamin Herrenschmidt
11915af50993SBenjamin Herrenschmidt /*
11925af50993SBenjamin Herrenschmidt * This will return a tentative server and actual
11935af50993SBenjamin Herrenschmidt * priority. The count for that new target will have
11945af50993SBenjamin Herrenschmidt * already been incremented.
11955af50993SBenjamin Herrenschmidt */
1196e8676ce5SCédric Le Goater rc = kvmppc_xive_select_target(kvm, &server, prio);
11975af50993SBenjamin Herrenschmidt
11985af50993SBenjamin Herrenschmidt /*
11995af50993SBenjamin Herrenschmidt * We failed to find a target ? Not much we can do
12005af50993SBenjamin Herrenschmidt * at least until we support the GIQ.
12015af50993SBenjamin Herrenschmidt */
12025af50993SBenjamin Herrenschmidt if (rc)
12035af50993SBenjamin Herrenschmidt return rc;
12045af50993SBenjamin Herrenschmidt
12055af50993SBenjamin Herrenschmidt /*
12065af50993SBenjamin Herrenschmidt * Increment the old queue pending count if there
12075af50993SBenjamin Herrenschmidt * was one so that the old queue count gets adjusted later
12085af50993SBenjamin Herrenschmidt * when observed to be empty.
12095af50993SBenjamin Herrenschmidt */
12105af50993SBenjamin Herrenschmidt if (state->act_priority != MASKED)
12115af50993SBenjamin Herrenschmidt xive_inc_q_pending(kvm,
12125af50993SBenjamin Herrenschmidt state->act_server,
12135af50993SBenjamin Herrenschmidt state->act_priority);
12145af50993SBenjamin Herrenschmidt /*
12155af50993SBenjamin Herrenschmidt * Update state and HW
12165af50993SBenjamin Herrenschmidt */
12175af50993SBenjamin Herrenschmidt state->act_priority = prio;
12185af50993SBenjamin Herrenschmidt state->act_server = server;
12195af50993SBenjamin Herrenschmidt
12205af50993SBenjamin Herrenschmidt /* Get the right irq */
12215af50993SBenjamin Herrenschmidt kvmppc_xive_select_irq(state, &hw_num, NULL);
12225af50993SBenjamin Herrenschmidt
12235af50993SBenjamin Herrenschmidt return xive_native_configure_irq(hw_num,
1224eacc56bbSCédric Le Goater kvmppc_xive_vp(xive, server),
12255af50993SBenjamin Herrenschmidt prio, state->number);
12265af50993SBenjamin Herrenschmidt }
12275af50993SBenjamin Herrenschmidt
12285af50993SBenjamin Herrenschmidt /*
12295af50993SBenjamin Herrenschmidt * Targetting rules: In order to avoid losing track of
12301fd02f66SJulia Lawall * pending interrupts across mask and unmask, which would
12315af50993SBenjamin Herrenschmidt * allow queue overflows, we implement the following rules:
12325af50993SBenjamin Herrenschmidt *
12335af50993SBenjamin Herrenschmidt * - Unless it was never enabled (or we run out of capacity)
12345af50993SBenjamin Herrenschmidt * an interrupt is always targetted at a valid server/queue
12355af50993SBenjamin Herrenschmidt * pair even when "masked" by the guest. This pair tends to
12365af50993SBenjamin Herrenschmidt * be the last one used but it can be changed under some
12375af50993SBenjamin Herrenschmidt * circumstances. That allows us to separate targetting
12385af50993SBenjamin Herrenschmidt * from masking, we only handle accounting during (re)targetting,
12395af50993SBenjamin Herrenschmidt * this also allows us to let an interrupt drain into its target
12405af50993SBenjamin Herrenschmidt * queue after masking, avoiding complex schemes to remove
12415af50993SBenjamin Herrenschmidt * interrupts out of remote processor queues.
12425af50993SBenjamin Herrenschmidt *
12435af50993SBenjamin Herrenschmidt * - When masking, we set PQ to 10 and save the previous value
12445af50993SBenjamin Herrenschmidt * of P and Q.
12455af50993SBenjamin Herrenschmidt *
12465af50993SBenjamin Herrenschmidt * - When unmasking, if saved Q was set, we set PQ to 11
12475af50993SBenjamin Herrenschmidt * otherwise we leave PQ to the HW state which will be either
12485af50993SBenjamin Herrenschmidt * 10 if nothing happened or 11 if the interrupt fired while
12495af50993SBenjamin Herrenschmidt * masked. Effectively we are OR'ing the previous Q into the
12505af50993SBenjamin Herrenschmidt * HW Q.
12515af50993SBenjamin Herrenschmidt *
12525af50993SBenjamin Herrenschmidt * Then if saved P is clear, we do an effective EOI (Q->P->Trigger)
12535af50993SBenjamin Herrenschmidt * which will unmask the interrupt and shoot a new one if Q was
12545af50993SBenjamin Herrenschmidt * set.
12555af50993SBenjamin Herrenschmidt *
12565af50993SBenjamin Herrenschmidt * Otherwise (saved P is set) we leave PQ unchanged (so 10 or 11,
12575af50993SBenjamin Herrenschmidt * effectively meaning an H_EOI from the guest is still expected
12585af50993SBenjamin Herrenschmidt * for that interrupt).
12595af50993SBenjamin Herrenschmidt *
12605af50993SBenjamin Herrenschmidt * - If H_EOI occurs while masked, we clear the saved P.
12615af50993SBenjamin Herrenschmidt *
12625af50993SBenjamin Herrenschmidt * - When changing target, we account on the new target and
12635af50993SBenjamin Herrenschmidt * increment a separate "pending" counter on the old one.
12645af50993SBenjamin Herrenschmidt * This pending counter will be used to decrement the old
12655af50993SBenjamin Herrenschmidt * target's count when its queue has been observed empty.
12665af50993SBenjamin Herrenschmidt */
12675af50993SBenjamin Herrenschmidt
kvmppc_xive_set_xive(struct kvm * kvm,u32 irq,u32 server,u32 priority)12685af50993SBenjamin Herrenschmidt int kvmppc_xive_set_xive(struct kvm *kvm, u32 irq, u32 server,
12695af50993SBenjamin Herrenschmidt u32 priority)
12705af50993SBenjamin Herrenschmidt {
12715af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = kvm->arch.xive;
12725af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb;
12735af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state;
12745af50993SBenjamin Herrenschmidt u8 new_act_prio;
12755af50993SBenjamin Herrenschmidt int rc = 0;
12765af50993SBenjamin Herrenschmidt u16 idx;
12775af50993SBenjamin Herrenschmidt
12785af50993SBenjamin Herrenschmidt if (!xive)
12795af50993SBenjamin Herrenschmidt return -ENODEV;
12805af50993SBenjamin Herrenschmidt
12815af50993SBenjamin Herrenschmidt pr_devel("set_xive ! irq 0x%x server 0x%x prio %d\n",
12825af50993SBenjamin Herrenschmidt irq, server, priority);
12835af50993SBenjamin Herrenschmidt
12845af50993SBenjamin Herrenschmidt /* First, check provisioning of queues */
12857e10b9a6SCédric Le Goater if (priority != MASKED) {
12867e10b9a6SCédric Le Goater mutex_lock(&xive->lock);
12875af50993SBenjamin Herrenschmidt rc = xive_check_provisioning(xive->kvm,
12885af50993SBenjamin Herrenschmidt xive_prio_from_guest(priority));
12897e10b9a6SCédric Le Goater mutex_unlock(&xive->lock);
12907e10b9a6SCédric Le Goater }
12915af50993SBenjamin Herrenschmidt if (rc) {
12925af50993SBenjamin Herrenschmidt pr_devel(" provisioning failure %d !\n", rc);
12935af50993SBenjamin Herrenschmidt return rc;
12945af50993SBenjamin Herrenschmidt }
12955af50993SBenjamin Herrenschmidt
12965af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, irq, &idx);
12975af50993SBenjamin Herrenschmidt if (!sb)
12985af50993SBenjamin Herrenschmidt return -EINVAL;
12995af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx];
13005af50993SBenjamin Herrenschmidt
13015af50993SBenjamin Herrenschmidt /*
13025af50993SBenjamin Herrenschmidt * We first handle masking/unmasking since the locking
13035af50993SBenjamin Herrenschmidt * might need to be retried due to EOIs, we'll handle
13045af50993SBenjamin Herrenschmidt * targetting changes later. These functions will return
13055af50993SBenjamin Herrenschmidt * with the SB lock held.
13065af50993SBenjamin Herrenschmidt *
13075af50993SBenjamin Herrenschmidt * xive_lock_and_mask() will also set state->guest_priority
13085af50993SBenjamin Herrenschmidt * but won't otherwise change other fields of the state.
13095af50993SBenjamin Herrenschmidt *
13105af50993SBenjamin Herrenschmidt * xive_lock_for_unmask will not actually unmask, this will
13115af50993SBenjamin Herrenschmidt * be done later by xive_finish_unmask() once the targetting
13125af50993SBenjamin Herrenschmidt * has been done, so we don't try to unmask an interrupt
13135af50993SBenjamin Herrenschmidt * that hasn't yet been targetted.
13145af50993SBenjamin Herrenschmidt */
13155af50993SBenjamin Herrenschmidt if (priority == MASKED)
13165af50993SBenjamin Herrenschmidt xive_lock_and_mask(xive, sb, state);
13175af50993SBenjamin Herrenschmidt else
13185af50993SBenjamin Herrenschmidt xive_lock_for_unmask(sb, state);
13195af50993SBenjamin Herrenschmidt
13205af50993SBenjamin Herrenschmidt
13215af50993SBenjamin Herrenschmidt /*
13225af50993SBenjamin Herrenschmidt * Then we handle targetting.
13235af50993SBenjamin Herrenschmidt *
13245af50993SBenjamin Herrenschmidt * First calculate a new "actual priority"
13255af50993SBenjamin Herrenschmidt */
13265af50993SBenjamin Herrenschmidt new_act_prio = state->act_priority;
13275af50993SBenjamin Herrenschmidt if (priority != MASKED)
13285af50993SBenjamin Herrenschmidt new_act_prio = xive_prio_from_guest(priority);
13295af50993SBenjamin Herrenschmidt
13305af50993SBenjamin Herrenschmidt pr_devel(" new_act_prio=%x act_server=%x act_prio=%x\n",
13315af50993SBenjamin Herrenschmidt new_act_prio, state->act_server, state->act_priority);
13325af50993SBenjamin Herrenschmidt
13335af50993SBenjamin Herrenschmidt /*
13345af50993SBenjamin Herrenschmidt * Then check if we actually need to change anything,
13355af50993SBenjamin Herrenschmidt *
13365af50993SBenjamin Herrenschmidt * The condition for re-targetting the interrupt is that
13375af50993SBenjamin Herrenschmidt * we have a valid new priority (new_act_prio is not 0xff)
13385af50993SBenjamin Herrenschmidt * and either the server or the priority changed.
13395af50993SBenjamin Herrenschmidt *
13405af50993SBenjamin Herrenschmidt * Note: If act_priority was ff and the new priority is
13415af50993SBenjamin Herrenschmidt * also ff, we don't do anything and leave the interrupt
13425af50993SBenjamin Herrenschmidt * untargetted. An attempt of doing an int_on on an
13435af50993SBenjamin Herrenschmidt * untargetted interrupt will fail. If that is a problem
13445af50993SBenjamin Herrenschmidt * we could initialize interrupts with valid default
13455af50993SBenjamin Herrenschmidt */
13465af50993SBenjamin Herrenschmidt
13475af50993SBenjamin Herrenschmidt if (new_act_prio != MASKED &&
13485af50993SBenjamin Herrenschmidt (state->act_server != server ||
13495af50993SBenjamin Herrenschmidt state->act_priority != new_act_prio))
13505af50993SBenjamin Herrenschmidt rc = xive_target_interrupt(kvm, state, server, new_act_prio);
13515af50993SBenjamin Herrenschmidt
13525af50993SBenjamin Herrenschmidt /*
13535af50993SBenjamin Herrenschmidt * Perform the final unmasking of the interrupt source
13545af50993SBenjamin Herrenschmidt * if necessary
13555af50993SBenjamin Herrenschmidt */
13565af50993SBenjamin Herrenschmidt if (priority != MASKED)
13575af50993SBenjamin Herrenschmidt xive_finish_unmask(xive, sb, state, priority);
13585af50993SBenjamin Herrenschmidt
13595af50993SBenjamin Herrenschmidt /*
13605af50993SBenjamin Herrenschmidt * Finally Update saved_priority to match. Only int_on/off
13615af50993SBenjamin Herrenschmidt * set this field to a different value.
13625af50993SBenjamin Herrenschmidt */
13635af50993SBenjamin Herrenschmidt state->saved_priority = priority;
13645af50993SBenjamin Herrenschmidt
13655af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock);
13665af50993SBenjamin Herrenschmidt return rc;
13675af50993SBenjamin Herrenschmidt }
13685af50993SBenjamin Herrenschmidt
kvmppc_xive_get_xive(struct kvm * kvm,u32 irq,u32 * server,u32 * priority)13695af50993SBenjamin Herrenschmidt int kvmppc_xive_get_xive(struct kvm *kvm, u32 irq, u32 *server,
13705af50993SBenjamin Herrenschmidt u32 *priority)
13715af50993SBenjamin Herrenschmidt {
13725af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = kvm->arch.xive;
13735af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb;
13745af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state;
13755af50993SBenjamin Herrenschmidt u16 idx;
13765af50993SBenjamin Herrenschmidt
13775af50993SBenjamin Herrenschmidt if (!xive)
13785af50993SBenjamin Herrenschmidt return -ENODEV;
13795af50993SBenjamin Herrenschmidt
13805af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, irq, &idx);
13815af50993SBenjamin Herrenschmidt if (!sb)
13825af50993SBenjamin Herrenschmidt return -EINVAL;
13835af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx];
13845af50993SBenjamin Herrenschmidt arch_spin_lock(&sb->lock);
13852fb1e946SSam Bobroff *server = state->act_server;
13865af50993SBenjamin Herrenschmidt *priority = state->guest_priority;
13875af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock);
13885af50993SBenjamin Herrenschmidt
13895af50993SBenjamin Herrenschmidt return 0;
13905af50993SBenjamin Herrenschmidt }
13915af50993SBenjamin Herrenschmidt
kvmppc_xive_int_on(struct kvm * kvm,u32 irq)13925af50993SBenjamin Herrenschmidt int kvmppc_xive_int_on(struct kvm *kvm, u32 irq)
13935af50993SBenjamin Herrenschmidt {
13945af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = kvm->arch.xive;
13955af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb;
13965af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state;
13975af50993SBenjamin Herrenschmidt u16 idx;
13985af50993SBenjamin Herrenschmidt
13995af50993SBenjamin Herrenschmidt if (!xive)
14005af50993SBenjamin Herrenschmidt return -ENODEV;
14015af50993SBenjamin Herrenschmidt
14025af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, irq, &idx);
14035af50993SBenjamin Herrenschmidt if (!sb)
14045af50993SBenjamin Herrenschmidt return -EINVAL;
14055af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx];
14065af50993SBenjamin Herrenschmidt
14075af50993SBenjamin Herrenschmidt pr_devel("int_on(irq=0x%x)\n", irq);
14085af50993SBenjamin Herrenschmidt
14095af50993SBenjamin Herrenschmidt /*
14105af50993SBenjamin Herrenschmidt * Check if interrupt was not targetted
14115af50993SBenjamin Herrenschmidt */
14125af50993SBenjamin Herrenschmidt if (state->act_priority == MASKED) {
14135af50993SBenjamin Herrenschmidt pr_devel("int_on on untargetted interrupt\n");
14145af50993SBenjamin Herrenschmidt return -EINVAL;
14155af50993SBenjamin Herrenschmidt }
14165af50993SBenjamin Herrenschmidt
14175af50993SBenjamin Herrenschmidt /* If saved_priority is 0xff, do nothing */
14185af50993SBenjamin Herrenschmidt if (state->saved_priority == MASKED)
14195af50993SBenjamin Herrenschmidt return 0;
14205af50993SBenjamin Herrenschmidt
14215af50993SBenjamin Herrenschmidt /*
14225af50993SBenjamin Herrenschmidt * Lock and unmask it.
14235af50993SBenjamin Herrenschmidt */
14245af50993SBenjamin Herrenschmidt xive_lock_for_unmask(sb, state);
14255af50993SBenjamin Herrenschmidt xive_finish_unmask(xive, sb, state, state->saved_priority);
14265af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock);
14275af50993SBenjamin Herrenschmidt
14285af50993SBenjamin Herrenschmidt return 0;
14295af50993SBenjamin Herrenschmidt }
14305af50993SBenjamin Herrenschmidt
kvmppc_xive_int_off(struct kvm * kvm,u32 irq)14315af50993SBenjamin Herrenschmidt int kvmppc_xive_int_off(struct kvm *kvm, u32 irq)
14325af50993SBenjamin Herrenschmidt {
14335af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = kvm->arch.xive;
14345af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb;
14355af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state;
14365af50993SBenjamin Herrenschmidt u16 idx;
14375af50993SBenjamin Herrenschmidt
14385af50993SBenjamin Herrenschmidt if (!xive)
14395af50993SBenjamin Herrenschmidt return -ENODEV;
14405af50993SBenjamin Herrenschmidt
14415af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, irq, &idx);
14425af50993SBenjamin Herrenschmidt if (!sb)
14435af50993SBenjamin Herrenschmidt return -EINVAL;
14445af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx];
14455af50993SBenjamin Herrenschmidt
14465af50993SBenjamin Herrenschmidt pr_devel("int_off(irq=0x%x)\n", irq);
14475af50993SBenjamin Herrenschmidt
14485af50993SBenjamin Herrenschmidt /*
14495af50993SBenjamin Herrenschmidt * Lock and mask
14505af50993SBenjamin Herrenschmidt */
14515af50993SBenjamin Herrenschmidt state->saved_priority = xive_lock_and_mask(xive, sb, state);
14525af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock);
14535af50993SBenjamin Herrenschmidt
14545af50993SBenjamin Herrenschmidt return 0;
14555af50993SBenjamin Herrenschmidt }
14565af50993SBenjamin Herrenschmidt
xive_restore_pending_irq(struct kvmppc_xive * xive,u32 irq)14575af50993SBenjamin Herrenschmidt static bool xive_restore_pending_irq(struct kvmppc_xive *xive, u32 irq)
14585af50993SBenjamin Herrenschmidt {
14595af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb;
14605af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state;
14615af50993SBenjamin Herrenschmidt u16 idx;
14625af50993SBenjamin Herrenschmidt
14635af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, irq, &idx);
14645af50993SBenjamin Herrenschmidt if (!sb)
14655af50993SBenjamin Herrenschmidt return false;
14665af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx];
14675af50993SBenjamin Herrenschmidt if (!state->valid)
14685af50993SBenjamin Herrenschmidt return false;
14695af50993SBenjamin Herrenschmidt
14705af50993SBenjamin Herrenschmidt /*
14715af50993SBenjamin Herrenschmidt * Trigger the IPI. This assumes we never restore a pass-through
14725af50993SBenjamin Herrenschmidt * interrupt which should be safe enough
14735af50993SBenjamin Herrenschmidt */
14745af50993SBenjamin Herrenschmidt xive_irq_trigger(&state->ipi_data);
14755af50993SBenjamin Herrenschmidt
14765af50993SBenjamin Herrenschmidt return true;
14775af50993SBenjamin Herrenschmidt }
14785af50993SBenjamin Herrenschmidt
kvmppc_xive_get_icp(struct kvm_vcpu * vcpu)14795af50993SBenjamin Herrenschmidt u64 kvmppc_xive_get_icp(struct kvm_vcpu *vcpu)
14805af50993SBenjamin Herrenschmidt {
14815af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
14825af50993SBenjamin Herrenschmidt
14835af50993SBenjamin Herrenschmidt if (!xc)
14845af50993SBenjamin Herrenschmidt return 0;
14855af50993SBenjamin Herrenschmidt
14865af50993SBenjamin Herrenschmidt /* Return the per-cpu state for state saving/migration */
14875af50993SBenjamin Herrenschmidt return (u64)xc->cppr << KVM_REG_PPC_ICP_CPPR_SHIFT |
14887333b5acSLaurent Vivier (u64)xc->mfrr << KVM_REG_PPC_ICP_MFRR_SHIFT |
14897333b5acSLaurent Vivier (u64)0xff << KVM_REG_PPC_ICP_PPRI_SHIFT;
14905af50993SBenjamin Herrenschmidt }
14915af50993SBenjamin Herrenschmidt
kvmppc_xive_set_icp(struct kvm_vcpu * vcpu,u64 icpval)14925af50993SBenjamin Herrenschmidt int kvmppc_xive_set_icp(struct kvm_vcpu *vcpu, u64 icpval)
14935af50993SBenjamin Herrenschmidt {
14945af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
14955af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
14965af50993SBenjamin Herrenschmidt u8 cppr, mfrr;
14975af50993SBenjamin Herrenschmidt u32 xisr;
14985af50993SBenjamin Herrenschmidt
14995af50993SBenjamin Herrenschmidt if (!xc || !xive)
15005af50993SBenjamin Herrenschmidt return -ENOENT;
15015af50993SBenjamin Herrenschmidt
15025af50993SBenjamin Herrenschmidt /* Grab individual state fields. We don't use pending_pri */
15035af50993SBenjamin Herrenschmidt cppr = icpval >> KVM_REG_PPC_ICP_CPPR_SHIFT;
15045af50993SBenjamin Herrenschmidt xisr = (icpval >> KVM_REG_PPC_ICP_XISR_SHIFT) &
15055af50993SBenjamin Herrenschmidt KVM_REG_PPC_ICP_XISR_MASK;
15065af50993SBenjamin Herrenschmidt mfrr = icpval >> KVM_REG_PPC_ICP_MFRR_SHIFT;
15075af50993SBenjamin Herrenschmidt
15085af50993SBenjamin Herrenschmidt pr_devel("set_icp vcpu %d cppr=0x%x mfrr=0x%x xisr=0x%x\n",
15095af50993SBenjamin Herrenschmidt xc->server_num, cppr, mfrr, xisr);
15105af50993SBenjamin Herrenschmidt
15115af50993SBenjamin Herrenschmidt /*
15125af50993SBenjamin Herrenschmidt * We can't update the state of a "pushed" VCPU, but that
15136f868405SPaul Mackerras * shouldn't happen because the vcpu->mutex makes running a
15146f868405SPaul Mackerras * vcpu mutually exclusive with doing one_reg get/set on it.
15155af50993SBenjamin Herrenschmidt */
15165af50993SBenjamin Herrenschmidt if (WARN_ON(vcpu->arch.xive_pushed))
15175af50993SBenjamin Herrenschmidt return -EIO;
15185af50993SBenjamin Herrenschmidt
15195af50993SBenjamin Herrenschmidt /* Update VCPU HW saved state */
15205af50993SBenjamin Herrenschmidt vcpu->arch.xive_saved_state.cppr = cppr;
15215af50993SBenjamin Herrenschmidt xc->hw_cppr = xc->cppr = cppr;
15225af50993SBenjamin Herrenschmidt
15235af50993SBenjamin Herrenschmidt /*
15245af50993SBenjamin Herrenschmidt * Update MFRR state. If it's not 0xff, we mark the VCPU as
15255af50993SBenjamin Herrenschmidt * having a pending MFRR change, which will re-evaluate the
15265af50993SBenjamin Herrenschmidt * target. The VCPU will thus potentially get a spurious
15275af50993SBenjamin Herrenschmidt * interrupt but that's not a big deal.
15285af50993SBenjamin Herrenschmidt */
15295af50993SBenjamin Herrenschmidt xc->mfrr = mfrr;
15305af50993SBenjamin Herrenschmidt if (mfrr < cppr)
15315af50993SBenjamin Herrenschmidt xive_irq_trigger(&xc->vp_ipi_data);
15325af50993SBenjamin Herrenschmidt
15335af50993SBenjamin Herrenschmidt /*
15345af50993SBenjamin Herrenschmidt * Now saved XIRR is "interesting". It means there's something in
15355af50993SBenjamin Herrenschmidt * the legacy "1 element" queue... for an IPI we simply ignore it,
15365af50993SBenjamin Herrenschmidt * as the MFRR restore will handle that. For anything else we need
15375af50993SBenjamin Herrenschmidt * to force a resend of the source.
15385af50993SBenjamin Herrenschmidt * However the source may not have been setup yet. If that's the
15395af50993SBenjamin Herrenschmidt * case, we keep that info and increment a counter in the xive to
15405af50993SBenjamin Herrenschmidt * tell subsequent xive_set_source() to go look.
15415af50993SBenjamin Herrenschmidt */
15425af50993SBenjamin Herrenschmidt if (xisr > XICS_IPI && !xive_restore_pending_irq(xive, xisr)) {
15435af50993SBenjamin Herrenschmidt xc->delayed_irq = xisr;
15445af50993SBenjamin Herrenschmidt xive->delayed_irqs++;
15455af50993SBenjamin Herrenschmidt pr_devel(" xisr restore delayed\n");
15465af50993SBenjamin Herrenschmidt }
15475af50993SBenjamin Herrenschmidt
15485af50993SBenjamin Herrenschmidt return 0;
15495af50993SBenjamin Herrenschmidt }
15505af50993SBenjamin Herrenschmidt
kvmppc_xive_set_mapped(struct kvm * kvm,unsigned long guest_irq,unsigned long host_irq)15515af50993SBenjamin Herrenschmidt int kvmppc_xive_set_mapped(struct kvm *kvm, unsigned long guest_irq,
1552e5e78b15SCédric Le Goater unsigned long host_irq)
15535af50993SBenjamin Herrenschmidt {
15545af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = kvm->arch.xive;
15555af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb;
15565af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state;
155751be9e51SCédric Le Goater struct irq_data *host_data =
155851be9e51SCédric Le Goater irq_domain_get_irq_data(irq_get_default_host(), host_irq);
15595af50993SBenjamin Herrenschmidt unsigned int hw_irq = (unsigned int)irqd_to_hwirq(host_data);
15605af50993SBenjamin Herrenschmidt u16 idx;
15615af50993SBenjamin Herrenschmidt u8 prio;
15625af50993SBenjamin Herrenschmidt int rc;
15635af50993SBenjamin Herrenschmidt
15645af50993SBenjamin Herrenschmidt if (!xive)
15655af50993SBenjamin Herrenschmidt return -ENODEV;
15665af50993SBenjamin Herrenschmidt
1567e5e78b15SCédric Le Goater pr_debug("%s: GIRQ 0x%lx host IRQ %ld XIVE HW IRQ 0x%x\n",
1568e5e78b15SCédric Le Goater __func__, guest_irq, host_irq, hw_irq);
15695af50993SBenjamin Herrenschmidt
15705af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, guest_irq, &idx);
15715af50993SBenjamin Herrenschmidt if (!sb)
15725af50993SBenjamin Herrenschmidt return -EINVAL;
15735af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx];
15745af50993SBenjamin Herrenschmidt
15755af50993SBenjamin Herrenschmidt /*
15765af50993SBenjamin Herrenschmidt * Mark the passed-through interrupt as going to a VCPU,
15775af50993SBenjamin Herrenschmidt * this will prevent further EOIs and similar operations
15785af50993SBenjamin Herrenschmidt * from the XIVE code. It will also mask the interrupt
15795af50993SBenjamin Herrenschmidt * to either PQ=10 or 11 state, the latter if the interrupt
15805af50993SBenjamin Herrenschmidt * is pending. This will allow us to unmask or retrigger it
15815af50993SBenjamin Herrenschmidt * after routing it to the guest with a simple EOI.
15825af50993SBenjamin Herrenschmidt *
15835af50993SBenjamin Herrenschmidt * The "state" argument is a "token", all it needs is to be
15845af50993SBenjamin Herrenschmidt * non-NULL to switch to passed-through or NULL for the
15855af50993SBenjamin Herrenschmidt * other way around. We may not yet have an actual VCPU
15865af50993SBenjamin Herrenschmidt * target here and we don't really care.
15875af50993SBenjamin Herrenschmidt */
15885af50993SBenjamin Herrenschmidt rc = irq_set_vcpu_affinity(host_irq, state);
15895af50993SBenjamin Herrenschmidt if (rc) {
1590e5e78b15SCédric Le Goater pr_err("Failed to set VCPU affinity for host IRQ %ld\n", host_irq);
15915af50993SBenjamin Herrenschmidt return rc;
15925af50993SBenjamin Herrenschmidt }
15935af50993SBenjamin Herrenschmidt
15945af50993SBenjamin Herrenschmidt /*
15955af50993SBenjamin Herrenschmidt * Mask and read state of IPI. We need to know if its P bit
15965af50993SBenjamin Herrenschmidt * is set as that means it's potentially already using a
15975af50993SBenjamin Herrenschmidt * queue entry in the target
15985af50993SBenjamin Herrenschmidt */
15995af50993SBenjamin Herrenschmidt prio = xive_lock_and_mask(xive, sb, state);
16005af50993SBenjamin Herrenschmidt pr_devel(" old IPI prio %02x P:%d Q:%d\n", prio,
16015af50993SBenjamin Herrenschmidt state->old_p, state->old_q);
16025af50993SBenjamin Herrenschmidt
16035af50993SBenjamin Herrenschmidt /* Turn the IPI hard off */
16045af50993SBenjamin Herrenschmidt xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
16055af50993SBenjamin Herrenschmidt
1606232b984bSCédric Le Goater /*
1607232b984bSCédric Le Goater * Reset ESB guest mapping. Needed when ESB pages are exposed
1608232b984bSCédric Le Goater * to the guest in XIVE native mode
1609232b984bSCédric Le Goater */
1610232b984bSCédric Le Goater if (xive->ops && xive->ops->reset_mapped)
1611232b984bSCédric Le Goater xive->ops->reset_mapped(kvm, guest_irq);
1612232b984bSCédric Le Goater
16135af50993SBenjamin Herrenschmidt /* Grab info about irq */
16145af50993SBenjamin Herrenschmidt state->pt_number = hw_irq;
16155af50993SBenjamin Herrenschmidt state->pt_data = irq_data_get_irq_handler_data(host_data);
16165af50993SBenjamin Herrenschmidt
16175af50993SBenjamin Herrenschmidt /*
16185af50993SBenjamin Herrenschmidt * Configure the IRQ to match the existing configuration of
16195af50993SBenjamin Herrenschmidt * the IPI if it was already targetted. Otherwise this will
16205af50993SBenjamin Herrenschmidt * mask the interrupt in a lossy way (act_priority is 0xff)
16215af50993SBenjamin Herrenschmidt * which is fine for a never started interrupt.
16225af50993SBenjamin Herrenschmidt */
16235af50993SBenjamin Herrenschmidt xive_native_configure_irq(hw_irq,
1624eacc56bbSCédric Le Goater kvmppc_xive_vp(xive, state->act_server),
16255af50993SBenjamin Herrenschmidt state->act_priority, state->number);
16265af50993SBenjamin Herrenschmidt
16275af50993SBenjamin Herrenschmidt /*
16285af50993SBenjamin Herrenschmidt * We do an EOI to enable the interrupt (and retrigger if needed)
16295af50993SBenjamin Herrenschmidt * if the guest has the interrupt unmasked and the P bit was *not*
16305af50993SBenjamin Herrenschmidt * set in the IPI. If it was set, we know a slot may still be in
16315af50993SBenjamin Herrenschmidt * use in the target queue thus we have to wait for a guest
16325af50993SBenjamin Herrenschmidt * originated EOI
16335af50993SBenjamin Herrenschmidt */
16345af50993SBenjamin Herrenschmidt if (prio != MASKED && !state->old_p)
16355af50993SBenjamin Herrenschmidt xive_vm_source_eoi(hw_irq, state->pt_data);
16365af50993SBenjamin Herrenschmidt
16375af50993SBenjamin Herrenschmidt /* Clear old_p/old_q as they are no longer relevant */
16385af50993SBenjamin Herrenschmidt state->old_p = state->old_q = false;
16395af50993SBenjamin Herrenschmidt
16405af50993SBenjamin Herrenschmidt /* Restore guest prio (unlocks EOI) */
16415af50993SBenjamin Herrenschmidt mb();
16425af50993SBenjamin Herrenschmidt state->guest_priority = prio;
16435af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock);
16445af50993SBenjamin Herrenschmidt
16455af50993SBenjamin Herrenschmidt return 0;
16465af50993SBenjamin Herrenschmidt }
16475af50993SBenjamin Herrenschmidt EXPORT_SYMBOL_GPL(kvmppc_xive_set_mapped);
16485af50993SBenjamin Herrenschmidt
kvmppc_xive_clr_mapped(struct kvm * kvm,unsigned long guest_irq,unsigned long host_irq)16495af50993SBenjamin Herrenschmidt int kvmppc_xive_clr_mapped(struct kvm *kvm, unsigned long guest_irq,
1650e5e78b15SCédric Le Goater unsigned long host_irq)
16515af50993SBenjamin Herrenschmidt {
16525af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = kvm->arch.xive;
16535af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb;
16545af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state;
16555af50993SBenjamin Herrenschmidt u16 idx;
16565af50993SBenjamin Herrenschmidt u8 prio;
16575af50993SBenjamin Herrenschmidt int rc;
16585af50993SBenjamin Herrenschmidt
16595af50993SBenjamin Herrenschmidt if (!xive)
16605af50993SBenjamin Herrenschmidt return -ENODEV;
16615af50993SBenjamin Herrenschmidt
1662e5e78b15SCédric Le Goater pr_debug("%s: GIRQ 0x%lx host IRQ %ld\n", __func__, guest_irq, host_irq);
16635af50993SBenjamin Herrenschmidt
16645af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, guest_irq, &idx);
16655af50993SBenjamin Herrenschmidt if (!sb)
16665af50993SBenjamin Herrenschmidt return -EINVAL;
16675af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx];
16685af50993SBenjamin Herrenschmidt
16695af50993SBenjamin Herrenschmidt /*
16705af50993SBenjamin Herrenschmidt * Mask and read state of IRQ. We need to know if its P bit
16715af50993SBenjamin Herrenschmidt * is set as that means it's potentially already using a
16725af50993SBenjamin Herrenschmidt * queue entry in the target
16735af50993SBenjamin Herrenschmidt */
16745af50993SBenjamin Herrenschmidt prio = xive_lock_and_mask(xive, sb, state);
16755af50993SBenjamin Herrenschmidt pr_devel(" old IRQ prio %02x P:%d Q:%d\n", prio,
16765af50993SBenjamin Herrenschmidt state->old_p, state->old_q);
16775af50993SBenjamin Herrenschmidt
16785af50993SBenjamin Herrenschmidt /*
16795af50993SBenjamin Herrenschmidt * If old_p is set, the interrupt is pending, we switch it to
16805af50993SBenjamin Herrenschmidt * PQ=11. This will force a resend in the host so the interrupt
16811fd02f66SJulia Lawall * isn't lost to whatever host driver may pick it up
16825af50993SBenjamin Herrenschmidt */
16835af50993SBenjamin Herrenschmidt if (state->old_p)
16845af50993SBenjamin Herrenschmidt xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_11);
16855af50993SBenjamin Herrenschmidt
16865af50993SBenjamin Herrenschmidt /* Release the passed-through interrupt to the host */
16875af50993SBenjamin Herrenschmidt rc = irq_set_vcpu_affinity(host_irq, NULL);
16885af50993SBenjamin Herrenschmidt if (rc) {
1689e5e78b15SCédric Le Goater pr_err("Failed to clr VCPU affinity for host IRQ %ld\n", host_irq);
16905af50993SBenjamin Herrenschmidt return rc;
16915af50993SBenjamin Herrenschmidt }
16925af50993SBenjamin Herrenschmidt
16935af50993SBenjamin Herrenschmidt /* Forget about the IRQ */
16945af50993SBenjamin Herrenschmidt state->pt_number = 0;
16955af50993SBenjamin Herrenschmidt state->pt_data = NULL;
16965af50993SBenjamin Herrenschmidt
1697232b984bSCédric Le Goater /*
1698232b984bSCédric Le Goater * Reset ESB guest mapping. Needed when ESB pages are exposed
1699232b984bSCédric Le Goater * to the guest in XIVE native mode
1700232b984bSCédric Le Goater */
1701232b984bSCédric Le Goater if (xive->ops && xive->ops->reset_mapped) {
1702232b984bSCédric Le Goater xive->ops->reset_mapped(kvm, guest_irq);
1703232b984bSCédric Le Goater }
1704232b984bSCédric Le Goater
17055af50993SBenjamin Herrenschmidt /* Reconfigure the IPI */
17065af50993SBenjamin Herrenschmidt xive_native_configure_irq(state->ipi_number,
1707eacc56bbSCédric Le Goater kvmppc_xive_vp(xive, state->act_server),
17085af50993SBenjamin Herrenschmidt state->act_priority, state->number);
17095af50993SBenjamin Herrenschmidt
17105af50993SBenjamin Herrenschmidt /*
17115af50993SBenjamin Herrenschmidt * If old_p is set (we have a queue entry potentially
17125af50993SBenjamin Herrenschmidt * occupied) or the interrupt is masked, we set the IPI
17135af50993SBenjamin Herrenschmidt * to PQ=10 state. Otherwise we just re-enable it (PQ=00).
17145af50993SBenjamin Herrenschmidt */
17155af50993SBenjamin Herrenschmidt if (prio == MASKED || state->old_p)
17165af50993SBenjamin Herrenschmidt xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_10);
17175af50993SBenjamin Herrenschmidt else
17185af50993SBenjamin Herrenschmidt xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_00);
17195af50993SBenjamin Herrenschmidt
17205af50993SBenjamin Herrenschmidt /* Restore guest prio (unlocks EOI) */
17215af50993SBenjamin Herrenschmidt mb();
17225af50993SBenjamin Herrenschmidt state->guest_priority = prio;
17235af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock);
17245af50993SBenjamin Herrenschmidt
17255af50993SBenjamin Herrenschmidt return 0;
17265af50993SBenjamin Herrenschmidt }
17275af50993SBenjamin Herrenschmidt EXPORT_SYMBOL_GPL(kvmppc_xive_clr_mapped);
17285af50993SBenjamin Herrenschmidt
kvmppc_xive_disable_vcpu_interrupts(struct kvm_vcpu * vcpu)1729eacc56bbSCédric Le Goater void kvmppc_xive_disable_vcpu_interrupts(struct kvm_vcpu *vcpu)
17305af50993SBenjamin Herrenschmidt {
17315af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
17325af50993SBenjamin Herrenschmidt struct kvm *kvm = vcpu->kvm;
17335af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = kvm->arch.xive;
17345af50993SBenjamin Herrenschmidt int i, j;
17355af50993SBenjamin Herrenschmidt
17365af50993SBenjamin Herrenschmidt for (i = 0; i <= xive->max_sbid; i++) {
17375af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
17385af50993SBenjamin Herrenschmidt
17395af50993SBenjamin Herrenschmidt if (!sb)
17405af50993SBenjamin Herrenschmidt continue;
17415af50993SBenjamin Herrenschmidt for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) {
17425af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state = &sb->irq_state[j];
17435af50993SBenjamin Herrenschmidt
17445af50993SBenjamin Herrenschmidt if (!state->valid)
17455af50993SBenjamin Herrenschmidt continue;
17465af50993SBenjamin Herrenschmidt if (state->act_priority == MASKED)
17475af50993SBenjamin Herrenschmidt continue;
17485af50993SBenjamin Herrenschmidt if (state->act_server != xc->server_num)
17495af50993SBenjamin Herrenschmidt continue;
17505af50993SBenjamin Herrenschmidt
17515af50993SBenjamin Herrenschmidt /* Clean it up */
17525af50993SBenjamin Herrenschmidt arch_spin_lock(&sb->lock);
17535af50993SBenjamin Herrenschmidt state->act_priority = MASKED;
17545af50993SBenjamin Herrenschmidt xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
17555af50993SBenjamin Herrenschmidt xive_native_configure_irq(state->ipi_number, 0, MASKED, 0);
17565af50993SBenjamin Herrenschmidt if (state->pt_number) {
17575af50993SBenjamin Herrenschmidt xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_01);
17585af50993SBenjamin Herrenschmidt xive_native_configure_irq(state->pt_number, 0, MASKED, 0);
17595af50993SBenjamin Herrenschmidt }
17605af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock);
17615af50993SBenjamin Herrenschmidt }
17625af50993SBenjamin Herrenschmidt }
17630caecf5bSPaul Mackerras
17640caecf5bSPaul Mackerras /* Disable vcpu's escalation interrupt */
17650caecf5bSPaul Mackerras if (vcpu->arch.xive_esc_on) {
17660caecf5bSPaul Mackerras __raw_readq((void __iomem *)(vcpu->arch.xive_esc_vaddr +
17670caecf5bSPaul Mackerras XIVE_ESB_SET_PQ_01));
17680caecf5bSPaul Mackerras vcpu->arch.xive_esc_on = false;
17690caecf5bSPaul Mackerras }
17700caecf5bSPaul Mackerras
17710caecf5bSPaul Mackerras /*
17720caecf5bSPaul Mackerras * Clear pointers to escalation interrupt ESB.
17730caecf5bSPaul Mackerras * This is safe because the vcpu->mutex is held, preventing
17740caecf5bSPaul Mackerras * any other CPU from concurrently executing a KVM_RUN ioctl.
17750caecf5bSPaul Mackerras */
17760caecf5bSPaul Mackerras vcpu->arch.xive_esc_vaddr = 0;
17770caecf5bSPaul Mackerras vcpu->arch.xive_esc_raddr = 0;
17785af50993SBenjamin Herrenschmidt }
17795af50993SBenjamin Herrenschmidt
1780da15c03bSPaul Mackerras /*
1781da15c03bSPaul Mackerras * In single escalation mode, the escalation interrupt is marked so
1782da15c03bSPaul Mackerras * that EOI doesn't re-enable it, but just sets the stale_p flag to
1783da15c03bSPaul Mackerras * indicate that the P bit has already been dealt with. However, the
1784da15c03bSPaul Mackerras * assembly code that enters the guest sets PQ to 00 without clearing
1785da15c03bSPaul Mackerras * stale_p (because it has no easy way to address it). Hence we have
1786da15c03bSPaul Mackerras * to adjust stale_p before shutting down the interrupt.
1787da15c03bSPaul Mackerras */
xive_cleanup_single_escalation(struct kvm_vcpu * vcpu,int irq)1788*932c6deaSDeming Wang void xive_cleanup_single_escalation(struct kvm_vcpu *vcpu, int irq)
1789da15c03bSPaul Mackerras {
1790da15c03bSPaul Mackerras struct irq_data *d = irq_get_irq_data(irq);
1791da15c03bSPaul Mackerras struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
1792da15c03bSPaul Mackerras
1793da15c03bSPaul Mackerras /*
1794da15c03bSPaul Mackerras * This slightly odd sequence gives the right result
1795da15c03bSPaul Mackerras * (i.e. stale_p set if xive_esc_on is false) even if
1796da15c03bSPaul Mackerras * we race with xive_esc_irq() and xive_irq_eoi().
1797da15c03bSPaul Mackerras */
1798da15c03bSPaul Mackerras xd->stale_p = false;
1799da15c03bSPaul Mackerras smp_mb(); /* paired with smb_wmb in xive_esc_irq */
1800da15c03bSPaul Mackerras if (!vcpu->arch.xive_esc_on)
1801da15c03bSPaul Mackerras xd->stale_p = true;
1802da15c03bSPaul Mackerras }
1803da15c03bSPaul Mackerras
kvmppc_xive_cleanup_vcpu(struct kvm_vcpu * vcpu)18045af50993SBenjamin Herrenschmidt void kvmppc_xive_cleanup_vcpu(struct kvm_vcpu *vcpu)
18055af50993SBenjamin Herrenschmidt {
18065af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
18075422e951SCédric Le Goater struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
18085af50993SBenjamin Herrenschmidt int i;
18095af50993SBenjamin Herrenschmidt
18105422e951SCédric Le Goater if (!kvmppc_xics_enabled(vcpu))
18115422e951SCédric Le Goater return;
18125422e951SCédric Le Goater
18135422e951SCédric Le Goater if (!xc)
18145422e951SCédric Le Goater return;
18155422e951SCédric Le Goater
18165af50993SBenjamin Herrenschmidt pr_devel("cleanup_vcpu(cpu=%d)\n", xc->server_num);
18175af50993SBenjamin Herrenschmidt
18185af50993SBenjamin Herrenschmidt /* Ensure no interrupt is still routed to that VP */
18195af50993SBenjamin Herrenschmidt xc->valid = false;
18205af50993SBenjamin Herrenschmidt kvmppc_xive_disable_vcpu_interrupts(vcpu);
18215af50993SBenjamin Herrenschmidt
18225af50993SBenjamin Herrenschmidt /* Mask the VP IPI */
18235af50993SBenjamin Herrenschmidt xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_01);
18245af50993SBenjamin Herrenschmidt
1825237aed48SCédric Le Goater /* Free escalations */
18265af50993SBenjamin Herrenschmidt for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
18275af50993SBenjamin Herrenschmidt if (xc->esc_virq[i]) {
1828b68c6646SCédric Le Goater if (kvmppc_xive_has_single_escalation(xc->xive))
1829*932c6deaSDeming Wang xive_cleanup_single_escalation(vcpu, xc->esc_virq[i]);
18305af50993SBenjamin Herrenschmidt free_irq(xc->esc_virq[i], vcpu);
18315af50993SBenjamin Herrenschmidt irq_dispose_mapping(xc->esc_virq[i]);
18325af50993SBenjamin Herrenschmidt kfree(xc->esc_virq_names[i]);
18335af50993SBenjamin Herrenschmidt }
1834237aed48SCédric Le Goater }
1835237aed48SCédric Le Goater
1836237aed48SCédric Le Goater /* Disable the VP */
1837237aed48SCédric Le Goater xive_native_disable_vp(xc->vp_id);
1838237aed48SCédric Le Goater
18398d4ba9c9SPaul Mackerras /* Clear the cam word so guest entry won't try to push context */
18408d4ba9c9SPaul Mackerras vcpu->arch.xive_cam_word = 0;
18418d4ba9c9SPaul Mackerras
1842237aed48SCédric Le Goater /* Free the queues */
1843237aed48SCédric Le Goater for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
1844237aed48SCédric Le Goater struct xive_q *q = &xc->queues[i];
1845237aed48SCédric Le Goater
18465af50993SBenjamin Herrenschmidt xive_native_disable_queue(xc->vp_id, q, i);
18475af50993SBenjamin Herrenschmidt if (q->qpage) {
18485af50993SBenjamin Herrenschmidt free_pages((unsigned long)q->qpage,
18495af50993SBenjamin Herrenschmidt xive->q_page_order);
18505af50993SBenjamin Herrenschmidt q->qpage = NULL;
18515af50993SBenjamin Herrenschmidt }
18525af50993SBenjamin Herrenschmidt }
18535af50993SBenjamin Herrenschmidt
18545af50993SBenjamin Herrenschmidt /* Free the IPI */
18555af50993SBenjamin Herrenschmidt if (xc->vp_ipi) {
18565af50993SBenjamin Herrenschmidt xive_cleanup_irq_data(&xc->vp_ipi_data);
18575af50993SBenjamin Herrenschmidt xive_native_free_irq(xc->vp_ipi);
18585af50993SBenjamin Herrenschmidt }
18595af50993SBenjamin Herrenschmidt /* Free the VP */
18605af50993SBenjamin Herrenschmidt kfree(xc);
18615422e951SCédric Le Goater
18625422e951SCédric Le Goater /* Cleanup the vcpu */
18635422e951SCédric Le Goater vcpu->arch.irq_type = KVMPPC_IRQ_DEFAULT;
18645422e951SCédric Le Goater vcpu->arch.xive_vcpu = NULL;
18655af50993SBenjamin Herrenschmidt }
18665af50993SBenjamin Herrenschmidt
kvmppc_xive_vcpu_id_valid(struct kvmppc_xive * xive,u32 cpu)18678db29ea2SGreg Kurz static bool kvmppc_xive_vcpu_id_valid(struct kvmppc_xive *xive, u32 cpu)
18688db29ea2SGreg Kurz {
1869062cfab7SGreg Kurz /* We have a block of xive->nr_servers VPs. We just need to check
1870f54db39fSGreg Kurz * packed vCPU ids are below that.
18718db29ea2SGreg Kurz */
1872f54db39fSGreg Kurz return kvmppc_pack_vcpu_id(xive->kvm, cpu) < xive->nr_servers;
18738db29ea2SGreg Kurz }
18748db29ea2SGreg Kurz
kvmppc_xive_compute_vp_id(struct kvmppc_xive * xive,u32 cpu,u32 * vp)18758db29ea2SGreg Kurz int kvmppc_xive_compute_vp_id(struct kvmppc_xive *xive, u32 cpu, u32 *vp)
18768db29ea2SGreg Kurz {
18778db29ea2SGreg Kurz u32 vp_id;
18788db29ea2SGreg Kurz
18798db29ea2SGreg Kurz if (!kvmppc_xive_vcpu_id_valid(xive, cpu)) {
18808db29ea2SGreg Kurz pr_devel("Out of bounds !\n");
18818db29ea2SGreg Kurz return -EINVAL;
18828db29ea2SGreg Kurz }
18838db29ea2SGreg Kurz
1884062cfab7SGreg Kurz if (xive->vp_base == XIVE_INVALID_VP) {
1885062cfab7SGreg Kurz xive->vp_base = xive_native_alloc_vp_block(xive->nr_servers);
1886062cfab7SGreg Kurz pr_devel("VP_Base=%x nr_servers=%d\n", xive->vp_base, xive->nr_servers);
1887062cfab7SGreg Kurz
1888062cfab7SGreg Kurz if (xive->vp_base == XIVE_INVALID_VP)
1889062cfab7SGreg Kurz return -ENOSPC;
1890062cfab7SGreg Kurz }
1891062cfab7SGreg Kurz
18928db29ea2SGreg Kurz vp_id = kvmppc_xive_vp(xive, cpu);
18938db29ea2SGreg Kurz if (kvmppc_xive_vp_in_use(xive->kvm, vp_id)) {
18948db29ea2SGreg Kurz pr_devel("Duplicate !\n");
18958db29ea2SGreg Kurz return -EEXIST;
18968db29ea2SGreg Kurz }
18978db29ea2SGreg Kurz
18988db29ea2SGreg Kurz *vp = vp_id;
18998db29ea2SGreg Kurz
19008db29ea2SGreg Kurz return 0;
19018db29ea2SGreg Kurz }
19028db29ea2SGreg Kurz
kvmppc_xive_connect_vcpu(struct kvm_device * dev,struct kvm_vcpu * vcpu,u32 cpu)19035af50993SBenjamin Herrenschmidt int kvmppc_xive_connect_vcpu(struct kvm_device *dev,
19045af50993SBenjamin Herrenschmidt struct kvm_vcpu *vcpu, u32 cpu)
19055af50993SBenjamin Herrenschmidt {
19065af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = dev->private;
19075af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc;
19085af50993SBenjamin Herrenschmidt int i, r = -EBUSY;
190912ade69cSGreg Kurz u32 vp_id;
19105af50993SBenjamin Herrenschmidt
19115af50993SBenjamin Herrenschmidt pr_devel("connect_vcpu(cpu=%d)\n", cpu);
19125af50993SBenjamin Herrenschmidt
19135af50993SBenjamin Herrenschmidt if (dev->ops != &kvm_xive_ops) {
19145af50993SBenjamin Herrenschmidt pr_devel("Wrong ops !\n");
19155af50993SBenjamin Herrenschmidt return -EPERM;
19165af50993SBenjamin Herrenschmidt }
19175af50993SBenjamin Herrenschmidt if (xive->kvm != vcpu->kvm)
19185af50993SBenjamin Herrenschmidt return -EPERM;
19195422e951SCédric Le Goater if (vcpu->arch.irq_type != KVMPPC_IRQ_DEFAULT)
19205af50993SBenjamin Herrenschmidt return -EBUSY;
19215af50993SBenjamin Herrenschmidt
19225af50993SBenjamin Herrenschmidt /* We need to synchronize with queue provisioning */
19237e10b9a6SCédric Le Goater mutex_lock(&xive->lock);
192412ade69cSGreg Kurz
19258db29ea2SGreg Kurz r = kvmppc_xive_compute_vp_id(xive, cpu, &vp_id);
19268db29ea2SGreg Kurz if (r)
192712ade69cSGreg Kurz goto bail;
192812ade69cSGreg Kurz
192912ade69cSGreg Kurz xc = kzalloc(sizeof(*xc), GFP_KERNEL);
193012ade69cSGreg Kurz if (!xc) {
193112ade69cSGreg Kurz r = -ENOMEM;
193212ade69cSGreg Kurz goto bail;
193312ade69cSGreg Kurz }
193412ade69cSGreg Kurz
19355af50993SBenjamin Herrenschmidt vcpu->arch.xive_vcpu = xc;
19365af50993SBenjamin Herrenschmidt xc->xive = xive;
19375af50993SBenjamin Herrenschmidt xc->vcpu = vcpu;
19385af50993SBenjamin Herrenschmidt xc->server_num = cpu;
193912ade69cSGreg Kurz xc->vp_id = vp_id;
19405af50993SBenjamin Herrenschmidt xc->mfrr = 0xff;
19415af50993SBenjamin Herrenschmidt xc->valid = true;
19425af50993SBenjamin Herrenschmidt
19435af50993SBenjamin Herrenschmidt r = xive_native_get_vp_info(xc->vp_id, &xc->vp_cam, &xc->vp_chip_id);
19445af50993SBenjamin Herrenschmidt if (r)
19455af50993SBenjamin Herrenschmidt goto bail;
19465af50993SBenjamin Herrenschmidt
1947f5af0a97SCédric Le Goater if (!kvmppc_xive_check_save_restore(vcpu)) {
1948f5af0a97SCédric Le Goater pr_err("inconsistent save-restore setup for VCPU %d\n", cpu);
1949f5af0a97SCédric Le Goater r = -EIO;
1950f5af0a97SCédric Le Goater goto bail;
1951f5af0a97SCédric Le Goater }
1952f5af0a97SCédric Le Goater
19535af50993SBenjamin Herrenschmidt /* Configure VCPU fields for use by assembly push/pull */
19545af50993SBenjamin Herrenschmidt vcpu->arch.xive_saved_state.w01 = cpu_to_be64(0xff000000);
19555af50993SBenjamin Herrenschmidt vcpu->arch.xive_cam_word = cpu_to_be32(xc->vp_cam | TM_QW1W2_VO);
19565af50993SBenjamin Herrenschmidt
19575af50993SBenjamin Herrenschmidt /* Allocate IPI */
19585af50993SBenjamin Herrenschmidt xc->vp_ipi = xive_native_alloc_irq();
19595af50993SBenjamin Herrenschmidt if (!xc->vp_ipi) {
1960bf4159daSBenjamin Herrenschmidt pr_err("Failed to allocate xive irq for VCPU IPI\n");
19615af50993SBenjamin Herrenschmidt r = -EIO;
19625af50993SBenjamin Herrenschmidt goto bail;
19635af50993SBenjamin Herrenschmidt }
19645af50993SBenjamin Herrenschmidt pr_devel(" IPI=0x%x\n", xc->vp_ipi);
19655af50993SBenjamin Herrenschmidt
19665af50993SBenjamin Herrenschmidt r = xive_native_populate_irq_data(xc->vp_ipi, &xc->vp_ipi_data);
19675af50993SBenjamin Herrenschmidt if (r)
19685af50993SBenjamin Herrenschmidt goto bail;
19695af50993SBenjamin Herrenschmidt
19705af50993SBenjamin Herrenschmidt /*
1971bf4159daSBenjamin Herrenschmidt * Enable the VP first as the single escalation mode will
1972bf4159daSBenjamin Herrenschmidt * affect escalation interrupts numbering
1973bf4159daSBenjamin Herrenschmidt */
1974b68c6646SCédric Le Goater r = xive_native_enable_vp(xc->vp_id, kvmppc_xive_has_single_escalation(xive));
1975bf4159daSBenjamin Herrenschmidt if (r) {
1976bf4159daSBenjamin Herrenschmidt pr_err("Failed to enable VP in OPAL, err %d\n", r);
1977bf4159daSBenjamin Herrenschmidt goto bail;
1978bf4159daSBenjamin Herrenschmidt }
1979bf4159daSBenjamin Herrenschmidt
1980bf4159daSBenjamin Herrenschmidt /*
19815af50993SBenjamin Herrenschmidt * Initialize queues. Initially we set them all for no queueing
19825af50993SBenjamin Herrenschmidt * and we enable escalation for queue 0 only which we'll use for
19835af50993SBenjamin Herrenschmidt * our mfrr change notifications. If the VCPU is hot-plugged, we
1984bf4159daSBenjamin Herrenschmidt * do handle provisioning however based on the existing "map"
1985bf4159daSBenjamin Herrenschmidt * of enabled queues.
19865af50993SBenjamin Herrenschmidt */
19875af50993SBenjamin Herrenschmidt for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
19885af50993SBenjamin Herrenschmidt struct xive_q *q = &xc->queues[i];
19895af50993SBenjamin Herrenschmidt
1990bf4159daSBenjamin Herrenschmidt /* Single escalation, no queue 7 */
1991b68c6646SCédric Le Goater if (i == 7 && kvmppc_xive_has_single_escalation(xive))
1992bf4159daSBenjamin Herrenschmidt break;
1993bf4159daSBenjamin Herrenschmidt
19945af50993SBenjamin Herrenschmidt /* Is queue already enabled ? Provision it */
19955af50993SBenjamin Herrenschmidt if (xive->qmap & (1 << i)) {
19965af50993SBenjamin Herrenschmidt r = xive_provision_queue(vcpu, i);
1997b68c6646SCédric Le Goater if (r == 0 && !kvmppc_xive_has_single_escalation(xive))
199813ce3297SCédric Le Goater kvmppc_xive_attach_escalation(
1999b68c6646SCédric Le Goater vcpu, i, kvmppc_xive_has_single_escalation(xive));
20005af50993SBenjamin Herrenschmidt if (r)
20015af50993SBenjamin Herrenschmidt goto bail;
20025af50993SBenjamin Herrenschmidt } else {
20035af50993SBenjamin Herrenschmidt r = xive_native_configure_queue(xc->vp_id,
20045af50993SBenjamin Herrenschmidt q, i, NULL, 0, true);
20055af50993SBenjamin Herrenschmidt if (r) {
20065af50993SBenjamin Herrenschmidt pr_err("Failed to configure queue %d for VCPU %d\n",
20075af50993SBenjamin Herrenschmidt i, cpu);
20085af50993SBenjamin Herrenschmidt goto bail;
20095af50993SBenjamin Herrenschmidt }
20105af50993SBenjamin Herrenschmidt }
20115af50993SBenjamin Herrenschmidt }
20125af50993SBenjamin Herrenschmidt
20135af50993SBenjamin Herrenschmidt /* If not done above, attach priority 0 escalation */
2014b68c6646SCédric Le Goater r = kvmppc_xive_attach_escalation(vcpu, 0, kvmppc_xive_has_single_escalation(xive));
20155af50993SBenjamin Herrenschmidt if (r)
20165af50993SBenjamin Herrenschmidt goto bail;
20175af50993SBenjamin Herrenschmidt
20185af50993SBenjamin Herrenschmidt /* Route the IPI */
20195af50993SBenjamin Herrenschmidt r = xive_native_configure_irq(xc->vp_ipi, xc->vp_id, 0, XICS_IPI);
20205af50993SBenjamin Herrenschmidt if (!r)
20215af50993SBenjamin Herrenschmidt xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_00);
20225af50993SBenjamin Herrenschmidt
20235af50993SBenjamin Herrenschmidt bail:
20247e10b9a6SCédric Le Goater mutex_unlock(&xive->lock);
20255af50993SBenjamin Herrenschmidt if (r) {
20265af50993SBenjamin Herrenschmidt kvmppc_xive_cleanup_vcpu(vcpu);
20275af50993SBenjamin Herrenschmidt return r;
20285af50993SBenjamin Herrenschmidt }
20295af50993SBenjamin Herrenschmidt
20305af50993SBenjamin Herrenschmidt vcpu->arch.irq_type = KVMPPC_IRQ_XICS;
20315af50993SBenjamin Herrenschmidt return 0;
20325af50993SBenjamin Herrenschmidt }
20335af50993SBenjamin Herrenschmidt
20345af50993SBenjamin Herrenschmidt /*
20355af50993SBenjamin Herrenschmidt * Scanning of queues before/after migration save
20365af50993SBenjamin Herrenschmidt */
xive_pre_save_set_queued(struct kvmppc_xive * xive,u32 irq)20375af50993SBenjamin Herrenschmidt static void xive_pre_save_set_queued(struct kvmppc_xive *xive, u32 irq)
20385af50993SBenjamin Herrenschmidt {
20395af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb;
20405af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state;
20415af50993SBenjamin Herrenschmidt u16 idx;
20425af50993SBenjamin Herrenschmidt
20435af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, irq, &idx);
20445af50993SBenjamin Herrenschmidt if (!sb)
20455af50993SBenjamin Herrenschmidt return;
20465af50993SBenjamin Herrenschmidt
20475af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx];
20485af50993SBenjamin Herrenschmidt
20495af50993SBenjamin Herrenschmidt /* Some sanity checking */
20505af50993SBenjamin Herrenschmidt if (!state->valid) {
20515af50993SBenjamin Herrenschmidt pr_err("invalid irq 0x%x in cpu queue!\n", irq);
20525af50993SBenjamin Herrenschmidt return;
20535af50993SBenjamin Herrenschmidt }
20545af50993SBenjamin Herrenschmidt
20555af50993SBenjamin Herrenschmidt /*
20565af50993SBenjamin Herrenschmidt * If the interrupt is in a queue it should have P set.
20575af50993SBenjamin Herrenschmidt * We warn so that gets reported. A backtrace isn't useful
20585af50993SBenjamin Herrenschmidt * so no need to use a WARN_ON.
20595af50993SBenjamin Herrenschmidt */
20605af50993SBenjamin Herrenschmidt if (!state->saved_p)
20615af50993SBenjamin Herrenschmidt pr_err("Interrupt 0x%x is marked in a queue but P not set !\n", irq);
20625af50993SBenjamin Herrenschmidt
20635af50993SBenjamin Herrenschmidt /* Set flag */
20645af50993SBenjamin Herrenschmidt state->in_queue = true;
20655af50993SBenjamin Herrenschmidt }
20665af50993SBenjamin Herrenschmidt
xive_pre_save_mask_irq(struct kvmppc_xive * xive,struct kvmppc_xive_src_block * sb,u32 irq)20675af50993SBenjamin Herrenschmidt static void xive_pre_save_mask_irq(struct kvmppc_xive *xive,
20685af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb,
20695af50993SBenjamin Herrenschmidt u32 irq)
20705af50993SBenjamin Herrenschmidt {
20715af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state = &sb->irq_state[irq];
20725af50993SBenjamin Herrenschmidt
20735af50993SBenjamin Herrenschmidt if (!state->valid)
20745af50993SBenjamin Herrenschmidt return;
20755af50993SBenjamin Herrenschmidt
20765af50993SBenjamin Herrenschmidt /* Mask and save state, this will also sync HW queues */
20775af50993SBenjamin Herrenschmidt state->saved_scan_prio = xive_lock_and_mask(xive, sb, state);
20785af50993SBenjamin Herrenschmidt
20795af50993SBenjamin Herrenschmidt /* Transfer P and Q */
20805af50993SBenjamin Herrenschmidt state->saved_p = state->old_p;
20815af50993SBenjamin Herrenschmidt state->saved_q = state->old_q;
20825af50993SBenjamin Herrenschmidt
20835af50993SBenjamin Herrenschmidt /* Unlock */
20845af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock);
20855af50993SBenjamin Herrenschmidt }
20865af50993SBenjamin Herrenschmidt
xive_pre_save_unmask_irq(struct kvmppc_xive * xive,struct kvmppc_xive_src_block * sb,u32 irq)20875af50993SBenjamin Herrenschmidt static void xive_pre_save_unmask_irq(struct kvmppc_xive *xive,
20885af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb,
20895af50993SBenjamin Herrenschmidt u32 irq)
20905af50993SBenjamin Herrenschmidt {
20915af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state = &sb->irq_state[irq];
20925af50993SBenjamin Herrenschmidt
20935af50993SBenjamin Herrenschmidt if (!state->valid)
20945af50993SBenjamin Herrenschmidt return;
20955af50993SBenjamin Herrenschmidt
20965af50993SBenjamin Herrenschmidt /*
20975af50993SBenjamin Herrenschmidt * Lock / exclude EOI (not technically necessary if the
20985af50993SBenjamin Herrenschmidt * guest isn't running concurrently. If this becomes a
20995af50993SBenjamin Herrenschmidt * performance issue we can probably remove the lock.
21005af50993SBenjamin Herrenschmidt */
21015af50993SBenjamin Herrenschmidt xive_lock_for_unmask(sb, state);
21025af50993SBenjamin Herrenschmidt
21035af50993SBenjamin Herrenschmidt /* Restore mask/prio if it wasn't masked */
21045af50993SBenjamin Herrenschmidt if (state->saved_scan_prio != MASKED)
21055af50993SBenjamin Herrenschmidt xive_finish_unmask(xive, sb, state, state->saved_scan_prio);
21065af50993SBenjamin Herrenschmidt
21075af50993SBenjamin Herrenschmidt /* Unlock */
21085af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock);
21095af50993SBenjamin Herrenschmidt }
21105af50993SBenjamin Herrenschmidt
xive_pre_save_queue(struct kvmppc_xive * xive,struct xive_q * q)21115af50993SBenjamin Herrenschmidt static void xive_pre_save_queue(struct kvmppc_xive *xive, struct xive_q *q)
21125af50993SBenjamin Herrenschmidt {
21135af50993SBenjamin Herrenschmidt u32 idx = q->idx;
21145af50993SBenjamin Herrenschmidt u32 toggle = q->toggle;
21155af50993SBenjamin Herrenschmidt u32 irq;
21165af50993SBenjamin Herrenschmidt
21175af50993SBenjamin Herrenschmidt do {
21185af50993SBenjamin Herrenschmidt irq = __xive_read_eq(q->qpage, q->msk, &idx, &toggle);
21195af50993SBenjamin Herrenschmidt if (irq > XICS_IPI)
21205af50993SBenjamin Herrenschmidt xive_pre_save_set_queued(xive, irq);
21215af50993SBenjamin Herrenschmidt } while(irq);
21225af50993SBenjamin Herrenschmidt }
21235af50993SBenjamin Herrenschmidt
xive_pre_save_scan(struct kvmppc_xive * xive)21245af50993SBenjamin Herrenschmidt static void xive_pre_save_scan(struct kvmppc_xive *xive)
21255af50993SBenjamin Herrenschmidt {
21265af50993SBenjamin Herrenschmidt struct kvm_vcpu *vcpu = NULL;
212746808a4cSMarc Zyngier unsigned long i;
212846808a4cSMarc Zyngier int j;
21295af50993SBenjamin Herrenschmidt
21305af50993SBenjamin Herrenschmidt /*
21315af50993SBenjamin Herrenschmidt * See comment in xive_get_source() about how this
21325af50993SBenjamin Herrenschmidt * work. Collect a stable state for all interrupts
21335af50993SBenjamin Herrenschmidt */
21345af50993SBenjamin Herrenschmidt for (i = 0; i <= xive->max_sbid; i++) {
21355af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
21365af50993SBenjamin Herrenschmidt if (!sb)
21375af50993SBenjamin Herrenschmidt continue;
21385af50993SBenjamin Herrenschmidt for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++)
21395af50993SBenjamin Herrenschmidt xive_pre_save_mask_irq(xive, sb, j);
21405af50993SBenjamin Herrenschmidt }
21415af50993SBenjamin Herrenschmidt
21425af50993SBenjamin Herrenschmidt /* Then scan the queues and update the "in_queue" flag */
21435af50993SBenjamin Herrenschmidt kvm_for_each_vcpu(i, vcpu, xive->kvm) {
21445af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
21455af50993SBenjamin Herrenschmidt if (!xc)
21465af50993SBenjamin Herrenschmidt continue;
21475af50993SBenjamin Herrenschmidt for (j = 0; j < KVMPPC_XIVE_Q_COUNT; j++) {
214800c14757SPaul Mackerras if (xc->queues[j].qpage)
214900c14757SPaul Mackerras xive_pre_save_queue(xive, &xc->queues[j]);
21505af50993SBenjamin Herrenschmidt }
21515af50993SBenjamin Herrenschmidt }
21525af50993SBenjamin Herrenschmidt
21535af50993SBenjamin Herrenschmidt /* Finally restore interrupt states */
21545af50993SBenjamin Herrenschmidt for (i = 0; i <= xive->max_sbid; i++) {
21555af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
21565af50993SBenjamin Herrenschmidt if (!sb)
21575af50993SBenjamin Herrenschmidt continue;
21585af50993SBenjamin Herrenschmidt for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++)
21595af50993SBenjamin Herrenschmidt xive_pre_save_unmask_irq(xive, sb, j);
21605af50993SBenjamin Herrenschmidt }
21615af50993SBenjamin Herrenschmidt }
21625af50993SBenjamin Herrenschmidt
xive_post_save_scan(struct kvmppc_xive * xive)21635af50993SBenjamin Herrenschmidt static void xive_post_save_scan(struct kvmppc_xive *xive)
21645af50993SBenjamin Herrenschmidt {
21655af50993SBenjamin Herrenschmidt u32 i, j;
21665af50993SBenjamin Herrenschmidt
21675af50993SBenjamin Herrenschmidt /* Clear all the in_queue flags */
21685af50993SBenjamin Herrenschmidt for (i = 0; i <= xive->max_sbid; i++) {
21695af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
21705af50993SBenjamin Herrenschmidt if (!sb)
21715af50993SBenjamin Herrenschmidt continue;
21725af50993SBenjamin Herrenschmidt for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++)
21735af50993SBenjamin Herrenschmidt sb->irq_state[j].in_queue = false;
21745af50993SBenjamin Herrenschmidt }
21755af50993SBenjamin Herrenschmidt
21765af50993SBenjamin Herrenschmidt /* Next get_source() will do a new scan */
21775af50993SBenjamin Herrenschmidt xive->saved_src_count = 0;
21785af50993SBenjamin Herrenschmidt }
21795af50993SBenjamin Herrenschmidt
21805af50993SBenjamin Herrenschmidt /*
21815af50993SBenjamin Herrenschmidt * This returns the source configuration and state to user space.
21825af50993SBenjamin Herrenschmidt */
xive_get_source(struct kvmppc_xive * xive,long irq,u64 addr)21835af50993SBenjamin Herrenschmidt static int xive_get_source(struct kvmppc_xive *xive, long irq, u64 addr)
21845af50993SBenjamin Herrenschmidt {
21855af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb;
21865af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state;
21875af50993SBenjamin Herrenschmidt u64 __user *ubufp = (u64 __user *) addr;
21885af50993SBenjamin Herrenschmidt u64 val, prio;
21895af50993SBenjamin Herrenschmidt u16 idx;
21905af50993SBenjamin Herrenschmidt
21915af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, irq, &idx);
21925af50993SBenjamin Herrenschmidt if (!sb)
21935af50993SBenjamin Herrenschmidt return -ENOENT;
21945af50993SBenjamin Herrenschmidt
21955af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx];
21965af50993SBenjamin Herrenschmidt
21975af50993SBenjamin Herrenschmidt if (!state->valid)
21985af50993SBenjamin Herrenschmidt return -ENOENT;
21995af50993SBenjamin Herrenschmidt
22005af50993SBenjamin Herrenschmidt pr_devel("get_source(%ld)...\n", irq);
22015af50993SBenjamin Herrenschmidt
22025af50993SBenjamin Herrenschmidt /*
22035af50993SBenjamin Herrenschmidt * So to properly save the state into something that looks like a
22045af50993SBenjamin Herrenschmidt * XICS migration stream we cannot treat interrupts individually.
22055af50993SBenjamin Herrenschmidt *
22065af50993SBenjamin Herrenschmidt * We need, instead, mask them all (& save their previous PQ state)
22075af50993SBenjamin Herrenschmidt * to get a stable state in the HW, then sync them to ensure that
22085af50993SBenjamin Herrenschmidt * any interrupt that had already fired hits its queue, and finally
22095af50993SBenjamin Herrenschmidt * scan all the queues to collect which interrupts are still present
22105af50993SBenjamin Herrenschmidt * in the queues, so we can set the "pending" flag on them and
22115af50993SBenjamin Herrenschmidt * they can be resent on restore.
22125af50993SBenjamin Herrenschmidt *
22135af50993SBenjamin Herrenschmidt * So we do it all when the "first" interrupt gets saved, all the
22145af50993SBenjamin Herrenschmidt * state is collected at that point, the rest of xive_get_source()
22155af50993SBenjamin Herrenschmidt * will merely collect and convert that state to the expected
22165af50993SBenjamin Herrenschmidt * userspace bit mask.
22175af50993SBenjamin Herrenschmidt */
22185af50993SBenjamin Herrenschmidt if (xive->saved_src_count == 0)
22195af50993SBenjamin Herrenschmidt xive_pre_save_scan(xive);
22205af50993SBenjamin Herrenschmidt xive->saved_src_count++;
22215af50993SBenjamin Herrenschmidt
22225af50993SBenjamin Herrenschmidt /* Convert saved state into something compatible with xics */
22232fb1e946SSam Bobroff val = state->act_server;
22245af50993SBenjamin Herrenschmidt prio = state->saved_scan_prio;
22255af50993SBenjamin Herrenschmidt
22265af50993SBenjamin Herrenschmidt if (prio == MASKED) {
22275af50993SBenjamin Herrenschmidt val |= KVM_XICS_MASKED;
22285af50993SBenjamin Herrenschmidt prio = state->saved_priority;
22295af50993SBenjamin Herrenschmidt }
22305af50993SBenjamin Herrenschmidt val |= prio << KVM_XICS_PRIORITY_SHIFT;
22315af50993SBenjamin Herrenschmidt if (state->lsi) {
22325af50993SBenjamin Herrenschmidt val |= KVM_XICS_LEVEL_SENSITIVE;
22335af50993SBenjamin Herrenschmidt if (state->saved_p)
22345af50993SBenjamin Herrenschmidt val |= KVM_XICS_PENDING;
22355af50993SBenjamin Herrenschmidt } else {
22365af50993SBenjamin Herrenschmidt if (state->saved_p)
22375af50993SBenjamin Herrenschmidt val |= KVM_XICS_PRESENTED;
22385af50993SBenjamin Herrenschmidt
22395af50993SBenjamin Herrenschmidt if (state->saved_q)
22405af50993SBenjamin Herrenschmidt val |= KVM_XICS_QUEUED;
22415af50993SBenjamin Herrenschmidt
22425af50993SBenjamin Herrenschmidt /*
22435af50993SBenjamin Herrenschmidt * We mark it pending (which will attempt a re-delivery)
22445af50993SBenjamin Herrenschmidt * if we are in a queue *or* we were masked and had
22455af50993SBenjamin Herrenschmidt * Q set which is equivalent to the XICS "masked pending"
22465af50993SBenjamin Herrenschmidt * state
22475af50993SBenjamin Herrenschmidt */
22485af50993SBenjamin Herrenschmidt if (state->in_queue || (prio == MASKED && state->saved_q))
22495af50993SBenjamin Herrenschmidt val |= KVM_XICS_PENDING;
22505af50993SBenjamin Herrenschmidt }
22515af50993SBenjamin Herrenschmidt
22525af50993SBenjamin Herrenschmidt /*
22535af50993SBenjamin Herrenschmidt * If that was the last interrupt saved, reset the
22545af50993SBenjamin Herrenschmidt * in_queue flags
22555af50993SBenjamin Herrenschmidt */
22565af50993SBenjamin Herrenschmidt if (xive->saved_src_count == xive->src_count)
22575af50993SBenjamin Herrenschmidt xive_post_save_scan(xive);
22585af50993SBenjamin Herrenschmidt
22595af50993SBenjamin Herrenschmidt /* Copy the result to userspace */
22605af50993SBenjamin Herrenschmidt if (put_user(val, ubufp))
22615af50993SBenjamin Herrenschmidt return -EFAULT;
22625af50993SBenjamin Herrenschmidt
22635af50993SBenjamin Herrenschmidt return 0;
22645af50993SBenjamin Herrenschmidt }
22655af50993SBenjamin Herrenschmidt
kvmppc_xive_create_src_block(struct kvmppc_xive * xive,int irq)22664131f83cSCédric Le Goater struct kvmppc_xive_src_block *kvmppc_xive_create_src_block(
22674131f83cSCédric Le Goater struct kvmppc_xive *xive, int irq)
22685af50993SBenjamin Herrenschmidt {
22695af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb;
22705af50993SBenjamin Herrenschmidt int i, bid;
22715af50993SBenjamin Herrenschmidt
22725af50993SBenjamin Herrenschmidt bid = irq >> KVMPPC_XICS_ICS_SHIFT;
22735af50993SBenjamin Herrenschmidt
22747e10b9a6SCédric Le Goater mutex_lock(&xive->lock);
22755af50993SBenjamin Herrenschmidt
22765af50993SBenjamin Herrenschmidt /* block already exists - somebody else got here first */
22775af50993SBenjamin Herrenschmidt if (xive->src_blocks[bid])
22785af50993SBenjamin Herrenschmidt goto out;
22795af50993SBenjamin Herrenschmidt
22805af50993SBenjamin Herrenschmidt /* Create the ICS */
22815af50993SBenjamin Herrenschmidt sb = kzalloc(sizeof(*sb), GFP_KERNEL);
22825af50993SBenjamin Herrenschmidt if (!sb)
22835af50993SBenjamin Herrenschmidt goto out;
22845af50993SBenjamin Herrenschmidt
22855af50993SBenjamin Herrenschmidt sb->id = bid;
22865af50993SBenjamin Herrenschmidt
22875af50993SBenjamin Herrenschmidt for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
22885af50993SBenjamin Herrenschmidt sb->irq_state[i].number = (bid << KVMPPC_XICS_ICS_SHIFT) | i;
2289e8676ce5SCédric Le Goater sb->irq_state[i].eisn = 0;
22905af50993SBenjamin Herrenschmidt sb->irq_state[i].guest_priority = MASKED;
22915af50993SBenjamin Herrenschmidt sb->irq_state[i].saved_priority = MASKED;
22925af50993SBenjamin Herrenschmidt sb->irq_state[i].act_priority = MASKED;
22935af50993SBenjamin Herrenschmidt }
22945af50993SBenjamin Herrenschmidt smp_wmb();
22955af50993SBenjamin Herrenschmidt xive->src_blocks[bid] = sb;
22965af50993SBenjamin Herrenschmidt
22975af50993SBenjamin Herrenschmidt if (bid > xive->max_sbid)
22985af50993SBenjamin Herrenschmidt xive->max_sbid = bid;
22995af50993SBenjamin Herrenschmidt
23005af50993SBenjamin Herrenschmidt out:
23017e10b9a6SCédric Le Goater mutex_unlock(&xive->lock);
23025af50993SBenjamin Herrenschmidt return xive->src_blocks[bid];
23035af50993SBenjamin Herrenschmidt }
23045af50993SBenjamin Herrenschmidt
xive_check_delayed_irq(struct kvmppc_xive * xive,u32 irq)23055af50993SBenjamin Herrenschmidt static bool xive_check_delayed_irq(struct kvmppc_xive *xive, u32 irq)
23065af50993SBenjamin Herrenschmidt {
23075af50993SBenjamin Herrenschmidt struct kvm *kvm = xive->kvm;
23085af50993SBenjamin Herrenschmidt struct kvm_vcpu *vcpu = NULL;
230946808a4cSMarc Zyngier unsigned long i;
23105af50993SBenjamin Herrenschmidt
23115af50993SBenjamin Herrenschmidt kvm_for_each_vcpu(i, vcpu, kvm) {
23125af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
23135af50993SBenjamin Herrenschmidt
23145af50993SBenjamin Herrenschmidt if (!xc)
23155af50993SBenjamin Herrenschmidt continue;
23165af50993SBenjamin Herrenschmidt
23175af50993SBenjamin Herrenschmidt if (xc->delayed_irq == irq) {
23185af50993SBenjamin Herrenschmidt xc->delayed_irq = 0;
23195af50993SBenjamin Herrenschmidt xive->delayed_irqs--;
23205af50993SBenjamin Herrenschmidt return true;
23215af50993SBenjamin Herrenschmidt }
23225af50993SBenjamin Herrenschmidt }
23235af50993SBenjamin Herrenschmidt return false;
23245af50993SBenjamin Herrenschmidt }
23255af50993SBenjamin Herrenschmidt
xive_set_source(struct kvmppc_xive * xive,long irq,u64 addr)23265af50993SBenjamin Herrenschmidt static int xive_set_source(struct kvmppc_xive *xive, long irq, u64 addr)
23275af50993SBenjamin Herrenschmidt {
23285af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb;
23295af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state;
23305af50993SBenjamin Herrenschmidt u64 __user *ubufp = (u64 __user *) addr;
23315af50993SBenjamin Herrenschmidt u16 idx;
23325af50993SBenjamin Herrenschmidt u64 val;
23335af50993SBenjamin Herrenschmidt u8 act_prio, guest_prio;
23345af50993SBenjamin Herrenschmidt u32 server;
23355af50993SBenjamin Herrenschmidt int rc = 0;
23365af50993SBenjamin Herrenschmidt
23375af50993SBenjamin Herrenschmidt if (irq < KVMPPC_XICS_FIRST_IRQ || irq >= KVMPPC_XICS_NR_IRQS)
23385af50993SBenjamin Herrenschmidt return -ENOENT;
23395af50993SBenjamin Herrenschmidt
23405af50993SBenjamin Herrenschmidt pr_devel("set_source(irq=0x%lx)\n", irq);
23415af50993SBenjamin Herrenschmidt
23425af50993SBenjamin Herrenschmidt /* Find the source */
23435af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, irq, &idx);
23445af50993SBenjamin Herrenschmidt if (!sb) {
23455af50993SBenjamin Herrenschmidt pr_devel("No source, creating source block...\n");
23464131f83cSCédric Le Goater sb = kvmppc_xive_create_src_block(xive, irq);
23475af50993SBenjamin Herrenschmidt if (!sb) {
23485af50993SBenjamin Herrenschmidt pr_devel("Failed to create block...\n");
23495af50993SBenjamin Herrenschmidt return -ENOMEM;
23505af50993SBenjamin Herrenschmidt }
23515af50993SBenjamin Herrenschmidt }
23525af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx];
23535af50993SBenjamin Herrenschmidt
23545af50993SBenjamin Herrenschmidt /* Read user passed data */
23555af50993SBenjamin Herrenschmidt if (get_user(val, ubufp)) {
23565af50993SBenjamin Herrenschmidt pr_devel("fault getting user info !\n");
23575af50993SBenjamin Herrenschmidt return -EFAULT;
23585af50993SBenjamin Herrenschmidt }
23595af50993SBenjamin Herrenschmidt
23605af50993SBenjamin Herrenschmidt server = val & KVM_XICS_DESTINATION_MASK;
23615af50993SBenjamin Herrenschmidt guest_prio = val >> KVM_XICS_PRIORITY_SHIFT;
23625af50993SBenjamin Herrenschmidt
23635af50993SBenjamin Herrenschmidt pr_devel(" val=0x016%llx (server=0x%x, guest_prio=%d)\n",
23645af50993SBenjamin Herrenschmidt val, server, guest_prio);
2365bf4159daSBenjamin Herrenschmidt
23665af50993SBenjamin Herrenschmidt /*
23675af50993SBenjamin Herrenschmidt * If the source doesn't already have an IPI, allocate
23685af50993SBenjamin Herrenschmidt * one and get the corresponding data
23695af50993SBenjamin Herrenschmidt */
23705af50993SBenjamin Herrenschmidt if (!state->ipi_number) {
23715af50993SBenjamin Herrenschmidt state->ipi_number = xive_native_alloc_irq();
23725af50993SBenjamin Herrenschmidt if (state->ipi_number == 0) {
23735af50993SBenjamin Herrenschmidt pr_devel("Failed to allocate IPI !\n");
23745af50993SBenjamin Herrenschmidt return -ENOMEM;
23755af50993SBenjamin Herrenschmidt }
23765af50993SBenjamin Herrenschmidt xive_native_populate_irq_data(state->ipi_number, &state->ipi_data);
23775af50993SBenjamin Herrenschmidt pr_devel(" src_ipi=0x%x\n", state->ipi_number);
23785af50993SBenjamin Herrenschmidt }
23795af50993SBenjamin Herrenschmidt
23805af50993SBenjamin Herrenschmidt /*
23815af50993SBenjamin Herrenschmidt * We use lock_and_mask() to set us in the right masked
23825af50993SBenjamin Herrenschmidt * state. We will override that state from the saved state
23835af50993SBenjamin Herrenschmidt * further down, but this will handle the cases of interrupts
23845af50993SBenjamin Herrenschmidt * that need FW masking. We set the initial guest_priority to
23855af50993SBenjamin Herrenschmidt * 0 before calling it to ensure it actually performs the masking.
23865af50993SBenjamin Herrenschmidt */
23875af50993SBenjamin Herrenschmidt state->guest_priority = 0;
23885af50993SBenjamin Herrenschmidt xive_lock_and_mask(xive, sb, state);
23895af50993SBenjamin Herrenschmidt
23905af50993SBenjamin Herrenschmidt /*
23915af50993SBenjamin Herrenschmidt * Now, we select a target if we have one. If we don't we
23925af50993SBenjamin Herrenschmidt * leave the interrupt untargetted. It means that an interrupt
23935af50993SBenjamin Herrenschmidt * can become "untargetted" across migration if it was masked
23945af50993SBenjamin Herrenschmidt * by set_xive() but there is little we can do about it.
23955af50993SBenjamin Herrenschmidt */
23965af50993SBenjamin Herrenschmidt
23975af50993SBenjamin Herrenschmidt /* First convert prio and mark interrupt as untargetted */
23985af50993SBenjamin Herrenschmidt act_prio = xive_prio_from_guest(guest_prio);
23995af50993SBenjamin Herrenschmidt state->act_priority = MASKED;
24005af50993SBenjamin Herrenschmidt
24015af50993SBenjamin Herrenschmidt /*
24025af50993SBenjamin Herrenschmidt * We need to drop the lock due to the mutex below. Hopefully
24035af50993SBenjamin Herrenschmidt * nothing is touching that interrupt yet since it hasn't been
24045af50993SBenjamin Herrenschmidt * advertized to a running guest yet
24055af50993SBenjamin Herrenschmidt */
24065af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock);
24075af50993SBenjamin Herrenschmidt
24085af50993SBenjamin Herrenschmidt /* If we have a priority target the interrupt */
24095af50993SBenjamin Herrenschmidt if (act_prio != MASKED) {
24105af50993SBenjamin Herrenschmidt /* First, check provisioning of queues */
24117e10b9a6SCédric Le Goater mutex_lock(&xive->lock);
24125af50993SBenjamin Herrenschmidt rc = xive_check_provisioning(xive->kvm, act_prio);
24137e10b9a6SCédric Le Goater mutex_unlock(&xive->lock);
24145af50993SBenjamin Herrenschmidt
24155af50993SBenjamin Herrenschmidt /* Target interrupt */
24165af50993SBenjamin Herrenschmidt if (rc == 0)
24175af50993SBenjamin Herrenschmidt rc = xive_target_interrupt(xive->kvm, state,
24185af50993SBenjamin Herrenschmidt server, act_prio);
24195af50993SBenjamin Herrenschmidt /*
24205af50993SBenjamin Herrenschmidt * If provisioning or targetting failed, leave it
24215af50993SBenjamin Herrenschmidt * alone and masked. It will remain disabled until
24225af50993SBenjamin Herrenschmidt * the guest re-targets it.
24235af50993SBenjamin Herrenschmidt */
24245af50993SBenjamin Herrenschmidt }
24255af50993SBenjamin Herrenschmidt
24265af50993SBenjamin Herrenschmidt /*
24275af50993SBenjamin Herrenschmidt * Find out if this was a delayed irq stashed in an ICP,
24285af50993SBenjamin Herrenschmidt * in which case, treat it as pending
24295af50993SBenjamin Herrenschmidt */
24305af50993SBenjamin Herrenschmidt if (xive->delayed_irqs && xive_check_delayed_irq(xive, irq)) {
24315af50993SBenjamin Herrenschmidt val |= KVM_XICS_PENDING;
24325af50993SBenjamin Herrenschmidt pr_devel(" Found delayed ! forcing PENDING !\n");
24335af50993SBenjamin Herrenschmidt }
24345af50993SBenjamin Herrenschmidt
24355af50993SBenjamin Herrenschmidt /* Cleanup the SW state */
24365af50993SBenjamin Herrenschmidt state->old_p = false;
24375af50993SBenjamin Herrenschmidt state->old_q = false;
24385af50993SBenjamin Herrenschmidt state->lsi = false;
24395af50993SBenjamin Herrenschmidt state->asserted = false;
24405af50993SBenjamin Herrenschmidt
24415af50993SBenjamin Herrenschmidt /* Restore LSI state */
24425af50993SBenjamin Herrenschmidt if (val & KVM_XICS_LEVEL_SENSITIVE) {
24435af50993SBenjamin Herrenschmidt state->lsi = true;
24445af50993SBenjamin Herrenschmidt if (val & KVM_XICS_PENDING)
24455af50993SBenjamin Herrenschmidt state->asserted = true;
24465af50993SBenjamin Herrenschmidt pr_devel(" LSI ! Asserted=%d\n", state->asserted);
24475af50993SBenjamin Herrenschmidt }
24485af50993SBenjamin Herrenschmidt
24495af50993SBenjamin Herrenschmidt /*
24505af50993SBenjamin Herrenschmidt * Restore P and Q. If the interrupt was pending, we
2451dc1c4165SCédric Le Goater * force Q and !P, which will trigger a resend.
24525af50993SBenjamin Herrenschmidt *
24535af50993SBenjamin Herrenschmidt * That means that a guest that had both an interrupt
24545af50993SBenjamin Herrenschmidt * pending (queued) and Q set will restore with only
24555af50993SBenjamin Herrenschmidt * one instance of that interrupt instead of 2, but that
24565af50993SBenjamin Herrenschmidt * is perfectly fine as coalescing interrupts that haven't
24575af50993SBenjamin Herrenschmidt * been presented yet is always allowed.
24585af50993SBenjamin Herrenschmidt */
2459dc1c4165SCédric Le Goater if (val & KVM_XICS_PRESENTED && !(val & KVM_XICS_PENDING))
24605af50993SBenjamin Herrenschmidt state->old_p = true;
24615af50993SBenjamin Herrenschmidt if (val & KVM_XICS_QUEUED || val & KVM_XICS_PENDING)
24625af50993SBenjamin Herrenschmidt state->old_q = true;
24635af50993SBenjamin Herrenschmidt
24645af50993SBenjamin Herrenschmidt pr_devel(" P=%d, Q=%d\n", state->old_p, state->old_q);
24655af50993SBenjamin Herrenschmidt
24665af50993SBenjamin Herrenschmidt /*
24675af50993SBenjamin Herrenschmidt * If the interrupt was unmasked, update guest priority and
24685af50993SBenjamin Herrenschmidt * perform the appropriate state transition and do a
24695af50993SBenjamin Herrenschmidt * re-trigger if necessary.
24705af50993SBenjamin Herrenschmidt */
24715af50993SBenjamin Herrenschmidt if (val & KVM_XICS_MASKED) {
24725af50993SBenjamin Herrenschmidt pr_devel(" masked, saving prio\n");
24735af50993SBenjamin Herrenschmidt state->guest_priority = MASKED;
24745af50993SBenjamin Herrenschmidt state->saved_priority = guest_prio;
24755af50993SBenjamin Herrenschmidt } else {
24765af50993SBenjamin Herrenschmidt pr_devel(" unmasked, restoring to prio %d\n", guest_prio);
24775af50993SBenjamin Herrenschmidt xive_finish_unmask(xive, sb, state, guest_prio);
24785af50993SBenjamin Herrenschmidt state->saved_priority = guest_prio;
24795af50993SBenjamin Herrenschmidt }
24805af50993SBenjamin Herrenschmidt
24815af50993SBenjamin Herrenschmidt /* Increment the number of valid sources and mark this one valid */
24825af50993SBenjamin Herrenschmidt if (!state->valid)
24835af50993SBenjamin Herrenschmidt xive->src_count++;
24845af50993SBenjamin Herrenschmidt state->valid = true;
24855af50993SBenjamin Herrenschmidt
24865af50993SBenjamin Herrenschmidt return 0;
24875af50993SBenjamin Herrenschmidt }
24885af50993SBenjamin Herrenschmidt
kvmppc_xive_set_irq(struct kvm * kvm,int irq_source_id,u32 irq,int level,bool line_status)24895af50993SBenjamin Herrenschmidt int kvmppc_xive_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
24905af50993SBenjamin Herrenschmidt bool line_status)
24915af50993SBenjamin Herrenschmidt {
24925af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = kvm->arch.xive;
24935af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb;
24945af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state;
24955af50993SBenjamin Herrenschmidt u16 idx;
24965af50993SBenjamin Herrenschmidt
24975af50993SBenjamin Herrenschmidt if (!xive)
24985af50993SBenjamin Herrenschmidt return -ENODEV;
24995af50993SBenjamin Herrenschmidt
25005af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, irq, &idx);
25015af50993SBenjamin Herrenschmidt if (!sb)
25025af50993SBenjamin Herrenschmidt return -EINVAL;
25035af50993SBenjamin Herrenschmidt
25045af50993SBenjamin Herrenschmidt /* Perform locklessly .... (we need to do some RCUisms here...) */
25055af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx];
25065af50993SBenjamin Herrenschmidt if (!state->valid)
25075af50993SBenjamin Herrenschmidt return -EINVAL;
25085af50993SBenjamin Herrenschmidt
25095af50993SBenjamin Herrenschmidt /* We don't allow a trigger on a passed-through interrupt */
25105af50993SBenjamin Herrenschmidt if (state->pt_number)
25115af50993SBenjamin Herrenschmidt return -EINVAL;
25125af50993SBenjamin Herrenschmidt
25135af50993SBenjamin Herrenschmidt if ((level == 1 && state->lsi) || level == KVM_INTERRUPT_SET_LEVEL)
2514c9df3f80SJiapeng Chong state->asserted = true;
25155af50993SBenjamin Herrenschmidt else if (level == 0 || level == KVM_INTERRUPT_UNSET) {
2516c9df3f80SJiapeng Chong state->asserted = false;
25175af50993SBenjamin Herrenschmidt return 0;
25185af50993SBenjamin Herrenschmidt }
25195af50993SBenjamin Herrenschmidt
25205af50993SBenjamin Herrenschmidt /* Trigger the IPI */
25215af50993SBenjamin Herrenschmidt xive_irq_trigger(&state->ipi_data);
25225af50993SBenjamin Herrenschmidt
25235af50993SBenjamin Herrenschmidt return 0;
25245af50993SBenjamin Herrenschmidt }
25255af50993SBenjamin Herrenschmidt
kvmppc_xive_set_nr_servers(struct kvmppc_xive * xive,u64 addr)2526062cfab7SGreg Kurz int kvmppc_xive_set_nr_servers(struct kvmppc_xive *xive, u64 addr)
2527062cfab7SGreg Kurz {
2528062cfab7SGreg Kurz u32 __user *ubufp = (u32 __user *) addr;
2529062cfab7SGreg Kurz u32 nr_servers;
2530062cfab7SGreg Kurz int rc = 0;
2531062cfab7SGreg Kurz
2532062cfab7SGreg Kurz if (get_user(nr_servers, ubufp))
2533062cfab7SGreg Kurz return -EFAULT;
2534062cfab7SGreg Kurz
2535062cfab7SGreg Kurz pr_devel("%s nr_servers=%u\n", __func__, nr_servers);
2536062cfab7SGreg Kurz
2537a1c42ddeSJuergen Gross if (!nr_servers || nr_servers > KVM_MAX_VCPU_IDS)
2538062cfab7SGreg Kurz return -EINVAL;
2539062cfab7SGreg Kurz
2540062cfab7SGreg Kurz mutex_lock(&xive->lock);
2541062cfab7SGreg Kurz if (xive->vp_base != XIVE_INVALID_VP)
2542062cfab7SGreg Kurz /* The VP block is allocated once and freed when the device
2543062cfab7SGreg Kurz * is released. Better not allow to change its size since its
2544062cfab7SGreg Kurz * used by connect_vcpu to validate vCPU ids are valid (eg,
2545062cfab7SGreg Kurz * setting it back to a higher value could allow connect_vcpu
2546062cfab7SGreg Kurz * to come up with a VP id that goes beyond the VP block, which
2547062cfab7SGreg Kurz * is likely to cause a crash in OPAL).
2548062cfab7SGreg Kurz */
2549062cfab7SGreg Kurz rc = -EBUSY;
2550062cfab7SGreg Kurz else if (nr_servers > KVM_MAX_VCPUS)
2551062cfab7SGreg Kurz /* We don't need more servers. Higher vCPU ids get packed
2552062cfab7SGreg Kurz * down below KVM_MAX_VCPUS by kvmppc_pack_vcpu_id().
2553062cfab7SGreg Kurz */
2554062cfab7SGreg Kurz xive->nr_servers = KVM_MAX_VCPUS;
2555062cfab7SGreg Kurz else
2556062cfab7SGreg Kurz xive->nr_servers = nr_servers;
2557062cfab7SGreg Kurz
2558062cfab7SGreg Kurz mutex_unlock(&xive->lock);
2559062cfab7SGreg Kurz
2560062cfab7SGreg Kurz return rc;
2561062cfab7SGreg Kurz }
2562062cfab7SGreg Kurz
xive_set_attr(struct kvm_device * dev,struct kvm_device_attr * attr)25635af50993SBenjamin Herrenschmidt static int xive_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
25645af50993SBenjamin Herrenschmidt {
25655af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = dev->private;
25665af50993SBenjamin Herrenschmidt
25675af50993SBenjamin Herrenschmidt /* We honor the existing XICS ioctl */
25685af50993SBenjamin Herrenschmidt switch (attr->group) {
25695af50993SBenjamin Herrenschmidt case KVM_DEV_XICS_GRP_SOURCES:
25705af50993SBenjamin Herrenschmidt return xive_set_source(xive, attr->attr, attr->addr);
2571efe5ddcaSGreg Kurz case KVM_DEV_XICS_GRP_CTRL:
2572efe5ddcaSGreg Kurz switch (attr->attr) {
2573efe5ddcaSGreg Kurz case KVM_DEV_XICS_NR_SERVERS:
2574efe5ddcaSGreg Kurz return kvmppc_xive_set_nr_servers(xive, attr->addr);
2575efe5ddcaSGreg Kurz }
25765af50993SBenjamin Herrenschmidt }
25775af50993SBenjamin Herrenschmidt return -ENXIO;
25785af50993SBenjamin Herrenschmidt }
25795af50993SBenjamin Herrenschmidt
xive_get_attr(struct kvm_device * dev,struct kvm_device_attr * attr)25805af50993SBenjamin Herrenschmidt static int xive_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
25815af50993SBenjamin Herrenschmidt {
25825af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = dev->private;
25835af50993SBenjamin Herrenschmidt
25845af50993SBenjamin Herrenschmidt /* We honor the existing XICS ioctl */
25855af50993SBenjamin Herrenschmidt switch (attr->group) {
25865af50993SBenjamin Herrenschmidt case KVM_DEV_XICS_GRP_SOURCES:
25875af50993SBenjamin Herrenschmidt return xive_get_source(xive, attr->attr, attr->addr);
25885af50993SBenjamin Herrenschmidt }
25895af50993SBenjamin Herrenschmidt return -ENXIO;
25905af50993SBenjamin Herrenschmidt }
25915af50993SBenjamin Herrenschmidt
xive_has_attr(struct kvm_device * dev,struct kvm_device_attr * attr)25925af50993SBenjamin Herrenschmidt static int xive_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
25935af50993SBenjamin Herrenschmidt {
25945af50993SBenjamin Herrenschmidt /* We honor the same limits as XICS, at least for now */
25955af50993SBenjamin Herrenschmidt switch (attr->group) {
25965af50993SBenjamin Herrenschmidt case KVM_DEV_XICS_GRP_SOURCES:
25975af50993SBenjamin Herrenschmidt if (attr->attr >= KVMPPC_XICS_FIRST_IRQ &&
25985af50993SBenjamin Herrenschmidt attr->attr < KVMPPC_XICS_NR_IRQS)
25995af50993SBenjamin Herrenschmidt return 0;
26005af50993SBenjamin Herrenschmidt break;
2601efe5ddcaSGreg Kurz case KVM_DEV_XICS_GRP_CTRL:
2602efe5ddcaSGreg Kurz switch (attr->attr) {
2603efe5ddcaSGreg Kurz case KVM_DEV_XICS_NR_SERVERS:
2604efe5ddcaSGreg Kurz return 0;
2605efe5ddcaSGreg Kurz }
26065af50993SBenjamin Herrenschmidt }
26075af50993SBenjamin Herrenschmidt return -ENXIO;
26085af50993SBenjamin Herrenschmidt }
26095af50993SBenjamin Herrenschmidt
kvmppc_xive_cleanup_irq(u32 hw_num,struct xive_irq_data * xd)26105af50993SBenjamin Herrenschmidt static void kvmppc_xive_cleanup_irq(u32 hw_num, struct xive_irq_data *xd)
26115af50993SBenjamin Herrenschmidt {
26125af50993SBenjamin Herrenschmidt xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01);
26135af50993SBenjamin Herrenschmidt xive_native_configure_irq(hw_num, 0, MASKED, 0);
26145af50993SBenjamin Herrenschmidt }
26155af50993SBenjamin Herrenschmidt
kvmppc_xive_free_sources(struct kvmppc_xive_src_block * sb)26164131f83cSCédric Le Goater void kvmppc_xive_free_sources(struct kvmppc_xive_src_block *sb)
26175af50993SBenjamin Herrenschmidt {
26185af50993SBenjamin Herrenschmidt int i;
26195af50993SBenjamin Herrenschmidt
26205af50993SBenjamin Herrenschmidt for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
26215af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state = &sb->irq_state[i];
26225af50993SBenjamin Herrenschmidt
26235af50993SBenjamin Herrenschmidt if (!state->valid)
26245af50993SBenjamin Herrenschmidt continue;
26255af50993SBenjamin Herrenschmidt
26265af50993SBenjamin Herrenschmidt kvmppc_xive_cleanup_irq(state->ipi_number, &state->ipi_data);
2627ef974020SCédric Le Goater xive_cleanup_irq_data(&state->ipi_data);
26285af50993SBenjamin Herrenschmidt xive_native_free_irq(state->ipi_number);
26295af50993SBenjamin Herrenschmidt
2630ef974020SCédric Le Goater /* Pass-through, cleanup too but keep IRQ hw data */
26315af50993SBenjamin Herrenschmidt if (state->pt_number)
26325af50993SBenjamin Herrenschmidt kvmppc_xive_cleanup_irq(state->pt_number, state->pt_data);
26335af50993SBenjamin Herrenschmidt
26345af50993SBenjamin Herrenschmidt state->valid = false;
26355af50993SBenjamin Herrenschmidt }
26365af50993SBenjamin Herrenschmidt }
26375af50993SBenjamin Herrenschmidt
26385422e951SCédric Le Goater /*
26396f868405SPaul Mackerras * Called when device fd is closed. kvm->lock is held.
26405422e951SCédric Le Goater */
kvmppc_xive_release(struct kvm_device * dev)26415422e951SCédric Le Goater static void kvmppc_xive_release(struct kvm_device *dev)
26425af50993SBenjamin Herrenschmidt {
26435af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = dev->private;
26445af50993SBenjamin Herrenschmidt struct kvm *kvm = xive->kvm;
26455422e951SCédric Le Goater struct kvm_vcpu *vcpu;
264646808a4cSMarc Zyngier unsigned long i;
26475af50993SBenjamin Herrenschmidt
26485422e951SCédric Le Goater pr_devel("Releasing xive device\n");
26495422e951SCédric Le Goater
26506f868405SPaul Mackerras /*
26516f868405SPaul Mackerras * Since this is the device release function, we know that
26526f868405SPaul Mackerras * userspace does not have any open fd referring to the
26536f868405SPaul Mackerras * device. Therefore there can not be any of the device
26546f868405SPaul Mackerras * attribute set/get functions being executed concurrently,
26556f868405SPaul Mackerras * and similarly, the connect_vcpu and set/clr_mapped
26566f868405SPaul Mackerras * functions also cannot be being executed.
26576f868405SPaul Mackerras */
2658c395fe1dSPaul Mackerras
2659c395fe1dSPaul Mackerras debugfs_remove(xive->dentry);
26606f868405SPaul Mackerras
26616f868405SPaul Mackerras /*
26626f868405SPaul Mackerras * We should clean up the vCPU interrupt presenters first.
26636f868405SPaul Mackerras */
26646f868405SPaul Mackerras kvm_for_each_vcpu(i, vcpu, kvm) {
26656f868405SPaul Mackerras /*
26666f868405SPaul Mackerras * Take vcpu->mutex to ensure that no one_reg get/set ioctl
26676f868405SPaul Mackerras * (i.e. kvmppc_xive_[gs]et_icp) can be done concurrently.
2668c395fe1dSPaul Mackerras * Holding the vcpu->mutex also means that the vcpu cannot
2669c395fe1dSPaul Mackerras * be executing the KVM_RUN ioctl, and therefore it cannot
2670c395fe1dSPaul Mackerras * be executing the XIVE push or pull code or accessing
2671c395fe1dSPaul Mackerras * the XIVE MMIO regions.
26726f868405SPaul Mackerras */
26736f868405SPaul Mackerras mutex_lock(&vcpu->mutex);
26746f868405SPaul Mackerras kvmppc_xive_cleanup_vcpu(vcpu);
26756f868405SPaul Mackerras mutex_unlock(&vcpu->mutex);
26766f868405SPaul Mackerras }
26776f868405SPaul Mackerras
2678c395fe1dSPaul Mackerras /*
2679c395fe1dSPaul Mackerras * Now that we have cleared vcpu->arch.xive_vcpu, vcpu->arch.irq_type
2680c395fe1dSPaul Mackerras * and vcpu->arch.xive_esc_[vr]addr on each vcpu, we are safe
2681c395fe1dSPaul Mackerras * against xive code getting called during vcpu execution or
2682c395fe1dSPaul Mackerras * set/get one_reg operations.
2683c395fe1dSPaul Mackerras */
26845af50993SBenjamin Herrenschmidt kvm->arch.xive = NULL;
26855af50993SBenjamin Herrenschmidt
26865af50993SBenjamin Herrenschmidt /* Mask and free interrupts */
26875af50993SBenjamin Herrenschmidt for (i = 0; i <= xive->max_sbid; i++) {
26885af50993SBenjamin Herrenschmidt if (xive->src_blocks[i])
26895af50993SBenjamin Herrenschmidt kvmppc_xive_free_sources(xive->src_blocks[i]);
26905af50993SBenjamin Herrenschmidt kfree(xive->src_blocks[i]);
26915af50993SBenjamin Herrenschmidt xive->src_blocks[i] = NULL;
26925af50993SBenjamin Herrenschmidt }
26935af50993SBenjamin Herrenschmidt
26945af50993SBenjamin Herrenschmidt if (xive->vp_base != XIVE_INVALID_VP)
26955af50993SBenjamin Herrenschmidt xive_native_free_vp_block(xive->vp_base);
26965af50993SBenjamin Herrenschmidt
26975422e951SCédric Le Goater /*
26985422e951SCédric Le Goater * A reference of the kvmppc_xive pointer is now kept under
26995422e951SCédric Le Goater * the xive_devices struct of the machine for reuse. It is
27005422e951SCédric Le Goater * freed when the VM is destroyed for now until we fix all the
27015422e951SCédric Le Goater * execution paths.
27025422e951SCédric Le Goater */
27035af50993SBenjamin Herrenschmidt
27045af50993SBenjamin Herrenschmidt kfree(dev);
27055af50993SBenjamin Herrenschmidt }
27065af50993SBenjamin Herrenschmidt
27075422e951SCédric Le Goater /*
27085422e951SCédric Le Goater * When the guest chooses the interrupt mode (XICS legacy or XIVE
27095422e951SCédric Le Goater * native), the VM will switch of KVM device. The previous device will
27105422e951SCédric Le Goater * be "released" before the new one is created.
27115422e951SCédric Le Goater *
27125422e951SCédric Le Goater * Until we are sure all execution paths are well protected, provide a
27135422e951SCédric Le Goater * fail safe (transitional) method for device destruction, in which
27145422e951SCédric Le Goater * the XIVE device pointer is recycled and not directly freed.
27155422e951SCédric Le Goater */
kvmppc_xive_get_device(struct kvm * kvm,u32 type)27165422e951SCédric Le Goater struct kvmppc_xive *kvmppc_xive_get_device(struct kvm *kvm, u32 type)
27175422e951SCédric Le Goater {
27185422e951SCédric Le Goater struct kvmppc_xive **kvm_xive_device = type == KVM_DEV_TYPE_XIVE ?
27195422e951SCédric Le Goater &kvm->arch.xive_devices.native :
27205422e951SCédric Le Goater &kvm->arch.xive_devices.xics_on_xive;
27215422e951SCédric Le Goater struct kvmppc_xive *xive = *kvm_xive_device;
27225422e951SCédric Le Goater
27235422e951SCédric Le Goater if (!xive) {
27245422e951SCédric Le Goater xive = kzalloc(sizeof(*xive), GFP_KERNEL);
27255422e951SCédric Le Goater *kvm_xive_device = xive;
27265422e951SCédric Le Goater } else {
27275422e951SCédric Le Goater memset(xive, 0, sizeof(*xive));
27285422e951SCédric Le Goater }
27295422e951SCédric Le Goater
27305422e951SCédric Le Goater return xive;
27315422e951SCédric Le Goater }
27325422e951SCédric Le Goater
27336f868405SPaul Mackerras /*
27346f868405SPaul Mackerras * Create a XICS device with XIVE backend. kvm->lock is held.
27356f868405SPaul Mackerras */
kvmppc_xive_create(struct kvm_device * dev,u32 type)27365af50993SBenjamin Herrenschmidt static int kvmppc_xive_create(struct kvm_device *dev, u32 type)
27375af50993SBenjamin Herrenschmidt {
27385af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive;
27395af50993SBenjamin Herrenschmidt struct kvm *kvm = dev->kvm;
27405af50993SBenjamin Herrenschmidt
27415af50993SBenjamin Herrenschmidt pr_devel("Creating xive for partition\n");
27425af50993SBenjamin Herrenschmidt
2743e7d71c94SGreg Kurz /* Already there ? */
2744e7d71c94SGreg Kurz if (kvm->arch.xive)
2745e7d71c94SGreg Kurz return -EEXIST;
2746e7d71c94SGreg Kurz
27475422e951SCédric Le Goater xive = kvmppc_xive_get_device(kvm, type);
27485af50993SBenjamin Herrenschmidt if (!xive)
27495af50993SBenjamin Herrenschmidt return -ENOMEM;
27505af50993SBenjamin Herrenschmidt
27515af50993SBenjamin Herrenschmidt dev->private = xive;
27525af50993SBenjamin Herrenschmidt xive->dev = dev;
27535af50993SBenjamin Herrenschmidt xive->kvm = kvm;
27547e10b9a6SCédric Le Goater mutex_init(&xive->lock);
27555af50993SBenjamin Herrenschmidt
27565af50993SBenjamin Herrenschmidt /* We use the default queue size set by the host */
27575af50993SBenjamin Herrenschmidt xive->q_order = xive_native_default_eq_shift();
27585af50993SBenjamin Herrenschmidt if (xive->q_order < PAGE_SHIFT)
27595af50993SBenjamin Herrenschmidt xive->q_page_order = 0;
27605af50993SBenjamin Herrenschmidt else
27615af50993SBenjamin Herrenschmidt xive->q_page_order = xive->q_order - PAGE_SHIFT;
27625af50993SBenjamin Herrenschmidt
2763062cfab7SGreg Kurz /* VP allocation is delayed to the first call to connect_vcpu */
2764062cfab7SGreg Kurz xive->vp_base = XIVE_INVALID_VP;
2765062cfab7SGreg Kurz /* KVM_MAX_VCPUS limits the number of VMs to roughly 64 per sockets
2766062cfab7SGreg Kurz * on a POWER9 system.
2767062cfab7SGreg Kurz */
2768062cfab7SGreg Kurz xive->nr_servers = KVM_MAX_VCPUS;
27695af50993SBenjamin Herrenschmidt
2770b68c6646SCédric Le Goater if (xive_native_has_single_escalation())
2771b68c6646SCédric Le Goater xive->flags |= KVMPPC_XIVE_FLAG_SINGLE_ESCALATION;
2772bf4159daSBenjamin Herrenschmidt
2773f5af0a97SCédric Le Goater if (xive_native_has_save_restore())
2774f5af0a97SCédric Le Goater xive->flags |= KVMPPC_XIVE_FLAG_SAVE_RESTORE;
2775f5af0a97SCédric Le Goater
2776e7d71c94SGreg Kurz kvm->arch.xive = xive;
27775af50993SBenjamin Herrenschmidt return 0;
27785af50993SBenjamin Herrenschmidt }
27795af50993SBenjamin Herrenschmidt
kvmppc_xive_xics_hcall(struct kvm_vcpu * vcpu,u32 req)27809dc2babcSNicholas Piggin int kvmppc_xive_xics_hcall(struct kvm_vcpu *vcpu, u32 req)
27819dc2babcSNicholas Piggin {
27829dc2babcSNicholas Piggin struct kvmppc_vcore *vc = vcpu->arch.vcore;
27839dc2babcSNicholas Piggin
27849dc2babcSNicholas Piggin /* The VM should have configured XICS mode before doing XICS hcalls. */
27859dc2babcSNicholas Piggin if (!kvmppc_xics_enabled(vcpu))
27869dc2babcSNicholas Piggin return H_TOO_HARD;
27879dc2babcSNicholas Piggin
27889dc2babcSNicholas Piggin switch (req) {
27899dc2babcSNicholas Piggin case H_XIRR:
27909dc2babcSNicholas Piggin return xive_vm_h_xirr(vcpu);
27919dc2babcSNicholas Piggin case H_CPPR:
27929dc2babcSNicholas Piggin return xive_vm_h_cppr(vcpu, kvmppc_get_gpr(vcpu, 4));
27939dc2babcSNicholas Piggin case H_EOI:
27949dc2babcSNicholas Piggin return xive_vm_h_eoi(vcpu, kvmppc_get_gpr(vcpu, 4));
27959dc2babcSNicholas Piggin case H_IPI:
27969dc2babcSNicholas Piggin return xive_vm_h_ipi(vcpu, kvmppc_get_gpr(vcpu, 4),
27979dc2babcSNicholas Piggin kvmppc_get_gpr(vcpu, 5));
27989dc2babcSNicholas Piggin case H_IPOLL:
27999dc2babcSNicholas Piggin return xive_vm_h_ipoll(vcpu, kvmppc_get_gpr(vcpu, 4));
28009dc2babcSNicholas Piggin case H_XIRR_X:
28019dc2babcSNicholas Piggin xive_vm_h_xirr(vcpu);
28029dc2babcSNicholas Piggin kvmppc_set_gpr(vcpu, 5, get_tb() + vc->tb_offset);
28039dc2babcSNicholas Piggin return H_SUCCESS;
28049dc2babcSNicholas Piggin }
28059dc2babcSNicholas Piggin
28069dc2babcSNicholas Piggin return H_UNSUPPORTED;
28079dc2babcSNicholas Piggin }
28089dc2babcSNicholas Piggin EXPORT_SYMBOL_GPL(kvmppc_xive_xics_hcall);
28099dc2babcSNicholas Piggin
kvmppc_xive_debug_show_queues(struct seq_file * m,struct kvm_vcpu * vcpu)2810eacc56bbSCédric Le Goater int kvmppc_xive_debug_show_queues(struct seq_file *m, struct kvm_vcpu *vcpu)
2811eacc56bbSCédric Le Goater {
2812eacc56bbSCédric Le Goater struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
2813eacc56bbSCédric Le Goater unsigned int i;
2814eacc56bbSCédric Le Goater
2815eacc56bbSCédric Le Goater for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
2816eacc56bbSCédric Le Goater struct xive_q *q = &xc->queues[i];
2817eacc56bbSCédric Le Goater u32 i0, i1, idx;
2818eacc56bbSCédric Le Goater
2819eacc56bbSCédric Le Goater if (!q->qpage && !xc->esc_virq[i])
2820eacc56bbSCédric Le Goater continue;
2821eacc56bbSCédric Le Goater
2822eacc56bbSCédric Le Goater if (q->qpage) {
282398983675SCédric Le Goater seq_printf(m, " q[%d]: ", i);
2824eacc56bbSCédric Le Goater idx = q->idx;
2825eacc56bbSCédric Le Goater i0 = be32_to_cpup(q->qpage + idx);
2826eacc56bbSCédric Le Goater idx = (idx + 1) & q->msk;
2827eacc56bbSCédric Le Goater i1 = be32_to_cpup(q->qpage + idx);
2828eacc56bbSCédric Le Goater seq_printf(m, "T=%d %08x %08x...\n", q->toggle,
2829eacc56bbSCédric Le Goater i0, i1);
2830eacc56bbSCédric Le Goater }
2831eacc56bbSCédric Le Goater if (xc->esc_virq[i]) {
2832eacc56bbSCédric Le Goater struct irq_data *d = irq_get_irq_data(xc->esc_virq[i]);
2833eacc56bbSCédric Le Goater struct xive_irq_data *xd =
2834eacc56bbSCédric Le Goater irq_data_get_irq_handler_data(d);
2835eacc56bbSCédric Le Goater u64 pq = xive_vm_esb_load(xd, XIVE_ESB_GET);
2836eacc56bbSCédric Le Goater
283798983675SCédric Le Goater seq_printf(m, " ESC %d %c%c EOI @%llx",
283898983675SCédric Le Goater xc->esc_virq[i],
283998983675SCédric Le Goater (pq & XIVE_ESB_VAL_P) ? 'P' : '-',
284098983675SCédric Le Goater (pq & XIVE_ESB_VAL_Q) ? 'Q' : '-',
284198983675SCédric Le Goater xd->eoi_page);
2842eacc56bbSCédric Le Goater seq_puts(m, "\n");
2843eacc56bbSCédric Le Goater }
2844eacc56bbSCédric Le Goater }
2845eacc56bbSCédric Le Goater return 0;
2846eacc56bbSCédric Le Goater }
28475af50993SBenjamin Herrenschmidt
kvmppc_xive_debug_show_sources(struct seq_file * m,struct kvmppc_xive_src_block * sb)284898983675SCédric Le Goater void kvmppc_xive_debug_show_sources(struct seq_file *m,
284998983675SCédric Le Goater struct kvmppc_xive_src_block *sb)
285098983675SCédric Le Goater {
285198983675SCédric Le Goater int i;
285298983675SCédric Le Goater
285398983675SCédric Le Goater seq_puts(m, " LISN HW/CHIP TYPE PQ EISN CPU/PRIO\n");
285498983675SCédric Le Goater for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
285598983675SCédric Le Goater struct kvmppc_xive_irq_state *state = &sb->irq_state[i];
285698983675SCédric Le Goater struct xive_irq_data *xd;
285798983675SCédric Le Goater u64 pq;
285898983675SCédric Le Goater u32 hw_num;
285998983675SCédric Le Goater
286098983675SCédric Le Goater if (!state->valid)
286198983675SCédric Le Goater continue;
286298983675SCédric Le Goater
286398983675SCédric Le Goater kvmppc_xive_select_irq(state, &hw_num, &xd);
286498983675SCédric Le Goater
286598983675SCédric Le Goater pq = xive_vm_esb_load(xd, XIVE_ESB_GET);
286698983675SCédric Le Goater
286798983675SCédric Le Goater seq_printf(m, "%08x %08x/%02x", state->number, hw_num,
286898983675SCédric Le Goater xd->src_chip);
286998983675SCédric Le Goater if (state->lsi)
287098983675SCédric Le Goater seq_printf(m, " %cLSI", state->asserted ? '^' : ' ');
287198983675SCédric Le Goater else
287298983675SCédric Le Goater seq_puts(m, " MSI");
287398983675SCédric Le Goater
287498983675SCédric Le Goater seq_printf(m, " %s %c%c %08x % 4d/%d",
287598983675SCédric Le Goater state->ipi_number == hw_num ? "IPI" : " PT",
287698983675SCédric Le Goater pq & XIVE_ESB_VAL_P ? 'P' : '-',
287798983675SCédric Le Goater pq & XIVE_ESB_VAL_Q ? 'Q' : '-',
287898983675SCédric Le Goater state->eisn, state->act_server,
287998983675SCédric Le Goater state->act_priority);
288098983675SCédric Le Goater
288198983675SCédric Le Goater seq_puts(m, "\n");
288298983675SCédric Le Goater }
288398983675SCédric Le Goater }
288498983675SCédric Le Goater
xive_debug_show(struct seq_file * m,void * private)28855af50993SBenjamin Herrenschmidt static int xive_debug_show(struct seq_file *m, void *private)
28865af50993SBenjamin Herrenschmidt {
28875af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = m->private;
28885af50993SBenjamin Herrenschmidt struct kvm *kvm = xive->kvm;
28895af50993SBenjamin Herrenschmidt struct kvm_vcpu *vcpu;
28905af50993SBenjamin Herrenschmidt u64 t_rm_h_xirr = 0;
28915af50993SBenjamin Herrenschmidt u64 t_rm_h_ipoll = 0;
28925af50993SBenjamin Herrenschmidt u64 t_rm_h_cppr = 0;
28935af50993SBenjamin Herrenschmidt u64 t_rm_h_eoi = 0;
28945af50993SBenjamin Herrenschmidt u64 t_rm_h_ipi = 0;
28955af50993SBenjamin Herrenschmidt u64 t_vm_h_xirr = 0;
28965af50993SBenjamin Herrenschmidt u64 t_vm_h_ipoll = 0;
28975af50993SBenjamin Herrenschmidt u64 t_vm_h_cppr = 0;
28985af50993SBenjamin Herrenschmidt u64 t_vm_h_eoi = 0;
28995af50993SBenjamin Herrenschmidt u64 t_vm_h_ipi = 0;
290046808a4cSMarc Zyngier unsigned long i;
29015af50993SBenjamin Herrenschmidt
29025af50993SBenjamin Herrenschmidt if (!kvm)
29035af50993SBenjamin Herrenschmidt return 0;
29045af50993SBenjamin Herrenschmidt
290598983675SCédric Le Goater seq_puts(m, "=========\nVCPU state\n=========\n");
29065af50993SBenjamin Herrenschmidt
29075af50993SBenjamin Herrenschmidt kvm_for_each_vcpu(i, vcpu, kvm) {
29085af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
29095af50993SBenjamin Herrenschmidt
29105af50993SBenjamin Herrenschmidt if (!xc)
29115af50993SBenjamin Herrenschmidt continue;
29125af50993SBenjamin Herrenschmidt
291398983675SCédric Le Goater seq_printf(m, "VCPU %d: VP:%#x/%02x\n"
291498983675SCédric Le Goater " CPPR:%#x HWCPPR:%#x MFRR:%#x PEND:%#x h_xirr: R=%lld V=%lld\n",
291598983675SCédric Le Goater xc->server_num, xc->vp_id, xc->vp_chip_id,
291698983675SCédric Le Goater xc->cppr, xc->hw_cppr,
29175af50993SBenjamin Herrenschmidt xc->mfrr, xc->pending,
29185af50993SBenjamin Herrenschmidt xc->stat_rm_h_xirr, xc->stat_vm_h_xirr);
2919c424c108SBenjamin Herrenschmidt
2920eacc56bbSCédric Le Goater kvmppc_xive_debug_show_queues(m, vcpu);
29215af50993SBenjamin Herrenschmidt
29225af50993SBenjamin Herrenschmidt t_rm_h_xirr += xc->stat_rm_h_xirr;
29235af50993SBenjamin Herrenschmidt t_rm_h_ipoll += xc->stat_rm_h_ipoll;
29245af50993SBenjamin Herrenschmidt t_rm_h_cppr += xc->stat_rm_h_cppr;
29255af50993SBenjamin Herrenschmidt t_rm_h_eoi += xc->stat_rm_h_eoi;
29265af50993SBenjamin Herrenschmidt t_rm_h_ipi += xc->stat_rm_h_ipi;
29275af50993SBenjamin Herrenschmidt t_vm_h_xirr += xc->stat_vm_h_xirr;
29285af50993SBenjamin Herrenschmidt t_vm_h_ipoll += xc->stat_vm_h_ipoll;
29295af50993SBenjamin Herrenschmidt t_vm_h_cppr += xc->stat_vm_h_cppr;
29305af50993SBenjamin Herrenschmidt t_vm_h_eoi += xc->stat_vm_h_eoi;
29315af50993SBenjamin Herrenschmidt t_vm_h_ipi += xc->stat_vm_h_ipi;
29325af50993SBenjamin Herrenschmidt }
29335af50993SBenjamin Herrenschmidt
293498983675SCédric Le Goater seq_puts(m, "Hcalls totals\n");
29355af50993SBenjamin Herrenschmidt seq_printf(m, " H_XIRR R=%10lld V=%10lld\n", t_rm_h_xirr, t_vm_h_xirr);
29365af50993SBenjamin Herrenschmidt seq_printf(m, " H_IPOLL R=%10lld V=%10lld\n", t_rm_h_ipoll, t_vm_h_ipoll);
29375af50993SBenjamin Herrenschmidt seq_printf(m, " H_CPPR R=%10lld V=%10lld\n", t_rm_h_cppr, t_vm_h_cppr);
29385af50993SBenjamin Herrenschmidt seq_printf(m, " H_EOI R=%10lld V=%10lld\n", t_rm_h_eoi, t_vm_h_eoi);
29395af50993SBenjamin Herrenschmidt seq_printf(m, " H_IPI R=%10lld V=%10lld\n", t_rm_h_ipi, t_vm_h_ipi);
29405af50993SBenjamin Herrenschmidt
294198983675SCédric Le Goater seq_puts(m, "=========\nSources\n=========\n");
294298983675SCédric Le Goater
294398983675SCédric Le Goater for (i = 0; i <= xive->max_sbid; i++) {
294498983675SCédric Le Goater struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
294598983675SCédric Le Goater
294698983675SCédric Le Goater if (sb) {
294798983675SCédric Le Goater arch_spin_lock(&sb->lock);
294898983675SCédric Le Goater kvmppc_xive_debug_show_sources(m, sb);
294998983675SCédric Le Goater arch_spin_unlock(&sb->lock);
295098983675SCédric Le Goater }
295198983675SCédric Le Goater }
295298983675SCédric Le Goater
29535af50993SBenjamin Herrenschmidt return 0;
29545af50993SBenjamin Herrenschmidt }
29555af50993SBenjamin Herrenschmidt
29560f6ddf34SYangtao Li DEFINE_SHOW_ATTRIBUTE(xive_debug);
29575af50993SBenjamin Herrenschmidt
xive_debugfs_init(struct kvmppc_xive * xive)29585af50993SBenjamin Herrenschmidt static void xive_debugfs_init(struct kvmppc_xive *xive)
29595af50993SBenjamin Herrenschmidt {
2960faf01aefSAlexey Kardashevskiy xive->dentry = debugfs_create_file("xive", S_IRUGO, xive->kvm->debugfs_dentry,
29615af50993SBenjamin Herrenschmidt xive, &xive_debug_fops);
29625af50993SBenjamin Herrenschmidt
2963faf01aefSAlexey Kardashevskiy pr_debug("%s: created\n", __func__);
29645af50993SBenjamin Herrenschmidt }
29655af50993SBenjamin Herrenschmidt
kvmppc_xive_init(struct kvm_device * dev)29665af50993SBenjamin Herrenschmidt static void kvmppc_xive_init(struct kvm_device *dev)
29675af50993SBenjamin Herrenschmidt {
2968e40b38a4SNour-eddine Taleb struct kvmppc_xive *xive = dev->private;
29695af50993SBenjamin Herrenschmidt
29705af50993SBenjamin Herrenschmidt /* Register some debug interfaces */
29715af50993SBenjamin Herrenschmidt xive_debugfs_init(xive);
29725af50993SBenjamin Herrenschmidt }
29735af50993SBenjamin Herrenschmidt
29745af50993SBenjamin Herrenschmidt struct kvm_device_ops kvm_xive_ops = {
29755af50993SBenjamin Herrenschmidt .name = "kvm-xive",
29765af50993SBenjamin Herrenschmidt .create = kvmppc_xive_create,
29775af50993SBenjamin Herrenschmidt .init = kvmppc_xive_init,
29785422e951SCédric Le Goater .release = kvmppc_xive_release,
29795af50993SBenjamin Herrenschmidt .set_attr = xive_set_attr,
29805af50993SBenjamin Herrenschmidt .get_attr = xive_get_attr,
29815af50993SBenjamin Herrenschmidt .has_attr = xive_has_attr,
29825af50993SBenjamin Herrenschmidt };
2983