15af50993SBenjamin Herrenschmidt /* 25af50993SBenjamin Herrenschmidt * Copyright 2017 Benjamin Herrenschmidt, IBM Corporation. 35af50993SBenjamin Herrenschmidt * 45af50993SBenjamin Herrenschmidt * This program is free software; you can redistribute it and/or modify 55af50993SBenjamin Herrenschmidt * it under the terms of the GNU General Public License, version 2, as 65af50993SBenjamin Herrenschmidt * published by the Free Software Foundation. 75af50993SBenjamin Herrenschmidt */ 85af50993SBenjamin Herrenschmidt 95af50993SBenjamin Herrenschmidt #define pr_fmt(fmt) "xive-kvm: " fmt 105af50993SBenjamin Herrenschmidt 115af50993SBenjamin Herrenschmidt #include <linux/kernel.h> 125af50993SBenjamin Herrenschmidt #include <linux/kvm_host.h> 135af50993SBenjamin Herrenschmidt #include <linux/err.h> 145af50993SBenjamin Herrenschmidt #include <linux/gfp.h> 155af50993SBenjamin Herrenschmidt #include <linux/spinlock.h> 165af50993SBenjamin Herrenschmidt #include <linux/delay.h> 175af50993SBenjamin Herrenschmidt #include <linux/percpu.h> 185af50993SBenjamin Herrenschmidt #include <linux/cpumask.h> 195bb866deSAl Viro #include <linux/uaccess.h> 205af50993SBenjamin Herrenschmidt #include <asm/kvm_book3s.h> 215af50993SBenjamin Herrenschmidt #include <asm/kvm_ppc.h> 225af50993SBenjamin Herrenschmidt #include <asm/hvcall.h> 235af50993SBenjamin Herrenschmidt #include <asm/xics.h> 245af50993SBenjamin Herrenschmidt #include <asm/xive.h> 255af50993SBenjamin Herrenschmidt #include <asm/xive-regs.h> 265af50993SBenjamin Herrenschmidt #include <asm/debug.h> 274415b335SPaolo Bonzini #include <asm/debugfs.h> 285af50993SBenjamin Herrenschmidt #include <asm/time.h> 295af50993SBenjamin Herrenschmidt #include <asm/opal.h> 305af50993SBenjamin Herrenschmidt 315af50993SBenjamin Herrenschmidt #include <linux/debugfs.h> 325af50993SBenjamin Herrenschmidt #include <linux/seq_file.h> 335af50993SBenjamin Herrenschmidt 345af50993SBenjamin Herrenschmidt #include "book3s_xive.h" 355af50993SBenjamin Herrenschmidt 365af50993SBenjamin Herrenschmidt 375af50993SBenjamin Herrenschmidt /* 385af50993SBenjamin Herrenschmidt * Virtual mode variants of the hcalls for use on radix/radix 395af50993SBenjamin Herrenschmidt * with AIL. They require the VCPU's VP to be "pushed" 405af50993SBenjamin Herrenschmidt * 413cc97beaSFinn Thain * We still instantiate them here because we use some of the 425af50993SBenjamin Herrenschmidt * generated utility functions as well in this file. 435af50993SBenjamin Herrenschmidt */ 445af50993SBenjamin Herrenschmidt #define XIVE_RUNTIME_CHECKS 455af50993SBenjamin Herrenschmidt #define X_PFX xive_vm_ 465af50993SBenjamin Herrenschmidt #define X_STATIC static 475af50993SBenjamin Herrenschmidt #define X_STAT_PFX stat_vm_ 485af50993SBenjamin Herrenschmidt #define __x_tima xive_tima 495af50993SBenjamin Herrenschmidt #define __x_eoi_page(xd) ((void __iomem *)((xd)->eoi_mmio)) 505af50993SBenjamin Herrenschmidt #define __x_trig_page(xd) ((void __iomem *)((xd)->trig_mmio)) 515af50993SBenjamin Herrenschmidt #define __x_writeb __raw_writeb 525af50993SBenjamin Herrenschmidt #define __x_readw __raw_readw 535af50993SBenjamin Herrenschmidt #define __x_readq __raw_readq 545af50993SBenjamin Herrenschmidt #define __x_writeq __raw_writeq 555af50993SBenjamin Herrenschmidt 565af50993SBenjamin Herrenschmidt #include "book3s_xive_template.c" 575af50993SBenjamin Herrenschmidt 585af50993SBenjamin Herrenschmidt /* 595af50993SBenjamin Herrenschmidt * We leave a gap of a couple of interrupts in the queue to 605af50993SBenjamin Herrenschmidt * account for the IPI and additional safety guard. 615af50993SBenjamin Herrenschmidt */ 625af50993SBenjamin Herrenschmidt #define XIVE_Q_GAP 2 635af50993SBenjamin Herrenschmidt 645af50993SBenjamin Herrenschmidt /* 6595a6432cSPaul Mackerras * Push a vcpu's context to the XIVE on guest entry. 6695a6432cSPaul Mackerras * This assumes we are in virtual mode (MMU on) 6795a6432cSPaul Mackerras */ 6895a6432cSPaul Mackerras void kvmppc_xive_push_vcpu(struct kvm_vcpu *vcpu) 6995a6432cSPaul Mackerras { 7095a6432cSPaul Mackerras void __iomem *tima = local_paca->kvm_hstate.xive_tima_virt; 7195a6432cSPaul Mackerras u64 pq; 7295a6432cSPaul Mackerras 7395a6432cSPaul Mackerras if (!tima) 7495a6432cSPaul Mackerras return; 7595a6432cSPaul Mackerras eieio(); 7695a6432cSPaul Mackerras __raw_writeq(vcpu->arch.xive_saved_state.w01, tima + TM_QW1_OS); 7795a6432cSPaul Mackerras __raw_writel(vcpu->arch.xive_cam_word, tima + TM_QW1_OS + TM_WORD2); 7895a6432cSPaul Mackerras vcpu->arch.xive_pushed = 1; 7995a6432cSPaul Mackerras eieio(); 8095a6432cSPaul Mackerras 8195a6432cSPaul Mackerras /* 8295a6432cSPaul Mackerras * We clear the irq_pending flag. There is a small chance of a 8395a6432cSPaul Mackerras * race vs. the escalation interrupt happening on another 8495a6432cSPaul Mackerras * processor setting it again, but the only consequence is to 8595a6432cSPaul Mackerras * cause a spurious wakeup on the next H_CEDE, which is not an 8695a6432cSPaul Mackerras * issue. 8795a6432cSPaul Mackerras */ 8895a6432cSPaul Mackerras vcpu->arch.irq_pending = 0; 8995a6432cSPaul Mackerras 9095a6432cSPaul Mackerras /* 9195a6432cSPaul Mackerras * In single escalation mode, if the escalation interrupt is 9295a6432cSPaul Mackerras * on, we mask it. 9395a6432cSPaul Mackerras */ 9495a6432cSPaul Mackerras if (vcpu->arch.xive_esc_on) { 9595a6432cSPaul Mackerras pq = __raw_readq((void __iomem *)(vcpu->arch.xive_esc_vaddr + 9695a6432cSPaul Mackerras XIVE_ESB_SET_PQ_01)); 9795a6432cSPaul Mackerras mb(); 9895a6432cSPaul Mackerras 9995a6432cSPaul Mackerras /* 10095a6432cSPaul Mackerras * We have a possible subtle race here: The escalation 10195a6432cSPaul Mackerras * interrupt might have fired and be on its way to the 10295a6432cSPaul Mackerras * host queue while we mask it, and if we unmask it 10395a6432cSPaul Mackerras * early enough (re-cede right away), there is a 10495a6432cSPaul Mackerras * theorical possibility that it fires again, thus 10595a6432cSPaul Mackerras * landing in the target queue more than once which is 10695a6432cSPaul Mackerras * a big no-no. 10795a6432cSPaul Mackerras * 10895a6432cSPaul Mackerras * Fortunately, solving this is rather easy. If the 10995a6432cSPaul Mackerras * above load setting PQ to 01 returns a previous 11095a6432cSPaul Mackerras * value where P is set, then we know the escalation 11195a6432cSPaul Mackerras * interrupt is somewhere on its way to the host. In 11295a6432cSPaul Mackerras * that case we simply don't clear the xive_esc_on 11395a6432cSPaul Mackerras * flag below. It will be eventually cleared by the 11495a6432cSPaul Mackerras * handler for the escalation interrupt. 11595a6432cSPaul Mackerras * 11695a6432cSPaul Mackerras * Then, when doing a cede, we check that flag again 11795a6432cSPaul Mackerras * before re-enabling the escalation interrupt, and if 11895a6432cSPaul Mackerras * set, we abort the cede. 11995a6432cSPaul Mackerras */ 12095a6432cSPaul Mackerras if (!(pq & XIVE_ESB_VAL_P)) 12195a6432cSPaul Mackerras /* Now P is 0, we can clear the flag */ 12295a6432cSPaul Mackerras vcpu->arch.xive_esc_on = 0; 12395a6432cSPaul Mackerras } 12495a6432cSPaul Mackerras } 12595a6432cSPaul Mackerras EXPORT_SYMBOL_GPL(kvmppc_xive_push_vcpu); 12695a6432cSPaul Mackerras 12795a6432cSPaul Mackerras /* 1285af50993SBenjamin Herrenschmidt * This is a simple trigger for a generic XIVE IRQ. This must 1295af50993SBenjamin Herrenschmidt * only be called for interrupts that support a trigger page 1305af50993SBenjamin Herrenschmidt */ 1315af50993SBenjamin Herrenschmidt static bool xive_irq_trigger(struct xive_irq_data *xd) 1325af50993SBenjamin Herrenschmidt { 1335af50993SBenjamin Herrenschmidt /* This should be only for MSIs */ 1345af50993SBenjamin Herrenschmidt if (WARN_ON(xd->flags & XIVE_IRQ_FLAG_LSI)) 1355af50993SBenjamin Herrenschmidt return false; 1365af50993SBenjamin Herrenschmidt 1375af50993SBenjamin Herrenschmidt /* Those interrupts should always have a trigger page */ 1385af50993SBenjamin Herrenschmidt if (WARN_ON(!xd->trig_mmio)) 1395af50993SBenjamin Herrenschmidt return false; 1405af50993SBenjamin Herrenschmidt 1415af50993SBenjamin Herrenschmidt out_be64(xd->trig_mmio, 0); 1425af50993SBenjamin Herrenschmidt 1435af50993SBenjamin Herrenschmidt return true; 1445af50993SBenjamin Herrenschmidt } 1455af50993SBenjamin Herrenschmidt 1465af50993SBenjamin Herrenschmidt static irqreturn_t xive_esc_irq(int irq, void *data) 1475af50993SBenjamin Herrenschmidt { 1485af50993SBenjamin Herrenschmidt struct kvm_vcpu *vcpu = data; 1495af50993SBenjamin Herrenschmidt 1502267ea76SBenjamin Herrenschmidt vcpu->arch.irq_pending = 1; 1515af50993SBenjamin Herrenschmidt smp_mb(); 1525af50993SBenjamin Herrenschmidt if (vcpu->arch.ceded) 1535af50993SBenjamin Herrenschmidt kvmppc_fast_vcpu_kick(vcpu); 1545af50993SBenjamin Herrenschmidt 1559b9b13a6SBenjamin Herrenschmidt /* Since we have the no-EOI flag, the interrupt is effectively 1569b9b13a6SBenjamin Herrenschmidt * disabled now. Clearing xive_esc_on means we won't bother 1579b9b13a6SBenjamin Herrenschmidt * doing so on the next entry. 1589b9b13a6SBenjamin Herrenschmidt * 1599b9b13a6SBenjamin Herrenschmidt * This also allows the entry code to know that if a PQ combination 1609b9b13a6SBenjamin Herrenschmidt * of 10 is observed while xive_esc_on is true, it means the queue 1619b9b13a6SBenjamin Herrenschmidt * contains an unprocessed escalation interrupt. We don't make use of 1629b9b13a6SBenjamin Herrenschmidt * that knowledge today but might (see comment in book3s_hv_rmhandler.S) 1639b9b13a6SBenjamin Herrenschmidt */ 1649b9b13a6SBenjamin Herrenschmidt vcpu->arch.xive_esc_on = false; 1659b9b13a6SBenjamin Herrenschmidt 1665af50993SBenjamin Herrenschmidt return IRQ_HANDLED; 1675af50993SBenjamin Herrenschmidt } 1685af50993SBenjamin Herrenschmidt 16913ce3297SCédric Le Goater int kvmppc_xive_attach_escalation(struct kvm_vcpu *vcpu, u8 prio, 17013ce3297SCédric Le Goater bool single_escalation) 1715af50993SBenjamin Herrenschmidt { 1725af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 1735af50993SBenjamin Herrenschmidt struct xive_q *q = &xc->queues[prio]; 1745af50993SBenjamin Herrenschmidt char *name = NULL; 1755af50993SBenjamin Herrenschmidt int rc; 1765af50993SBenjamin Herrenschmidt 1775af50993SBenjamin Herrenschmidt /* Already there ? */ 1785af50993SBenjamin Herrenschmidt if (xc->esc_virq[prio]) 1795af50993SBenjamin Herrenschmidt return 0; 1805af50993SBenjamin Herrenschmidt 1815af50993SBenjamin Herrenschmidt /* Hook up the escalation interrupt */ 1825af50993SBenjamin Herrenschmidt xc->esc_virq[prio] = irq_create_mapping(NULL, q->esc_irq); 1835af50993SBenjamin Herrenschmidt if (!xc->esc_virq[prio]) { 1845af50993SBenjamin Herrenschmidt pr_err("Failed to map escalation interrupt for queue %d of VCPU %d\n", 1855af50993SBenjamin Herrenschmidt prio, xc->server_num); 1865af50993SBenjamin Herrenschmidt return -EIO; 1875af50993SBenjamin Herrenschmidt } 1885af50993SBenjamin Herrenschmidt 18913ce3297SCédric Le Goater if (single_escalation) 190bf4159daSBenjamin Herrenschmidt name = kasprintf(GFP_KERNEL, "kvm-%d-%d", 191bf4159daSBenjamin Herrenschmidt vcpu->kvm->arch.lpid, xc->server_num); 192bf4159daSBenjamin Herrenschmidt else 1935af50993SBenjamin Herrenschmidt name = kasprintf(GFP_KERNEL, "kvm-%d-%d-%d", 1945af50993SBenjamin Herrenschmidt vcpu->kvm->arch.lpid, xc->server_num, prio); 1955af50993SBenjamin Herrenschmidt if (!name) { 1965af50993SBenjamin Herrenschmidt pr_err("Failed to allocate escalation irq name for queue %d of VCPU %d\n", 1975af50993SBenjamin Herrenschmidt prio, xc->server_num); 1985af50993SBenjamin Herrenschmidt rc = -ENOMEM; 1995af50993SBenjamin Herrenschmidt goto error; 2005af50993SBenjamin Herrenschmidt } 201bf4159daSBenjamin Herrenschmidt 202bf4159daSBenjamin Herrenschmidt pr_devel("Escalation %s irq %d (prio %d)\n", name, xc->esc_virq[prio], prio); 203bf4159daSBenjamin Herrenschmidt 2045af50993SBenjamin Herrenschmidt rc = request_irq(xc->esc_virq[prio], xive_esc_irq, 2055af50993SBenjamin Herrenschmidt IRQF_NO_THREAD, name, vcpu); 2065af50993SBenjamin Herrenschmidt if (rc) { 2075af50993SBenjamin Herrenschmidt pr_err("Failed to request escalation interrupt for queue %d of VCPU %d\n", 2085af50993SBenjamin Herrenschmidt prio, xc->server_num); 2095af50993SBenjamin Herrenschmidt goto error; 2105af50993SBenjamin Herrenschmidt } 2115af50993SBenjamin Herrenschmidt xc->esc_virq_names[prio] = name; 2129b9b13a6SBenjamin Herrenschmidt 2139b9b13a6SBenjamin Herrenschmidt /* In single escalation mode, we grab the ESB MMIO of the 2149b9b13a6SBenjamin Herrenschmidt * interrupt and mask it. Also populate the VCPU v/raddr 2159b9b13a6SBenjamin Herrenschmidt * of the ESB page for use by asm entry/exit code. Finally 2169b9b13a6SBenjamin Herrenschmidt * set the XIVE_IRQ_NO_EOI flag which will prevent the 2179b9b13a6SBenjamin Herrenschmidt * core code from performing an EOI on the escalation 2189b9b13a6SBenjamin Herrenschmidt * interrupt, thus leaving it effectively masked after 2199b9b13a6SBenjamin Herrenschmidt * it fires once. 2209b9b13a6SBenjamin Herrenschmidt */ 22113ce3297SCédric Le Goater if (single_escalation) { 2229b9b13a6SBenjamin Herrenschmidt struct irq_data *d = irq_get_irq_data(xc->esc_virq[prio]); 2239b9b13a6SBenjamin Herrenschmidt struct xive_irq_data *xd = irq_data_get_irq_handler_data(d); 2249b9b13a6SBenjamin Herrenschmidt 2259b9b13a6SBenjamin Herrenschmidt xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01); 2269b9b13a6SBenjamin Herrenschmidt vcpu->arch.xive_esc_raddr = xd->eoi_page; 2279b9b13a6SBenjamin Herrenschmidt vcpu->arch.xive_esc_vaddr = (__force u64)xd->eoi_mmio; 2289b9b13a6SBenjamin Herrenschmidt xd->flags |= XIVE_IRQ_NO_EOI; 2299b9b13a6SBenjamin Herrenschmidt } 2309b9b13a6SBenjamin Herrenschmidt 2315af50993SBenjamin Herrenschmidt return 0; 2325af50993SBenjamin Herrenschmidt error: 2335af50993SBenjamin Herrenschmidt irq_dispose_mapping(xc->esc_virq[prio]); 2345af50993SBenjamin Herrenschmidt xc->esc_virq[prio] = 0; 2355af50993SBenjamin Herrenschmidt kfree(name); 2365af50993SBenjamin Herrenschmidt return rc; 2375af50993SBenjamin Herrenschmidt } 2385af50993SBenjamin Herrenschmidt 2395af50993SBenjamin Herrenschmidt static int xive_provision_queue(struct kvm_vcpu *vcpu, u8 prio) 2405af50993SBenjamin Herrenschmidt { 2415af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 2425af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = xc->xive; 2435af50993SBenjamin Herrenschmidt struct xive_q *q = &xc->queues[prio]; 2445af50993SBenjamin Herrenschmidt void *qpage; 2455af50993SBenjamin Herrenschmidt int rc; 2465af50993SBenjamin Herrenschmidt 2475af50993SBenjamin Herrenschmidt if (WARN_ON(q->qpage)) 2485af50993SBenjamin Herrenschmidt return 0; 2495af50993SBenjamin Herrenschmidt 2505af50993SBenjamin Herrenschmidt /* Allocate the queue and retrieve infos on current node for now */ 2515af50993SBenjamin Herrenschmidt qpage = (__be32 *)__get_free_pages(GFP_KERNEL, xive->q_page_order); 2525af50993SBenjamin Herrenschmidt if (!qpage) { 2535af50993SBenjamin Herrenschmidt pr_err("Failed to allocate queue %d for VCPU %d\n", 2545af50993SBenjamin Herrenschmidt prio, xc->server_num); 255ed7158baSIngo Molnar return -ENOMEM; 2565af50993SBenjamin Herrenschmidt } 2575af50993SBenjamin Herrenschmidt memset(qpage, 0, 1 << xive->q_order); 2585af50993SBenjamin Herrenschmidt 2595af50993SBenjamin Herrenschmidt /* 2605af50993SBenjamin Herrenschmidt * Reconfigure the queue. This will set q->qpage only once the 2615af50993SBenjamin Herrenschmidt * queue is fully configured. This is a requirement for prio 0 2625af50993SBenjamin Herrenschmidt * as we will stop doing EOIs for every IPI as soon as we observe 2635af50993SBenjamin Herrenschmidt * qpage being non-NULL, and instead will only EOI when we receive 2645af50993SBenjamin Herrenschmidt * corresponding queue 0 entries 2655af50993SBenjamin Herrenschmidt */ 2665af50993SBenjamin Herrenschmidt rc = xive_native_configure_queue(xc->vp_id, q, prio, qpage, 2675af50993SBenjamin Herrenschmidt xive->q_order, true); 2685af50993SBenjamin Herrenschmidt if (rc) 2695af50993SBenjamin Herrenschmidt pr_err("Failed to configure queue %d for VCPU %d\n", 2705af50993SBenjamin Herrenschmidt prio, xc->server_num); 2715af50993SBenjamin Herrenschmidt return rc; 2725af50993SBenjamin Herrenschmidt } 2735af50993SBenjamin Herrenschmidt 2747e10b9a6SCédric Le Goater /* Called with xive->lock held */ 2755af50993SBenjamin Herrenschmidt static int xive_check_provisioning(struct kvm *kvm, u8 prio) 2765af50993SBenjamin Herrenschmidt { 2775af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = kvm->arch.xive; 2785af50993SBenjamin Herrenschmidt struct kvm_vcpu *vcpu; 2795af50993SBenjamin Herrenschmidt int i, rc; 2805af50993SBenjamin Herrenschmidt 2817e10b9a6SCédric Le Goater lockdep_assert_held(&xive->lock); 2825af50993SBenjamin Herrenschmidt 2835af50993SBenjamin Herrenschmidt /* Already provisioned ? */ 2845af50993SBenjamin Herrenschmidt if (xive->qmap & (1 << prio)) 2855af50993SBenjamin Herrenschmidt return 0; 2865af50993SBenjamin Herrenschmidt 2875af50993SBenjamin Herrenschmidt pr_devel("Provisioning prio... %d\n", prio); 2885af50993SBenjamin Herrenschmidt 289bf4159daSBenjamin Herrenschmidt /* Provision each VCPU and enable escalations if needed */ 2905af50993SBenjamin Herrenschmidt kvm_for_each_vcpu(i, vcpu, kvm) { 2915af50993SBenjamin Herrenschmidt if (!vcpu->arch.xive_vcpu) 2925af50993SBenjamin Herrenschmidt continue; 2935af50993SBenjamin Herrenschmidt rc = xive_provision_queue(vcpu, prio); 294bf4159daSBenjamin Herrenschmidt if (rc == 0 && !xive->single_escalation) 29513ce3297SCédric Le Goater kvmppc_xive_attach_escalation(vcpu, prio, 29613ce3297SCédric Le Goater xive->single_escalation); 2975af50993SBenjamin Herrenschmidt if (rc) 2985af50993SBenjamin Herrenschmidt return rc; 2995af50993SBenjamin Herrenschmidt } 3005af50993SBenjamin Herrenschmidt 3015af50993SBenjamin Herrenschmidt /* Order previous stores and mark it as provisioned */ 3025af50993SBenjamin Herrenschmidt mb(); 3035af50993SBenjamin Herrenschmidt xive->qmap |= (1 << prio); 3045af50993SBenjamin Herrenschmidt return 0; 3055af50993SBenjamin Herrenschmidt } 3065af50993SBenjamin Herrenschmidt 3075af50993SBenjamin Herrenschmidt static void xive_inc_q_pending(struct kvm *kvm, u32 server, u8 prio) 3085af50993SBenjamin Herrenschmidt { 3095af50993SBenjamin Herrenschmidt struct kvm_vcpu *vcpu; 3105af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc; 3115af50993SBenjamin Herrenschmidt struct xive_q *q; 3125af50993SBenjamin Herrenschmidt 3135af50993SBenjamin Herrenschmidt /* Locate target server */ 3145af50993SBenjamin Herrenschmidt vcpu = kvmppc_xive_find_server(kvm, server); 3155af50993SBenjamin Herrenschmidt if (!vcpu) { 3165af50993SBenjamin Herrenschmidt pr_warn("%s: Can't find server %d\n", __func__, server); 3175af50993SBenjamin Herrenschmidt return; 3185af50993SBenjamin Herrenschmidt } 3195af50993SBenjamin Herrenschmidt xc = vcpu->arch.xive_vcpu; 3205af50993SBenjamin Herrenschmidt if (WARN_ON(!xc)) 3215af50993SBenjamin Herrenschmidt return; 3225af50993SBenjamin Herrenschmidt 3235af50993SBenjamin Herrenschmidt q = &xc->queues[prio]; 3245af50993SBenjamin Herrenschmidt atomic_inc(&q->pending_count); 3255af50993SBenjamin Herrenschmidt } 3265af50993SBenjamin Herrenschmidt 3275af50993SBenjamin Herrenschmidt static int xive_try_pick_queue(struct kvm_vcpu *vcpu, u8 prio) 3285af50993SBenjamin Herrenschmidt { 3295af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 3305af50993SBenjamin Herrenschmidt struct xive_q *q; 3315af50993SBenjamin Herrenschmidt u32 max; 3325af50993SBenjamin Herrenschmidt 3335af50993SBenjamin Herrenschmidt if (WARN_ON(!xc)) 3345af50993SBenjamin Herrenschmidt return -ENXIO; 3355af50993SBenjamin Herrenschmidt if (!xc->valid) 3365af50993SBenjamin Herrenschmidt return -ENXIO; 3375af50993SBenjamin Herrenschmidt 3385af50993SBenjamin Herrenschmidt q = &xc->queues[prio]; 3395af50993SBenjamin Herrenschmidt if (WARN_ON(!q->qpage)) 3405af50993SBenjamin Herrenschmidt return -ENXIO; 3415af50993SBenjamin Herrenschmidt 3425af50993SBenjamin Herrenschmidt /* Calculate max number of interrupts in that queue. */ 3435af50993SBenjamin Herrenschmidt max = (q->msk + 1) - XIVE_Q_GAP; 3445af50993SBenjamin Herrenschmidt return atomic_add_unless(&q->count, 1, max) ? 0 : -EBUSY; 3455af50993SBenjamin Herrenschmidt } 3465af50993SBenjamin Herrenschmidt 347e8676ce5SCédric Le Goater int kvmppc_xive_select_target(struct kvm *kvm, u32 *server, u8 prio) 3485af50993SBenjamin Herrenschmidt { 3495af50993SBenjamin Herrenschmidt struct kvm_vcpu *vcpu; 3505af50993SBenjamin Herrenschmidt int i, rc; 3515af50993SBenjamin Herrenschmidt 3525af50993SBenjamin Herrenschmidt /* Locate target server */ 3535af50993SBenjamin Herrenschmidt vcpu = kvmppc_xive_find_server(kvm, *server); 3545af50993SBenjamin Herrenschmidt if (!vcpu) { 3555af50993SBenjamin Herrenschmidt pr_devel("Can't find server %d\n", *server); 3565af50993SBenjamin Herrenschmidt return -EINVAL; 3575af50993SBenjamin Herrenschmidt } 3585af50993SBenjamin Herrenschmidt 3595af50993SBenjamin Herrenschmidt pr_devel("Finding irq target on 0x%x/%d...\n", *server, prio); 3605af50993SBenjamin Herrenschmidt 3615af50993SBenjamin Herrenschmidt /* Try pick it */ 3625af50993SBenjamin Herrenschmidt rc = xive_try_pick_queue(vcpu, prio); 3635af50993SBenjamin Herrenschmidt if (rc == 0) 3645af50993SBenjamin Herrenschmidt return rc; 3655af50993SBenjamin Herrenschmidt 3665af50993SBenjamin Herrenschmidt pr_devel(" .. failed, looking up candidate...\n"); 3675af50993SBenjamin Herrenschmidt 3685af50993SBenjamin Herrenschmidt /* Failed, pick another VCPU */ 3695af50993SBenjamin Herrenschmidt kvm_for_each_vcpu(i, vcpu, kvm) { 3705af50993SBenjamin Herrenschmidt if (!vcpu->arch.xive_vcpu) 3715af50993SBenjamin Herrenschmidt continue; 3725af50993SBenjamin Herrenschmidt rc = xive_try_pick_queue(vcpu, prio); 3735af50993SBenjamin Herrenschmidt if (rc == 0) { 3745af50993SBenjamin Herrenschmidt *server = vcpu->arch.xive_vcpu->server_num; 3755af50993SBenjamin Herrenschmidt pr_devel(" found on 0x%x/%d\n", *server, prio); 3765af50993SBenjamin Herrenschmidt return rc; 3775af50993SBenjamin Herrenschmidt } 3785af50993SBenjamin Herrenschmidt } 3795af50993SBenjamin Herrenschmidt pr_devel(" no available target !\n"); 3805af50993SBenjamin Herrenschmidt 3815af50993SBenjamin Herrenschmidt /* No available target ! */ 3825af50993SBenjamin Herrenschmidt return -EBUSY; 3835af50993SBenjamin Herrenschmidt } 3845af50993SBenjamin Herrenschmidt 3855af50993SBenjamin Herrenschmidt static u8 xive_lock_and_mask(struct kvmppc_xive *xive, 3865af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb, 3875af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state) 3885af50993SBenjamin Herrenschmidt { 3895af50993SBenjamin Herrenschmidt struct xive_irq_data *xd; 3905af50993SBenjamin Herrenschmidt u32 hw_num; 3915af50993SBenjamin Herrenschmidt u8 old_prio; 3925af50993SBenjamin Herrenschmidt u64 val; 3935af50993SBenjamin Herrenschmidt 3945af50993SBenjamin Herrenschmidt /* 3955af50993SBenjamin Herrenschmidt * Take the lock, set masked, try again if racing 3965af50993SBenjamin Herrenschmidt * with H_EOI 3975af50993SBenjamin Herrenschmidt */ 3985af50993SBenjamin Herrenschmidt for (;;) { 3995af50993SBenjamin Herrenschmidt arch_spin_lock(&sb->lock); 4005af50993SBenjamin Herrenschmidt old_prio = state->guest_priority; 4015af50993SBenjamin Herrenschmidt state->guest_priority = MASKED; 4025af50993SBenjamin Herrenschmidt mb(); 4035af50993SBenjamin Herrenschmidt if (!state->in_eoi) 4045af50993SBenjamin Herrenschmidt break; 4055af50993SBenjamin Herrenschmidt state->guest_priority = old_prio; 4065af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock); 4075af50993SBenjamin Herrenschmidt } 4085af50993SBenjamin Herrenschmidt 4095af50993SBenjamin Herrenschmidt /* No change ? Bail */ 4105af50993SBenjamin Herrenschmidt if (old_prio == MASKED) 4115af50993SBenjamin Herrenschmidt return old_prio; 4125af50993SBenjamin Herrenschmidt 4135af50993SBenjamin Herrenschmidt /* Get the right irq */ 4145af50993SBenjamin Herrenschmidt kvmppc_xive_select_irq(state, &hw_num, &xd); 4155af50993SBenjamin Herrenschmidt 4165af50993SBenjamin Herrenschmidt /* 4175af50993SBenjamin Herrenschmidt * If the interrupt is marked as needing masking via 4185af50993SBenjamin Herrenschmidt * firmware, we do it here. Firmware masking however 4195af50993SBenjamin Herrenschmidt * is "lossy", it won't return the old p and q bits 4205af50993SBenjamin Herrenschmidt * and won't set the interrupt to a state where it will 4215af50993SBenjamin Herrenschmidt * record queued ones. If this is an issue we should do 4225af50993SBenjamin Herrenschmidt * lazy masking instead. 4235af50993SBenjamin Herrenschmidt * 4245af50993SBenjamin Herrenschmidt * For now, we work around this in unmask by forcing 4255af50993SBenjamin Herrenschmidt * an interrupt whenever we unmask a non-LSI via FW 4265af50993SBenjamin Herrenschmidt * (if ever). 4275af50993SBenjamin Herrenschmidt */ 4285af50993SBenjamin Herrenschmidt if (xd->flags & OPAL_XIVE_IRQ_MASK_VIA_FW) { 4295af50993SBenjamin Herrenschmidt xive_native_configure_irq(hw_num, 430eacc56bbSCédric Le Goater kvmppc_xive_vp(xive, state->act_server), 4315af50993SBenjamin Herrenschmidt MASKED, state->number); 4325af50993SBenjamin Herrenschmidt /* set old_p so we can track if an H_EOI was done */ 4335af50993SBenjamin Herrenschmidt state->old_p = true; 4345af50993SBenjamin Herrenschmidt state->old_q = false; 4355af50993SBenjamin Herrenschmidt } else { 4365af50993SBenjamin Herrenschmidt /* Set PQ to 10, return old P and old Q and remember them */ 4375af50993SBenjamin Herrenschmidt val = xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_10); 4385af50993SBenjamin Herrenschmidt state->old_p = !!(val & 2); 4395af50993SBenjamin Herrenschmidt state->old_q = !!(val & 1); 4405af50993SBenjamin Herrenschmidt 4415af50993SBenjamin Herrenschmidt /* 4425af50993SBenjamin Herrenschmidt * Synchronize hardware to sensure the queues are updated 4435af50993SBenjamin Herrenschmidt * when masking 4445af50993SBenjamin Herrenschmidt */ 4455af50993SBenjamin Herrenschmidt xive_native_sync_source(hw_num); 4465af50993SBenjamin Herrenschmidt } 4475af50993SBenjamin Herrenschmidt 4485af50993SBenjamin Herrenschmidt return old_prio; 4495af50993SBenjamin Herrenschmidt } 4505af50993SBenjamin Herrenschmidt 4515af50993SBenjamin Herrenschmidt static void xive_lock_for_unmask(struct kvmppc_xive_src_block *sb, 4525af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state) 4535af50993SBenjamin Herrenschmidt { 4545af50993SBenjamin Herrenschmidt /* 4555af50993SBenjamin Herrenschmidt * Take the lock try again if racing with H_EOI 4565af50993SBenjamin Herrenschmidt */ 4575af50993SBenjamin Herrenschmidt for (;;) { 4585af50993SBenjamin Herrenschmidt arch_spin_lock(&sb->lock); 4595af50993SBenjamin Herrenschmidt if (!state->in_eoi) 4605af50993SBenjamin Herrenschmidt break; 4615af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock); 4625af50993SBenjamin Herrenschmidt } 4635af50993SBenjamin Herrenschmidt } 4645af50993SBenjamin Herrenschmidt 4655af50993SBenjamin Herrenschmidt static void xive_finish_unmask(struct kvmppc_xive *xive, 4665af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb, 4675af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state, 4685af50993SBenjamin Herrenschmidt u8 prio) 4695af50993SBenjamin Herrenschmidt { 4705af50993SBenjamin Herrenschmidt struct xive_irq_data *xd; 4715af50993SBenjamin Herrenschmidt u32 hw_num; 4725af50993SBenjamin Herrenschmidt 4735af50993SBenjamin Herrenschmidt /* If we aren't changing a thing, move on */ 4745af50993SBenjamin Herrenschmidt if (state->guest_priority != MASKED) 4755af50993SBenjamin Herrenschmidt goto bail; 4765af50993SBenjamin Herrenschmidt 4775af50993SBenjamin Herrenschmidt /* Get the right irq */ 4785af50993SBenjamin Herrenschmidt kvmppc_xive_select_irq(state, &hw_num, &xd); 4795af50993SBenjamin Herrenschmidt 4805af50993SBenjamin Herrenschmidt /* 4815af50993SBenjamin Herrenschmidt * See command in xive_lock_and_mask() concerning masking 4825af50993SBenjamin Herrenschmidt * via firmware. 4835af50993SBenjamin Herrenschmidt */ 4845af50993SBenjamin Herrenschmidt if (xd->flags & OPAL_XIVE_IRQ_MASK_VIA_FW) { 4855af50993SBenjamin Herrenschmidt xive_native_configure_irq(hw_num, 486eacc56bbSCédric Le Goater kvmppc_xive_vp(xive, state->act_server), 4875af50993SBenjamin Herrenschmidt state->act_priority, state->number); 4885af50993SBenjamin Herrenschmidt /* If an EOI is needed, do it here */ 4895af50993SBenjamin Herrenschmidt if (!state->old_p) 4905af50993SBenjamin Herrenschmidt xive_vm_source_eoi(hw_num, xd); 4915af50993SBenjamin Herrenschmidt /* If this is not an LSI, force a trigger */ 4925af50993SBenjamin Herrenschmidt if (!(xd->flags & OPAL_XIVE_IRQ_LSI)) 4935af50993SBenjamin Herrenschmidt xive_irq_trigger(xd); 4945af50993SBenjamin Herrenschmidt goto bail; 4955af50993SBenjamin Herrenschmidt } 4965af50993SBenjamin Herrenschmidt 4975af50993SBenjamin Herrenschmidt /* Old Q set, set PQ to 11 */ 4985af50993SBenjamin Herrenschmidt if (state->old_q) 4995af50993SBenjamin Herrenschmidt xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_11); 5005af50993SBenjamin Herrenschmidt 5015af50993SBenjamin Herrenschmidt /* 5025af50993SBenjamin Herrenschmidt * If not old P, then perform an "effective" EOI, 5035af50993SBenjamin Herrenschmidt * on the source. This will handle the cases where 5045af50993SBenjamin Herrenschmidt * FW EOI is needed. 5055af50993SBenjamin Herrenschmidt */ 5065af50993SBenjamin Herrenschmidt if (!state->old_p) 5075af50993SBenjamin Herrenschmidt xive_vm_source_eoi(hw_num, xd); 5085af50993SBenjamin Herrenschmidt 5095af50993SBenjamin Herrenschmidt /* Synchronize ordering and mark unmasked */ 5105af50993SBenjamin Herrenschmidt mb(); 5115af50993SBenjamin Herrenschmidt bail: 5125af50993SBenjamin Herrenschmidt state->guest_priority = prio; 5135af50993SBenjamin Herrenschmidt } 5145af50993SBenjamin Herrenschmidt 5155af50993SBenjamin Herrenschmidt /* 5165af50993SBenjamin Herrenschmidt * Target an interrupt to a given server/prio, this will fallback 5175af50993SBenjamin Herrenschmidt * to another server if necessary and perform the HW targetting 5185af50993SBenjamin Herrenschmidt * updates as needed 5195af50993SBenjamin Herrenschmidt * 5205af50993SBenjamin Herrenschmidt * NOTE: Must be called with the state lock held 5215af50993SBenjamin Herrenschmidt */ 5225af50993SBenjamin Herrenschmidt static int xive_target_interrupt(struct kvm *kvm, 5235af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state, 5245af50993SBenjamin Herrenschmidt u32 server, u8 prio) 5255af50993SBenjamin Herrenschmidt { 5265af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = kvm->arch.xive; 5275af50993SBenjamin Herrenschmidt u32 hw_num; 5285af50993SBenjamin Herrenschmidt int rc; 5295af50993SBenjamin Herrenschmidt 5305af50993SBenjamin Herrenschmidt /* 5315af50993SBenjamin Herrenschmidt * This will return a tentative server and actual 5325af50993SBenjamin Herrenschmidt * priority. The count for that new target will have 5335af50993SBenjamin Herrenschmidt * already been incremented. 5345af50993SBenjamin Herrenschmidt */ 535e8676ce5SCédric Le Goater rc = kvmppc_xive_select_target(kvm, &server, prio); 5365af50993SBenjamin Herrenschmidt 5375af50993SBenjamin Herrenschmidt /* 5385af50993SBenjamin Herrenschmidt * We failed to find a target ? Not much we can do 5395af50993SBenjamin Herrenschmidt * at least until we support the GIQ. 5405af50993SBenjamin Herrenschmidt */ 5415af50993SBenjamin Herrenschmidt if (rc) 5425af50993SBenjamin Herrenschmidt return rc; 5435af50993SBenjamin Herrenschmidt 5445af50993SBenjamin Herrenschmidt /* 5455af50993SBenjamin Herrenschmidt * Increment the old queue pending count if there 5465af50993SBenjamin Herrenschmidt * was one so that the old queue count gets adjusted later 5475af50993SBenjamin Herrenschmidt * when observed to be empty. 5485af50993SBenjamin Herrenschmidt */ 5495af50993SBenjamin Herrenschmidt if (state->act_priority != MASKED) 5505af50993SBenjamin Herrenschmidt xive_inc_q_pending(kvm, 5515af50993SBenjamin Herrenschmidt state->act_server, 5525af50993SBenjamin Herrenschmidt state->act_priority); 5535af50993SBenjamin Herrenschmidt /* 5545af50993SBenjamin Herrenschmidt * Update state and HW 5555af50993SBenjamin Herrenschmidt */ 5565af50993SBenjamin Herrenschmidt state->act_priority = prio; 5575af50993SBenjamin Herrenschmidt state->act_server = server; 5585af50993SBenjamin Herrenschmidt 5595af50993SBenjamin Herrenschmidt /* Get the right irq */ 5605af50993SBenjamin Herrenschmidt kvmppc_xive_select_irq(state, &hw_num, NULL); 5615af50993SBenjamin Herrenschmidt 5625af50993SBenjamin Herrenschmidt return xive_native_configure_irq(hw_num, 563eacc56bbSCédric Le Goater kvmppc_xive_vp(xive, server), 5645af50993SBenjamin Herrenschmidt prio, state->number); 5655af50993SBenjamin Herrenschmidt } 5665af50993SBenjamin Herrenschmidt 5675af50993SBenjamin Herrenschmidt /* 5685af50993SBenjamin Herrenschmidt * Targetting rules: In order to avoid losing track of 5695af50993SBenjamin Herrenschmidt * pending interrupts accross mask and unmask, which would 5705af50993SBenjamin Herrenschmidt * allow queue overflows, we implement the following rules: 5715af50993SBenjamin Herrenschmidt * 5725af50993SBenjamin Herrenschmidt * - Unless it was never enabled (or we run out of capacity) 5735af50993SBenjamin Herrenschmidt * an interrupt is always targetted at a valid server/queue 5745af50993SBenjamin Herrenschmidt * pair even when "masked" by the guest. This pair tends to 5755af50993SBenjamin Herrenschmidt * be the last one used but it can be changed under some 5765af50993SBenjamin Herrenschmidt * circumstances. That allows us to separate targetting 5775af50993SBenjamin Herrenschmidt * from masking, we only handle accounting during (re)targetting, 5785af50993SBenjamin Herrenschmidt * this also allows us to let an interrupt drain into its target 5795af50993SBenjamin Herrenschmidt * queue after masking, avoiding complex schemes to remove 5805af50993SBenjamin Herrenschmidt * interrupts out of remote processor queues. 5815af50993SBenjamin Herrenschmidt * 5825af50993SBenjamin Herrenschmidt * - When masking, we set PQ to 10 and save the previous value 5835af50993SBenjamin Herrenschmidt * of P and Q. 5845af50993SBenjamin Herrenschmidt * 5855af50993SBenjamin Herrenschmidt * - When unmasking, if saved Q was set, we set PQ to 11 5865af50993SBenjamin Herrenschmidt * otherwise we leave PQ to the HW state which will be either 5875af50993SBenjamin Herrenschmidt * 10 if nothing happened or 11 if the interrupt fired while 5885af50993SBenjamin Herrenschmidt * masked. Effectively we are OR'ing the previous Q into the 5895af50993SBenjamin Herrenschmidt * HW Q. 5905af50993SBenjamin Herrenschmidt * 5915af50993SBenjamin Herrenschmidt * Then if saved P is clear, we do an effective EOI (Q->P->Trigger) 5925af50993SBenjamin Herrenschmidt * which will unmask the interrupt and shoot a new one if Q was 5935af50993SBenjamin Herrenschmidt * set. 5945af50993SBenjamin Herrenschmidt * 5955af50993SBenjamin Herrenschmidt * Otherwise (saved P is set) we leave PQ unchanged (so 10 or 11, 5965af50993SBenjamin Herrenschmidt * effectively meaning an H_EOI from the guest is still expected 5975af50993SBenjamin Herrenschmidt * for that interrupt). 5985af50993SBenjamin Herrenschmidt * 5995af50993SBenjamin Herrenschmidt * - If H_EOI occurs while masked, we clear the saved P. 6005af50993SBenjamin Herrenschmidt * 6015af50993SBenjamin Herrenschmidt * - When changing target, we account on the new target and 6025af50993SBenjamin Herrenschmidt * increment a separate "pending" counter on the old one. 6035af50993SBenjamin Herrenschmidt * This pending counter will be used to decrement the old 6045af50993SBenjamin Herrenschmidt * target's count when its queue has been observed empty. 6055af50993SBenjamin Herrenschmidt */ 6065af50993SBenjamin Herrenschmidt 6075af50993SBenjamin Herrenschmidt int kvmppc_xive_set_xive(struct kvm *kvm, u32 irq, u32 server, 6085af50993SBenjamin Herrenschmidt u32 priority) 6095af50993SBenjamin Herrenschmidt { 6105af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = kvm->arch.xive; 6115af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb; 6125af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state; 6135af50993SBenjamin Herrenschmidt u8 new_act_prio; 6145af50993SBenjamin Herrenschmidt int rc = 0; 6155af50993SBenjamin Herrenschmidt u16 idx; 6165af50993SBenjamin Herrenschmidt 6175af50993SBenjamin Herrenschmidt if (!xive) 6185af50993SBenjamin Herrenschmidt return -ENODEV; 6195af50993SBenjamin Herrenschmidt 6205af50993SBenjamin Herrenschmidt pr_devel("set_xive ! irq 0x%x server 0x%x prio %d\n", 6215af50993SBenjamin Herrenschmidt irq, server, priority); 6225af50993SBenjamin Herrenschmidt 6235af50993SBenjamin Herrenschmidt /* First, check provisioning of queues */ 6247e10b9a6SCédric Le Goater if (priority != MASKED) { 6257e10b9a6SCédric Le Goater mutex_lock(&xive->lock); 6265af50993SBenjamin Herrenschmidt rc = xive_check_provisioning(xive->kvm, 6275af50993SBenjamin Herrenschmidt xive_prio_from_guest(priority)); 6287e10b9a6SCédric Le Goater mutex_unlock(&xive->lock); 6297e10b9a6SCédric Le Goater } 6305af50993SBenjamin Herrenschmidt if (rc) { 6315af50993SBenjamin Herrenschmidt pr_devel(" provisioning failure %d !\n", rc); 6325af50993SBenjamin Herrenschmidt return rc; 6335af50993SBenjamin Herrenschmidt } 6345af50993SBenjamin Herrenschmidt 6355af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, irq, &idx); 6365af50993SBenjamin Herrenschmidt if (!sb) 6375af50993SBenjamin Herrenschmidt return -EINVAL; 6385af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx]; 6395af50993SBenjamin Herrenschmidt 6405af50993SBenjamin Herrenschmidt /* 6415af50993SBenjamin Herrenschmidt * We first handle masking/unmasking since the locking 6425af50993SBenjamin Herrenschmidt * might need to be retried due to EOIs, we'll handle 6435af50993SBenjamin Herrenschmidt * targetting changes later. These functions will return 6445af50993SBenjamin Herrenschmidt * with the SB lock held. 6455af50993SBenjamin Herrenschmidt * 6465af50993SBenjamin Herrenschmidt * xive_lock_and_mask() will also set state->guest_priority 6475af50993SBenjamin Herrenschmidt * but won't otherwise change other fields of the state. 6485af50993SBenjamin Herrenschmidt * 6495af50993SBenjamin Herrenschmidt * xive_lock_for_unmask will not actually unmask, this will 6505af50993SBenjamin Herrenschmidt * be done later by xive_finish_unmask() once the targetting 6515af50993SBenjamin Herrenschmidt * has been done, so we don't try to unmask an interrupt 6525af50993SBenjamin Herrenschmidt * that hasn't yet been targetted. 6535af50993SBenjamin Herrenschmidt */ 6545af50993SBenjamin Herrenschmidt if (priority == MASKED) 6555af50993SBenjamin Herrenschmidt xive_lock_and_mask(xive, sb, state); 6565af50993SBenjamin Herrenschmidt else 6575af50993SBenjamin Herrenschmidt xive_lock_for_unmask(sb, state); 6585af50993SBenjamin Herrenschmidt 6595af50993SBenjamin Herrenschmidt 6605af50993SBenjamin Herrenschmidt /* 6615af50993SBenjamin Herrenschmidt * Then we handle targetting. 6625af50993SBenjamin Herrenschmidt * 6635af50993SBenjamin Herrenschmidt * First calculate a new "actual priority" 6645af50993SBenjamin Herrenschmidt */ 6655af50993SBenjamin Herrenschmidt new_act_prio = state->act_priority; 6665af50993SBenjamin Herrenschmidt if (priority != MASKED) 6675af50993SBenjamin Herrenschmidt new_act_prio = xive_prio_from_guest(priority); 6685af50993SBenjamin Herrenschmidt 6695af50993SBenjamin Herrenschmidt pr_devel(" new_act_prio=%x act_server=%x act_prio=%x\n", 6705af50993SBenjamin Herrenschmidt new_act_prio, state->act_server, state->act_priority); 6715af50993SBenjamin Herrenschmidt 6725af50993SBenjamin Herrenschmidt /* 6735af50993SBenjamin Herrenschmidt * Then check if we actually need to change anything, 6745af50993SBenjamin Herrenschmidt * 6755af50993SBenjamin Herrenschmidt * The condition for re-targetting the interrupt is that 6765af50993SBenjamin Herrenschmidt * we have a valid new priority (new_act_prio is not 0xff) 6775af50993SBenjamin Herrenschmidt * and either the server or the priority changed. 6785af50993SBenjamin Herrenschmidt * 6795af50993SBenjamin Herrenschmidt * Note: If act_priority was ff and the new priority is 6805af50993SBenjamin Herrenschmidt * also ff, we don't do anything and leave the interrupt 6815af50993SBenjamin Herrenschmidt * untargetted. An attempt of doing an int_on on an 6825af50993SBenjamin Herrenschmidt * untargetted interrupt will fail. If that is a problem 6835af50993SBenjamin Herrenschmidt * we could initialize interrupts with valid default 6845af50993SBenjamin Herrenschmidt */ 6855af50993SBenjamin Herrenschmidt 6865af50993SBenjamin Herrenschmidt if (new_act_prio != MASKED && 6875af50993SBenjamin Herrenschmidt (state->act_server != server || 6885af50993SBenjamin Herrenschmidt state->act_priority != new_act_prio)) 6895af50993SBenjamin Herrenschmidt rc = xive_target_interrupt(kvm, state, server, new_act_prio); 6905af50993SBenjamin Herrenschmidt 6915af50993SBenjamin Herrenschmidt /* 6925af50993SBenjamin Herrenschmidt * Perform the final unmasking of the interrupt source 6935af50993SBenjamin Herrenschmidt * if necessary 6945af50993SBenjamin Herrenschmidt */ 6955af50993SBenjamin Herrenschmidt if (priority != MASKED) 6965af50993SBenjamin Herrenschmidt xive_finish_unmask(xive, sb, state, priority); 6975af50993SBenjamin Herrenschmidt 6985af50993SBenjamin Herrenschmidt /* 6995af50993SBenjamin Herrenschmidt * Finally Update saved_priority to match. Only int_on/off 7005af50993SBenjamin Herrenschmidt * set this field to a different value. 7015af50993SBenjamin Herrenschmidt */ 7025af50993SBenjamin Herrenschmidt state->saved_priority = priority; 7035af50993SBenjamin Herrenschmidt 7045af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock); 7055af50993SBenjamin Herrenschmidt return rc; 7065af50993SBenjamin Herrenschmidt } 7075af50993SBenjamin Herrenschmidt 7085af50993SBenjamin Herrenschmidt int kvmppc_xive_get_xive(struct kvm *kvm, u32 irq, u32 *server, 7095af50993SBenjamin Herrenschmidt u32 *priority) 7105af50993SBenjamin Herrenschmidt { 7115af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = kvm->arch.xive; 7125af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb; 7135af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state; 7145af50993SBenjamin Herrenschmidt u16 idx; 7155af50993SBenjamin Herrenschmidt 7165af50993SBenjamin Herrenschmidt if (!xive) 7175af50993SBenjamin Herrenschmidt return -ENODEV; 7185af50993SBenjamin Herrenschmidt 7195af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, irq, &idx); 7205af50993SBenjamin Herrenschmidt if (!sb) 7215af50993SBenjamin Herrenschmidt return -EINVAL; 7225af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx]; 7235af50993SBenjamin Herrenschmidt arch_spin_lock(&sb->lock); 7242fb1e946SSam Bobroff *server = state->act_server; 7255af50993SBenjamin Herrenschmidt *priority = state->guest_priority; 7265af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock); 7275af50993SBenjamin Herrenschmidt 7285af50993SBenjamin Herrenschmidt return 0; 7295af50993SBenjamin Herrenschmidt } 7305af50993SBenjamin Herrenschmidt 7315af50993SBenjamin Herrenschmidt int kvmppc_xive_int_on(struct kvm *kvm, u32 irq) 7325af50993SBenjamin Herrenschmidt { 7335af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = kvm->arch.xive; 7345af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb; 7355af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state; 7365af50993SBenjamin Herrenschmidt u16 idx; 7375af50993SBenjamin Herrenschmidt 7385af50993SBenjamin Herrenschmidt if (!xive) 7395af50993SBenjamin Herrenschmidt return -ENODEV; 7405af50993SBenjamin Herrenschmidt 7415af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, irq, &idx); 7425af50993SBenjamin Herrenschmidt if (!sb) 7435af50993SBenjamin Herrenschmidt return -EINVAL; 7445af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx]; 7455af50993SBenjamin Herrenschmidt 7465af50993SBenjamin Herrenschmidt pr_devel("int_on(irq=0x%x)\n", irq); 7475af50993SBenjamin Herrenschmidt 7485af50993SBenjamin Herrenschmidt /* 7495af50993SBenjamin Herrenschmidt * Check if interrupt was not targetted 7505af50993SBenjamin Herrenschmidt */ 7515af50993SBenjamin Herrenschmidt if (state->act_priority == MASKED) { 7525af50993SBenjamin Herrenschmidt pr_devel("int_on on untargetted interrupt\n"); 7535af50993SBenjamin Herrenschmidt return -EINVAL; 7545af50993SBenjamin Herrenschmidt } 7555af50993SBenjamin Herrenschmidt 7565af50993SBenjamin Herrenschmidt /* If saved_priority is 0xff, do nothing */ 7575af50993SBenjamin Herrenschmidt if (state->saved_priority == MASKED) 7585af50993SBenjamin Herrenschmidt return 0; 7595af50993SBenjamin Herrenschmidt 7605af50993SBenjamin Herrenschmidt /* 7615af50993SBenjamin Herrenschmidt * Lock and unmask it. 7625af50993SBenjamin Herrenschmidt */ 7635af50993SBenjamin Herrenschmidt xive_lock_for_unmask(sb, state); 7645af50993SBenjamin Herrenschmidt xive_finish_unmask(xive, sb, state, state->saved_priority); 7655af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock); 7665af50993SBenjamin Herrenschmidt 7675af50993SBenjamin Herrenschmidt return 0; 7685af50993SBenjamin Herrenschmidt } 7695af50993SBenjamin Herrenschmidt 7705af50993SBenjamin Herrenschmidt int kvmppc_xive_int_off(struct kvm *kvm, u32 irq) 7715af50993SBenjamin Herrenschmidt { 7725af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = kvm->arch.xive; 7735af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb; 7745af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state; 7755af50993SBenjamin Herrenschmidt u16 idx; 7765af50993SBenjamin Herrenschmidt 7775af50993SBenjamin Herrenschmidt if (!xive) 7785af50993SBenjamin Herrenschmidt return -ENODEV; 7795af50993SBenjamin Herrenschmidt 7805af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, irq, &idx); 7815af50993SBenjamin Herrenschmidt if (!sb) 7825af50993SBenjamin Herrenschmidt return -EINVAL; 7835af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx]; 7845af50993SBenjamin Herrenschmidt 7855af50993SBenjamin Herrenschmidt pr_devel("int_off(irq=0x%x)\n", irq); 7865af50993SBenjamin Herrenschmidt 7875af50993SBenjamin Herrenschmidt /* 7885af50993SBenjamin Herrenschmidt * Lock and mask 7895af50993SBenjamin Herrenschmidt */ 7905af50993SBenjamin Herrenschmidt state->saved_priority = xive_lock_and_mask(xive, sb, state); 7915af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock); 7925af50993SBenjamin Herrenschmidt 7935af50993SBenjamin Herrenschmidt return 0; 7945af50993SBenjamin Herrenschmidt } 7955af50993SBenjamin Herrenschmidt 7965af50993SBenjamin Herrenschmidt static bool xive_restore_pending_irq(struct kvmppc_xive *xive, u32 irq) 7975af50993SBenjamin Herrenschmidt { 7985af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb; 7995af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state; 8005af50993SBenjamin Herrenschmidt u16 idx; 8015af50993SBenjamin Herrenschmidt 8025af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, irq, &idx); 8035af50993SBenjamin Herrenschmidt if (!sb) 8045af50993SBenjamin Herrenschmidt return false; 8055af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx]; 8065af50993SBenjamin Herrenschmidt if (!state->valid) 8075af50993SBenjamin Herrenschmidt return false; 8085af50993SBenjamin Herrenschmidt 8095af50993SBenjamin Herrenschmidt /* 8105af50993SBenjamin Herrenschmidt * Trigger the IPI. This assumes we never restore a pass-through 8115af50993SBenjamin Herrenschmidt * interrupt which should be safe enough 8125af50993SBenjamin Herrenschmidt */ 8135af50993SBenjamin Herrenschmidt xive_irq_trigger(&state->ipi_data); 8145af50993SBenjamin Herrenschmidt 8155af50993SBenjamin Herrenschmidt return true; 8165af50993SBenjamin Herrenschmidt } 8175af50993SBenjamin Herrenschmidt 8185af50993SBenjamin Herrenschmidt u64 kvmppc_xive_get_icp(struct kvm_vcpu *vcpu) 8195af50993SBenjamin Herrenschmidt { 8205af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 8215af50993SBenjamin Herrenschmidt 8225af50993SBenjamin Herrenschmidt if (!xc) 8235af50993SBenjamin Herrenschmidt return 0; 8245af50993SBenjamin Herrenschmidt 8255af50993SBenjamin Herrenschmidt /* Return the per-cpu state for state saving/migration */ 8265af50993SBenjamin Herrenschmidt return (u64)xc->cppr << KVM_REG_PPC_ICP_CPPR_SHIFT | 8277333b5acSLaurent Vivier (u64)xc->mfrr << KVM_REG_PPC_ICP_MFRR_SHIFT | 8287333b5acSLaurent Vivier (u64)0xff << KVM_REG_PPC_ICP_PPRI_SHIFT; 8295af50993SBenjamin Herrenschmidt } 8305af50993SBenjamin Herrenschmidt 8315af50993SBenjamin Herrenschmidt int kvmppc_xive_set_icp(struct kvm_vcpu *vcpu, u64 icpval) 8325af50993SBenjamin Herrenschmidt { 8335af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 8345af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = vcpu->kvm->arch.xive; 8355af50993SBenjamin Herrenschmidt u8 cppr, mfrr; 8365af50993SBenjamin Herrenschmidt u32 xisr; 8375af50993SBenjamin Herrenschmidt 8385af50993SBenjamin Herrenschmidt if (!xc || !xive) 8395af50993SBenjamin Herrenschmidt return -ENOENT; 8405af50993SBenjamin Herrenschmidt 8415af50993SBenjamin Herrenschmidt /* Grab individual state fields. We don't use pending_pri */ 8425af50993SBenjamin Herrenschmidt cppr = icpval >> KVM_REG_PPC_ICP_CPPR_SHIFT; 8435af50993SBenjamin Herrenschmidt xisr = (icpval >> KVM_REG_PPC_ICP_XISR_SHIFT) & 8445af50993SBenjamin Herrenschmidt KVM_REG_PPC_ICP_XISR_MASK; 8455af50993SBenjamin Herrenschmidt mfrr = icpval >> KVM_REG_PPC_ICP_MFRR_SHIFT; 8465af50993SBenjamin Herrenschmidt 8475af50993SBenjamin Herrenschmidt pr_devel("set_icp vcpu %d cppr=0x%x mfrr=0x%x xisr=0x%x\n", 8485af50993SBenjamin Herrenschmidt xc->server_num, cppr, mfrr, xisr); 8495af50993SBenjamin Herrenschmidt 8505af50993SBenjamin Herrenschmidt /* 8515af50993SBenjamin Herrenschmidt * We can't update the state of a "pushed" VCPU, but that 8526f868405SPaul Mackerras * shouldn't happen because the vcpu->mutex makes running a 8536f868405SPaul Mackerras * vcpu mutually exclusive with doing one_reg get/set on it. 8545af50993SBenjamin Herrenschmidt */ 8555af50993SBenjamin Herrenschmidt if (WARN_ON(vcpu->arch.xive_pushed)) 8565af50993SBenjamin Herrenschmidt return -EIO; 8575af50993SBenjamin Herrenschmidt 8585af50993SBenjamin Herrenschmidt /* Update VCPU HW saved state */ 8595af50993SBenjamin Herrenschmidt vcpu->arch.xive_saved_state.cppr = cppr; 8605af50993SBenjamin Herrenschmidt xc->hw_cppr = xc->cppr = cppr; 8615af50993SBenjamin Herrenschmidt 8625af50993SBenjamin Herrenschmidt /* 8635af50993SBenjamin Herrenschmidt * Update MFRR state. If it's not 0xff, we mark the VCPU as 8645af50993SBenjamin Herrenschmidt * having a pending MFRR change, which will re-evaluate the 8655af50993SBenjamin Herrenschmidt * target. The VCPU will thus potentially get a spurious 8665af50993SBenjamin Herrenschmidt * interrupt but that's not a big deal. 8675af50993SBenjamin Herrenschmidt */ 8685af50993SBenjamin Herrenschmidt xc->mfrr = mfrr; 8695af50993SBenjamin Herrenschmidt if (mfrr < cppr) 8705af50993SBenjamin Herrenschmidt xive_irq_trigger(&xc->vp_ipi_data); 8715af50993SBenjamin Herrenschmidt 8725af50993SBenjamin Herrenschmidt /* 8735af50993SBenjamin Herrenschmidt * Now saved XIRR is "interesting". It means there's something in 8745af50993SBenjamin Herrenschmidt * the legacy "1 element" queue... for an IPI we simply ignore it, 8755af50993SBenjamin Herrenschmidt * as the MFRR restore will handle that. For anything else we need 8765af50993SBenjamin Herrenschmidt * to force a resend of the source. 8775af50993SBenjamin Herrenschmidt * However the source may not have been setup yet. If that's the 8785af50993SBenjamin Herrenschmidt * case, we keep that info and increment a counter in the xive to 8795af50993SBenjamin Herrenschmidt * tell subsequent xive_set_source() to go look. 8805af50993SBenjamin Herrenschmidt */ 8815af50993SBenjamin Herrenschmidt if (xisr > XICS_IPI && !xive_restore_pending_irq(xive, xisr)) { 8825af50993SBenjamin Herrenschmidt xc->delayed_irq = xisr; 8835af50993SBenjamin Herrenschmidt xive->delayed_irqs++; 8845af50993SBenjamin Herrenschmidt pr_devel(" xisr restore delayed\n"); 8855af50993SBenjamin Herrenschmidt } 8865af50993SBenjamin Herrenschmidt 8875af50993SBenjamin Herrenschmidt return 0; 8885af50993SBenjamin Herrenschmidt } 8895af50993SBenjamin Herrenschmidt 8905af50993SBenjamin Herrenschmidt int kvmppc_xive_set_mapped(struct kvm *kvm, unsigned long guest_irq, 8915af50993SBenjamin Herrenschmidt struct irq_desc *host_desc) 8925af50993SBenjamin Herrenschmidt { 8935af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = kvm->arch.xive; 8945af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb; 8955af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state; 8965af50993SBenjamin Herrenschmidt struct irq_data *host_data = irq_desc_get_irq_data(host_desc); 8975af50993SBenjamin Herrenschmidt unsigned int host_irq = irq_desc_get_irq(host_desc); 8985af50993SBenjamin Herrenschmidt unsigned int hw_irq = (unsigned int)irqd_to_hwirq(host_data); 8995af50993SBenjamin Herrenschmidt u16 idx; 9005af50993SBenjamin Herrenschmidt u8 prio; 9015af50993SBenjamin Herrenschmidt int rc; 9025af50993SBenjamin Herrenschmidt 9035af50993SBenjamin Herrenschmidt if (!xive) 9045af50993SBenjamin Herrenschmidt return -ENODEV; 9055af50993SBenjamin Herrenschmidt 9065af50993SBenjamin Herrenschmidt pr_devel("set_mapped girq 0x%lx host HW irq 0x%x...\n",guest_irq, hw_irq); 9075af50993SBenjamin Herrenschmidt 9085af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, guest_irq, &idx); 9095af50993SBenjamin Herrenschmidt if (!sb) 9105af50993SBenjamin Herrenschmidt return -EINVAL; 9115af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx]; 9125af50993SBenjamin Herrenschmidt 9135af50993SBenjamin Herrenschmidt /* 9145af50993SBenjamin Herrenschmidt * Mark the passed-through interrupt as going to a VCPU, 9155af50993SBenjamin Herrenschmidt * this will prevent further EOIs and similar operations 9165af50993SBenjamin Herrenschmidt * from the XIVE code. It will also mask the interrupt 9175af50993SBenjamin Herrenschmidt * to either PQ=10 or 11 state, the latter if the interrupt 9185af50993SBenjamin Herrenschmidt * is pending. This will allow us to unmask or retrigger it 9195af50993SBenjamin Herrenschmidt * after routing it to the guest with a simple EOI. 9205af50993SBenjamin Herrenschmidt * 9215af50993SBenjamin Herrenschmidt * The "state" argument is a "token", all it needs is to be 9225af50993SBenjamin Herrenschmidt * non-NULL to switch to passed-through or NULL for the 9235af50993SBenjamin Herrenschmidt * other way around. We may not yet have an actual VCPU 9245af50993SBenjamin Herrenschmidt * target here and we don't really care. 9255af50993SBenjamin Herrenschmidt */ 9265af50993SBenjamin Herrenschmidt rc = irq_set_vcpu_affinity(host_irq, state); 9275af50993SBenjamin Herrenschmidt if (rc) { 9285af50993SBenjamin Herrenschmidt pr_err("Failed to set VCPU affinity for irq %d\n", host_irq); 9295af50993SBenjamin Herrenschmidt return rc; 9305af50993SBenjamin Herrenschmidt } 9315af50993SBenjamin Herrenschmidt 9325af50993SBenjamin Herrenschmidt /* 9335af50993SBenjamin Herrenschmidt * Mask and read state of IPI. We need to know if its P bit 9345af50993SBenjamin Herrenschmidt * is set as that means it's potentially already using a 9355af50993SBenjamin Herrenschmidt * queue entry in the target 9365af50993SBenjamin Herrenschmidt */ 9375af50993SBenjamin Herrenschmidt prio = xive_lock_and_mask(xive, sb, state); 9385af50993SBenjamin Herrenschmidt pr_devel(" old IPI prio %02x P:%d Q:%d\n", prio, 9395af50993SBenjamin Herrenschmidt state->old_p, state->old_q); 9405af50993SBenjamin Herrenschmidt 9415af50993SBenjamin Herrenschmidt /* Turn the IPI hard off */ 9425af50993SBenjamin Herrenschmidt xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01); 9435af50993SBenjamin Herrenschmidt 944232b984bSCédric Le Goater /* 945232b984bSCédric Le Goater * Reset ESB guest mapping. Needed when ESB pages are exposed 946232b984bSCédric Le Goater * to the guest in XIVE native mode 947232b984bSCédric Le Goater */ 948232b984bSCédric Le Goater if (xive->ops && xive->ops->reset_mapped) 949232b984bSCédric Le Goater xive->ops->reset_mapped(kvm, guest_irq); 950232b984bSCédric Le Goater 9515af50993SBenjamin Herrenschmidt /* Grab info about irq */ 9525af50993SBenjamin Herrenschmidt state->pt_number = hw_irq; 9535af50993SBenjamin Herrenschmidt state->pt_data = irq_data_get_irq_handler_data(host_data); 9545af50993SBenjamin Herrenschmidt 9555af50993SBenjamin Herrenschmidt /* 9565af50993SBenjamin Herrenschmidt * Configure the IRQ to match the existing configuration of 9575af50993SBenjamin Herrenschmidt * the IPI if it was already targetted. Otherwise this will 9585af50993SBenjamin Herrenschmidt * mask the interrupt in a lossy way (act_priority is 0xff) 9595af50993SBenjamin Herrenschmidt * which is fine for a never started interrupt. 9605af50993SBenjamin Herrenschmidt */ 9615af50993SBenjamin Herrenschmidt xive_native_configure_irq(hw_irq, 962eacc56bbSCédric Le Goater kvmppc_xive_vp(xive, state->act_server), 9635af50993SBenjamin Herrenschmidt state->act_priority, state->number); 9645af50993SBenjamin Herrenschmidt 9655af50993SBenjamin Herrenschmidt /* 9665af50993SBenjamin Herrenschmidt * We do an EOI to enable the interrupt (and retrigger if needed) 9675af50993SBenjamin Herrenschmidt * if the guest has the interrupt unmasked and the P bit was *not* 9685af50993SBenjamin Herrenschmidt * set in the IPI. If it was set, we know a slot may still be in 9695af50993SBenjamin Herrenschmidt * use in the target queue thus we have to wait for a guest 9705af50993SBenjamin Herrenschmidt * originated EOI 9715af50993SBenjamin Herrenschmidt */ 9725af50993SBenjamin Herrenschmidt if (prio != MASKED && !state->old_p) 9735af50993SBenjamin Herrenschmidt xive_vm_source_eoi(hw_irq, state->pt_data); 9745af50993SBenjamin Herrenschmidt 9755af50993SBenjamin Herrenschmidt /* Clear old_p/old_q as they are no longer relevant */ 9765af50993SBenjamin Herrenschmidt state->old_p = state->old_q = false; 9775af50993SBenjamin Herrenschmidt 9785af50993SBenjamin Herrenschmidt /* Restore guest prio (unlocks EOI) */ 9795af50993SBenjamin Herrenschmidt mb(); 9805af50993SBenjamin Herrenschmidt state->guest_priority = prio; 9815af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock); 9825af50993SBenjamin Herrenschmidt 9835af50993SBenjamin Herrenschmidt return 0; 9845af50993SBenjamin Herrenschmidt } 9855af50993SBenjamin Herrenschmidt EXPORT_SYMBOL_GPL(kvmppc_xive_set_mapped); 9865af50993SBenjamin Herrenschmidt 9875af50993SBenjamin Herrenschmidt int kvmppc_xive_clr_mapped(struct kvm *kvm, unsigned long guest_irq, 9885af50993SBenjamin Herrenschmidt struct irq_desc *host_desc) 9895af50993SBenjamin Herrenschmidt { 9905af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = kvm->arch.xive; 9915af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb; 9925af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state; 9935af50993SBenjamin Herrenschmidt unsigned int host_irq = irq_desc_get_irq(host_desc); 9945af50993SBenjamin Herrenschmidt u16 idx; 9955af50993SBenjamin Herrenschmidt u8 prio; 9965af50993SBenjamin Herrenschmidt int rc; 9975af50993SBenjamin Herrenschmidt 9985af50993SBenjamin Herrenschmidt if (!xive) 9995af50993SBenjamin Herrenschmidt return -ENODEV; 10005af50993SBenjamin Herrenschmidt 10015af50993SBenjamin Herrenschmidt pr_devel("clr_mapped girq 0x%lx...\n", guest_irq); 10025af50993SBenjamin Herrenschmidt 10035af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, guest_irq, &idx); 10045af50993SBenjamin Herrenschmidt if (!sb) 10055af50993SBenjamin Herrenschmidt return -EINVAL; 10065af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx]; 10075af50993SBenjamin Herrenschmidt 10085af50993SBenjamin Herrenschmidt /* 10095af50993SBenjamin Herrenschmidt * Mask and read state of IRQ. We need to know if its P bit 10105af50993SBenjamin Herrenschmidt * is set as that means it's potentially already using a 10115af50993SBenjamin Herrenschmidt * queue entry in the target 10125af50993SBenjamin Herrenschmidt */ 10135af50993SBenjamin Herrenschmidt prio = xive_lock_and_mask(xive, sb, state); 10145af50993SBenjamin Herrenschmidt pr_devel(" old IRQ prio %02x P:%d Q:%d\n", prio, 10155af50993SBenjamin Herrenschmidt state->old_p, state->old_q); 10165af50993SBenjamin Herrenschmidt 10175af50993SBenjamin Herrenschmidt /* 10185af50993SBenjamin Herrenschmidt * If old_p is set, the interrupt is pending, we switch it to 10195af50993SBenjamin Herrenschmidt * PQ=11. This will force a resend in the host so the interrupt 10205af50993SBenjamin Herrenschmidt * isn't lost to whatver host driver may pick it up 10215af50993SBenjamin Herrenschmidt */ 10225af50993SBenjamin Herrenschmidt if (state->old_p) 10235af50993SBenjamin Herrenschmidt xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_11); 10245af50993SBenjamin Herrenschmidt 10255af50993SBenjamin Herrenschmidt /* Release the passed-through interrupt to the host */ 10265af50993SBenjamin Herrenschmidt rc = irq_set_vcpu_affinity(host_irq, NULL); 10275af50993SBenjamin Herrenschmidt if (rc) { 10285af50993SBenjamin Herrenschmidt pr_err("Failed to clr VCPU affinity for irq %d\n", host_irq); 10295af50993SBenjamin Herrenschmidt return rc; 10305af50993SBenjamin Herrenschmidt } 10315af50993SBenjamin Herrenschmidt 10325af50993SBenjamin Herrenschmidt /* Forget about the IRQ */ 10335af50993SBenjamin Herrenschmidt state->pt_number = 0; 10345af50993SBenjamin Herrenschmidt state->pt_data = NULL; 10355af50993SBenjamin Herrenschmidt 1036232b984bSCédric Le Goater /* 1037232b984bSCédric Le Goater * Reset ESB guest mapping. Needed when ESB pages are exposed 1038232b984bSCédric Le Goater * to the guest in XIVE native mode 1039232b984bSCédric Le Goater */ 1040232b984bSCédric Le Goater if (xive->ops && xive->ops->reset_mapped) { 1041232b984bSCédric Le Goater xive->ops->reset_mapped(kvm, guest_irq); 1042232b984bSCédric Le Goater } 1043232b984bSCédric Le Goater 10445af50993SBenjamin Herrenschmidt /* Reconfigure the IPI */ 10455af50993SBenjamin Herrenschmidt xive_native_configure_irq(state->ipi_number, 1046eacc56bbSCédric Le Goater kvmppc_xive_vp(xive, state->act_server), 10475af50993SBenjamin Herrenschmidt state->act_priority, state->number); 10485af50993SBenjamin Herrenschmidt 10495af50993SBenjamin Herrenschmidt /* 10505af50993SBenjamin Herrenschmidt * If old_p is set (we have a queue entry potentially 10515af50993SBenjamin Herrenschmidt * occupied) or the interrupt is masked, we set the IPI 10525af50993SBenjamin Herrenschmidt * to PQ=10 state. Otherwise we just re-enable it (PQ=00). 10535af50993SBenjamin Herrenschmidt */ 10545af50993SBenjamin Herrenschmidt if (prio == MASKED || state->old_p) 10555af50993SBenjamin Herrenschmidt xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_10); 10565af50993SBenjamin Herrenschmidt else 10575af50993SBenjamin Herrenschmidt xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_00); 10585af50993SBenjamin Herrenschmidt 10595af50993SBenjamin Herrenschmidt /* Restore guest prio (unlocks EOI) */ 10605af50993SBenjamin Herrenschmidt mb(); 10615af50993SBenjamin Herrenschmidt state->guest_priority = prio; 10625af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock); 10635af50993SBenjamin Herrenschmidt 10645af50993SBenjamin Herrenschmidt return 0; 10655af50993SBenjamin Herrenschmidt } 10665af50993SBenjamin Herrenschmidt EXPORT_SYMBOL_GPL(kvmppc_xive_clr_mapped); 10675af50993SBenjamin Herrenschmidt 1068eacc56bbSCédric Le Goater void kvmppc_xive_disable_vcpu_interrupts(struct kvm_vcpu *vcpu) 10695af50993SBenjamin Herrenschmidt { 10705af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 10715af50993SBenjamin Herrenschmidt struct kvm *kvm = vcpu->kvm; 10725af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = kvm->arch.xive; 10735af50993SBenjamin Herrenschmidt int i, j; 10745af50993SBenjamin Herrenschmidt 10755af50993SBenjamin Herrenschmidt for (i = 0; i <= xive->max_sbid; i++) { 10765af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb = xive->src_blocks[i]; 10775af50993SBenjamin Herrenschmidt 10785af50993SBenjamin Herrenschmidt if (!sb) 10795af50993SBenjamin Herrenschmidt continue; 10805af50993SBenjamin Herrenschmidt for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) { 10815af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state = &sb->irq_state[j]; 10825af50993SBenjamin Herrenschmidt 10835af50993SBenjamin Herrenschmidt if (!state->valid) 10845af50993SBenjamin Herrenschmidt continue; 10855af50993SBenjamin Herrenschmidt if (state->act_priority == MASKED) 10865af50993SBenjamin Herrenschmidt continue; 10875af50993SBenjamin Herrenschmidt if (state->act_server != xc->server_num) 10885af50993SBenjamin Herrenschmidt continue; 10895af50993SBenjamin Herrenschmidt 10905af50993SBenjamin Herrenschmidt /* Clean it up */ 10915af50993SBenjamin Herrenschmidt arch_spin_lock(&sb->lock); 10925af50993SBenjamin Herrenschmidt state->act_priority = MASKED; 10935af50993SBenjamin Herrenschmidt xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01); 10945af50993SBenjamin Herrenschmidt xive_native_configure_irq(state->ipi_number, 0, MASKED, 0); 10955af50993SBenjamin Herrenschmidt if (state->pt_number) { 10965af50993SBenjamin Herrenschmidt xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_01); 10975af50993SBenjamin Herrenschmidt xive_native_configure_irq(state->pt_number, 0, MASKED, 0); 10985af50993SBenjamin Herrenschmidt } 10995af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock); 11005af50993SBenjamin Herrenschmidt } 11015af50993SBenjamin Herrenschmidt } 11020caecf5bSPaul Mackerras 11030caecf5bSPaul Mackerras /* Disable vcpu's escalation interrupt */ 11040caecf5bSPaul Mackerras if (vcpu->arch.xive_esc_on) { 11050caecf5bSPaul Mackerras __raw_readq((void __iomem *)(vcpu->arch.xive_esc_vaddr + 11060caecf5bSPaul Mackerras XIVE_ESB_SET_PQ_01)); 11070caecf5bSPaul Mackerras vcpu->arch.xive_esc_on = false; 11080caecf5bSPaul Mackerras } 11090caecf5bSPaul Mackerras 11100caecf5bSPaul Mackerras /* 11110caecf5bSPaul Mackerras * Clear pointers to escalation interrupt ESB. 11120caecf5bSPaul Mackerras * This is safe because the vcpu->mutex is held, preventing 11130caecf5bSPaul Mackerras * any other CPU from concurrently executing a KVM_RUN ioctl. 11140caecf5bSPaul Mackerras */ 11150caecf5bSPaul Mackerras vcpu->arch.xive_esc_vaddr = 0; 11160caecf5bSPaul Mackerras vcpu->arch.xive_esc_raddr = 0; 11175af50993SBenjamin Herrenschmidt } 11185af50993SBenjamin Herrenschmidt 11195af50993SBenjamin Herrenschmidt void kvmppc_xive_cleanup_vcpu(struct kvm_vcpu *vcpu) 11205af50993SBenjamin Herrenschmidt { 11215af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 11225422e951SCédric Le Goater struct kvmppc_xive *xive = vcpu->kvm->arch.xive; 11235af50993SBenjamin Herrenschmidt int i; 11245af50993SBenjamin Herrenschmidt 11255422e951SCédric Le Goater if (!kvmppc_xics_enabled(vcpu)) 11265422e951SCédric Le Goater return; 11275422e951SCédric Le Goater 11285422e951SCédric Le Goater if (!xc) 11295422e951SCédric Le Goater return; 11305422e951SCédric Le Goater 11315af50993SBenjamin Herrenschmidt pr_devel("cleanup_vcpu(cpu=%d)\n", xc->server_num); 11325af50993SBenjamin Herrenschmidt 11335af50993SBenjamin Herrenschmidt /* Ensure no interrupt is still routed to that VP */ 11345af50993SBenjamin Herrenschmidt xc->valid = false; 11355af50993SBenjamin Herrenschmidt kvmppc_xive_disable_vcpu_interrupts(vcpu); 11365af50993SBenjamin Herrenschmidt 11375af50993SBenjamin Herrenschmidt /* Mask the VP IPI */ 11385af50993SBenjamin Herrenschmidt xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_01); 11395af50993SBenjamin Herrenschmidt 11405af50993SBenjamin Herrenschmidt /* Disable the VP */ 11415af50993SBenjamin Herrenschmidt xive_native_disable_vp(xc->vp_id); 11425af50993SBenjamin Herrenschmidt 11435af50993SBenjamin Herrenschmidt /* Free the queues & associated interrupts */ 11445af50993SBenjamin Herrenschmidt for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) { 11455af50993SBenjamin Herrenschmidt struct xive_q *q = &xc->queues[i]; 11465af50993SBenjamin Herrenschmidt 11475af50993SBenjamin Herrenschmidt /* Free the escalation irq */ 11485af50993SBenjamin Herrenschmidt if (xc->esc_virq[i]) { 11495af50993SBenjamin Herrenschmidt free_irq(xc->esc_virq[i], vcpu); 11505af50993SBenjamin Herrenschmidt irq_dispose_mapping(xc->esc_virq[i]); 11515af50993SBenjamin Herrenschmidt kfree(xc->esc_virq_names[i]); 11525af50993SBenjamin Herrenschmidt } 11535af50993SBenjamin Herrenschmidt /* Free the queue */ 11545af50993SBenjamin Herrenschmidt xive_native_disable_queue(xc->vp_id, q, i); 11555af50993SBenjamin Herrenschmidt if (q->qpage) { 11565af50993SBenjamin Herrenschmidt free_pages((unsigned long)q->qpage, 11575af50993SBenjamin Herrenschmidt xive->q_page_order); 11585af50993SBenjamin Herrenschmidt q->qpage = NULL; 11595af50993SBenjamin Herrenschmidt } 11605af50993SBenjamin Herrenschmidt } 11615af50993SBenjamin Herrenschmidt 11625af50993SBenjamin Herrenschmidt /* Free the IPI */ 11635af50993SBenjamin Herrenschmidt if (xc->vp_ipi) { 11645af50993SBenjamin Herrenschmidt xive_cleanup_irq_data(&xc->vp_ipi_data); 11655af50993SBenjamin Herrenschmidt xive_native_free_irq(xc->vp_ipi); 11665af50993SBenjamin Herrenschmidt } 11675af50993SBenjamin Herrenschmidt /* Free the VP */ 11685af50993SBenjamin Herrenschmidt kfree(xc); 11695422e951SCédric Le Goater 11705422e951SCédric Le Goater /* Cleanup the vcpu */ 11715422e951SCédric Le Goater vcpu->arch.irq_type = KVMPPC_IRQ_DEFAULT; 11725422e951SCédric Le Goater vcpu->arch.xive_vcpu = NULL; 11735af50993SBenjamin Herrenschmidt } 11745af50993SBenjamin Herrenschmidt 11755af50993SBenjamin Herrenschmidt int kvmppc_xive_connect_vcpu(struct kvm_device *dev, 11765af50993SBenjamin Herrenschmidt struct kvm_vcpu *vcpu, u32 cpu) 11775af50993SBenjamin Herrenschmidt { 11785af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = dev->private; 11795af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc; 11805af50993SBenjamin Herrenschmidt int i, r = -EBUSY; 11815af50993SBenjamin Herrenschmidt 11825af50993SBenjamin Herrenschmidt pr_devel("connect_vcpu(cpu=%d)\n", cpu); 11835af50993SBenjamin Herrenschmidt 11845af50993SBenjamin Herrenschmidt if (dev->ops != &kvm_xive_ops) { 11855af50993SBenjamin Herrenschmidt pr_devel("Wrong ops !\n"); 11865af50993SBenjamin Herrenschmidt return -EPERM; 11875af50993SBenjamin Herrenschmidt } 11885af50993SBenjamin Herrenschmidt if (xive->kvm != vcpu->kvm) 11895af50993SBenjamin Herrenschmidt return -EPERM; 11905422e951SCédric Le Goater if (vcpu->arch.irq_type != KVMPPC_IRQ_DEFAULT) 11915af50993SBenjamin Herrenschmidt return -EBUSY; 11925af50993SBenjamin Herrenschmidt if (kvmppc_xive_find_server(vcpu->kvm, cpu)) { 11935af50993SBenjamin Herrenschmidt pr_devel("Duplicate !\n"); 11945af50993SBenjamin Herrenschmidt return -EEXIST; 11955af50993SBenjamin Herrenschmidt } 11961e175d2eSSam Bobroff if (cpu >= (KVM_MAX_VCPUS * vcpu->kvm->arch.emul_smt_mode)) { 11975af50993SBenjamin Herrenschmidt pr_devel("Out of bounds !\n"); 11985af50993SBenjamin Herrenschmidt return -EINVAL; 11995af50993SBenjamin Herrenschmidt } 12005af50993SBenjamin Herrenschmidt xc = kzalloc(sizeof(*xc), GFP_KERNEL); 12015af50993SBenjamin Herrenschmidt if (!xc) 12025af50993SBenjamin Herrenschmidt return -ENOMEM; 12035af50993SBenjamin Herrenschmidt 12045af50993SBenjamin Herrenschmidt /* We need to synchronize with queue provisioning */ 12057e10b9a6SCédric Le Goater mutex_lock(&xive->lock); 12065af50993SBenjamin Herrenschmidt vcpu->arch.xive_vcpu = xc; 12075af50993SBenjamin Herrenschmidt xc->xive = xive; 12085af50993SBenjamin Herrenschmidt xc->vcpu = vcpu; 12095af50993SBenjamin Herrenschmidt xc->server_num = cpu; 1210eacc56bbSCédric Le Goater xc->vp_id = kvmppc_xive_vp(xive, cpu); 12115af50993SBenjamin Herrenschmidt xc->mfrr = 0xff; 12125af50993SBenjamin Herrenschmidt xc->valid = true; 12135af50993SBenjamin Herrenschmidt 12145af50993SBenjamin Herrenschmidt r = xive_native_get_vp_info(xc->vp_id, &xc->vp_cam, &xc->vp_chip_id); 12155af50993SBenjamin Herrenschmidt if (r) 12165af50993SBenjamin Herrenschmidt goto bail; 12175af50993SBenjamin Herrenschmidt 12185af50993SBenjamin Herrenschmidt /* Configure VCPU fields for use by assembly push/pull */ 12195af50993SBenjamin Herrenschmidt vcpu->arch.xive_saved_state.w01 = cpu_to_be64(0xff000000); 12205af50993SBenjamin Herrenschmidt vcpu->arch.xive_cam_word = cpu_to_be32(xc->vp_cam | TM_QW1W2_VO); 12215af50993SBenjamin Herrenschmidt 12225af50993SBenjamin Herrenschmidt /* Allocate IPI */ 12235af50993SBenjamin Herrenschmidt xc->vp_ipi = xive_native_alloc_irq(); 12245af50993SBenjamin Herrenschmidt if (!xc->vp_ipi) { 1225bf4159daSBenjamin Herrenschmidt pr_err("Failed to allocate xive irq for VCPU IPI\n"); 12265af50993SBenjamin Herrenschmidt r = -EIO; 12275af50993SBenjamin Herrenschmidt goto bail; 12285af50993SBenjamin Herrenschmidt } 12295af50993SBenjamin Herrenschmidt pr_devel(" IPI=0x%x\n", xc->vp_ipi); 12305af50993SBenjamin Herrenschmidt 12315af50993SBenjamin Herrenschmidt r = xive_native_populate_irq_data(xc->vp_ipi, &xc->vp_ipi_data); 12325af50993SBenjamin Herrenschmidt if (r) 12335af50993SBenjamin Herrenschmidt goto bail; 12345af50993SBenjamin Herrenschmidt 12355af50993SBenjamin Herrenschmidt /* 1236bf4159daSBenjamin Herrenschmidt * Enable the VP first as the single escalation mode will 1237bf4159daSBenjamin Herrenschmidt * affect escalation interrupts numbering 1238bf4159daSBenjamin Herrenschmidt */ 1239bf4159daSBenjamin Herrenschmidt r = xive_native_enable_vp(xc->vp_id, xive->single_escalation); 1240bf4159daSBenjamin Herrenschmidt if (r) { 1241bf4159daSBenjamin Herrenschmidt pr_err("Failed to enable VP in OPAL, err %d\n", r); 1242bf4159daSBenjamin Herrenschmidt goto bail; 1243bf4159daSBenjamin Herrenschmidt } 1244bf4159daSBenjamin Herrenschmidt 1245bf4159daSBenjamin Herrenschmidt /* 12465af50993SBenjamin Herrenschmidt * Initialize queues. Initially we set them all for no queueing 12475af50993SBenjamin Herrenschmidt * and we enable escalation for queue 0 only which we'll use for 12485af50993SBenjamin Herrenschmidt * our mfrr change notifications. If the VCPU is hot-plugged, we 1249bf4159daSBenjamin Herrenschmidt * do handle provisioning however based on the existing "map" 1250bf4159daSBenjamin Herrenschmidt * of enabled queues. 12515af50993SBenjamin Herrenschmidt */ 12525af50993SBenjamin Herrenschmidt for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) { 12535af50993SBenjamin Herrenschmidt struct xive_q *q = &xc->queues[i]; 12545af50993SBenjamin Herrenschmidt 1255bf4159daSBenjamin Herrenschmidt /* Single escalation, no queue 7 */ 1256bf4159daSBenjamin Herrenschmidt if (i == 7 && xive->single_escalation) 1257bf4159daSBenjamin Herrenschmidt break; 1258bf4159daSBenjamin Herrenschmidt 12595af50993SBenjamin Herrenschmidt /* Is queue already enabled ? Provision it */ 12605af50993SBenjamin Herrenschmidt if (xive->qmap & (1 << i)) { 12615af50993SBenjamin Herrenschmidt r = xive_provision_queue(vcpu, i); 1262bf4159daSBenjamin Herrenschmidt if (r == 0 && !xive->single_escalation) 126313ce3297SCédric Le Goater kvmppc_xive_attach_escalation( 126413ce3297SCédric Le Goater vcpu, i, xive->single_escalation); 12655af50993SBenjamin Herrenschmidt if (r) 12665af50993SBenjamin Herrenschmidt goto bail; 12675af50993SBenjamin Herrenschmidt } else { 12685af50993SBenjamin Herrenschmidt r = xive_native_configure_queue(xc->vp_id, 12695af50993SBenjamin Herrenschmidt q, i, NULL, 0, true); 12705af50993SBenjamin Herrenschmidt if (r) { 12715af50993SBenjamin Herrenschmidt pr_err("Failed to configure queue %d for VCPU %d\n", 12725af50993SBenjamin Herrenschmidt i, cpu); 12735af50993SBenjamin Herrenschmidt goto bail; 12745af50993SBenjamin Herrenschmidt } 12755af50993SBenjamin Herrenschmidt } 12765af50993SBenjamin Herrenschmidt } 12775af50993SBenjamin Herrenschmidt 12785af50993SBenjamin Herrenschmidt /* If not done above, attach priority 0 escalation */ 127913ce3297SCédric Le Goater r = kvmppc_xive_attach_escalation(vcpu, 0, xive->single_escalation); 12805af50993SBenjamin Herrenschmidt if (r) 12815af50993SBenjamin Herrenschmidt goto bail; 12825af50993SBenjamin Herrenschmidt 12835af50993SBenjamin Herrenschmidt /* Route the IPI */ 12845af50993SBenjamin Herrenschmidt r = xive_native_configure_irq(xc->vp_ipi, xc->vp_id, 0, XICS_IPI); 12855af50993SBenjamin Herrenschmidt if (!r) 12865af50993SBenjamin Herrenschmidt xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_00); 12875af50993SBenjamin Herrenschmidt 12885af50993SBenjamin Herrenschmidt bail: 12897e10b9a6SCédric Le Goater mutex_unlock(&xive->lock); 12905af50993SBenjamin Herrenschmidt if (r) { 12915af50993SBenjamin Herrenschmidt kvmppc_xive_cleanup_vcpu(vcpu); 12925af50993SBenjamin Herrenschmidt return r; 12935af50993SBenjamin Herrenschmidt } 12945af50993SBenjamin Herrenschmidt 12955af50993SBenjamin Herrenschmidt vcpu->arch.irq_type = KVMPPC_IRQ_XICS; 12965af50993SBenjamin Herrenschmidt return 0; 12975af50993SBenjamin Herrenschmidt } 12985af50993SBenjamin Herrenschmidt 12995af50993SBenjamin Herrenschmidt /* 13005af50993SBenjamin Herrenschmidt * Scanning of queues before/after migration save 13015af50993SBenjamin Herrenschmidt */ 13025af50993SBenjamin Herrenschmidt static void xive_pre_save_set_queued(struct kvmppc_xive *xive, u32 irq) 13035af50993SBenjamin Herrenschmidt { 13045af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb; 13055af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state; 13065af50993SBenjamin Herrenschmidt u16 idx; 13075af50993SBenjamin Herrenschmidt 13085af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, irq, &idx); 13095af50993SBenjamin Herrenschmidt if (!sb) 13105af50993SBenjamin Herrenschmidt return; 13115af50993SBenjamin Herrenschmidt 13125af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx]; 13135af50993SBenjamin Herrenschmidt 13145af50993SBenjamin Herrenschmidt /* Some sanity checking */ 13155af50993SBenjamin Herrenschmidt if (!state->valid) { 13165af50993SBenjamin Herrenschmidt pr_err("invalid irq 0x%x in cpu queue!\n", irq); 13175af50993SBenjamin Herrenschmidt return; 13185af50993SBenjamin Herrenschmidt } 13195af50993SBenjamin Herrenschmidt 13205af50993SBenjamin Herrenschmidt /* 13215af50993SBenjamin Herrenschmidt * If the interrupt is in a queue it should have P set. 13225af50993SBenjamin Herrenschmidt * We warn so that gets reported. A backtrace isn't useful 13235af50993SBenjamin Herrenschmidt * so no need to use a WARN_ON. 13245af50993SBenjamin Herrenschmidt */ 13255af50993SBenjamin Herrenschmidt if (!state->saved_p) 13265af50993SBenjamin Herrenschmidt pr_err("Interrupt 0x%x is marked in a queue but P not set !\n", irq); 13275af50993SBenjamin Herrenschmidt 13285af50993SBenjamin Herrenschmidt /* Set flag */ 13295af50993SBenjamin Herrenschmidt state->in_queue = true; 13305af50993SBenjamin Herrenschmidt } 13315af50993SBenjamin Herrenschmidt 13325af50993SBenjamin Herrenschmidt static void xive_pre_save_mask_irq(struct kvmppc_xive *xive, 13335af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb, 13345af50993SBenjamin Herrenschmidt u32 irq) 13355af50993SBenjamin Herrenschmidt { 13365af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state = &sb->irq_state[irq]; 13375af50993SBenjamin Herrenschmidt 13385af50993SBenjamin Herrenschmidt if (!state->valid) 13395af50993SBenjamin Herrenschmidt return; 13405af50993SBenjamin Herrenschmidt 13415af50993SBenjamin Herrenschmidt /* Mask and save state, this will also sync HW queues */ 13425af50993SBenjamin Herrenschmidt state->saved_scan_prio = xive_lock_and_mask(xive, sb, state); 13435af50993SBenjamin Herrenschmidt 13445af50993SBenjamin Herrenschmidt /* Transfer P and Q */ 13455af50993SBenjamin Herrenschmidt state->saved_p = state->old_p; 13465af50993SBenjamin Herrenschmidt state->saved_q = state->old_q; 13475af50993SBenjamin Herrenschmidt 13485af50993SBenjamin Herrenschmidt /* Unlock */ 13495af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock); 13505af50993SBenjamin Herrenschmidt } 13515af50993SBenjamin Herrenschmidt 13525af50993SBenjamin Herrenschmidt static void xive_pre_save_unmask_irq(struct kvmppc_xive *xive, 13535af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb, 13545af50993SBenjamin Herrenschmidt u32 irq) 13555af50993SBenjamin Herrenschmidt { 13565af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state = &sb->irq_state[irq]; 13575af50993SBenjamin Herrenschmidt 13585af50993SBenjamin Herrenschmidt if (!state->valid) 13595af50993SBenjamin Herrenschmidt return; 13605af50993SBenjamin Herrenschmidt 13615af50993SBenjamin Herrenschmidt /* 13625af50993SBenjamin Herrenschmidt * Lock / exclude EOI (not technically necessary if the 13635af50993SBenjamin Herrenschmidt * guest isn't running concurrently. If this becomes a 13645af50993SBenjamin Herrenschmidt * performance issue we can probably remove the lock. 13655af50993SBenjamin Herrenschmidt */ 13665af50993SBenjamin Herrenschmidt xive_lock_for_unmask(sb, state); 13675af50993SBenjamin Herrenschmidt 13685af50993SBenjamin Herrenschmidt /* Restore mask/prio if it wasn't masked */ 13695af50993SBenjamin Herrenschmidt if (state->saved_scan_prio != MASKED) 13705af50993SBenjamin Herrenschmidt xive_finish_unmask(xive, sb, state, state->saved_scan_prio); 13715af50993SBenjamin Herrenschmidt 13725af50993SBenjamin Herrenschmidt /* Unlock */ 13735af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock); 13745af50993SBenjamin Herrenschmidt } 13755af50993SBenjamin Herrenschmidt 13765af50993SBenjamin Herrenschmidt static void xive_pre_save_queue(struct kvmppc_xive *xive, struct xive_q *q) 13775af50993SBenjamin Herrenschmidt { 13785af50993SBenjamin Herrenschmidt u32 idx = q->idx; 13795af50993SBenjamin Herrenschmidt u32 toggle = q->toggle; 13805af50993SBenjamin Herrenschmidt u32 irq; 13815af50993SBenjamin Herrenschmidt 13825af50993SBenjamin Herrenschmidt do { 13835af50993SBenjamin Herrenschmidt irq = __xive_read_eq(q->qpage, q->msk, &idx, &toggle); 13845af50993SBenjamin Herrenschmidt if (irq > XICS_IPI) 13855af50993SBenjamin Herrenschmidt xive_pre_save_set_queued(xive, irq); 13865af50993SBenjamin Herrenschmidt } while(irq); 13875af50993SBenjamin Herrenschmidt } 13885af50993SBenjamin Herrenschmidt 13895af50993SBenjamin Herrenschmidt static void xive_pre_save_scan(struct kvmppc_xive *xive) 13905af50993SBenjamin Herrenschmidt { 13915af50993SBenjamin Herrenschmidt struct kvm_vcpu *vcpu = NULL; 13925af50993SBenjamin Herrenschmidt int i, j; 13935af50993SBenjamin Herrenschmidt 13945af50993SBenjamin Herrenschmidt /* 13955af50993SBenjamin Herrenschmidt * See comment in xive_get_source() about how this 13965af50993SBenjamin Herrenschmidt * work. Collect a stable state for all interrupts 13975af50993SBenjamin Herrenschmidt */ 13985af50993SBenjamin Herrenschmidt for (i = 0; i <= xive->max_sbid; i++) { 13995af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb = xive->src_blocks[i]; 14005af50993SBenjamin Herrenschmidt if (!sb) 14015af50993SBenjamin Herrenschmidt continue; 14025af50993SBenjamin Herrenschmidt for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) 14035af50993SBenjamin Herrenschmidt xive_pre_save_mask_irq(xive, sb, j); 14045af50993SBenjamin Herrenschmidt } 14055af50993SBenjamin Herrenschmidt 14065af50993SBenjamin Herrenschmidt /* Then scan the queues and update the "in_queue" flag */ 14075af50993SBenjamin Herrenschmidt kvm_for_each_vcpu(i, vcpu, xive->kvm) { 14085af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 14095af50993SBenjamin Herrenschmidt if (!xc) 14105af50993SBenjamin Herrenschmidt continue; 14115af50993SBenjamin Herrenschmidt for (j = 0; j < KVMPPC_XIVE_Q_COUNT; j++) { 141200c14757SPaul Mackerras if (xc->queues[j].qpage) 141300c14757SPaul Mackerras xive_pre_save_queue(xive, &xc->queues[j]); 14145af50993SBenjamin Herrenschmidt } 14155af50993SBenjamin Herrenschmidt } 14165af50993SBenjamin Herrenschmidt 14175af50993SBenjamin Herrenschmidt /* Finally restore interrupt states */ 14185af50993SBenjamin Herrenschmidt for (i = 0; i <= xive->max_sbid; i++) { 14195af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb = xive->src_blocks[i]; 14205af50993SBenjamin Herrenschmidt if (!sb) 14215af50993SBenjamin Herrenschmidt continue; 14225af50993SBenjamin Herrenschmidt for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) 14235af50993SBenjamin Herrenschmidt xive_pre_save_unmask_irq(xive, sb, j); 14245af50993SBenjamin Herrenschmidt } 14255af50993SBenjamin Herrenschmidt } 14265af50993SBenjamin Herrenschmidt 14275af50993SBenjamin Herrenschmidt static void xive_post_save_scan(struct kvmppc_xive *xive) 14285af50993SBenjamin Herrenschmidt { 14295af50993SBenjamin Herrenschmidt u32 i, j; 14305af50993SBenjamin Herrenschmidt 14315af50993SBenjamin Herrenschmidt /* Clear all the in_queue flags */ 14325af50993SBenjamin Herrenschmidt for (i = 0; i <= xive->max_sbid; i++) { 14335af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb = xive->src_blocks[i]; 14345af50993SBenjamin Herrenschmidt if (!sb) 14355af50993SBenjamin Herrenschmidt continue; 14365af50993SBenjamin Herrenschmidt for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) 14375af50993SBenjamin Herrenschmidt sb->irq_state[j].in_queue = false; 14385af50993SBenjamin Herrenschmidt } 14395af50993SBenjamin Herrenschmidt 14405af50993SBenjamin Herrenschmidt /* Next get_source() will do a new scan */ 14415af50993SBenjamin Herrenschmidt xive->saved_src_count = 0; 14425af50993SBenjamin Herrenschmidt } 14435af50993SBenjamin Herrenschmidt 14445af50993SBenjamin Herrenschmidt /* 14455af50993SBenjamin Herrenschmidt * This returns the source configuration and state to user space. 14465af50993SBenjamin Herrenschmidt */ 14475af50993SBenjamin Herrenschmidt static int xive_get_source(struct kvmppc_xive *xive, long irq, u64 addr) 14485af50993SBenjamin Herrenschmidt { 14495af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb; 14505af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state; 14515af50993SBenjamin Herrenschmidt u64 __user *ubufp = (u64 __user *) addr; 14525af50993SBenjamin Herrenschmidt u64 val, prio; 14535af50993SBenjamin Herrenschmidt u16 idx; 14545af50993SBenjamin Herrenschmidt 14555af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, irq, &idx); 14565af50993SBenjamin Herrenschmidt if (!sb) 14575af50993SBenjamin Herrenschmidt return -ENOENT; 14585af50993SBenjamin Herrenschmidt 14595af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx]; 14605af50993SBenjamin Herrenschmidt 14615af50993SBenjamin Herrenschmidt if (!state->valid) 14625af50993SBenjamin Herrenschmidt return -ENOENT; 14635af50993SBenjamin Herrenschmidt 14645af50993SBenjamin Herrenschmidt pr_devel("get_source(%ld)...\n", irq); 14655af50993SBenjamin Herrenschmidt 14665af50993SBenjamin Herrenschmidt /* 14675af50993SBenjamin Herrenschmidt * So to properly save the state into something that looks like a 14685af50993SBenjamin Herrenschmidt * XICS migration stream we cannot treat interrupts individually. 14695af50993SBenjamin Herrenschmidt * 14705af50993SBenjamin Herrenschmidt * We need, instead, mask them all (& save their previous PQ state) 14715af50993SBenjamin Herrenschmidt * to get a stable state in the HW, then sync them to ensure that 14725af50993SBenjamin Herrenschmidt * any interrupt that had already fired hits its queue, and finally 14735af50993SBenjamin Herrenschmidt * scan all the queues to collect which interrupts are still present 14745af50993SBenjamin Herrenschmidt * in the queues, so we can set the "pending" flag on them and 14755af50993SBenjamin Herrenschmidt * they can be resent on restore. 14765af50993SBenjamin Herrenschmidt * 14775af50993SBenjamin Herrenschmidt * So we do it all when the "first" interrupt gets saved, all the 14785af50993SBenjamin Herrenschmidt * state is collected at that point, the rest of xive_get_source() 14795af50993SBenjamin Herrenschmidt * will merely collect and convert that state to the expected 14805af50993SBenjamin Herrenschmidt * userspace bit mask. 14815af50993SBenjamin Herrenschmidt */ 14825af50993SBenjamin Herrenschmidt if (xive->saved_src_count == 0) 14835af50993SBenjamin Herrenschmidt xive_pre_save_scan(xive); 14845af50993SBenjamin Herrenschmidt xive->saved_src_count++; 14855af50993SBenjamin Herrenschmidt 14865af50993SBenjamin Herrenschmidt /* Convert saved state into something compatible with xics */ 14872fb1e946SSam Bobroff val = state->act_server; 14885af50993SBenjamin Herrenschmidt prio = state->saved_scan_prio; 14895af50993SBenjamin Herrenschmidt 14905af50993SBenjamin Herrenschmidt if (prio == MASKED) { 14915af50993SBenjamin Herrenschmidt val |= KVM_XICS_MASKED; 14925af50993SBenjamin Herrenschmidt prio = state->saved_priority; 14935af50993SBenjamin Herrenschmidt } 14945af50993SBenjamin Herrenschmidt val |= prio << KVM_XICS_PRIORITY_SHIFT; 14955af50993SBenjamin Herrenschmidt if (state->lsi) { 14965af50993SBenjamin Herrenschmidt val |= KVM_XICS_LEVEL_SENSITIVE; 14975af50993SBenjamin Herrenschmidt if (state->saved_p) 14985af50993SBenjamin Herrenschmidt val |= KVM_XICS_PENDING; 14995af50993SBenjamin Herrenschmidt } else { 15005af50993SBenjamin Herrenschmidt if (state->saved_p) 15015af50993SBenjamin Herrenschmidt val |= KVM_XICS_PRESENTED; 15025af50993SBenjamin Herrenschmidt 15035af50993SBenjamin Herrenschmidt if (state->saved_q) 15045af50993SBenjamin Herrenschmidt val |= KVM_XICS_QUEUED; 15055af50993SBenjamin Herrenschmidt 15065af50993SBenjamin Herrenschmidt /* 15075af50993SBenjamin Herrenschmidt * We mark it pending (which will attempt a re-delivery) 15085af50993SBenjamin Herrenschmidt * if we are in a queue *or* we were masked and had 15095af50993SBenjamin Herrenschmidt * Q set which is equivalent to the XICS "masked pending" 15105af50993SBenjamin Herrenschmidt * state 15115af50993SBenjamin Herrenschmidt */ 15125af50993SBenjamin Herrenschmidt if (state->in_queue || (prio == MASKED && state->saved_q)) 15135af50993SBenjamin Herrenschmidt val |= KVM_XICS_PENDING; 15145af50993SBenjamin Herrenschmidt } 15155af50993SBenjamin Herrenschmidt 15165af50993SBenjamin Herrenschmidt /* 15175af50993SBenjamin Herrenschmidt * If that was the last interrupt saved, reset the 15185af50993SBenjamin Herrenschmidt * in_queue flags 15195af50993SBenjamin Herrenschmidt */ 15205af50993SBenjamin Herrenschmidt if (xive->saved_src_count == xive->src_count) 15215af50993SBenjamin Herrenschmidt xive_post_save_scan(xive); 15225af50993SBenjamin Herrenschmidt 15235af50993SBenjamin Herrenschmidt /* Copy the result to userspace */ 15245af50993SBenjamin Herrenschmidt if (put_user(val, ubufp)) 15255af50993SBenjamin Herrenschmidt return -EFAULT; 15265af50993SBenjamin Herrenschmidt 15275af50993SBenjamin Herrenschmidt return 0; 15285af50993SBenjamin Herrenschmidt } 15295af50993SBenjamin Herrenschmidt 15304131f83cSCédric Le Goater struct kvmppc_xive_src_block *kvmppc_xive_create_src_block( 15314131f83cSCédric Le Goater struct kvmppc_xive *xive, int irq) 15325af50993SBenjamin Herrenschmidt { 15335af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb; 15345af50993SBenjamin Herrenschmidt int i, bid; 15355af50993SBenjamin Herrenschmidt 15365af50993SBenjamin Herrenschmidt bid = irq >> KVMPPC_XICS_ICS_SHIFT; 15375af50993SBenjamin Herrenschmidt 15387e10b9a6SCédric Le Goater mutex_lock(&xive->lock); 15395af50993SBenjamin Herrenschmidt 15405af50993SBenjamin Herrenschmidt /* block already exists - somebody else got here first */ 15415af50993SBenjamin Herrenschmidt if (xive->src_blocks[bid]) 15425af50993SBenjamin Herrenschmidt goto out; 15435af50993SBenjamin Herrenschmidt 15445af50993SBenjamin Herrenschmidt /* Create the ICS */ 15455af50993SBenjamin Herrenschmidt sb = kzalloc(sizeof(*sb), GFP_KERNEL); 15465af50993SBenjamin Herrenschmidt if (!sb) 15475af50993SBenjamin Herrenschmidt goto out; 15485af50993SBenjamin Herrenschmidt 15495af50993SBenjamin Herrenschmidt sb->id = bid; 15505af50993SBenjamin Herrenschmidt 15515af50993SBenjamin Herrenschmidt for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) { 15525af50993SBenjamin Herrenschmidt sb->irq_state[i].number = (bid << KVMPPC_XICS_ICS_SHIFT) | i; 1553e8676ce5SCédric Le Goater sb->irq_state[i].eisn = 0; 15545af50993SBenjamin Herrenschmidt sb->irq_state[i].guest_priority = MASKED; 15555af50993SBenjamin Herrenschmidt sb->irq_state[i].saved_priority = MASKED; 15565af50993SBenjamin Herrenschmidt sb->irq_state[i].act_priority = MASKED; 15575af50993SBenjamin Herrenschmidt } 15585af50993SBenjamin Herrenschmidt smp_wmb(); 15595af50993SBenjamin Herrenschmidt xive->src_blocks[bid] = sb; 15605af50993SBenjamin Herrenschmidt 15615af50993SBenjamin Herrenschmidt if (bid > xive->max_sbid) 15625af50993SBenjamin Herrenschmidt xive->max_sbid = bid; 15635af50993SBenjamin Herrenschmidt 15645af50993SBenjamin Herrenschmidt out: 15657e10b9a6SCédric Le Goater mutex_unlock(&xive->lock); 15665af50993SBenjamin Herrenschmidt return xive->src_blocks[bid]; 15675af50993SBenjamin Herrenschmidt } 15685af50993SBenjamin Herrenschmidt 15695af50993SBenjamin Herrenschmidt static bool xive_check_delayed_irq(struct kvmppc_xive *xive, u32 irq) 15705af50993SBenjamin Herrenschmidt { 15715af50993SBenjamin Herrenschmidt struct kvm *kvm = xive->kvm; 15725af50993SBenjamin Herrenschmidt struct kvm_vcpu *vcpu = NULL; 15735af50993SBenjamin Herrenschmidt int i; 15745af50993SBenjamin Herrenschmidt 15755af50993SBenjamin Herrenschmidt kvm_for_each_vcpu(i, vcpu, kvm) { 15765af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 15775af50993SBenjamin Herrenschmidt 15785af50993SBenjamin Herrenschmidt if (!xc) 15795af50993SBenjamin Herrenschmidt continue; 15805af50993SBenjamin Herrenschmidt 15815af50993SBenjamin Herrenschmidt if (xc->delayed_irq == irq) { 15825af50993SBenjamin Herrenschmidt xc->delayed_irq = 0; 15835af50993SBenjamin Herrenschmidt xive->delayed_irqs--; 15845af50993SBenjamin Herrenschmidt return true; 15855af50993SBenjamin Herrenschmidt } 15865af50993SBenjamin Herrenschmidt } 15875af50993SBenjamin Herrenschmidt return false; 15885af50993SBenjamin Herrenschmidt } 15895af50993SBenjamin Herrenschmidt 15905af50993SBenjamin Herrenschmidt static int xive_set_source(struct kvmppc_xive *xive, long irq, u64 addr) 15915af50993SBenjamin Herrenschmidt { 15925af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb; 15935af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state; 15945af50993SBenjamin Herrenschmidt u64 __user *ubufp = (u64 __user *) addr; 15955af50993SBenjamin Herrenschmidt u16 idx; 15965af50993SBenjamin Herrenschmidt u64 val; 15975af50993SBenjamin Herrenschmidt u8 act_prio, guest_prio; 15985af50993SBenjamin Herrenschmidt u32 server; 15995af50993SBenjamin Herrenschmidt int rc = 0; 16005af50993SBenjamin Herrenschmidt 16015af50993SBenjamin Herrenschmidt if (irq < KVMPPC_XICS_FIRST_IRQ || irq >= KVMPPC_XICS_NR_IRQS) 16025af50993SBenjamin Herrenschmidt return -ENOENT; 16035af50993SBenjamin Herrenschmidt 16045af50993SBenjamin Herrenschmidt pr_devel("set_source(irq=0x%lx)\n", irq); 16055af50993SBenjamin Herrenschmidt 16065af50993SBenjamin Herrenschmidt /* Find the source */ 16075af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, irq, &idx); 16085af50993SBenjamin Herrenschmidt if (!sb) { 16095af50993SBenjamin Herrenschmidt pr_devel("No source, creating source block...\n"); 16104131f83cSCédric Le Goater sb = kvmppc_xive_create_src_block(xive, irq); 16115af50993SBenjamin Herrenschmidt if (!sb) { 16125af50993SBenjamin Herrenschmidt pr_devel("Failed to create block...\n"); 16135af50993SBenjamin Herrenschmidt return -ENOMEM; 16145af50993SBenjamin Herrenschmidt } 16155af50993SBenjamin Herrenschmidt } 16165af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx]; 16175af50993SBenjamin Herrenschmidt 16185af50993SBenjamin Herrenschmidt /* Read user passed data */ 16195af50993SBenjamin Herrenschmidt if (get_user(val, ubufp)) { 16205af50993SBenjamin Herrenschmidt pr_devel("fault getting user info !\n"); 16215af50993SBenjamin Herrenschmidt return -EFAULT; 16225af50993SBenjamin Herrenschmidt } 16235af50993SBenjamin Herrenschmidt 16245af50993SBenjamin Herrenschmidt server = val & KVM_XICS_DESTINATION_MASK; 16255af50993SBenjamin Herrenschmidt guest_prio = val >> KVM_XICS_PRIORITY_SHIFT; 16265af50993SBenjamin Herrenschmidt 16275af50993SBenjamin Herrenschmidt pr_devel(" val=0x016%llx (server=0x%x, guest_prio=%d)\n", 16285af50993SBenjamin Herrenschmidt val, server, guest_prio); 1629bf4159daSBenjamin Herrenschmidt 16305af50993SBenjamin Herrenschmidt /* 16315af50993SBenjamin Herrenschmidt * If the source doesn't already have an IPI, allocate 16325af50993SBenjamin Herrenschmidt * one and get the corresponding data 16335af50993SBenjamin Herrenschmidt */ 16345af50993SBenjamin Herrenschmidt if (!state->ipi_number) { 16355af50993SBenjamin Herrenschmidt state->ipi_number = xive_native_alloc_irq(); 16365af50993SBenjamin Herrenschmidt if (state->ipi_number == 0) { 16375af50993SBenjamin Herrenschmidt pr_devel("Failed to allocate IPI !\n"); 16385af50993SBenjamin Herrenschmidt return -ENOMEM; 16395af50993SBenjamin Herrenschmidt } 16405af50993SBenjamin Herrenschmidt xive_native_populate_irq_data(state->ipi_number, &state->ipi_data); 16415af50993SBenjamin Herrenschmidt pr_devel(" src_ipi=0x%x\n", state->ipi_number); 16425af50993SBenjamin Herrenschmidt } 16435af50993SBenjamin Herrenschmidt 16445af50993SBenjamin Herrenschmidt /* 16455af50993SBenjamin Herrenschmidt * We use lock_and_mask() to set us in the right masked 16465af50993SBenjamin Herrenschmidt * state. We will override that state from the saved state 16475af50993SBenjamin Herrenschmidt * further down, but this will handle the cases of interrupts 16485af50993SBenjamin Herrenschmidt * that need FW masking. We set the initial guest_priority to 16495af50993SBenjamin Herrenschmidt * 0 before calling it to ensure it actually performs the masking. 16505af50993SBenjamin Herrenschmidt */ 16515af50993SBenjamin Herrenschmidt state->guest_priority = 0; 16525af50993SBenjamin Herrenschmidt xive_lock_and_mask(xive, sb, state); 16535af50993SBenjamin Herrenschmidt 16545af50993SBenjamin Herrenschmidt /* 16555af50993SBenjamin Herrenschmidt * Now, we select a target if we have one. If we don't we 16565af50993SBenjamin Herrenschmidt * leave the interrupt untargetted. It means that an interrupt 16575af50993SBenjamin Herrenschmidt * can become "untargetted" accross migration if it was masked 16585af50993SBenjamin Herrenschmidt * by set_xive() but there is little we can do about it. 16595af50993SBenjamin Herrenschmidt */ 16605af50993SBenjamin Herrenschmidt 16615af50993SBenjamin Herrenschmidt /* First convert prio and mark interrupt as untargetted */ 16625af50993SBenjamin Herrenschmidt act_prio = xive_prio_from_guest(guest_prio); 16635af50993SBenjamin Herrenschmidt state->act_priority = MASKED; 16645af50993SBenjamin Herrenschmidt 16655af50993SBenjamin Herrenschmidt /* 16665af50993SBenjamin Herrenschmidt * We need to drop the lock due to the mutex below. Hopefully 16675af50993SBenjamin Herrenschmidt * nothing is touching that interrupt yet since it hasn't been 16685af50993SBenjamin Herrenschmidt * advertized to a running guest yet 16695af50993SBenjamin Herrenschmidt */ 16705af50993SBenjamin Herrenschmidt arch_spin_unlock(&sb->lock); 16715af50993SBenjamin Herrenschmidt 16725af50993SBenjamin Herrenschmidt /* If we have a priority target the interrupt */ 16735af50993SBenjamin Herrenschmidt if (act_prio != MASKED) { 16745af50993SBenjamin Herrenschmidt /* First, check provisioning of queues */ 16757e10b9a6SCédric Le Goater mutex_lock(&xive->lock); 16765af50993SBenjamin Herrenschmidt rc = xive_check_provisioning(xive->kvm, act_prio); 16777e10b9a6SCédric Le Goater mutex_unlock(&xive->lock); 16785af50993SBenjamin Herrenschmidt 16795af50993SBenjamin Herrenschmidt /* Target interrupt */ 16805af50993SBenjamin Herrenschmidt if (rc == 0) 16815af50993SBenjamin Herrenschmidt rc = xive_target_interrupt(xive->kvm, state, 16825af50993SBenjamin Herrenschmidt server, act_prio); 16835af50993SBenjamin Herrenschmidt /* 16845af50993SBenjamin Herrenschmidt * If provisioning or targetting failed, leave it 16855af50993SBenjamin Herrenschmidt * alone and masked. It will remain disabled until 16865af50993SBenjamin Herrenschmidt * the guest re-targets it. 16875af50993SBenjamin Herrenschmidt */ 16885af50993SBenjamin Herrenschmidt } 16895af50993SBenjamin Herrenschmidt 16905af50993SBenjamin Herrenschmidt /* 16915af50993SBenjamin Herrenschmidt * Find out if this was a delayed irq stashed in an ICP, 16925af50993SBenjamin Herrenschmidt * in which case, treat it as pending 16935af50993SBenjamin Herrenschmidt */ 16945af50993SBenjamin Herrenschmidt if (xive->delayed_irqs && xive_check_delayed_irq(xive, irq)) { 16955af50993SBenjamin Herrenschmidt val |= KVM_XICS_PENDING; 16965af50993SBenjamin Herrenschmidt pr_devel(" Found delayed ! forcing PENDING !\n"); 16975af50993SBenjamin Herrenschmidt } 16985af50993SBenjamin Herrenschmidt 16995af50993SBenjamin Herrenschmidt /* Cleanup the SW state */ 17005af50993SBenjamin Herrenschmidt state->old_p = false; 17015af50993SBenjamin Herrenschmidt state->old_q = false; 17025af50993SBenjamin Herrenschmidt state->lsi = false; 17035af50993SBenjamin Herrenschmidt state->asserted = false; 17045af50993SBenjamin Herrenschmidt 17055af50993SBenjamin Herrenschmidt /* Restore LSI state */ 17065af50993SBenjamin Herrenschmidt if (val & KVM_XICS_LEVEL_SENSITIVE) { 17075af50993SBenjamin Herrenschmidt state->lsi = true; 17085af50993SBenjamin Herrenschmidt if (val & KVM_XICS_PENDING) 17095af50993SBenjamin Herrenschmidt state->asserted = true; 17105af50993SBenjamin Herrenschmidt pr_devel(" LSI ! Asserted=%d\n", state->asserted); 17115af50993SBenjamin Herrenschmidt } 17125af50993SBenjamin Herrenschmidt 17135af50993SBenjamin Herrenschmidt /* 17145af50993SBenjamin Herrenschmidt * Restore P and Q. If the interrupt was pending, we 1715dc1c4165SCédric Le Goater * force Q and !P, which will trigger a resend. 17165af50993SBenjamin Herrenschmidt * 17175af50993SBenjamin Herrenschmidt * That means that a guest that had both an interrupt 17185af50993SBenjamin Herrenschmidt * pending (queued) and Q set will restore with only 17195af50993SBenjamin Herrenschmidt * one instance of that interrupt instead of 2, but that 17205af50993SBenjamin Herrenschmidt * is perfectly fine as coalescing interrupts that haven't 17215af50993SBenjamin Herrenschmidt * been presented yet is always allowed. 17225af50993SBenjamin Herrenschmidt */ 1723dc1c4165SCédric Le Goater if (val & KVM_XICS_PRESENTED && !(val & KVM_XICS_PENDING)) 17245af50993SBenjamin Herrenschmidt state->old_p = true; 17255af50993SBenjamin Herrenschmidt if (val & KVM_XICS_QUEUED || val & KVM_XICS_PENDING) 17265af50993SBenjamin Herrenschmidt state->old_q = true; 17275af50993SBenjamin Herrenschmidt 17285af50993SBenjamin Herrenschmidt pr_devel(" P=%d, Q=%d\n", state->old_p, state->old_q); 17295af50993SBenjamin Herrenschmidt 17305af50993SBenjamin Herrenschmidt /* 17315af50993SBenjamin Herrenschmidt * If the interrupt was unmasked, update guest priority and 17325af50993SBenjamin Herrenschmidt * perform the appropriate state transition and do a 17335af50993SBenjamin Herrenschmidt * re-trigger if necessary. 17345af50993SBenjamin Herrenschmidt */ 17355af50993SBenjamin Herrenschmidt if (val & KVM_XICS_MASKED) { 17365af50993SBenjamin Herrenschmidt pr_devel(" masked, saving prio\n"); 17375af50993SBenjamin Herrenschmidt state->guest_priority = MASKED; 17385af50993SBenjamin Herrenschmidt state->saved_priority = guest_prio; 17395af50993SBenjamin Herrenschmidt } else { 17405af50993SBenjamin Herrenschmidt pr_devel(" unmasked, restoring to prio %d\n", guest_prio); 17415af50993SBenjamin Herrenschmidt xive_finish_unmask(xive, sb, state, guest_prio); 17425af50993SBenjamin Herrenschmidt state->saved_priority = guest_prio; 17435af50993SBenjamin Herrenschmidt } 17445af50993SBenjamin Herrenschmidt 17455af50993SBenjamin Herrenschmidt /* Increment the number of valid sources and mark this one valid */ 17465af50993SBenjamin Herrenschmidt if (!state->valid) 17475af50993SBenjamin Herrenschmidt xive->src_count++; 17485af50993SBenjamin Herrenschmidt state->valid = true; 17495af50993SBenjamin Herrenschmidt 17505af50993SBenjamin Herrenschmidt return 0; 17515af50993SBenjamin Herrenschmidt } 17525af50993SBenjamin Herrenschmidt 17535af50993SBenjamin Herrenschmidt int kvmppc_xive_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level, 17545af50993SBenjamin Herrenschmidt bool line_status) 17555af50993SBenjamin Herrenschmidt { 17565af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = kvm->arch.xive; 17575af50993SBenjamin Herrenschmidt struct kvmppc_xive_src_block *sb; 17585af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state; 17595af50993SBenjamin Herrenschmidt u16 idx; 17605af50993SBenjamin Herrenschmidt 17615af50993SBenjamin Herrenschmidt if (!xive) 17625af50993SBenjamin Herrenschmidt return -ENODEV; 17635af50993SBenjamin Herrenschmidt 17645af50993SBenjamin Herrenschmidt sb = kvmppc_xive_find_source(xive, irq, &idx); 17655af50993SBenjamin Herrenschmidt if (!sb) 17665af50993SBenjamin Herrenschmidt return -EINVAL; 17675af50993SBenjamin Herrenschmidt 17685af50993SBenjamin Herrenschmidt /* Perform locklessly .... (we need to do some RCUisms here...) */ 17695af50993SBenjamin Herrenschmidt state = &sb->irq_state[idx]; 17705af50993SBenjamin Herrenschmidt if (!state->valid) 17715af50993SBenjamin Herrenschmidt return -EINVAL; 17725af50993SBenjamin Herrenschmidt 17735af50993SBenjamin Herrenschmidt /* We don't allow a trigger on a passed-through interrupt */ 17745af50993SBenjamin Herrenschmidt if (state->pt_number) 17755af50993SBenjamin Herrenschmidt return -EINVAL; 17765af50993SBenjamin Herrenschmidt 17775af50993SBenjamin Herrenschmidt if ((level == 1 && state->lsi) || level == KVM_INTERRUPT_SET_LEVEL) 17785af50993SBenjamin Herrenschmidt state->asserted = 1; 17795af50993SBenjamin Herrenschmidt else if (level == 0 || level == KVM_INTERRUPT_UNSET) { 17805af50993SBenjamin Herrenschmidt state->asserted = 0; 17815af50993SBenjamin Herrenschmidt return 0; 17825af50993SBenjamin Herrenschmidt } 17835af50993SBenjamin Herrenschmidt 17845af50993SBenjamin Herrenschmidt /* Trigger the IPI */ 17855af50993SBenjamin Herrenschmidt xive_irq_trigger(&state->ipi_data); 17865af50993SBenjamin Herrenschmidt 17875af50993SBenjamin Herrenschmidt return 0; 17885af50993SBenjamin Herrenschmidt } 17895af50993SBenjamin Herrenschmidt 17905af50993SBenjamin Herrenschmidt static int xive_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 17915af50993SBenjamin Herrenschmidt { 17925af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = dev->private; 17935af50993SBenjamin Herrenschmidt 17945af50993SBenjamin Herrenschmidt /* We honor the existing XICS ioctl */ 17955af50993SBenjamin Herrenschmidt switch (attr->group) { 17965af50993SBenjamin Herrenschmidt case KVM_DEV_XICS_GRP_SOURCES: 17975af50993SBenjamin Herrenschmidt return xive_set_source(xive, attr->attr, attr->addr); 17985af50993SBenjamin Herrenschmidt } 17995af50993SBenjamin Herrenschmidt return -ENXIO; 18005af50993SBenjamin Herrenschmidt } 18015af50993SBenjamin Herrenschmidt 18025af50993SBenjamin Herrenschmidt static int xive_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 18035af50993SBenjamin Herrenschmidt { 18045af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = dev->private; 18055af50993SBenjamin Herrenschmidt 18065af50993SBenjamin Herrenschmidt /* We honor the existing XICS ioctl */ 18075af50993SBenjamin Herrenschmidt switch (attr->group) { 18085af50993SBenjamin Herrenschmidt case KVM_DEV_XICS_GRP_SOURCES: 18095af50993SBenjamin Herrenschmidt return xive_get_source(xive, attr->attr, attr->addr); 18105af50993SBenjamin Herrenschmidt } 18115af50993SBenjamin Herrenschmidt return -ENXIO; 18125af50993SBenjamin Herrenschmidt } 18135af50993SBenjamin Herrenschmidt 18145af50993SBenjamin Herrenschmidt static int xive_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 18155af50993SBenjamin Herrenschmidt { 18165af50993SBenjamin Herrenschmidt /* We honor the same limits as XICS, at least for now */ 18175af50993SBenjamin Herrenschmidt switch (attr->group) { 18185af50993SBenjamin Herrenschmidt case KVM_DEV_XICS_GRP_SOURCES: 18195af50993SBenjamin Herrenschmidt if (attr->attr >= KVMPPC_XICS_FIRST_IRQ && 18205af50993SBenjamin Herrenschmidt attr->attr < KVMPPC_XICS_NR_IRQS) 18215af50993SBenjamin Herrenschmidt return 0; 18225af50993SBenjamin Herrenschmidt break; 18235af50993SBenjamin Herrenschmidt } 18245af50993SBenjamin Herrenschmidt return -ENXIO; 18255af50993SBenjamin Herrenschmidt } 18265af50993SBenjamin Herrenschmidt 18275af50993SBenjamin Herrenschmidt static void kvmppc_xive_cleanup_irq(u32 hw_num, struct xive_irq_data *xd) 18285af50993SBenjamin Herrenschmidt { 18295af50993SBenjamin Herrenschmidt xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01); 18305af50993SBenjamin Herrenschmidt xive_native_configure_irq(hw_num, 0, MASKED, 0); 18315af50993SBenjamin Herrenschmidt } 18325af50993SBenjamin Herrenschmidt 18334131f83cSCédric Le Goater void kvmppc_xive_free_sources(struct kvmppc_xive_src_block *sb) 18345af50993SBenjamin Herrenschmidt { 18355af50993SBenjamin Herrenschmidt int i; 18365af50993SBenjamin Herrenschmidt 18375af50993SBenjamin Herrenschmidt for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) { 18385af50993SBenjamin Herrenschmidt struct kvmppc_xive_irq_state *state = &sb->irq_state[i]; 18395af50993SBenjamin Herrenschmidt 18405af50993SBenjamin Herrenschmidt if (!state->valid) 18415af50993SBenjamin Herrenschmidt continue; 18425af50993SBenjamin Herrenschmidt 18435af50993SBenjamin Herrenschmidt kvmppc_xive_cleanup_irq(state->ipi_number, &state->ipi_data); 1844ef974020SCédric Le Goater xive_cleanup_irq_data(&state->ipi_data); 18455af50993SBenjamin Herrenschmidt xive_native_free_irq(state->ipi_number); 18465af50993SBenjamin Herrenschmidt 1847ef974020SCédric Le Goater /* Pass-through, cleanup too but keep IRQ hw data */ 18485af50993SBenjamin Herrenschmidt if (state->pt_number) 18495af50993SBenjamin Herrenschmidt kvmppc_xive_cleanup_irq(state->pt_number, state->pt_data); 18505af50993SBenjamin Herrenschmidt 18515af50993SBenjamin Herrenschmidt state->valid = false; 18525af50993SBenjamin Herrenschmidt } 18535af50993SBenjamin Herrenschmidt } 18545af50993SBenjamin Herrenschmidt 18555422e951SCédric Le Goater /* 18566f868405SPaul Mackerras * Called when device fd is closed. kvm->lock is held. 18575422e951SCédric Le Goater */ 18585422e951SCédric Le Goater static void kvmppc_xive_release(struct kvm_device *dev) 18595af50993SBenjamin Herrenschmidt { 18605af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = dev->private; 18615af50993SBenjamin Herrenschmidt struct kvm *kvm = xive->kvm; 18625422e951SCédric Le Goater struct kvm_vcpu *vcpu; 18635af50993SBenjamin Herrenschmidt int i; 18645af50993SBenjamin Herrenschmidt 18655422e951SCédric Le Goater pr_devel("Releasing xive device\n"); 18665422e951SCédric Le Goater 18676f868405SPaul Mackerras /* 18686f868405SPaul Mackerras * Since this is the device release function, we know that 18696f868405SPaul Mackerras * userspace does not have any open fd referring to the 18706f868405SPaul Mackerras * device. Therefore there can not be any of the device 18716f868405SPaul Mackerras * attribute set/get functions being executed concurrently, 18726f868405SPaul Mackerras * and similarly, the connect_vcpu and set/clr_mapped 18736f868405SPaul Mackerras * functions also cannot be being executed. 18746f868405SPaul Mackerras */ 1875c395fe1dSPaul Mackerras 1876c395fe1dSPaul Mackerras debugfs_remove(xive->dentry); 18776f868405SPaul Mackerras 18786f868405SPaul Mackerras /* 18796f868405SPaul Mackerras * We should clean up the vCPU interrupt presenters first. 18806f868405SPaul Mackerras */ 18816f868405SPaul Mackerras kvm_for_each_vcpu(i, vcpu, kvm) { 18826f868405SPaul Mackerras /* 18836f868405SPaul Mackerras * Take vcpu->mutex to ensure that no one_reg get/set ioctl 18846f868405SPaul Mackerras * (i.e. kvmppc_xive_[gs]et_icp) can be done concurrently. 1885c395fe1dSPaul Mackerras * Holding the vcpu->mutex also means that the vcpu cannot 1886c395fe1dSPaul Mackerras * be executing the KVM_RUN ioctl, and therefore it cannot 1887c395fe1dSPaul Mackerras * be executing the XIVE push or pull code or accessing 1888c395fe1dSPaul Mackerras * the XIVE MMIO regions. 18896f868405SPaul Mackerras */ 18906f868405SPaul Mackerras mutex_lock(&vcpu->mutex); 18916f868405SPaul Mackerras kvmppc_xive_cleanup_vcpu(vcpu); 18926f868405SPaul Mackerras mutex_unlock(&vcpu->mutex); 18936f868405SPaul Mackerras } 18946f868405SPaul Mackerras 1895c395fe1dSPaul Mackerras /* 1896c395fe1dSPaul Mackerras * Now that we have cleared vcpu->arch.xive_vcpu, vcpu->arch.irq_type 1897c395fe1dSPaul Mackerras * and vcpu->arch.xive_esc_[vr]addr on each vcpu, we are safe 1898c395fe1dSPaul Mackerras * against xive code getting called during vcpu execution or 1899c395fe1dSPaul Mackerras * set/get one_reg operations. 1900c395fe1dSPaul Mackerras */ 19015af50993SBenjamin Herrenschmidt kvm->arch.xive = NULL; 19025af50993SBenjamin Herrenschmidt 19035af50993SBenjamin Herrenschmidt /* Mask and free interrupts */ 19045af50993SBenjamin Herrenschmidt for (i = 0; i <= xive->max_sbid; i++) { 19055af50993SBenjamin Herrenschmidt if (xive->src_blocks[i]) 19065af50993SBenjamin Herrenschmidt kvmppc_xive_free_sources(xive->src_blocks[i]); 19075af50993SBenjamin Herrenschmidt kfree(xive->src_blocks[i]); 19085af50993SBenjamin Herrenschmidt xive->src_blocks[i] = NULL; 19095af50993SBenjamin Herrenschmidt } 19105af50993SBenjamin Herrenschmidt 19115af50993SBenjamin Herrenschmidt if (xive->vp_base != XIVE_INVALID_VP) 19125af50993SBenjamin Herrenschmidt xive_native_free_vp_block(xive->vp_base); 19135af50993SBenjamin Herrenschmidt 19145422e951SCédric Le Goater /* 19155422e951SCédric Le Goater * A reference of the kvmppc_xive pointer is now kept under 19165422e951SCédric Le Goater * the xive_devices struct of the machine for reuse. It is 19175422e951SCédric Le Goater * freed when the VM is destroyed for now until we fix all the 19185422e951SCédric Le Goater * execution paths. 19195422e951SCédric Le Goater */ 19205af50993SBenjamin Herrenschmidt 19215af50993SBenjamin Herrenschmidt kfree(dev); 19225af50993SBenjamin Herrenschmidt } 19235af50993SBenjamin Herrenschmidt 19245422e951SCédric Le Goater /* 19255422e951SCédric Le Goater * When the guest chooses the interrupt mode (XICS legacy or XIVE 19265422e951SCédric Le Goater * native), the VM will switch of KVM device. The previous device will 19275422e951SCédric Le Goater * be "released" before the new one is created. 19285422e951SCédric Le Goater * 19295422e951SCédric Le Goater * Until we are sure all execution paths are well protected, provide a 19305422e951SCédric Le Goater * fail safe (transitional) method for device destruction, in which 19315422e951SCédric Le Goater * the XIVE device pointer is recycled and not directly freed. 19325422e951SCédric Le Goater */ 19335422e951SCédric Le Goater struct kvmppc_xive *kvmppc_xive_get_device(struct kvm *kvm, u32 type) 19345422e951SCédric Le Goater { 19355422e951SCédric Le Goater struct kvmppc_xive **kvm_xive_device = type == KVM_DEV_TYPE_XIVE ? 19365422e951SCédric Le Goater &kvm->arch.xive_devices.native : 19375422e951SCédric Le Goater &kvm->arch.xive_devices.xics_on_xive; 19385422e951SCédric Le Goater struct kvmppc_xive *xive = *kvm_xive_device; 19395422e951SCédric Le Goater 19405422e951SCédric Le Goater if (!xive) { 19415422e951SCédric Le Goater xive = kzalloc(sizeof(*xive), GFP_KERNEL); 19425422e951SCédric Le Goater *kvm_xive_device = xive; 19435422e951SCédric Le Goater } else { 19445422e951SCédric Le Goater memset(xive, 0, sizeof(*xive)); 19455422e951SCédric Le Goater } 19465422e951SCédric Le Goater 19475422e951SCédric Le Goater return xive; 19485422e951SCédric Le Goater } 19495422e951SCédric Le Goater 19506f868405SPaul Mackerras /* 19516f868405SPaul Mackerras * Create a XICS device with XIVE backend. kvm->lock is held. 19526f868405SPaul Mackerras */ 19535af50993SBenjamin Herrenschmidt static int kvmppc_xive_create(struct kvm_device *dev, u32 type) 19545af50993SBenjamin Herrenschmidt { 19555af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive; 19565af50993SBenjamin Herrenschmidt struct kvm *kvm = dev->kvm; 19575af50993SBenjamin Herrenschmidt int ret = 0; 19585af50993SBenjamin Herrenschmidt 19595af50993SBenjamin Herrenschmidt pr_devel("Creating xive for partition\n"); 19605af50993SBenjamin Herrenschmidt 19615422e951SCédric Le Goater xive = kvmppc_xive_get_device(kvm, type); 19625af50993SBenjamin Herrenschmidt if (!xive) 19635af50993SBenjamin Herrenschmidt return -ENOMEM; 19645af50993SBenjamin Herrenschmidt 19655af50993SBenjamin Herrenschmidt dev->private = xive; 19665af50993SBenjamin Herrenschmidt xive->dev = dev; 19675af50993SBenjamin Herrenschmidt xive->kvm = kvm; 19687e10b9a6SCédric Le Goater mutex_init(&xive->lock); 19695af50993SBenjamin Herrenschmidt 19705af50993SBenjamin Herrenschmidt /* Already there ? */ 19715af50993SBenjamin Herrenschmidt if (kvm->arch.xive) 19725af50993SBenjamin Herrenschmidt ret = -EEXIST; 19735af50993SBenjamin Herrenschmidt else 19745af50993SBenjamin Herrenschmidt kvm->arch.xive = xive; 19755af50993SBenjamin Herrenschmidt 19765af50993SBenjamin Herrenschmidt /* We use the default queue size set by the host */ 19775af50993SBenjamin Herrenschmidt xive->q_order = xive_native_default_eq_shift(); 19785af50993SBenjamin Herrenschmidt if (xive->q_order < PAGE_SHIFT) 19795af50993SBenjamin Herrenschmidt xive->q_page_order = 0; 19805af50993SBenjamin Herrenschmidt else 19815af50993SBenjamin Herrenschmidt xive->q_page_order = xive->q_order - PAGE_SHIFT; 19825af50993SBenjamin Herrenschmidt 19835af50993SBenjamin Herrenschmidt /* Allocate a bunch of VPs */ 19845af50993SBenjamin Herrenschmidt xive->vp_base = xive_native_alloc_vp_block(KVM_MAX_VCPUS); 19855af50993SBenjamin Herrenschmidt pr_devel("VP_Base=%x\n", xive->vp_base); 19865af50993SBenjamin Herrenschmidt 19875af50993SBenjamin Herrenschmidt if (xive->vp_base == XIVE_INVALID_VP) 19885af50993SBenjamin Herrenschmidt ret = -ENOMEM; 19895af50993SBenjamin Herrenschmidt 1990bf4159daSBenjamin Herrenschmidt xive->single_escalation = xive_native_has_single_escalation(); 1991bf4159daSBenjamin Herrenschmidt 19925af50993SBenjamin Herrenschmidt if (ret) { 19935af50993SBenjamin Herrenschmidt kfree(xive); 19945af50993SBenjamin Herrenschmidt return ret; 19955af50993SBenjamin Herrenschmidt } 19965af50993SBenjamin Herrenschmidt 19975af50993SBenjamin Herrenschmidt return 0; 19985af50993SBenjamin Herrenschmidt } 19995af50993SBenjamin Herrenschmidt 2000eacc56bbSCédric Le Goater int kvmppc_xive_debug_show_queues(struct seq_file *m, struct kvm_vcpu *vcpu) 2001eacc56bbSCédric Le Goater { 2002eacc56bbSCédric Le Goater struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 2003eacc56bbSCédric Le Goater unsigned int i; 2004eacc56bbSCédric Le Goater 2005eacc56bbSCédric Le Goater for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) { 2006eacc56bbSCédric Le Goater struct xive_q *q = &xc->queues[i]; 2007eacc56bbSCédric Le Goater u32 i0, i1, idx; 2008eacc56bbSCédric Le Goater 2009eacc56bbSCédric Le Goater if (!q->qpage && !xc->esc_virq[i]) 2010eacc56bbSCédric Le Goater continue; 2011eacc56bbSCédric Le Goater 2012eacc56bbSCédric Le Goater seq_printf(m, " [q%d]: ", i); 2013eacc56bbSCédric Le Goater 2014eacc56bbSCédric Le Goater if (q->qpage) { 2015eacc56bbSCédric Le Goater idx = q->idx; 2016eacc56bbSCédric Le Goater i0 = be32_to_cpup(q->qpage + idx); 2017eacc56bbSCédric Le Goater idx = (idx + 1) & q->msk; 2018eacc56bbSCédric Le Goater i1 = be32_to_cpup(q->qpage + idx); 2019eacc56bbSCédric Le Goater seq_printf(m, "T=%d %08x %08x...\n", q->toggle, 2020eacc56bbSCédric Le Goater i0, i1); 2021eacc56bbSCédric Le Goater } 2022eacc56bbSCédric Le Goater if (xc->esc_virq[i]) { 2023eacc56bbSCédric Le Goater struct irq_data *d = irq_get_irq_data(xc->esc_virq[i]); 2024eacc56bbSCédric Le Goater struct xive_irq_data *xd = 2025eacc56bbSCédric Le Goater irq_data_get_irq_handler_data(d); 2026eacc56bbSCédric Le Goater u64 pq = xive_vm_esb_load(xd, XIVE_ESB_GET); 2027eacc56bbSCédric Le Goater 2028eacc56bbSCédric Le Goater seq_printf(m, "E:%c%c I(%d:%llx:%llx)", 2029eacc56bbSCédric Le Goater (pq & XIVE_ESB_VAL_P) ? 'P' : 'p', 2030eacc56bbSCédric Le Goater (pq & XIVE_ESB_VAL_Q) ? 'Q' : 'q', 2031eacc56bbSCédric Le Goater xc->esc_virq[i], pq, xd->eoi_page); 2032eacc56bbSCédric Le Goater seq_puts(m, "\n"); 2033eacc56bbSCédric Le Goater } 2034eacc56bbSCédric Le Goater } 2035eacc56bbSCédric Le Goater return 0; 2036eacc56bbSCédric Le Goater } 20375af50993SBenjamin Herrenschmidt 20385af50993SBenjamin Herrenschmidt static int xive_debug_show(struct seq_file *m, void *private) 20395af50993SBenjamin Herrenschmidt { 20405af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = m->private; 20415af50993SBenjamin Herrenschmidt struct kvm *kvm = xive->kvm; 20425af50993SBenjamin Herrenschmidt struct kvm_vcpu *vcpu; 20435af50993SBenjamin Herrenschmidt u64 t_rm_h_xirr = 0; 20445af50993SBenjamin Herrenschmidt u64 t_rm_h_ipoll = 0; 20455af50993SBenjamin Herrenschmidt u64 t_rm_h_cppr = 0; 20465af50993SBenjamin Herrenschmidt u64 t_rm_h_eoi = 0; 20475af50993SBenjamin Herrenschmidt u64 t_rm_h_ipi = 0; 20485af50993SBenjamin Herrenschmidt u64 t_vm_h_xirr = 0; 20495af50993SBenjamin Herrenschmidt u64 t_vm_h_ipoll = 0; 20505af50993SBenjamin Herrenschmidt u64 t_vm_h_cppr = 0; 20515af50993SBenjamin Herrenschmidt u64 t_vm_h_eoi = 0; 20525af50993SBenjamin Herrenschmidt u64 t_vm_h_ipi = 0; 20535af50993SBenjamin Herrenschmidt unsigned int i; 20545af50993SBenjamin Herrenschmidt 20555af50993SBenjamin Herrenschmidt if (!kvm) 20565af50993SBenjamin Herrenschmidt return 0; 20575af50993SBenjamin Herrenschmidt 20585af50993SBenjamin Herrenschmidt seq_printf(m, "=========\nVCPU state\n=========\n"); 20595af50993SBenjamin Herrenschmidt 20605af50993SBenjamin Herrenschmidt kvm_for_each_vcpu(i, vcpu, kvm) { 20615af50993SBenjamin Herrenschmidt struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 20625af50993SBenjamin Herrenschmidt 20635af50993SBenjamin Herrenschmidt if (!xc) 20645af50993SBenjamin Herrenschmidt continue; 20655af50993SBenjamin Herrenschmidt 20665af50993SBenjamin Herrenschmidt seq_printf(m, "cpu server %#x CPPR:%#x HWCPPR:%#x" 20675af50993SBenjamin Herrenschmidt " MFRR:%#x PEND:%#x h_xirr: R=%lld V=%lld\n", 20685af50993SBenjamin Herrenschmidt xc->server_num, xc->cppr, xc->hw_cppr, 20695af50993SBenjamin Herrenschmidt xc->mfrr, xc->pending, 20705af50993SBenjamin Herrenschmidt xc->stat_rm_h_xirr, xc->stat_vm_h_xirr); 2071c424c108SBenjamin Herrenschmidt 2072eacc56bbSCédric Le Goater kvmppc_xive_debug_show_queues(m, vcpu); 20735af50993SBenjamin Herrenschmidt 20745af50993SBenjamin Herrenschmidt t_rm_h_xirr += xc->stat_rm_h_xirr; 20755af50993SBenjamin Herrenschmidt t_rm_h_ipoll += xc->stat_rm_h_ipoll; 20765af50993SBenjamin Herrenschmidt t_rm_h_cppr += xc->stat_rm_h_cppr; 20775af50993SBenjamin Herrenschmidt t_rm_h_eoi += xc->stat_rm_h_eoi; 20785af50993SBenjamin Herrenschmidt t_rm_h_ipi += xc->stat_rm_h_ipi; 20795af50993SBenjamin Herrenschmidt t_vm_h_xirr += xc->stat_vm_h_xirr; 20805af50993SBenjamin Herrenschmidt t_vm_h_ipoll += xc->stat_vm_h_ipoll; 20815af50993SBenjamin Herrenschmidt t_vm_h_cppr += xc->stat_vm_h_cppr; 20825af50993SBenjamin Herrenschmidt t_vm_h_eoi += xc->stat_vm_h_eoi; 20835af50993SBenjamin Herrenschmidt t_vm_h_ipi += xc->stat_vm_h_ipi; 20845af50993SBenjamin Herrenschmidt } 20855af50993SBenjamin Herrenschmidt 20865af50993SBenjamin Herrenschmidt seq_printf(m, "Hcalls totals\n"); 20875af50993SBenjamin Herrenschmidt seq_printf(m, " H_XIRR R=%10lld V=%10lld\n", t_rm_h_xirr, t_vm_h_xirr); 20885af50993SBenjamin Herrenschmidt seq_printf(m, " H_IPOLL R=%10lld V=%10lld\n", t_rm_h_ipoll, t_vm_h_ipoll); 20895af50993SBenjamin Herrenschmidt seq_printf(m, " H_CPPR R=%10lld V=%10lld\n", t_rm_h_cppr, t_vm_h_cppr); 20905af50993SBenjamin Herrenschmidt seq_printf(m, " H_EOI R=%10lld V=%10lld\n", t_rm_h_eoi, t_vm_h_eoi); 20915af50993SBenjamin Herrenschmidt seq_printf(m, " H_IPI R=%10lld V=%10lld\n", t_rm_h_ipi, t_vm_h_ipi); 20925af50993SBenjamin Herrenschmidt 20935af50993SBenjamin Herrenschmidt return 0; 20945af50993SBenjamin Herrenschmidt } 20955af50993SBenjamin Herrenschmidt 20960f6ddf34SYangtao Li DEFINE_SHOW_ATTRIBUTE(xive_debug); 20975af50993SBenjamin Herrenschmidt 20985af50993SBenjamin Herrenschmidt static void xive_debugfs_init(struct kvmppc_xive *xive) 20995af50993SBenjamin Herrenschmidt { 21005af50993SBenjamin Herrenschmidt char *name; 21015af50993SBenjamin Herrenschmidt 21025af50993SBenjamin Herrenschmidt name = kasprintf(GFP_KERNEL, "kvm-xive-%p", xive); 21035af50993SBenjamin Herrenschmidt if (!name) { 21045af50993SBenjamin Herrenschmidt pr_err("%s: no memory for name\n", __func__); 21055af50993SBenjamin Herrenschmidt return; 21065af50993SBenjamin Herrenschmidt } 21075af50993SBenjamin Herrenschmidt 21085af50993SBenjamin Herrenschmidt xive->dentry = debugfs_create_file(name, S_IRUGO, powerpc_debugfs_root, 21095af50993SBenjamin Herrenschmidt xive, &xive_debug_fops); 21105af50993SBenjamin Herrenschmidt 21115af50993SBenjamin Herrenschmidt pr_debug("%s: created %s\n", __func__, name); 21125af50993SBenjamin Herrenschmidt kfree(name); 21135af50993SBenjamin Herrenschmidt } 21145af50993SBenjamin Herrenschmidt 21155af50993SBenjamin Herrenschmidt static void kvmppc_xive_init(struct kvm_device *dev) 21165af50993SBenjamin Herrenschmidt { 21175af50993SBenjamin Herrenschmidt struct kvmppc_xive *xive = (struct kvmppc_xive *)dev->private; 21185af50993SBenjamin Herrenschmidt 21195af50993SBenjamin Herrenschmidt /* Register some debug interfaces */ 21205af50993SBenjamin Herrenschmidt xive_debugfs_init(xive); 21215af50993SBenjamin Herrenschmidt } 21225af50993SBenjamin Herrenschmidt 21235af50993SBenjamin Herrenschmidt struct kvm_device_ops kvm_xive_ops = { 21245af50993SBenjamin Herrenschmidt .name = "kvm-xive", 21255af50993SBenjamin Herrenschmidt .create = kvmppc_xive_create, 21265af50993SBenjamin Herrenschmidt .init = kvmppc_xive_init, 21275422e951SCédric Le Goater .release = kvmppc_xive_release, 21285af50993SBenjamin Herrenschmidt .set_attr = xive_set_attr, 21295af50993SBenjamin Herrenschmidt .get_attr = xive_get_attr, 21305af50993SBenjamin Herrenschmidt .has_attr = xive_has_attr, 21315af50993SBenjamin Herrenschmidt }; 21325af50993SBenjamin Herrenschmidt 21335af50993SBenjamin Herrenschmidt void kvmppc_xive_init_module(void) 21345af50993SBenjamin Herrenschmidt { 21355af50993SBenjamin Herrenschmidt __xive_vm_h_xirr = xive_vm_h_xirr; 21365af50993SBenjamin Herrenschmidt __xive_vm_h_ipoll = xive_vm_h_ipoll; 21375af50993SBenjamin Herrenschmidt __xive_vm_h_ipi = xive_vm_h_ipi; 21385af50993SBenjamin Herrenschmidt __xive_vm_h_cppr = xive_vm_h_cppr; 21395af50993SBenjamin Herrenschmidt __xive_vm_h_eoi = xive_vm_h_eoi; 21405af50993SBenjamin Herrenschmidt } 21415af50993SBenjamin Herrenschmidt 21425af50993SBenjamin Herrenschmidt void kvmppc_xive_exit_module(void) 21435af50993SBenjamin Herrenschmidt { 21445af50993SBenjamin Herrenschmidt __xive_vm_h_xirr = NULL; 21455af50993SBenjamin Herrenschmidt __xive_vm_h_ipoll = NULL; 21465af50993SBenjamin Herrenschmidt __xive_vm_h_ipi = NULL; 21475af50993SBenjamin Herrenschmidt __xive_vm_h_cppr = NULL; 21485af50993SBenjamin Herrenschmidt __xive_vm_h_eoi = NULL; 21495af50993SBenjamin Herrenschmidt } 2150