1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Multiplex several virtual IPIs over a single HW IPI. 4 * 5 * Copyright The Asahi Linux Contributors 6 * Copyright (c) 2022 Ventana Micro Systems Inc. 7 */ 8 9 #define pr_fmt(fmt) "ipi-mux: " fmt 10 #include <linux/cpu.h> 11 #include <linux/init.h> 12 #include <linux/irq.h> 13 #include <linux/irqchip.h> 14 #include <linux/irqchip/chained_irq.h> 15 #include <linux/irqdomain.h> 16 #include <linux/jump_label.h> 17 #include <linux/percpu.h> 18 #include <linux/smp.h> 19 20 struct ipi_mux_cpu { 21 atomic_t enable; 22 atomic_t bits; 23 }; 24 25 static struct ipi_mux_cpu __percpu *ipi_mux_pcpu; 26 static struct irq_domain *ipi_mux_domain; 27 static void (*ipi_mux_send)(unsigned int cpu); 28 29 static void ipi_mux_mask(struct irq_data *d) 30 { 31 struct ipi_mux_cpu *icpu = this_cpu_ptr(ipi_mux_pcpu); 32 33 atomic_andnot(BIT(irqd_to_hwirq(d)), &icpu->enable); 34 } 35 36 static void ipi_mux_unmask(struct irq_data *d) 37 { 38 struct ipi_mux_cpu *icpu = this_cpu_ptr(ipi_mux_pcpu); 39 u32 ibit = BIT(irqd_to_hwirq(d)); 40 41 atomic_or(ibit, &icpu->enable); 42 43 /* 44 * The atomic_or() above must complete before the atomic_read() 45 * below to avoid racing ipi_mux_send_mask(). 46 */ 47 smp_mb__after_atomic(); 48 49 /* If a pending IPI was unmasked, raise a parent IPI immediately. */ 50 if (atomic_read(&icpu->bits) & ibit) 51 ipi_mux_send(smp_processor_id()); 52 } 53 54 static void ipi_mux_send_mask(struct irq_data *d, const struct cpumask *mask) 55 { 56 struct ipi_mux_cpu *icpu = this_cpu_ptr(ipi_mux_pcpu); 57 u32 ibit = BIT(irqd_to_hwirq(d)); 58 unsigned long pending; 59 int cpu; 60 61 for_each_cpu(cpu, mask) { 62 icpu = per_cpu_ptr(ipi_mux_pcpu, cpu); 63 64 /* 65 * This sequence is the mirror of the one in ipi_mux_unmask(); 66 * see the comment there. Additionally, release semantics 67 * ensure that the vIPI flag set is ordered after any shared 68 * memory accesses that precede it. This therefore also pairs 69 * with the atomic_fetch_andnot in ipi_mux_process(). 70 */ 71 pending = atomic_fetch_or_release(ibit, &icpu->bits); 72 73 /* 74 * The atomic_fetch_or_release() above must complete 75 * before the atomic_read() below to avoid racing with 76 * ipi_mux_unmask(). 77 */ 78 smp_mb__after_atomic(); 79 80 /* 81 * The flag writes must complete before the physical IPI is 82 * issued to another CPU. This is implied by the control 83 * dependency on the result of atomic_read() below, which is 84 * itself already ordered after the vIPI flag write. 85 */ 86 if (!(pending & ibit) && (atomic_read(&icpu->enable) & ibit)) 87 ipi_mux_send(cpu); 88 } 89 } 90 91 static const struct irq_chip ipi_mux_chip = { 92 .name = "IPI Mux", 93 .irq_mask = ipi_mux_mask, 94 .irq_unmask = ipi_mux_unmask, 95 .ipi_send_mask = ipi_mux_send_mask, 96 }; 97 98 static int ipi_mux_domain_alloc(struct irq_domain *d, unsigned int virq, 99 unsigned int nr_irqs, void *arg) 100 { 101 int i; 102 103 for (i = 0; i < nr_irqs; i++) { 104 irq_set_percpu_devid(virq + i); 105 irq_domain_set_info(d, virq + i, i, &ipi_mux_chip, NULL, 106 handle_percpu_devid_irq, NULL, NULL); 107 } 108 109 return 0; 110 } 111 112 static const struct irq_domain_ops ipi_mux_domain_ops = { 113 .alloc = ipi_mux_domain_alloc, 114 .free = irq_domain_free_irqs_top, 115 }; 116 117 /** 118 * ipi_mux_process - Process multiplexed virtual IPIs 119 */ 120 void ipi_mux_process(void) 121 { 122 struct ipi_mux_cpu *icpu = this_cpu_ptr(ipi_mux_pcpu); 123 irq_hw_number_t hwirq; 124 unsigned long ipis; 125 unsigned int en; 126 127 /* 128 * Reading enable mask does not need to be ordered as long as 129 * this function is called from interrupt handler because only 130 * the CPU itself can change it's own enable mask. 131 */ 132 en = atomic_read(&icpu->enable); 133 134 /* 135 * Clear the IPIs we are about to handle. This pairs with the 136 * atomic_fetch_or_release() in ipi_mux_send_mask(). 137 */ 138 ipis = atomic_fetch_andnot(en, &icpu->bits) & en; 139 140 for_each_set_bit(hwirq, &ipis, BITS_PER_TYPE(int)) 141 generic_handle_domain_irq(ipi_mux_domain, hwirq); 142 } 143 144 /** 145 * ipi_mux_create - Create virtual IPIs multiplexed on top of a single 146 * parent IPI. 147 * @nr_ipi: number of virtual IPIs to create. This should 148 * be <= BITS_PER_TYPE(int) 149 * @mux_send: callback to trigger parent IPI for a particular CPU 150 * 151 * Returns first virq of the newly created virtual IPIs upon success 152 * or <=0 upon failure 153 */ 154 int ipi_mux_create(unsigned int nr_ipi, void (*mux_send)(unsigned int cpu)) 155 { 156 struct fwnode_handle *fwnode; 157 struct irq_domain *domain; 158 int rc; 159 160 if (ipi_mux_domain) 161 return -EEXIST; 162 163 if (BITS_PER_TYPE(int) < nr_ipi || !mux_send) 164 return -EINVAL; 165 166 ipi_mux_pcpu = alloc_percpu(typeof(*ipi_mux_pcpu)); 167 if (!ipi_mux_pcpu) 168 return -ENOMEM; 169 170 fwnode = irq_domain_alloc_named_fwnode("IPI-Mux"); 171 if (!fwnode) { 172 pr_err("unable to create IPI Mux fwnode\n"); 173 rc = -ENOMEM; 174 goto fail_free_cpu; 175 } 176 177 domain = irq_domain_create_linear(fwnode, nr_ipi, 178 &ipi_mux_domain_ops, NULL); 179 if (!domain) { 180 pr_err("unable to add IPI Mux domain\n"); 181 rc = -ENOMEM; 182 goto fail_free_fwnode; 183 } 184 185 domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; 186 irq_domain_update_bus_token(domain, DOMAIN_BUS_IPI); 187 188 rc = irq_domain_alloc_irqs(domain, nr_ipi, NUMA_NO_NODE, NULL); 189 if (rc <= 0) { 190 pr_err("unable to alloc IRQs from IPI Mux domain\n"); 191 goto fail_free_domain; 192 } 193 194 ipi_mux_domain = domain; 195 ipi_mux_send = mux_send; 196 197 return rc; 198 199 fail_free_domain: 200 irq_domain_remove(domain); 201 fail_free_fwnode: 202 irq_domain_free_fwnode(fwnode); 203 fail_free_cpu: 204 free_percpu(ipi_mux_pcpu); 205 return rc; 206 } 207