1 /* 2 * linux/kernel/irq/ipi.c 3 * 4 * Copyright (C) 2015 Imagination Technologies Ltd 5 * Author: Qais Yousef <qais.yousef@imgtec.com> 6 * 7 * This file contains driver APIs to the IPI subsystem. 8 */ 9 10 #define pr_fmt(fmt) "genirq/ipi: " fmt 11 12 #include <linux/irqdomain.h> 13 #include <linux/irq.h> 14 15 /** 16 * irq_reserve_ipi() - Setup an IPI to destination cpumask 17 * @domain: IPI domain 18 * @dest: cpumask of cpus which can receive the IPI 19 * 20 * Allocate a virq that can be used to send IPI to any CPU in dest mask. 21 * 22 * On success it'll return linux irq number and 0 on failure 23 */ 24 unsigned int irq_reserve_ipi(struct irq_domain *domain, 25 const struct cpumask *dest) 26 { 27 unsigned int nr_irqs, offset; 28 struct irq_data *data; 29 int virq, i; 30 31 if (!domain ||!irq_domain_is_ipi(domain)) { 32 pr_warn("Reservation on a non IPI domain\n"); 33 return 0; 34 } 35 36 if (!cpumask_subset(dest, cpu_possible_mask)) { 37 pr_warn("Reservation is not in possible_cpu_mask\n"); 38 return 0; 39 } 40 41 nr_irqs = cpumask_weight(dest); 42 if (!nr_irqs) { 43 pr_warn("Reservation for empty destination mask\n"); 44 return 0; 45 } 46 47 if (irq_domain_is_ipi_single(domain)) { 48 /* 49 * If the underlying implementation uses a single HW irq on 50 * all cpus then we only need a single Linux irq number for 51 * it. We have no restrictions vs. the destination mask. The 52 * underlying implementation can deal with holes nicely. 53 */ 54 nr_irqs = 1; 55 offset = 0; 56 } else { 57 unsigned int next; 58 59 /* 60 * The IPI requires a seperate HW irq on each CPU. We require 61 * that the destination mask is consecutive. If an 62 * implementation needs to support holes, it can reserve 63 * several IPI ranges. 64 */ 65 offset = cpumask_first(dest); 66 /* 67 * Find a hole and if found look for another set bit after the 68 * hole. For now we don't support this scenario. 69 */ 70 next = cpumask_next_zero(offset, dest); 71 if (next < nr_cpu_ids) 72 next = cpumask_next(next, dest); 73 if (next < nr_cpu_ids) { 74 pr_warn("Destination mask has holes\n"); 75 return 0; 76 } 77 } 78 79 virq = irq_domain_alloc_descs(-1, nr_irqs, 0, NUMA_NO_NODE); 80 if (virq <= 0) { 81 pr_warn("Can't reserve IPI, failed to alloc descs\n"); 82 return 0; 83 } 84 85 virq = __irq_domain_alloc_irqs(domain, virq, nr_irqs, NUMA_NO_NODE, 86 (void *) dest, true); 87 88 if (virq <= 0) { 89 pr_warn("Can't reserve IPI, failed to alloc hw irqs\n"); 90 goto free_descs; 91 } 92 93 for (i = 0; i < nr_irqs; i++) { 94 data = irq_get_irq_data(virq + i); 95 cpumask_copy(data->common->affinity, dest); 96 data->common->ipi_offset = offset; 97 irq_set_status_flags(virq + i, IRQ_NO_BALANCING); 98 } 99 return virq; 100 101 free_descs: 102 irq_free_descs(virq, nr_irqs); 103 return 0; 104 } 105 106 /** 107 * irq_destroy_ipi() - unreserve an IPI that was previously allocated 108 * @irq: linux irq number to be destroyed 109 * 110 * Return the IPIs allocated with irq_reserve_ipi() to the system destroying 111 * all virqs associated with them. 112 */ 113 void irq_destroy_ipi(unsigned int irq) 114 { 115 struct irq_data *data = irq_get_irq_data(irq); 116 struct cpumask *ipimask = data ? irq_data_get_affinity_mask(data) : NULL; 117 struct irq_domain *domain; 118 unsigned int nr_irqs; 119 120 if (!irq || !data || !ipimask) 121 return; 122 123 domain = data->domain; 124 if (WARN_ON(domain == NULL)) 125 return; 126 127 if (!irq_domain_is_ipi(domain)) { 128 pr_warn("Trying to destroy a non IPI domain!\n"); 129 return; 130 } 131 132 if (irq_domain_is_ipi_per_cpu(domain)) 133 nr_irqs = cpumask_weight(ipimask); 134 else 135 nr_irqs = 1; 136 137 irq_domain_free_irqs(irq, nr_irqs); 138 } 139 140 /** 141 * ipi_get_hwirq - Get the hwirq associated with an IPI to a cpu 142 * @irq: linux irq number 143 * @cpu: the target cpu 144 * 145 * When dealing with coprocessors IPI, we need to inform the coprocessor of 146 * the hwirq it needs to use to receive and send IPIs. 147 * 148 * Returns hwirq value on success and INVALID_HWIRQ on failure. 149 */ 150 irq_hw_number_t ipi_get_hwirq(unsigned int irq, unsigned int cpu) 151 { 152 struct irq_data *data = irq_get_irq_data(irq); 153 struct cpumask *ipimask = data ? irq_data_get_affinity_mask(data) : NULL; 154 155 if (!data || !ipimask || cpu > nr_cpu_ids) 156 return INVALID_HWIRQ; 157 158 if (!cpumask_test_cpu(cpu, ipimask)) 159 return INVALID_HWIRQ; 160 161 /* 162 * Get the real hardware irq number if the underlying implementation 163 * uses a seperate irq per cpu. If the underlying implementation uses 164 * a single hardware irq for all cpus then the IPI send mechanism 165 * needs to take care of the cpu destinations. 166 */ 167 if (irq_domain_is_ipi_per_cpu(data->domain)) 168 data = irq_get_irq_data(irq + cpu - data->common->ipi_offset); 169 170 return data ? irqd_to_hwirq(data) : INVALID_HWIRQ; 171 } 172 EXPORT_SYMBOL_GPL(ipi_get_hwirq); 173 174 static int ipi_send_verify(struct irq_chip *chip, struct irq_data *data, 175 const struct cpumask *dest, unsigned int cpu) 176 { 177 struct cpumask *ipimask = irq_data_get_affinity_mask(data); 178 179 if (!chip || !ipimask) 180 return -EINVAL; 181 182 if (!chip->ipi_send_single && !chip->ipi_send_mask) 183 return -EINVAL; 184 185 if (cpu > nr_cpu_ids) 186 return -EINVAL; 187 188 if (dest) { 189 if (!cpumask_subset(dest, ipimask)) 190 return -EINVAL; 191 } else { 192 if (!cpumask_test_cpu(cpu, ipimask)) 193 return -EINVAL; 194 } 195 return 0; 196 } 197 198 /** 199 * __ipi_send_single - send an IPI to a target Linux SMP CPU 200 * @desc: pointer to irq_desc of the IRQ 201 * @cpu: destination CPU, must in the destination mask passed to 202 * irq_reserve_ipi() 203 * 204 * This function is for architecture or core code to speed up IPI sending. Not 205 * usable from driver code. 206 * 207 * Returns zero on success and negative error number on failure. 208 */ 209 int __ipi_send_single(struct irq_desc *desc, unsigned int cpu) 210 { 211 struct irq_data *data = irq_desc_get_irq_data(desc); 212 struct irq_chip *chip = irq_data_get_irq_chip(data); 213 214 #ifdef DEBUG 215 /* 216 * Minimise the overhead by omitting the checks for Linux SMP IPIs. 217 * Since the callers should be arch or core code which is generally 218 * trusted, only check for errors when debugging. 219 */ 220 if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu))) 221 return -EINVAL; 222 #endif 223 if (!chip->ipi_send_single) { 224 chip->ipi_send_mask(data, cpumask_of(cpu)); 225 return 0; 226 } 227 228 /* FIXME: Store this information in irqdata flags */ 229 if (irq_domain_is_ipi_per_cpu(data->domain) && 230 cpu != data->common->ipi_offset) { 231 /* use the correct data for that cpu */ 232 unsigned irq = data->irq + cpu - data->common->ipi_offset; 233 234 data = irq_get_irq_data(irq); 235 } 236 chip->ipi_send_single(data, cpu); 237 return 0; 238 } 239 240 /** 241 * ipi_send_mask - send an IPI to target Linux SMP CPU(s) 242 * @desc: pointer to irq_desc of the IRQ 243 * @dest: dest CPU(s), must be a subset of the mask passed to 244 * irq_reserve_ipi() 245 * 246 * This function is for architecture or core code to speed up IPI sending. Not 247 * usable from driver code. 248 * 249 * Returns zero on success and negative error number on failure. 250 */ 251 int __ipi_send_mask(struct irq_desc *desc, const struct cpumask *dest) 252 { 253 struct irq_data *data = irq_desc_get_irq_data(desc); 254 struct irq_chip *chip = irq_data_get_irq_chip(data); 255 unsigned int cpu; 256 257 #ifdef DEBUG 258 /* 259 * Minimise the overhead by omitting the checks for Linux SMP IPIs. 260 * Since the callers should be arch or core code which is generally 261 * trusted, only check for errors when debugging. 262 */ 263 if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0))) 264 return -EINVAL; 265 #endif 266 if (chip->ipi_send_mask) { 267 chip->ipi_send_mask(data, dest); 268 return 0; 269 } 270 271 if (irq_domain_is_ipi_per_cpu(data->domain)) { 272 unsigned int base = data->irq; 273 274 for_each_cpu(cpu, dest) { 275 unsigned irq = base + cpu - data->common->ipi_offset; 276 277 data = irq_get_irq_data(irq); 278 chip->ipi_send_single(data, cpu); 279 } 280 } else { 281 for_each_cpu(cpu, dest) 282 chip->ipi_send_single(data, cpu); 283 } 284 return 0; 285 } 286 287 /** 288 * ipi_send_single - Send an IPI to a single CPU 289 * @virq: linux irq number from irq_reserve_ipi() 290 * @cpu: destination CPU, must in the destination mask passed to 291 * irq_reserve_ipi() 292 * 293 * Returns zero on success and negative error number on failure. 294 */ 295 int ipi_send_single(unsigned int virq, unsigned int cpu) 296 { 297 struct irq_desc *desc = irq_to_desc(virq); 298 struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL; 299 struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL; 300 301 if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu))) 302 return -EINVAL; 303 304 return __ipi_send_single(desc, cpu); 305 } 306 EXPORT_SYMBOL_GPL(ipi_send_single); 307 308 /** 309 * ipi_send_mask - Send an IPI to target CPU(s) 310 * @virq: linux irq number from irq_reserve_ipi() 311 * @dest: dest CPU(s), must be a subset of the mask passed to 312 * irq_reserve_ipi() 313 * 314 * Returns zero on success and negative error number on failure. 315 */ 316 int ipi_send_mask(unsigned int virq, const struct cpumask *dest) 317 { 318 struct irq_desc *desc = irq_to_desc(virq); 319 struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL; 320 struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL; 321 322 if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0))) 323 return -EINVAL; 324 325 return __ipi_send_mask(desc, dest); 326 } 327 EXPORT_SYMBOL_GPL(ipi_send_mask); 328