1 // SPDX-License-Identifier: GPL-2.0 2 #define KMSG_COMPONENT "zpci" 3 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 4 5 #include <linux/kernel.h> 6 #include <linux/irq.h> 7 #include <linux/kernel_stat.h> 8 #include <linux/pci.h> 9 #include <linux/msi.h> 10 #include <linux/smp.h> 11 12 #include <asm/isc.h> 13 #include <asm/airq.h> 14 15 static enum {FLOATING, DIRECTED} irq_delivery; 16 17 #define SIC_IRQ_MODE_ALL 0 18 #define SIC_IRQ_MODE_SINGLE 1 19 #define SIC_IRQ_MODE_DIRECT 4 20 #define SIC_IRQ_MODE_D_ALL 16 21 #define SIC_IRQ_MODE_D_SINGLE 17 22 #define SIC_IRQ_MODE_SET_CPU 18 23 24 /* 25 * summary bit vector 26 * FLOATING - summary bit per function 27 * DIRECTED - summary bit per cpu (only used in fallback path) 28 */ 29 static struct airq_iv *zpci_sbv; 30 31 /* 32 * interrupt bit vectors 33 * FLOATING - interrupt bit vector per function 34 * DIRECTED - interrupt bit vector per cpu 35 */ 36 static struct airq_iv **zpci_ibv; 37 38 /* Modify PCI: Register adapter interruptions */ 39 static int zpci_set_airq(struct zpci_dev *zdev) 40 { 41 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT); 42 struct zpci_fib fib = {0}; 43 u8 status; 44 45 fib.fmt0.isc = PCI_ISC; 46 fib.fmt0.sum = 1; /* enable summary notifications */ 47 fib.fmt0.noi = airq_iv_end(zdev->aibv); 48 fib.fmt0.aibv = (unsigned long) zdev->aibv->vector; 49 fib.fmt0.aibvo = 0; /* each zdev has its own interrupt vector */ 50 fib.fmt0.aisb = (unsigned long) zpci_sbv->vector + (zdev->aisb/64)*8; 51 fib.fmt0.aisbo = zdev->aisb & 63; 52 53 return zpci_mod_fc(req, &fib, &status) ? -EIO : 0; 54 } 55 56 /* Modify PCI: Unregister adapter interruptions */ 57 static int zpci_clear_airq(struct zpci_dev *zdev) 58 { 59 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_DEREG_INT); 60 struct zpci_fib fib = {0}; 61 u8 cc, status; 62 63 cc = zpci_mod_fc(req, &fib, &status); 64 if (cc == 3 || (cc == 1 && status == 24)) 65 /* Function already gone or IRQs already deregistered. */ 66 cc = 0; 67 68 return cc ? -EIO : 0; 69 } 70 71 /* Modify PCI: Register CPU directed interruptions */ 72 static int zpci_set_directed_irq(struct zpci_dev *zdev) 73 { 74 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT_D); 75 struct zpci_fib fib = {0}; 76 u8 status; 77 78 fib.fmt = 1; 79 fib.fmt1.noi = zdev->msi_nr_irqs; 80 fib.fmt1.dibvo = zdev->msi_first_bit; 81 82 return zpci_mod_fc(req, &fib, &status) ? -EIO : 0; 83 } 84 85 /* Modify PCI: Unregister CPU directed interruptions */ 86 static int zpci_clear_directed_irq(struct zpci_dev *zdev) 87 { 88 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_DEREG_INT_D); 89 struct zpci_fib fib = {0}; 90 u8 cc, status; 91 92 fib.fmt = 1; 93 cc = zpci_mod_fc(req, &fib, &status); 94 if (cc == 3 || (cc == 1 && status == 24)) 95 /* Function already gone or IRQs already deregistered. */ 96 cc = 0; 97 98 return cc ? -EIO : 0; 99 } 100 101 static int zpci_set_irq_affinity(struct irq_data *data, const struct cpumask *dest, 102 bool force) 103 { 104 struct msi_desc *entry = irq_get_msi_desc(data->irq); 105 struct msi_msg msg = entry->msg; 106 107 msg.address_lo &= 0xff0000ff; 108 msg.address_lo |= (cpumask_first(dest) << 8); 109 pci_write_msi_msg(data->irq, &msg); 110 111 return IRQ_SET_MASK_OK; 112 } 113 114 static struct irq_chip zpci_irq_chip = { 115 .name = "PCI-MSI", 116 .irq_unmask = pci_msi_unmask_irq, 117 .irq_mask = pci_msi_mask_irq, 118 }; 119 120 static void zpci_handle_cpu_local_irq(bool rescan) 121 { 122 struct airq_iv *dibv = zpci_ibv[smp_processor_id()]; 123 unsigned long bit; 124 int irqs_on = 0; 125 126 for (bit = 0;;) { 127 /* Scan the directed IRQ bit vector */ 128 bit = airq_iv_scan(dibv, bit, airq_iv_end(dibv)); 129 if (bit == -1UL) { 130 if (!rescan || irqs_on++) 131 /* End of second scan with interrupts on. */ 132 break; 133 /* First scan complete, reenable interrupts. */ 134 if (zpci_set_irq_ctrl(SIC_IRQ_MODE_D_SINGLE, PCI_ISC)) 135 break; 136 bit = 0; 137 continue; 138 } 139 inc_irq_stat(IRQIO_MSI); 140 generic_handle_irq(airq_iv_get_data(dibv, bit)); 141 } 142 } 143 144 struct cpu_irq_data { 145 call_single_data_t csd; 146 atomic_t scheduled; 147 }; 148 static DEFINE_PER_CPU_SHARED_ALIGNED(struct cpu_irq_data, irq_data); 149 150 static void zpci_handle_remote_irq(void *data) 151 { 152 atomic_t *scheduled = data; 153 154 do { 155 zpci_handle_cpu_local_irq(false); 156 } while (atomic_dec_return(scheduled)); 157 } 158 159 static void zpci_handle_fallback_irq(void) 160 { 161 struct cpu_irq_data *cpu_data; 162 unsigned long cpu; 163 int irqs_on = 0; 164 165 for (cpu = 0;;) { 166 cpu = airq_iv_scan(zpci_sbv, cpu, airq_iv_end(zpci_sbv)); 167 if (cpu == -1UL) { 168 if (irqs_on++) 169 /* End of second scan with interrupts on. */ 170 break; 171 /* First scan complete, reenable interrupts. */ 172 if (zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, PCI_ISC)) 173 break; 174 cpu = 0; 175 continue; 176 } 177 cpu_data = &per_cpu(irq_data, cpu); 178 if (atomic_inc_return(&cpu_data->scheduled) > 1) 179 continue; 180 181 cpu_data->csd.func = zpci_handle_remote_irq; 182 cpu_data->csd.info = &cpu_data->scheduled; 183 cpu_data->csd.flags = 0; 184 smp_call_function_single_async(cpu, &cpu_data->csd); 185 } 186 } 187 188 static void zpci_directed_irq_handler(struct airq_struct *airq, bool floating) 189 { 190 if (floating) { 191 inc_irq_stat(IRQIO_PCF); 192 zpci_handle_fallback_irq(); 193 } else { 194 inc_irq_stat(IRQIO_PCD); 195 zpci_handle_cpu_local_irq(true); 196 } 197 } 198 199 static void zpci_floating_irq_handler(struct airq_struct *airq, bool floating) 200 { 201 unsigned long si, ai; 202 struct airq_iv *aibv; 203 int irqs_on = 0; 204 205 inc_irq_stat(IRQIO_PCF); 206 for (si = 0;;) { 207 /* Scan adapter summary indicator bit vector */ 208 si = airq_iv_scan(zpci_sbv, si, airq_iv_end(zpci_sbv)); 209 if (si == -1UL) { 210 if (irqs_on++) 211 /* End of second scan with interrupts on. */ 212 break; 213 /* First scan complete, reenable interrupts. */ 214 if (zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, PCI_ISC)) 215 break; 216 si = 0; 217 continue; 218 } 219 220 /* Scan the adapter interrupt vector for this device. */ 221 aibv = zpci_ibv[si]; 222 for (ai = 0;;) { 223 ai = airq_iv_scan(aibv, ai, airq_iv_end(aibv)); 224 if (ai == -1UL) 225 break; 226 inc_irq_stat(IRQIO_MSI); 227 airq_iv_lock(aibv, ai); 228 generic_handle_irq(airq_iv_get_data(aibv, ai)); 229 airq_iv_unlock(aibv, ai); 230 } 231 } 232 } 233 234 int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) 235 { 236 struct zpci_dev *zdev = to_zpci(pdev); 237 unsigned int hwirq, msi_vecs, cpu; 238 unsigned long bit; 239 struct msi_desc *msi; 240 struct msi_msg msg; 241 int rc, irq; 242 243 zdev->aisb = -1UL; 244 zdev->msi_first_bit = -1U; 245 if (type == PCI_CAP_ID_MSI && nvec > 1) 246 return 1; 247 msi_vecs = min_t(unsigned int, nvec, zdev->max_msi); 248 249 if (irq_delivery == DIRECTED) { 250 /* Allocate cpu vector bits */ 251 bit = airq_iv_alloc(zpci_ibv[0], msi_vecs); 252 if (bit == -1UL) 253 return -EIO; 254 } else { 255 /* Allocate adapter summary indicator bit */ 256 bit = airq_iv_alloc_bit(zpci_sbv); 257 if (bit == -1UL) 258 return -EIO; 259 zdev->aisb = bit; 260 261 /* Create adapter interrupt vector */ 262 zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA | AIRQ_IV_BITLOCK); 263 if (!zdev->aibv) 264 return -ENOMEM; 265 266 /* Wire up shortcut pointer */ 267 zpci_ibv[bit] = zdev->aibv; 268 /* Each function has its own interrupt vector */ 269 bit = 0; 270 } 271 272 /* Request MSI interrupts */ 273 hwirq = bit; 274 for_each_pci_msi_entry(msi, pdev) { 275 rc = -EIO; 276 if (hwirq - bit >= msi_vecs) 277 break; 278 irq = __irq_alloc_descs(-1, 0, 1, 0, THIS_MODULE, 279 (irq_delivery == DIRECTED) ? 280 msi->affinity : NULL); 281 if (irq < 0) 282 return -ENOMEM; 283 rc = irq_set_msi_desc(irq, msi); 284 if (rc) 285 return rc; 286 irq_set_chip_and_handler(irq, &zpci_irq_chip, 287 handle_percpu_irq); 288 msg.data = hwirq - bit; 289 if (irq_delivery == DIRECTED) { 290 msg.address_lo = zdev->msi_addr & 0xff0000ff; 291 msg.address_lo |= msi->affinity ? 292 (cpumask_first(&msi->affinity->mask) << 8) : 0; 293 for_each_possible_cpu(cpu) { 294 airq_iv_set_data(zpci_ibv[cpu], hwirq, irq); 295 } 296 } else { 297 msg.address_lo = zdev->msi_addr & 0xffffffff; 298 airq_iv_set_data(zdev->aibv, hwirq, irq); 299 } 300 msg.address_hi = zdev->msi_addr >> 32; 301 pci_write_msi_msg(irq, &msg); 302 hwirq++; 303 } 304 305 zdev->msi_first_bit = bit; 306 zdev->msi_nr_irqs = msi_vecs; 307 308 if (irq_delivery == DIRECTED) 309 rc = zpci_set_directed_irq(zdev); 310 else 311 rc = zpci_set_airq(zdev); 312 if (rc) 313 return rc; 314 315 return (msi_vecs == nvec) ? 0 : msi_vecs; 316 } 317 318 void arch_teardown_msi_irqs(struct pci_dev *pdev) 319 { 320 struct zpci_dev *zdev = to_zpci(pdev); 321 struct msi_desc *msi; 322 int rc; 323 324 /* Disable interrupts */ 325 if (irq_delivery == DIRECTED) 326 rc = zpci_clear_directed_irq(zdev); 327 else 328 rc = zpci_clear_airq(zdev); 329 if (rc) 330 return; 331 332 /* Release MSI interrupts */ 333 for_each_pci_msi_entry(msi, pdev) { 334 if (!msi->irq) 335 continue; 336 if (msi->msi_attrib.is_msix) 337 __pci_msix_desc_mask_irq(msi, 1); 338 else 339 __pci_msi_desc_mask_irq(msi, 1, 1); 340 irq_set_msi_desc(msi->irq, NULL); 341 irq_free_desc(msi->irq); 342 msi->msg.address_lo = 0; 343 msi->msg.address_hi = 0; 344 msi->msg.data = 0; 345 msi->irq = 0; 346 } 347 348 if (zdev->aisb != -1UL) { 349 zpci_ibv[zdev->aisb] = NULL; 350 airq_iv_free_bit(zpci_sbv, zdev->aisb); 351 zdev->aisb = -1UL; 352 } 353 if (zdev->aibv) { 354 airq_iv_release(zdev->aibv); 355 zdev->aibv = NULL; 356 } 357 358 if ((irq_delivery == DIRECTED) && zdev->msi_first_bit != -1U) 359 airq_iv_free(zpci_ibv[0], zdev->msi_first_bit, zdev->msi_nr_irqs); 360 } 361 362 static struct airq_struct zpci_airq = { 363 .handler = zpci_floating_irq_handler, 364 .isc = PCI_ISC, 365 }; 366 367 static void __init cpu_enable_directed_irq(void *unused) 368 { 369 union zpci_sic_iib iib = {{0}}; 370 371 iib.cdiib.dibv_addr = (u64) zpci_ibv[smp_processor_id()]->vector; 372 373 __zpci_set_irq_ctrl(SIC_IRQ_MODE_SET_CPU, 0, &iib); 374 zpci_set_irq_ctrl(SIC_IRQ_MODE_D_SINGLE, PCI_ISC); 375 } 376 377 static int __init zpci_directed_irq_init(void) 378 { 379 union zpci_sic_iib iib = {{0}}; 380 unsigned int cpu; 381 382 zpci_sbv = airq_iv_create(num_possible_cpus(), 0); 383 if (!zpci_sbv) 384 return -ENOMEM; 385 386 iib.diib.isc = PCI_ISC; 387 iib.diib.nr_cpus = num_possible_cpus(); 388 iib.diib.disb_addr = (u64) zpci_sbv->vector; 389 __zpci_set_irq_ctrl(SIC_IRQ_MODE_DIRECT, 0, &iib); 390 391 zpci_ibv = kcalloc(num_possible_cpus(), sizeof(*zpci_ibv), 392 GFP_KERNEL); 393 if (!zpci_ibv) 394 return -ENOMEM; 395 396 for_each_possible_cpu(cpu) { 397 /* 398 * Per CPU IRQ vectors look the same but bit-allocation 399 * is only done on the first vector. 400 */ 401 zpci_ibv[cpu] = airq_iv_create(cache_line_size() * BITS_PER_BYTE, 402 AIRQ_IV_DATA | 403 AIRQ_IV_CACHELINE | 404 (!cpu ? AIRQ_IV_ALLOC : 0)); 405 if (!zpci_ibv[cpu]) 406 return -ENOMEM; 407 } 408 on_each_cpu(cpu_enable_directed_irq, NULL, 1); 409 410 zpci_irq_chip.irq_set_affinity = zpci_set_irq_affinity; 411 412 return 0; 413 } 414 415 static int __init zpci_floating_irq_init(void) 416 { 417 zpci_ibv = kcalloc(ZPCI_NR_DEVICES, sizeof(*zpci_ibv), GFP_KERNEL); 418 if (!zpci_ibv) 419 return -ENOMEM; 420 421 zpci_sbv = airq_iv_create(ZPCI_NR_DEVICES, AIRQ_IV_ALLOC); 422 if (!zpci_sbv) 423 goto out_free; 424 425 return 0; 426 427 out_free: 428 kfree(zpci_ibv); 429 return -ENOMEM; 430 } 431 432 int __init zpci_irq_init(void) 433 { 434 int rc; 435 436 irq_delivery = sclp.has_dirq ? DIRECTED : FLOATING; 437 if (s390_pci_force_floating) 438 irq_delivery = FLOATING; 439 440 if (irq_delivery == DIRECTED) 441 zpci_airq.handler = zpci_directed_irq_handler; 442 443 rc = register_adapter_interrupt(&zpci_airq); 444 if (rc) 445 goto out; 446 /* Set summary to 1 to be called every time for the ISC. */ 447 *zpci_airq.lsi_ptr = 1; 448 449 switch (irq_delivery) { 450 case FLOATING: 451 rc = zpci_floating_irq_init(); 452 break; 453 case DIRECTED: 454 rc = zpci_directed_irq_init(); 455 break; 456 } 457 458 if (rc) 459 goto out_airq; 460 461 /* 462 * Enable floating IRQs (with suppression after one IRQ). When using 463 * directed IRQs this enables the fallback path. 464 */ 465 zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, PCI_ISC); 466 467 return 0; 468 out_airq: 469 unregister_adapter_interrupt(&zpci_airq); 470 out: 471 return rc; 472 } 473 474 void __init zpci_irq_exit(void) 475 { 476 unsigned int cpu; 477 478 if (irq_delivery == DIRECTED) { 479 for_each_possible_cpu(cpu) { 480 airq_iv_release(zpci_ibv[cpu]); 481 } 482 } 483 kfree(zpci_ibv); 484 if (zpci_sbv) 485 airq_iv_release(zpci_sbv); 486 unregister_adapter_interrupt(&zpci_airq); 487 } 488