1 // SPDX-License-Identifier: GPL-2.0 2 #define KMSG_COMPONENT "zpci" 3 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 4 5 #include <linux/kernel.h> 6 #include <linux/irq.h> 7 #include <linux/kernel_stat.h> 8 #include <linux/pci.h> 9 #include <linux/msi.h> 10 #include <linux/smp.h> 11 12 #include <asm/isc.h> 13 #include <asm/airq.h> 14 15 static enum {FLOATING, DIRECTED} irq_delivery; 16 17 #define SIC_IRQ_MODE_ALL 0 18 #define SIC_IRQ_MODE_SINGLE 1 19 #define SIC_IRQ_MODE_DIRECT 4 20 #define SIC_IRQ_MODE_D_ALL 16 21 #define SIC_IRQ_MODE_D_SINGLE 17 22 #define SIC_IRQ_MODE_SET_CPU 18 23 24 /* 25 * summary bit vector 26 * FLOATING - summary bit per function 27 * DIRECTED - summary bit per cpu (only used in fallback path) 28 */ 29 static struct airq_iv *zpci_sbv; 30 31 /* 32 * interrupt bit vectors 33 * FLOATING - interrupt bit vector per function 34 * DIRECTED - interrupt bit vector per cpu 35 */ 36 static struct airq_iv **zpci_ibv; 37 38 /* Modify PCI: Register adapter interruptions */ 39 static int zpci_set_airq(struct zpci_dev *zdev) 40 { 41 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT); 42 struct zpci_fib fib = {0}; 43 u8 status; 44 45 fib.fmt0.isc = PCI_ISC; 46 fib.fmt0.sum = 1; /* enable summary notifications */ 47 fib.fmt0.noi = airq_iv_end(zdev->aibv); 48 fib.fmt0.aibv = (unsigned long) zdev->aibv->vector; 49 fib.fmt0.aibvo = 0; /* each zdev has its own interrupt vector */ 50 fib.fmt0.aisb = (unsigned long) zpci_sbv->vector + (zdev->aisb/64)*8; 51 fib.fmt0.aisbo = zdev->aisb & 63; 52 53 return zpci_mod_fc(req, &fib, &status) ? -EIO : 0; 54 } 55 56 /* Modify PCI: Unregister adapter interruptions */ 57 static int zpci_clear_airq(struct zpci_dev *zdev) 58 { 59 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_DEREG_INT); 60 struct zpci_fib fib = {0}; 61 u8 cc, status; 62 63 cc = zpci_mod_fc(req, &fib, &status); 64 if (cc == 3 || (cc == 1 && status == 24)) 65 /* Function already gone or IRQs already deregistered. */ 66 cc = 0; 67 68 return cc ? -EIO : 0; 69 } 70 71 /* Modify PCI: Register CPU directed interruptions */ 72 static int zpci_set_directed_irq(struct zpci_dev *zdev) 73 { 74 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT_D); 75 struct zpci_fib fib = {0}; 76 u8 status; 77 78 fib.fmt = 1; 79 fib.fmt1.noi = zdev->msi_nr_irqs; 80 fib.fmt1.dibvo = zdev->msi_first_bit; 81 82 return zpci_mod_fc(req, &fib, &status) ? -EIO : 0; 83 } 84 85 /* Modify PCI: Unregister CPU directed interruptions */ 86 static int zpci_clear_directed_irq(struct zpci_dev *zdev) 87 { 88 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_DEREG_INT_D); 89 struct zpci_fib fib = {0}; 90 u8 cc, status; 91 92 fib.fmt = 1; 93 cc = zpci_mod_fc(req, &fib, &status); 94 if (cc == 3 || (cc == 1 && status == 24)) 95 /* Function already gone or IRQs already deregistered. */ 96 cc = 0; 97 98 return cc ? -EIO : 0; 99 } 100 101 static int zpci_set_irq_affinity(struct irq_data *data, const struct cpumask *dest, 102 bool force) 103 { 104 struct msi_desc *entry = irq_get_msi_desc(data->irq); 105 struct msi_msg msg = entry->msg; 106 int cpu_addr = smp_cpu_get_cpu_address(cpumask_first(dest)); 107 108 msg.address_lo &= 0xff0000ff; 109 msg.address_lo |= (cpu_addr << 8); 110 pci_write_msi_msg(data->irq, &msg); 111 112 return IRQ_SET_MASK_OK; 113 } 114 115 static struct irq_chip zpci_irq_chip = { 116 .name = "PCI-MSI", 117 .irq_unmask = pci_msi_unmask_irq, 118 .irq_mask = pci_msi_mask_irq, 119 }; 120 121 static void zpci_handle_cpu_local_irq(bool rescan) 122 { 123 struct airq_iv *dibv = zpci_ibv[smp_processor_id()]; 124 unsigned long bit; 125 int irqs_on = 0; 126 127 for (bit = 0;;) { 128 /* Scan the directed IRQ bit vector */ 129 bit = airq_iv_scan(dibv, bit, airq_iv_end(dibv)); 130 if (bit == -1UL) { 131 if (!rescan || irqs_on++) 132 /* End of second scan with interrupts on. */ 133 break; 134 /* First scan complete, reenable interrupts. */ 135 if (zpci_set_irq_ctrl(SIC_IRQ_MODE_D_SINGLE, PCI_ISC)) 136 break; 137 bit = 0; 138 continue; 139 } 140 inc_irq_stat(IRQIO_MSI); 141 generic_handle_irq(airq_iv_get_data(dibv, bit)); 142 } 143 } 144 145 struct cpu_irq_data { 146 call_single_data_t csd; 147 atomic_t scheduled; 148 }; 149 static DEFINE_PER_CPU_SHARED_ALIGNED(struct cpu_irq_data, irq_data); 150 151 static void zpci_handle_remote_irq(void *data) 152 { 153 atomic_t *scheduled = data; 154 155 do { 156 zpci_handle_cpu_local_irq(false); 157 } while (atomic_dec_return(scheduled)); 158 } 159 160 static void zpci_handle_fallback_irq(void) 161 { 162 struct cpu_irq_data *cpu_data; 163 unsigned long cpu; 164 int irqs_on = 0; 165 166 for (cpu = 0;;) { 167 cpu = airq_iv_scan(zpci_sbv, cpu, airq_iv_end(zpci_sbv)); 168 if (cpu == -1UL) { 169 if (irqs_on++) 170 /* End of second scan with interrupts on. */ 171 break; 172 /* First scan complete, reenable interrupts. */ 173 if (zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, PCI_ISC)) 174 break; 175 cpu = 0; 176 continue; 177 } 178 cpu_data = &per_cpu(irq_data, cpu); 179 if (atomic_inc_return(&cpu_data->scheduled) > 1) 180 continue; 181 182 INIT_CSD(&cpu_data->csd, zpci_handle_remote_irq, &cpu_data->scheduled); 183 smp_call_function_single_async(cpu, &cpu_data->csd); 184 } 185 } 186 187 static void zpci_directed_irq_handler(struct airq_struct *airq, bool floating) 188 { 189 if (floating) { 190 inc_irq_stat(IRQIO_PCF); 191 zpci_handle_fallback_irq(); 192 } else { 193 inc_irq_stat(IRQIO_PCD); 194 zpci_handle_cpu_local_irq(true); 195 } 196 } 197 198 static void zpci_floating_irq_handler(struct airq_struct *airq, bool floating) 199 { 200 unsigned long si, ai; 201 struct airq_iv *aibv; 202 int irqs_on = 0; 203 204 inc_irq_stat(IRQIO_PCF); 205 for (si = 0;;) { 206 /* Scan adapter summary indicator bit vector */ 207 si = airq_iv_scan(zpci_sbv, si, airq_iv_end(zpci_sbv)); 208 if (si == -1UL) { 209 if (irqs_on++) 210 /* End of second scan with interrupts on. */ 211 break; 212 /* First scan complete, reenable interrupts. */ 213 if (zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, PCI_ISC)) 214 break; 215 si = 0; 216 continue; 217 } 218 219 /* Scan the adapter interrupt vector for this device. */ 220 aibv = zpci_ibv[si]; 221 for (ai = 0;;) { 222 ai = airq_iv_scan(aibv, ai, airq_iv_end(aibv)); 223 if (ai == -1UL) 224 break; 225 inc_irq_stat(IRQIO_MSI); 226 airq_iv_lock(aibv, ai); 227 generic_handle_irq(airq_iv_get_data(aibv, ai)); 228 airq_iv_unlock(aibv, ai); 229 } 230 } 231 } 232 233 int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) 234 { 235 struct zpci_dev *zdev = to_zpci(pdev); 236 unsigned int hwirq, msi_vecs, cpu; 237 unsigned long bit; 238 struct msi_desc *msi; 239 struct msi_msg msg; 240 int cpu_addr; 241 int rc, irq; 242 243 zdev->aisb = -1UL; 244 zdev->msi_first_bit = -1U; 245 if (type == PCI_CAP_ID_MSI && nvec > 1) 246 return 1; 247 msi_vecs = min_t(unsigned int, nvec, zdev->max_msi); 248 249 if (irq_delivery == DIRECTED) { 250 /* Allocate cpu vector bits */ 251 bit = airq_iv_alloc(zpci_ibv[0], msi_vecs); 252 if (bit == -1UL) 253 return -EIO; 254 } else { 255 /* Allocate adapter summary indicator bit */ 256 bit = airq_iv_alloc_bit(zpci_sbv); 257 if (bit == -1UL) 258 return -EIO; 259 zdev->aisb = bit; 260 261 /* Create adapter interrupt vector */ 262 zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA | AIRQ_IV_BITLOCK); 263 if (!zdev->aibv) 264 return -ENOMEM; 265 266 /* Wire up shortcut pointer */ 267 zpci_ibv[bit] = zdev->aibv; 268 /* Each function has its own interrupt vector */ 269 bit = 0; 270 } 271 272 /* Request MSI interrupts */ 273 hwirq = bit; 274 for_each_pci_msi_entry(msi, pdev) { 275 rc = -EIO; 276 if (hwirq - bit >= msi_vecs) 277 break; 278 irq = __irq_alloc_descs(-1, 0, 1, 0, THIS_MODULE, 279 (irq_delivery == DIRECTED) ? 280 msi->affinity : NULL); 281 if (irq < 0) 282 return -ENOMEM; 283 rc = irq_set_msi_desc(irq, msi); 284 if (rc) 285 return rc; 286 irq_set_chip_and_handler(irq, &zpci_irq_chip, 287 handle_percpu_irq); 288 msg.data = hwirq - bit; 289 if (irq_delivery == DIRECTED) { 290 if (msi->affinity) 291 cpu = cpumask_first(&msi->affinity->mask); 292 else 293 cpu = 0; 294 cpu_addr = smp_cpu_get_cpu_address(cpu); 295 296 msg.address_lo = zdev->msi_addr & 0xff0000ff; 297 msg.address_lo |= (cpu_addr << 8); 298 299 for_each_possible_cpu(cpu) { 300 airq_iv_set_data(zpci_ibv[cpu], hwirq, irq); 301 } 302 } else { 303 msg.address_lo = zdev->msi_addr & 0xffffffff; 304 airq_iv_set_data(zdev->aibv, hwirq, irq); 305 } 306 msg.address_hi = zdev->msi_addr >> 32; 307 pci_write_msi_msg(irq, &msg); 308 hwirq++; 309 } 310 311 zdev->msi_first_bit = bit; 312 zdev->msi_nr_irqs = msi_vecs; 313 314 if (irq_delivery == DIRECTED) 315 rc = zpci_set_directed_irq(zdev); 316 else 317 rc = zpci_set_airq(zdev); 318 if (rc) 319 return rc; 320 321 return (msi_vecs == nvec) ? 0 : msi_vecs; 322 } 323 324 void arch_teardown_msi_irqs(struct pci_dev *pdev) 325 { 326 struct zpci_dev *zdev = to_zpci(pdev); 327 struct msi_desc *msi; 328 int rc; 329 330 /* Disable interrupts */ 331 if (irq_delivery == DIRECTED) 332 rc = zpci_clear_directed_irq(zdev); 333 else 334 rc = zpci_clear_airq(zdev); 335 if (rc) 336 return; 337 338 /* Release MSI interrupts */ 339 for_each_pci_msi_entry(msi, pdev) { 340 if (!msi->irq) 341 continue; 342 if (msi->msi_attrib.is_msix) 343 __pci_msix_desc_mask_irq(msi, 1); 344 else 345 __pci_msi_desc_mask_irq(msi, 1, 1); 346 irq_set_msi_desc(msi->irq, NULL); 347 irq_free_desc(msi->irq); 348 msi->msg.address_lo = 0; 349 msi->msg.address_hi = 0; 350 msi->msg.data = 0; 351 msi->irq = 0; 352 } 353 354 if (zdev->aisb != -1UL) { 355 zpci_ibv[zdev->aisb] = NULL; 356 airq_iv_free_bit(zpci_sbv, zdev->aisb); 357 zdev->aisb = -1UL; 358 } 359 if (zdev->aibv) { 360 airq_iv_release(zdev->aibv); 361 zdev->aibv = NULL; 362 } 363 364 if ((irq_delivery == DIRECTED) && zdev->msi_first_bit != -1U) 365 airq_iv_free(zpci_ibv[0], zdev->msi_first_bit, zdev->msi_nr_irqs); 366 } 367 368 static struct airq_struct zpci_airq = { 369 .handler = zpci_floating_irq_handler, 370 .isc = PCI_ISC, 371 }; 372 373 static void __init cpu_enable_directed_irq(void *unused) 374 { 375 union zpci_sic_iib iib = {{0}}; 376 377 iib.cdiib.dibv_addr = (u64) zpci_ibv[smp_processor_id()]->vector; 378 379 __zpci_set_irq_ctrl(SIC_IRQ_MODE_SET_CPU, 0, &iib); 380 zpci_set_irq_ctrl(SIC_IRQ_MODE_D_SINGLE, PCI_ISC); 381 } 382 383 static int __init zpci_directed_irq_init(void) 384 { 385 union zpci_sic_iib iib = {{0}}; 386 unsigned int cpu; 387 388 zpci_sbv = airq_iv_create(num_possible_cpus(), 0); 389 if (!zpci_sbv) 390 return -ENOMEM; 391 392 iib.diib.isc = PCI_ISC; 393 iib.diib.nr_cpus = num_possible_cpus(); 394 iib.diib.disb_addr = (u64) zpci_sbv->vector; 395 __zpci_set_irq_ctrl(SIC_IRQ_MODE_DIRECT, 0, &iib); 396 397 zpci_ibv = kcalloc(num_possible_cpus(), sizeof(*zpci_ibv), 398 GFP_KERNEL); 399 if (!zpci_ibv) 400 return -ENOMEM; 401 402 for_each_possible_cpu(cpu) { 403 /* 404 * Per CPU IRQ vectors look the same but bit-allocation 405 * is only done on the first vector. 406 */ 407 zpci_ibv[cpu] = airq_iv_create(cache_line_size() * BITS_PER_BYTE, 408 AIRQ_IV_DATA | 409 AIRQ_IV_CACHELINE | 410 (!cpu ? AIRQ_IV_ALLOC : 0)); 411 if (!zpci_ibv[cpu]) 412 return -ENOMEM; 413 } 414 on_each_cpu(cpu_enable_directed_irq, NULL, 1); 415 416 zpci_irq_chip.irq_set_affinity = zpci_set_irq_affinity; 417 418 return 0; 419 } 420 421 static int __init zpci_floating_irq_init(void) 422 { 423 zpci_ibv = kcalloc(ZPCI_NR_DEVICES, sizeof(*zpci_ibv), GFP_KERNEL); 424 if (!zpci_ibv) 425 return -ENOMEM; 426 427 zpci_sbv = airq_iv_create(ZPCI_NR_DEVICES, AIRQ_IV_ALLOC); 428 if (!zpci_sbv) 429 goto out_free; 430 431 return 0; 432 433 out_free: 434 kfree(zpci_ibv); 435 return -ENOMEM; 436 } 437 438 int __init zpci_irq_init(void) 439 { 440 int rc; 441 442 irq_delivery = sclp.has_dirq ? DIRECTED : FLOATING; 443 if (s390_pci_force_floating) 444 irq_delivery = FLOATING; 445 446 if (irq_delivery == DIRECTED) 447 zpci_airq.handler = zpci_directed_irq_handler; 448 449 rc = register_adapter_interrupt(&zpci_airq); 450 if (rc) 451 goto out; 452 /* Set summary to 1 to be called every time for the ISC. */ 453 *zpci_airq.lsi_ptr = 1; 454 455 switch (irq_delivery) { 456 case FLOATING: 457 rc = zpci_floating_irq_init(); 458 break; 459 case DIRECTED: 460 rc = zpci_directed_irq_init(); 461 break; 462 } 463 464 if (rc) 465 goto out_airq; 466 467 /* 468 * Enable floating IRQs (with suppression after one IRQ). When using 469 * directed IRQs this enables the fallback path. 470 */ 471 zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, PCI_ISC); 472 473 return 0; 474 out_airq: 475 unregister_adapter_interrupt(&zpci_airq); 476 out: 477 return rc; 478 } 479 480 void __init zpci_irq_exit(void) 481 { 482 unsigned int cpu; 483 484 if (irq_delivery == DIRECTED) { 485 for_each_possible_cpu(cpu) { 486 airq_iv_release(zpci_ibv[cpu]); 487 } 488 } 489 kfree(zpci_ibv); 490 if (zpci_sbv) 491 airq_iv_release(zpci_sbv); 492 unregister_adapter_interrupt(&zpci_airq); 493 } 494