1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 2004-2014 Cavium, Inc. 7 */ 8 9 #include <linux/of_address.h> 10 #include <linux/interrupt.h> 11 #include <linux/irqdomain.h> 12 #include <linux/bitops.h> 13 #include <linux/of_irq.h> 14 #include <linux/percpu.h> 15 #include <linux/slab.h> 16 #include <linux/irq.h> 17 #include <linux/smp.h> 18 #include <linux/of.h> 19 20 #include <asm/octeon/octeon.h> 21 #include <asm/octeon/cvmx-ciu2-defs.h> 22 23 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu0_en_mirror); 24 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu1_en_mirror); 25 static DEFINE_PER_CPU(raw_spinlock_t, octeon_irq_ciu_spinlock); 26 27 struct octeon_irq_ciu_domain_data { 28 int num_sum; /* number of sum registers (2 or 3). */ 29 }; 30 31 static __read_mostly u8 octeon_irq_ciu_to_irq[8][64]; 32 33 struct octeon_ciu_chip_data { 34 union { 35 struct { /* only used for ciu3 */ 36 u64 ciu3_addr; 37 unsigned int intsn; 38 }; 39 struct { /* only used for ciu/ciu2 */ 40 u8 line; 41 u8 bit; 42 u8 gpio_line; 43 }; 44 }; 45 int current_cpu; /* Next CPU expected to take this irq */ 46 }; 47 48 struct octeon_core_chip_data { 49 struct mutex core_irq_mutex; 50 bool current_en; 51 bool desired_en; 52 u8 bit; 53 }; 54 55 #define MIPS_CORE_IRQ_LINES 8 56 57 static struct octeon_core_chip_data octeon_irq_core_chip_data[MIPS_CORE_IRQ_LINES]; 58 59 static int octeon_irq_set_ciu_mapping(int irq, int line, int bit, int gpio_line, 60 struct irq_chip *chip, 61 irq_flow_handler_t handler) 62 { 63 struct octeon_ciu_chip_data *cd; 64 65 cd = kzalloc(sizeof(*cd), GFP_KERNEL); 66 if (!cd) 67 return -ENOMEM; 68 69 irq_set_chip_and_handler(irq, chip, handler); 70 71 cd->line = line; 72 cd->bit = bit; 73 cd->gpio_line = gpio_line; 74 75 irq_set_chip_data(irq, cd); 76 octeon_irq_ciu_to_irq[line][bit] = irq; 77 return 0; 78 } 79 80 static void octeon_irq_free_cd(struct irq_domain *d, unsigned int irq) 81 { 82 struct irq_data *data = irq_get_irq_data(irq); 83 struct octeon_ciu_chip_data *cd = irq_data_get_irq_chip_data(data); 84 85 irq_set_chip_data(irq, NULL); 86 kfree(cd); 87 } 88 89 static int octeon_irq_force_ciu_mapping(struct irq_domain *domain, 90 int irq, int line, int bit) 91 { 92 return irq_domain_associate(domain, irq, line << 6 | bit); 93 } 94 95 static int octeon_coreid_for_cpu(int cpu) 96 { 97 #ifdef CONFIG_SMP 98 return cpu_logical_map(cpu); 99 #else 100 return cvmx_get_core_num(); 101 #endif 102 } 103 104 static int octeon_cpu_for_coreid(int coreid) 105 { 106 #ifdef CONFIG_SMP 107 return cpu_number_map(coreid); 108 #else 109 return smp_processor_id(); 110 #endif 111 } 112 113 static void octeon_irq_core_ack(struct irq_data *data) 114 { 115 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 116 unsigned int bit = cd->bit; 117 118 /* 119 * We don't need to disable IRQs to make these atomic since 120 * they are already disabled earlier in the low level 121 * interrupt code. 122 */ 123 clear_c0_status(0x100 << bit); 124 /* The two user interrupts must be cleared manually. */ 125 if (bit < 2) 126 clear_c0_cause(0x100 << bit); 127 } 128 129 static void octeon_irq_core_eoi(struct irq_data *data) 130 { 131 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 132 133 /* 134 * We don't need to disable IRQs to make these atomic since 135 * they are already disabled earlier in the low level 136 * interrupt code. 137 */ 138 set_c0_status(0x100 << cd->bit); 139 } 140 141 static void octeon_irq_core_set_enable_local(void *arg) 142 { 143 struct irq_data *data = arg; 144 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 145 unsigned int mask = 0x100 << cd->bit; 146 147 /* 148 * Interrupts are already disabled, so these are atomic. 149 */ 150 if (cd->desired_en) 151 set_c0_status(mask); 152 else 153 clear_c0_status(mask); 154 155 } 156 157 static void octeon_irq_core_disable(struct irq_data *data) 158 { 159 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 160 cd->desired_en = false; 161 } 162 163 static void octeon_irq_core_enable(struct irq_data *data) 164 { 165 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 166 cd->desired_en = true; 167 } 168 169 static void octeon_irq_core_bus_lock(struct irq_data *data) 170 { 171 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 172 173 mutex_lock(&cd->core_irq_mutex); 174 } 175 176 static void octeon_irq_core_bus_sync_unlock(struct irq_data *data) 177 { 178 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 179 180 if (cd->desired_en != cd->current_en) { 181 on_each_cpu(octeon_irq_core_set_enable_local, data, 1); 182 183 cd->current_en = cd->desired_en; 184 } 185 186 mutex_unlock(&cd->core_irq_mutex); 187 } 188 189 static struct irq_chip octeon_irq_chip_core = { 190 .name = "Core", 191 .irq_enable = octeon_irq_core_enable, 192 .irq_disable = octeon_irq_core_disable, 193 .irq_ack = octeon_irq_core_ack, 194 .irq_eoi = octeon_irq_core_eoi, 195 .irq_bus_lock = octeon_irq_core_bus_lock, 196 .irq_bus_sync_unlock = octeon_irq_core_bus_sync_unlock, 197 198 .irq_cpu_online = octeon_irq_core_eoi, 199 .irq_cpu_offline = octeon_irq_core_ack, 200 .flags = IRQCHIP_ONOFFLINE_ENABLED, 201 }; 202 203 static void __init octeon_irq_init_core(void) 204 { 205 int i; 206 int irq; 207 struct octeon_core_chip_data *cd; 208 209 for (i = 0; i < MIPS_CORE_IRQ_LINES; i++) { 210 cd = &octeon_irq_core_chip_data[i]; 211 cd->current_en = false; 212 cd->desired_en = false; 213 cd->bit = i; 214 mutex_init(&cd->core_irq_mutex); 215 216 irq = OCTEON_IRQ_SW0 + i; 217 irq_set_chip_data(irq, cd); 218 irq_set_chip_and_handler(irq, &octeon_irq_chip_core, 219 handle_percpu_irq); 220 } 221 } 222 223 static int next_cpu_for_irq(struct irq_data *data) 224 { 225 226 #ifdef CONFIG_SMP 227 int cpu; 228 struct cpumask *mask = irq_data_get_affinity_mask(data); 229 int weight = cpumask_weight(mask); 230 struct octeon_ciu_chip_data *cd = irq_data_get_irq_chip_data(data); 231 232 if (weight > 1) { 233 cpu = cd->current_cpu; 234 for (;;) { 235 cpu = cpumask_next(cpu, mask); 236 if (cpu >= nr_cpu_ids) { 237 cpu = -1; 238 continue; 239 } else if (cpumask_test_cpu(cpu, cpu_online_mask)) { 240 break; 241 } 242 } 243 } else if (weight == 1) { 244 cpu = cpumask_first(mask); 245 } else { 246 cpu = smp_processor_id(); 247 } 248 cd->current_cpu = cpu; 249 return cpu; 250 #else 251 return smp_processor_id(); 252 #endif 253 } 254 255 static void octeon_irq_ciu_enable(struct irq_data *data) 256 { 257 int cpu = next_cpu_for_irq(data); 258 int coreid = octeon_coreid_for_cpu(cpu); 259 unsigned long *pen; 260 unsigned long flags; 261 struct octeon_ciu_chip_data *cd; 262 raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 263 264 cd = irq_data_get_irq_chip_data(data); 265 266 raw_spin_lock_irqsave(lock, flags); 267 if (cd->line == 0) { 268 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 269 __set_bit(cd->bit, pen); 270 /* 271 * Must be visible to octeon_irq_ip{2,3}_ciu() before 272 * enabling the irq. 273 */ 274 wmb(); 275 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 276 } else { 277 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 278 __set_bit(cd->bit, pen); 279 /* 280 * Must be visible to octeon_irq_ip{2,3}_ciu() before 281 * enabling the irq. 282 */ 283 wmb(); 284 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 285 } 286 raw_spin_unlock_irqrestore(lock, flags); 287 } 288 289 static void octeon_irq_ciu_enable_local(struct irq_data *data) 290 { 291 unsigned long *pen; 292 unsigned long flags; 293 struct octeon_ciu_chip_data *cd; 294 raw_spinlock_t *lock = this_cpu_ptr(&octeon_irq_ciu_spinlock); 295 296 cd = irq_data_get_irq_chip_data(data); 297 298 raw_spin_lock_irqsave(lock, flags); 299 if (cd->line == 0) { 300 pen = this_cpu_ptr(&octeon_irq_ciu0_en_mirror); 301 __set_bit(cd->bit, pen); 302 /* 303 * Must be visible to octeon_irq_ip{2,3}_ciu() before 304 * enabling the irq. 305 */ 306 wmb(); 307 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen); 308 } else { 309 pen = this_cpu_ptr(&octeon_irq_ciu1_en_mirror); 310 __set_bit(cd->bit, pen); 311 /* 312 * Must be visible to octeon_irq_ip{2,3}_ciu() before 313 * enabling the irq. 314 */ 315 wmb(); 316 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen); 317 } 318 raw_spin_unlock_irqrestore(lock, flags); 319 } 320 321 static void octeon_irq_ciu_disable_local(struct irq_data *data) 322 { 323 unsigned long *pen; 324 unsigned long flags; 325 struct octeon_ciu_chip_data *cd; 326 raw_spinlock_t *lock = this_cpu_ptr(&octeon_irq_ciu_spinlock); 327 328 cd = irq_data_get_irq_chip_data(data); 329 330 raw_spin_lock_irqsave(lock, flags); 331 if (cd->line == 0) { 332 pen = this_cpu_ptr(&octeon_irq_ciu0_en_mirror); 333 __clear_bit(cd->bit, pen); 334 /* 335 * Must be visible to octeon_irq_ip{2,3}_ciu() before 336 * enabling the irq. 337 */ 338 wmb(); 339 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen); 340 } else { 341 pen = this_cpu_ptr(&octeon_irq_ciu1_en_mirror); 342 __clear_bit(cd->bit, pen); 343 /* 344 * Must be visible to octeon_irq_ip{2,3}_ciu() before 345 * enabling the irq. 346 */ 347 wmb(); 348 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen); 349 } 350 raw_spin_unlock_irqrestore(lock, flags); 351 } 352 353 static void octeon_irq_ciu_disable_all(struct irq_data *data) 354 { 355 unsigned long flags; 356 unsigned long *pen; 357 int cpu; 358 struct octeon_ciu_chip_data *cd; 359 raw_spinlock_t *lock; 360 361 cd = irq_data_get_irq_chip_data(data); 362 363 for_each_online_cpu(cpu) { 364 int coreid = octeon_coreid_for_cpu(cpu); 365 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 366 if (cd->line == 0) 367 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 368 else 369 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 370 371 raw_spin_lock_irqsave(lock, flags); 372 __clear_bit(cd->bit, pen); 373 /* 374 * Must be visible to octeon_irq_ip{2,3}_ciu() before 375 * enabling the irq. 376 */ 377 wmb(); 378 if (cd->line == 0) 379 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 380 else 381 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 382 raw_spin_unlock_irqrestore(lock, flags); 383 } 384 } 385 386 static void octeon_irq_ciu_enable_all(struct irq_data *data) 387 { 388 unsigned long flags; 389 unsigned long *pen; 390 int cpu; 391 struct octeon_ciu_chip_data *cd; 392 raw_spinlock_t *lock; 393 394 cd = irq_data_get_irq_chip_data(data); 395 396 for_each_online_cpu(cpu) { 397 int coreid = octeon_coreid_for_cpu(cpu); 398 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 399 if (cd->line == 0) 400 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 401 else 402 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 403 404 raw_spin_lock_irqsave(lock, flags); 405 __set_bit(cd->bit, pen); 406 /* 407 * Must be visible to octeon_irq_ip{2,3}_ciu() before 408 * enabling the irq. 409 */ 410 wmb(); 411 if (cd->line == 0) 412 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 413 else 414 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 415 raw_spin_unlock_irqrestore(lock, flags); 416 } 417 } 418 419 /* 420 * Enable the irq on the next core in the affinity set for chips that 421 * have the EN*_W1{S,C} registers. 422 */ 423 static void octeon_irq_ciu_enable_v2(struct irq_data *data) 424 { 425 u64 mask; 426 int cpu = next_cpu_for_irq(data); 427 struct octeon_ciu_chip_data *cd; 428 429 cd = irq_data_get_irq_chip_data(data); 430 mask = 1ull << (cd->bit); 431 432 /* 433 * Called under the desc lock, so these should never get out 434 * of sync. 435 */ 436 if (cd->line == 0) { 437 int index = octeon_coreid_for_cpu(cpu) * 2; 438 set_bit(cd->bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu)); 439 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 440 } else { 441 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 442 set_bit(cd->bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 443 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 444 } 445 } 446 447 /* 448 * Enable the irq in the sum2 registers. 449 */ 450 static void octeon_irq_ciu_enable_sum2(struct irq_data *data) 451 { 452 u64 mask; 453 int cpu = next_cpu_for_irq(data); 454 int index = octeon_coreid_for_cpu(cpu); 455 struct octeon_ciu_chip_data *cd; 456 457 cd = irq_data_get_irq_chip_data(data); 458 mask = 1ull << (cd->bit); 459 460 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1S(index), mask); 461 } 462 463 /* 464 * Disable the irq in the sum2 registers. 465 */ 466 static void octeon_irq_ciu_disable_local_sum2(struct irq_data *data) 467 { 468 u64 mask; 469 int cpu = next_cpu_for_irq(data); 470 int index = octeon_coreid_for_cpu(cpu); 471 struct octeon_ciu_chip_data *cd; 472 473 cd = irq_data_get_irq_chip_data(data); 474 mask = 1ull << (cd->bit); 475 476 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(index), mask); 477 } 478 479 static void octeon_irq_ciu_ack_sum2(struct irq_data *data) 480 { 481 u64 mask; 482 int cpu = next_cpu_for_irq(data); 483 int index = octeon_coreid_for_cpu(cpu); 484 struct octeon_ciu_chip_data *cd; 485 486 cd = irq_data_get_irq_chip_data(data); 487 mask = 1ull << (cd->bit); 488 489 cvmx_write_csr(CVMX_CIU_SUM2_PPX_IP4(index), mask); 490 } 491 492 static void octeon_irq_ciu_disable_all_sum2(struct irq_data *data) 493 { 494 int cpu; 495 struct octeon_ciu_chip_data *cd; 496 u64 mask; 497 498 cd = irq_data_get_irq_chip_data(data); 499 mask = 1ull << (cd->bit); 500 501 for_each_online_cpu(cpu) { 502 int coreid = octeon_coreid_for_cpu(cpu); 503 504 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(coreid), mask); 505 } 506 } 507 508 /* 509 * Enable the irq on the current CPU for chips that 510 * have the EN*_W1{S,C} registers. 511 */ 512 static void octeon_irq_ciu_enable_local_v2(struct irq_data *data) 513 { 514 u64 mask; 515 struct octeon_ciu_chip_data *cd; 516 517 cd = irq_data_get_irq_chip_data(data); 518 mask = 1ull << (cd->bit); 519 520 if (cd->line == 0) { 521 int index = cvmx_get_core_num() * 2; 522 set_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu0_en_mirror)); 523 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 524 } else { 525 int index = cvmx_get_core_num() * 2 + 1; 526 set_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu1_en_mirror)); 527 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 528 } 529 } 530 531 static void octeon_irq_ciu_disable_local_v2(struct irq_data *data) 532 { 533 u64 mask; 534 struct octeon_ciu_chip_data *cd; 535 536 cd = irq_data_get_irq_chip_data(data); 537 mask = 1ull << (cd->bit); 538 539 if (cd->line == 0) { 540 int index = cvmx_get_core_num() * 2; 541 clear_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu0_en_mirror)); 542 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); 543 } else { 544 int index = cvmx_get_core_num() * 2 + 1; 545 clear_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu1_en_mirror)); 546 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask); 547 } 548 } 549 550 /* 551 * Write to the W1C bit in CVMX_CIU_INTX_SUM0 to clear the irq. 552 */ 553 static void octeon_irq_ciu_ack(struct irq_data *data) 554 { 555 u64 mask; 556 struct octeon_ciu_chip_data *cd; 557 558 cd = irq_data_get_irq_chip_data(data); 559 mask = 1ull << (cd->bit); 560 561 if (cd->line == 0) { 562 int index = cvmx_get_core_num() * 2; 563 cvmx_write_csr(CVMX_CIU_INTX_SUM0(index), mask); 564 } else { 565 cvmx_write_csr(CVMX_CIU_INT_SUM1, mask); 566 } 567 } 568 569 /* 570 * Disable the irq on the all cores for chips that have the EN*_W1{S,C} 571 * registers. 572 */ 573 static void octeon_irq_ciu_disable_all_v2(struct irq_data *data) 574 { 575 int cpu; 576 u64 mask; 577 struct octeon_ciu_chip_data *cd; 578 579 cd = irq_data_get_irq_chip_data(data); 580 mask = 1ull << (cd->bit); 581 582 if (cd->line == 0) { 583 for_each_online_cpu(cpu) { 584 int index = octeon_coreid_for_cpu(cpu) * 2; 585 clear_bit(cd->bit, 586 &per_cpu(octeon_irq_ciu0_en_mirror, cpu)); 587 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); 588 } 589 } else { 590 for_each_online_cpu(cpu) { 591 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 592 clear_bit(cd->bit, 593 &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 594 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask); 595 } 596 } 597 } 598 599 /* 600 * Enable the irq on the all cores for chips that have the EN*_W1{S,C} 601 * registers. 602 */ 603 static void octeon_irq_ciu_enable_all_v2(struct irq_data *data) 604 { 605 int cpu; 606 u64 mask; 607 struct octeon_ciu_chip_data *cd; 608 609 cd = irq_data_get_irq_chip_data(data); 610 mask = 1ull << (cd->bit); 611 612 if (cd->line == 0) { 613 for_each_online_cpu(cpu) { 614 int index = octeon_coreid_for_cpu(cpu) * 2; 615 set_bit(cd->bit, 616 &per_cpu(octeon_irq_ciu0_en_mirror, cpu)); 617 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 618 } 619 } else { 620 for_each_online_cpu(cpu) { 621 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 622 set_bit(cd->bit, 623 &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 624 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 625 } 626 } 627 } 628 629 static void octeon_irq_gpio_setup(struct irq_data *data) 630 { 631 union cvmx_gpio_bit_cfgx cfg; 632 struct octeon_ciu_chip_data *cd; 633 u32 t = irqd_get_trigger_type(data); 634 635 cd = irq_data_get_irq_chip_data(data); 636 637 cfg.u64 = 0; 638 cfg.s.int_en = 1; 639 cfg.s.int_type = (t & IRQ_TYPE_EDGE_BOTH) != 0; 640 cfg.s.rx_xor = (t & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_EDGE_FALLING)) != 0; 641 642 /* 140 nS glitch filter*/ 643 cfg.s.fil_cnt = 7; 644 cfg.s.fil_sel = 3; 645 646 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), cfg.u64); 647 } 648 649 static void octeon_irq_ciu_enable_gpio_v2(struct irq_data *data) 650 { 651 octeon_irq_gpio_setup(data); 652 octeon_irq_ciu_enable_v2(data); 653 } 654 655 static void octeon_irq_ciu_enable_gpio(struct irq_data *data) 656 { 657 octeon_irq_gpio_setup(data); 658 octeon_irq_ciu_enable(data); 659 } 660 661 static int octeon_irq_ciu_gpio_set_type(struct irq_data *data, unsigned int t) 662 { 663 irqd_set_trigger_type(data, t); 664 octeon_irq_gpio_setup(data); 665 666 if (irqd_get_trigger_type(data) & IRQ_TYPE_EDGE_BOTH) 667 irq_set_handler_locked(data, handle_edge_irq); 668 else 669 irq_set_handler_locked(data, handle_level_irq); 670 671 return IRQ_SET_MASK_OK; 672 } 673 674 static void octeon_irq_ciu_disable_gpio_v2(struct irq_data *data) 675 { 676 struct octeon_ciu_chip_data *cd; 677 678 cd = irq_data_get_irq_chip_data(data); 679 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0); 680 681 octeon_irq_ciu_disable_all_v2(data); 682 } 683 684 static void octeon_irq_ciu_disable_gpio(struct irq_data *data) 685 { 686 struct octeon_ciu_chip_data *cd; 687 688 cd = irq_data_get_irq_chip_data(data); 689 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0); 690 691 octeon_irq_ciu_disable_all(data); 692 } 693 694 static void octeon_irq_ciu_gpio_ack(struct irq_data *data) 695 { 696 struct octeon_ciu_chip_data *cd; 697 u64 mask; 698 699 cd = irq_data_get_irq_chip_data(data); 700 mask = 1ull << (cd->gpio_line); 701 702 cvmx_write_csr(CVMX_GPIO_INT_CLR, mask); 703 } 704 705 #ifdef CONFIG_SMP 706 707 static void octeon_irq_cpu_offline_ciu(struct irq_data *data) 708 { 709 int cpu = smp_processor_id(); 710 cpumask_t new_affinity; 711 struct cpumask *mask = irq_data_get_affinity_mask(data); 712 713 if (!cpumask_test_cpu(cpu, mask)) 714 return; 715 716 if (cpumask_weight(mask) > 1) { 717 /* 718 * It has multi CPU affinity, just remove this CPU 719 * from the affinity set. 720 */ 721 cpumask_copy(&new_affinity, mask); 722 cpumask_clear_cpu(cpu, &new_affinity); 723 } else { 724 /* Otherwise, put it on lowest numbered online CPU. */ 725 cpumask_clear(&new_affinity); 726 cpumask_set_cpu(cpumask_first(cpu_online_mask), &new_affinity); 727 } 728 irq_set_affinity_locked(data, &new_affinity, false); 729 } 730 731 static int octeon_irq_ciu_set_affinity(struct irq_data *data, 732 const struct cpumask *dest, bool force) 733 { 734 int cpu; 735 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 736 unsigned long flags; 737 struct octeon_ciu_chip_data *cd; 738 unsigned long *pen; 739 raw_spinlock_t *lock; 740 741 cd = irq_data_get_irq_chip_data(data); 742 743 /* 744 * For non-v2 CIU, we will allow only single CPU affinity. 745 * This removes the need to do locking in the .ack/.eoi 746 * functions. 747 */ 748 if (cpumask_weight(dest) != 1) 749 return -EINVAL; 750 751 if (!enable_one) 752 return 0; 753 754 755 for_each_online_cpu(cpu) { 756 int coreid = octeon_coreid_for_cpu(cpu); 757 758 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 759 raw_spin_lock_irqsave(lock, flags); 760 761 if (cd->line == 0) 762 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 763 else 764 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 765 766 if (cpumask_test_cpu(cpu, dest) && enable_one) { 767 enable_one = 0; 768 __set_bit(cd->bit, pen); 769 } else { 770 __clear_bit(cd->bit, pen); 771 } 772 /* 773 * Must be visible to octeon_irq_ip{2,3}_ciu() before 774 * enabling the irq. 775 */ 776 wmb(); 777 778 if (cd->line == 0) 779 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 780 else 781 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 782 783 raw_spin_unlock_irqrestore(lock, flags); 784 } 785 return 0; 786 } 787 788 /* 789 * Set affinity for the irq for chips that have the EN*_W1{S,C} 790 * registers. 791 */ 792 static int octeon_irq_ciu_set_affinity_v2(struct irq_data *data, 793 const struct cpumask *dest, 794 bool force) 795 { 796 int cpu; 797 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 798 u64 mask; 799 struct octeon_ciu_chip_data *cd; 800 801 if (!enable_one) 802 return 0; 803 804 cd = irq_data_get_irq_chip_data(data); 805 mask = 1ull << cd->bit; 806 807 if (cd->line == 0) { 808 for_each_online_cpu(cpu) { 809 unsigned long *pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 810 int index = octeon_coreid_for_cpu(cpu) * 2; 811 if (cpumask_test_cpu(cpu, dest) && enable_one) { 812 enable_one = false; 813 set_bit(cd->bit, pen); 814 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 815 } else { 816 clear_bit(cd->bit, pen); 817 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); 818 } 819 } 820 } else { 821 for_each_online_cpu(cpu) { 822 unsigned long *pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 823 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 824 if (cpumask_test_cpu(cpu, dest) && enable_one) { 825 enable_one = false; 826 set_bit(cd->bit, pen); 827 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 828 } else { 829 clear_bit(cd->bit, pen); 830 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask); 831 } 832 } 833 } 834 return 0; 835 } 836 837 static int octeon_irq_ciu_set_affinity_sum2(struct irq_data *data, 838 const struct cpumask *dest, 839 bool force) 840 { 841 int cpu; 842 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 843 u64 mask; 844 struct octeon_ciu_chip_data *cd; 845 846 if (!enable_one) 847 return 0; 848 849 cd = irq_data_get_irq_chip_data(data); 850 mask = 1ull << cd->bit; 851 852 for_each_online_cpu(cpu) { 853 int index = octeon_coreid_for_cpu(cpu); 854 855 if (cpumask_test_cpu(cpu, dest) && enable_one) { 856 enable_one = false; 857 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1S(index), mask); 858 } else { 859 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(index), mask); 860 } 861 } 862 return 0; 863 } 864 #endif 865 866 /* 867 * Newer octeon chips have support for lockless CIU operation. 868 */ 869 static struct irq_chip octeon_irq_chip_ciu_v2 = { 870 .name = "CIU", 871 .irq_enable = octeon_irq_ciu_enable_v2, 872 .irq_disable = octeon_irq_ciu_disable_all_v2, 873 .irq_mask = octeon_irq_ciu_disable_local_v2, 874 .irq_unmask = octeon_irq_ciu_enable_v2, 875 #ifdef CONFIG_SMP 876 .irq_set_affinity = octeon_irq_ciu_set_affinity_v2, 877 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 878 #endif 879 }; 880 881 static struct irq_chip octeon_irq_chip_ciu_v2_edge = { 882 .name = "CIU", 883 .irq_enable = octeon_irq_ciu_enable_v2, 884 .irq_disable = octeon_irq_ciu_disable_all_v2, 885 .irq_ack = octeon_irq_ciu_ack, 886 .irq_mask = octeon_irq_ciu_disable_local_v2, 887 .irq_unmask = octeon_irq_ciu_enable_v2, 888 #ifdef CONFIG_SMP 889 .irq_set_affinity = octeon_irq_ciu_set_affinity_v2, 890 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 891 #endif 892 }; 893 894 /* 895 * Newer octeon chips have support for lockless CIU operation. 896 */ 897 static struct irq_chip octeon_irq_chip_ciu_sum2 = { 898 .name = "CIU", 899 .irq_enable = octeon_irq_ciu_enable_sum2, 900 .irq_disable = octeon_irq_ciu_disable_all_sum2, 901 .irq_mask = octeon_irq_ciu_disable_local_sum2, 902 .irq_unmask = octeon_irq_ciu_enable_sum2, 903 #ifdef CONFIG_SMP 904 .irq_set_affinity = octeon_irq_ciu_set_affinity_sum2, 905 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 906 #endif 907 }; 908 909 static struct irq_chip octeon_irq_chip_ciu_sum2_edge = { 910 .name = "CIU", 911 .irq_enable = octeon_irq_ciu_enable_sum2, 912 .irq_disable = octeon_irq_ciu_disable_all_sum2, 913 .irq_ack = octeon_irq_ciu_ack_sum2, 914 .irq_mask = octeon_irq_ciu_disable_local_sum2, 915 .irq_unmask = octeon_irq_ciu_enable_sum2, 916 #ifdef CONFIG_SMP 917 .irq_set_affinity = octeon_irq_ciu_set_affinity_sum2, 918 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 919 #endif 920 }; 921 922 static struct irq_chip octeon_irq_chip_ciu = { 923 .name = "CIU", 924 .irq_enable = octeon_irq_ciu_enable, 925 .irq_disable = octeon_irq_ciu_disable_all, 926 .irq_mask = octeon_irq_ciu_disable_local, 927 .irq_unmask = octeon_irq_ciu_enable, 928 #ifdef CONFIG_SMP 929 .irq_set_affinity = octeon_irq_ciu_set_affinity, 930 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 931 #endif 932 }; 933 934 static struct irq_chip octeon_irq_chip_ciu_edge = { 935 .name = "CIU", 936 .irq_enable = octeon_irq_ciu_enable, 937 .irq_disable = octeon_irq_ciu_disable_all, 938 .irq_ack = octeon_irq_ciu_ack, 939 .irq_mask = octeon_irq_ciu_disable_local, 940 .irq_unmask = octeon_irq_ciu_enable, 941 #ifdef CONFIG_SMP 942 .irq_set_affinity = octeon_irq_ciu_set_affinity, 943 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 944 #endif 945 }; 946 947 /* The mbox versions don't do any affinity or round-robin. */ 948 static struct irq_chip octeon_irq_chip_ciu_mbox_v2 = { 949 .name = "CIU-M", 950 .irq_enable = octeon_irq_ciu_enable_all_v2, 951 .irq_disable = octeon_irq_ciu_disable_all_v2, 952 .irq_ack = octeon_irq_ciu_disable_local_v2, 953 .irq_eoi = octeon_irq_ciu_enable_local_v2, 954 955 .irq_cpu_online = octeon_irq_ciu_enable_local_v2, 956 .irq_cpu_offline = octeon_irq_ciu_disable_local_v2, 957 .flags = IRQCHIP_ONOFFLINE_ENABLED, 958 }; 959 960 static struct irq_chip octeon_irq_chip_ciu_mbox = { 961 .name = "CIU-M", 962 .irq_enable = octeon_irq_ciu_enable_all, 963 .irq_disable = octeon_irq_ciu_disable_all, 964 .irq_ack = octeon_irq_ciu_disable_local, 965 .irq_eoi = octeon_irq_ciu_enable_local, 966 967 .irq_cpu_online = octeon_irq_ciu_enable_local, 968 .irq_cpu_offline = octeon_irq_ciu_disable_local, 969 .flags = IRQCHIP_ONOFFLINE_ENABLED, 970 }; 971 972 static struct irq_chip octeon_irq_chip_ciu_gpio_v2 = { 973 .name = "CIU-GPIO", 974 .irq_enable = octeon_irq_ciu_enable_gpio_v2, 975 .irq_disable = octeon_irq_ciu_disable_gpio_v2, 976 .irq_ack = octeon_irq_ciu_gpio_ack, 977 .irq_mask = octeon_irq_ciu_disable_local_v2, 978 .irq_unmask = octeon_irq_ciu_enable_v2, 979 .irq_set_type = octeon_irq_ciu_gpio_set_type, 980 #ifdef CONFIG_SMP 981 .irq_set_affinity = octeon_irq_ciu_set_affinity_v2, 982 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 983 #endif 984 .flags = IRQCHIP_SET_TYPE_MASKED, 985 }; 986 987 static struct irq_chip octeon_irq_chip_ciu_gpio = { 988 .name = "CIU-GPIO", 989 .irq_enable = octeon_irq_ciu_enable_gpio, 990 .irq_disable = octeon_irq_ciu_disable_gpio, 991 .irq_mask = octeon_irq_ciu_disable_local, 992 .irq_unmask = octeon_irq_ciu_enable, 993 .irq_ack = octeon_irq_ciu_gpio_ack, 994 .irq_set_type = octeon_irq_ciu_gpio_set_type, 995 #ifdef CONFIG_SMP 996 .irq_set_affinity = octeon_irq_ciu_set_affinity, 997 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 998 #endif 999 .flags = IRQCHIP_SET_TYPE_MASKED, 1000 }; 1001 1002 /* 1003 * Watchdog interrupts are special. They are associated with a single 1004 * core, so we hardwire the affinity to that core. 1005 */ 1006 static void octeon_irq_ciu_wd_enable(struct irq_data *data) 1007 { 1008 unsigned long flags; 1009 unsigned long *pen; 1010 int coreid = data->irq - OCTEON_IRQ_WDOG0; /* Bit 0-63 of EN1 */ 1011 int cpu = octeon_cpu_for_coreid(coreid); 1012 raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 1013 1014 raw_spin_lock_irqsave(lock, flags); 1015 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 1016 __set_bit(coreid, pen); 1017 /* 1018 * Must be visible to octeon_irq_ip{2,3}_ciu() before enabling 1019 * the irq. 1020 */ 1021 wmb(); 1022 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 1023 raw_spin_unlock_irqrestore(lock, flags); 1024 } 1025 1026 /* 1027 * Watchdog interrupts are special. They are associated with a single 1028 * core, so we hardwire the affinity to that core. 1029 */ 1030 static void octeon_irq_ciu1_wd_enable_v2(struct irq_data *data) 1031 { 1032 int coreid = data->irq - OCTEON_IRQ_WDOG0; 1033 int cpu = octeon_cpu_for_coreid(coreid); 1034 1035 set_bit(coreid, &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 1036 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(coreid * 2 + 1), 1ull << coreid); 1037 } 1038 1039 1040 static struct irq_chip octeon_irq_chip_ciu_wd_v2 = { 1041 .name = "CIU-W", 1042 .irq_enable = octeon_irq_ciu1_wd_enable_v2, 1043 .irq_disable = octeon_irq_ciu_disable_all_v2, 1044 .irq_mask = octeon_irq_ciu_disable_local_v2, 1045 .irq_unmask = octeon_irq_ciu_enable_local_v2, 1046 }; 1047 1048 static struct irq_chip octeon_irq_chip_ciu_wd = { 1049 .name = "CIU-W", 1050 .irq_enable = octeon_irq_ciu_wd_enable, 1051 .irq_disable = octeon_irq_ciu_disable_all, 1052 .irq_mask = octeon_irq_ciu_disable_local, 1053 .irq_unmask = octeon_irq_ciu_enable_local, 1054 }; 1055 1056 static bool octeon_irq_ciu_is_edge(unsigned int line, unsigned int bit) 1057 { 1058 bool edge = false; 1059 1060 if (line == 0) 1061 switch (bit) { 1062 case 48 ... 49: /* GMX DRP */ 1063 case 50: /* IPD_DRP */ 1064 case 52 ... 55: /* Timers */ 1065 case 58: /* MPI */ 1066 edge = true; 1067 break; 1068 default: 1069 break; 1070 } 1071 else /* line == 1 */ 1072 switch (bit) { 1073 case 47: /* PTP */ 1074 edge = true; 1075 break; 1076 default: 1077 break; 1078 } 1079 return edge; 1080 } 1081 1082 struct octeon_irq_gpio_domain_data { 1083 unsigned int base_hwirq; 1084 }; 1085 1086 static int octeon_irq_gpio_xlat(struct irq_domain *d, 1087 struct device_node *node, 1088 const u32 *intspec, 1089 unsigned int intsize, 1090 unsigned long *out_hwirq, 1091 unsigned int *out_type) 1092 { 1093 unsigned int type; 1094 unsigned int pin; 1095 unsigned int trigger; 1096 1097 if (irq_domain_get_of_node(d) != node) 1098 return -EINVAL; 1099 1100 if (intsize < 2) 1101 return -EINVAL; 1102 1103 pin = intspec[0]; 1104 if (pin >= 16) 1105 return -EINVAL; 1106 1107 trigger = intspec[1]; 1108 1109 switch (trigger) { 1110 case 1: 1111 type = IRQ_TYPE_EDGE_RISING; 1112 break; 1113 case 2: 1114 type = IRQ_TYPE_EDGE_FALLING; 1115 break; 1116 case 4: 1117 type = IRQ_TYPE_LEVEL_HIGH; 1118 break; 1119 case 8: 1120 type = IRQ_TYPE_LEVEL_LOW; 1121 break; 1122 default: 1123 pr_err("Error: (%s) Invalid irq trigger specification: %x\n", 1124 node->name, 1125 trigger); 1126 type = IRQ_TYPE_LEVEL_LOW; 1127 break; 1128 } 1129 *out_type = type; 1130 *out_hwirq = pin; 1131 1132 return 0; 1133 } 1134 1135 static int octeon_irq_ciu_xlat(struct irq_domain *d, 1136 struct device_node *node, 1137 const u32 *intspec, 1138 unsigned int intsize, 1139 unsigned long *out_hwirq, 1140 unsigned int *out_type) 1141 { 1142 unsigned int ciu, bit; 1143 struct octeon_irq_ciu_domain_data *dd = d->host_data; 1144 1145 ciu = intspec[0]; 1146 bit = intspec[1]; 1147 1148 if (ciu >= dd->num_sum || bit > 63) 1149 return -EINVAL; 1150 1151 *out_hwirq = (ciu << 6) | bit; 1152 *out_type = 0; 1153 1154 return 0; 1155 } 1156 1157 static struct irq_chip *octeon_irq_ciu_chip; 1158 static struct irq_chip *octeon_irq_ciu_chip_edge; 1159 static struct irq_chip *octeon_irq_gpio_chip; 1160 1161 static bool octeon_irq_virq_in_range(unsigned int virq) 1162 { 1163 /* We cannot let it overflow the mapping array. */ 1164 if (virq < (1ul << 8 * sizeof(octeon_irq_ciu_to_irq[0][0]))) 1165 return true; 1166 1167 WARN_ONCE(true, "virq out of range %u.\n", virq); 1168 return false; 1169 } 1170 1171 static int octeon_irq_ciu_map(struct irq_domain *d, 1172 unsigned int virq, irq_hw_number_t hw) 1173 { 1174 int rv; 1175 unsigned int line = hw >> 6; 1176 unsigned int bit = hw & 63; 1177 struct octeon_irq_ciu_domain_data *dd = d->host_data; 1178 1179 if (!octeon_irq_virq_in_range(virq)) 1180 return -EINVAL; 1181 1182 /* Don't map irq if it is reserved for GPIO. */ 1183 if (line == 0 && bit >= 16 && bit <32) 1184 return 0; 1185 1186 if (line >= dd->num_sum || octeon_irq_ciu_to_irq[line][bit] != 0) 1187 return -EINVAL; 1188 1189 if (line == 2) { 1190 if (octeon_irq_ciu_is_edge(line, bit)) 1191 rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1192 &octeon_irq_chip_ciu_sum2_edge, 1193 handle_edge_irq); 1194 else 1195 rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1196 &octeon_irq_chip_ciu_sum2, 1197 handle_level_irq); 1198 } else { 1199 if (octeon_irq_ciu_is_edge(line, bit)) 1200 rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1201 octeon_irq_ciu_chip_edge, 1202 handle_edge_irq); 1203 else 1204 rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1205 octeon_irq_ciu_chip, 1206 handle_level_irq); 1207 } 1208 return rv; 1209 } 1210 1211 static int octeon_irq_gpio_map(struct irq_domain *d, 1212 unsigned int virq, irq_hw_number_t hw) 1213 { 1214 struct octeon_irq_gpio_domain_data *gpiod = d->host_data; 1215 unsigned int line, bit; 1216 int r; 1217 1218 if (!octeon_irq_virq_in_range(virq)) 1219 return -EINVAL; 1220 1221 line = (hw + gpiod->base_hwirq) >> 6; 1222 bit = (hw + gpiod->base_hwirq) & 63; 1223 if (line > ARRAY_SIZE(octeon_irq_ciu_to_irq) || 1224 octeon_irq_ciu_to_irq[line][bit] != 0) 1225 return -EINVAL; 1226 1227 /* 1228 * Default to handle_level_irq. If the DT contains a different 1229 * trigger type, it will call the irq_set_type callback and 1230 * the handler gets updated. 1231 */ 1232 r = octeon_irq_set_ciu_mapping(virq, line, bit, hw, 1233 octeon_irq_gpio_chip, handle_level_irq); 1234 return r; 1235 } 1236 1237 static struct irq_domain_ops octeon_irq_domain_ciu_ops = { 1238 .map = octeon_irq_ciu_map, 1239 .unmap = octeon_irq_free_cd, 1240 .xlate = octeon_irq_ciu_xlat, 1241 }; 1242 1243 static struct irq_domain_ops octeon_irq_domain_gpio_ops = { 1244 .map = octeon_irq_gpio_map, 1245 .unmap = octeon_irq_free_cd, 1246 .xlate = octeon_irq_gpio_xlat, 1247 }; 1248 1249 static void octeon_irq_ip2_ciu(void) 1250 { 1251 const unsigned long core_id = cvmx_get_core_num(); 1252 u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INTX_SUM0(core_id * 2)); 1253 1254 ciu_sum &= __this_cpu_read(octeon_irq_ciu0_en_mirror); 1255 if (likely(ciu_sum)) { 1256 int bit = fls64(ciu_sum) - 1; 1257 int irq = octeon_irq_ciu_to_irq[0][bit]; 1258 if (likely(irq)) 1259 do_IRQ(irq); 1260 else 1261 spurious_interrupt(); 1262 } else { 1263 spurious_interrupt(); 1264 } 1265 } 1266 1267 static void octeon_irq_ip3_ciu(void) 1268 { 1269 u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INT_SUM1); 1270 1271 ciu_sum &= __this_cpu_read(octeon_irq_ciu1_en_mirror); 1272 if (likely(ciu_sum)) { 1273 int bit = fls64(ciu_sum) - 1; 1274 int irq = octeon_irq_ciu_to_irq[1][bit]; 1275 if (likely(irq)) 1276 do_IRQ(irq); 1277 else 1278 spurious_interrupt(); 1279 } else { 1280 spurious_interrupt(); 1281 } 1282 } 1283 1284 static void octeon_irq_ip4_ciu(void) 1285 { 1286 int coreid = cvmx_get_core_num(); 1287 u64 ciu_sum = cvmx_read_csr(CVMX_CIU_SUM2_PPX_IP4(coreid)); 1288 u64 ciu_en = cvmx_read_csr(CVMX_CIU_EN2_PPX_IP4(coreid)); 1289 1290 ciu_sum &= ciu_en; 1291 if (likely(ciu_sum)) { 1292 int bit = fls64(ciu_sum) - 1; 1293 int irq = octeon_irq_ciu_to_irq[2][bit]; 1294 1295 if (likely(irq)) 1296 do_IRQ(irq); 1297 else 1298 spurious_interrupt(); 1299 } else { 1300 spurious_interrupt(); 1301 } 1302 } 1303 1304 static bool octeon_irq_use_ip4; 1305 1306 static void octeon_irq_local_enable_ip4(void *arg) 1307 { 1308 set_c0_status(STATUSF_IP4); 1309 } 1310 1311 static void octeon_irq_ip4_mask(void) 1312 { 1313 clear_c0_status(STATUSF_IP4); 1314 spurious_interrupt(); 1315 } 1316 1317 static void (*octeon_irq_ip2)(void); 1318 static void (*octeon_irq_ip3)(void); 1319 static void (*octeon_irq_ip4)(void); 1320 1321 void (*octeon_irq_setup_secondary)(void); 1322 1323 void octeon_irq_set_ip4_handler(octeon_irq_ip4_handler_t h) 1324 { 1325 octeon_irq_ip4 = h; 1326 octeon_irq_use_ip4 = true; 1327 on_each_cpu(octeon_irq_local_enable_ip4, NULL, 1); 1328 } 1329 1330 static void octeon_irq_percpu_enable(void) 1331 { 1332 irq_cpu_online(); 1333 } 1334 1335 static void octeon_irq_init_ciu_percpu(void) 1336 { 1337 int coreid = cvmx_get_core_num(); 1338 1339 1340 __this_cpu_write(octeon_irq_ciu0_en_mirror, 0); 1341 __this_cpu_write(octeon_irq_ciu1_en_mirror, 0); 1342 wmb(); 1343 raw_spin_lock_init(this_cpu_ptr(&octeon_irq_ciu_spinlock)); 1344 /* 1345 * Disable All CIU Interrupts. The ones we need will be 1346 * enabled later. Read the SUM register so we know the write 1347 * completed. 1348 */ 1349 cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2)), 0); 1350 cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2 + 1)), 0); 1351 cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2)), 0); 1352 cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2 + 1)), 0); 1353 cvmx_read_csr(CVMX_CIU_INTX_SUM0((coreid * 2))); 1354 } 1355 1356 static void octeon_irq_init_ciu2_percpu(void) 1357 { 1358 u64 regx, ipx; 1359 int coreid = cvmx_get_core_num(); 1360 u64 base = CVMX_CIU2_EN_PPX_IP2_WRKQ(coreid); 1361 1362 /* 1363 * Disable All CIU2 Interrupts. The ones we need will be 1364 * enabled later. Read the SUM register so we know the write 1365 * completed. 1366 * 1367 * There are 9 registers and 3 IPX levels with strides 0x1000 1368 * and 0x200 respectivly. Use loops to clear them. 1369 */ 1370 for (regx = 0; regx <= 0x8000; regx += 0x1000) { 1371 for (ipx = 0; ipx <= 0x400; ipx += 0x200) 1372 cvmx_write_csr(base + regx + ipx, 0); 1373 } 1374 1375 cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(coreid)); 1376 } 1377 1378 static void octeon_irq_setup_secondary_ciu(void) 1379 { 1380 octeon_irq_init_ciu_percpu(); 1381 octeon_irq_percpu_enable(); 1382 1383 /* Enable the CIU lines */ 1384 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 1385 if (octeon_irq_use_ip4) 1386 set_c0_status(STATUSF_IP4); 1387 else 1388 clear_c0_status(STATUSF_IP4); 1389 } 1390 1391 static void octeon_irq_setup_secondary_ciu2(void) 1392 { 1393 octeon_irq_init_ciu2_percpu(); 1394 octeon_irq_percpu_enable(); 1395 1396 /* Enable the CIU lines */ 1397 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 1398 if (octeon_irq_use_ip4) 1399 set_c0_status(STATUSF_IP4); 1400 else 1401 clear_c0_status(STATUSF_IP4); 1402 } 1403 1404 static int __init octeon_irq_init_ciu( 1405 struct device_node *ciu_node, struct device_node *parent) 1406 { 1407 unsigned int i, r; 1408 struct irq_chip *chip; 1409 struct irq_chip *chip_edge; 1410 struct irq_chip *chip_mbox; 1411 struct irq_chip *chip_wd; 1412 struct irq_domain *ciu_domain = NULL; 1413 struct octeon_irq_ciu_domain_data *dd; 1414 1415 dd = kzalloc(sizeof(*dd), GFP_KERNEL); 1416 if (!dd) 1417 return -ENOMEM; 1418 1419 octeon_irq_init_ciu_percpu(); 1420 octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu; 1421 1422 octeon_irq_ip2 = octeon_irq_ip2_ciu; 1423 octeon_irq_ip3 = octeon_irq_ip3_ciu; 1424 if ((OCTEON_IS_OCTEON2() || OCTEON_IS_OCTEON3()) 1425 && !OCTEON_IS_MODEL(OCTEON_CN63XX)) { 1426 octeon_irq_ip4 = octeon_irq_ip4_ciu; 1427 dd->num_sum = 3; 1428 octeon_irq_use_ip4 = true; 1429 } else { 1430 octeon_irq_ip4 = octeon_irq_ip4_mask; 1431 dd->num_sum = 2; 1432 octeon_irq_use_ip4 = false; 1433 } 1434 if (OCTEON_IS_MODEL(OCTEON_CN58XX_PASS2_X) || 1435 OCTEON_IS_MODEL(OCTEON_CN56XX_PASS2_X) || 1436 OCTEON_IS_MODEL(OCTEON_CN52XX_PASS2_X) || 1437 OCTEON_IS_OCTEON2() || OCTEON_IS_OCTEON3()) { 1438 chip = &octeon_irq_chip_ciu_v2; 1439 chip_edge = &octeon_irq_chip_ciu_v2_edge; 1440 chip_mbox = &octeon_irq_chip_ciu_mbox_v2; 1441 chip_wd = &octeon_irq_chip_ciu_wd_v2; 1442 octeon_irq_gpio_chip = &octeon_irq_chip_ciu_gpio_v2; 1443 } else { 1444 chip = &octeon_irq_chip_ciu; 1445 chip_edge = &octeon_irq_chip_ciu_edge; 1446 chip_mbox = &octeon_irq_chip_ciu_mbox; 1447 chip_wd = &octeon_irq_chip_ciu_wd; 1448 octeon_irq_gpio_chip = &octeon_irq_chip_ciu_gpio; 1449 } 1450 octeon_irq_ciu_chip = chip; 1451 octeon_irq_ciu_chip_edge = chip_edge; 1452 1453 /* Mips internal */ 1454 octeon_irq_init_core(); 1455 1456 ciu_domain = irq_domain_add_tree( 1457 ciu_node, &octeon_irq_domain_ciu_ops, dd); 1458 irq_set_default_host(ciu_domain); 1459 1460 /* CIU_0 */ 1461 for (i = 0; i < 16; i++) { 1462 r = octeon_irq_force_ciu_mapping( 1463 ciu_domain, i + OCTEON_IRQ_WORKQ0, 0, i + 0); 1464 if (r) 1465 goto err; 1466 } 1467 1468 r = octeon_irq_set_ciu_mapping( 1469 OCTEON_IRQ_MBOX0, 0, 32, 0, chip_mbox, handle_percpu_irq); 1470 if (r) 1471 goto err; 1472 r = octeon_irq_set_ciu_mapping( 1473 OCTEON_IRQ_MBOX1, 0, 33, 0, chip_mbox, handle_percpu_irq); 1474 if (r) 1475 goto err; 1476 1477 for (i = 0; i < 4; i++) { 1478 r = octeon_irq_force_ciu_mapping( 1479 ciu_domain, i + OCTEON_IRQ_PCI_INT0, 0, i + 36); 1480 if (r) 1481 goto err; 1482 } 1483 for (i = 0; i < 4; i++) { 1484 r = octeon_irq_force_ciu_mapping( 1485 ciu_domain, i + OCTEON_IRQ_PCI_MSI0, 0, i + 40); 1486 if (r) 1487 goto err; 1488 } 1489 1490 r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_TWSI, 0, 45); 1491 if (r) 1492 goto err; 1493 1494 r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_RML, 0, 46); 1495 if (r) 1496 goto err; 1497 1498 for (i = 0; i < 4; i++) { 1499 r = octeon_irq_force_ciu_mapping( 1500 ciu_domain, i + OCTEON_IRQ_TIMER0, 0, i + 52); 1501 if (r) 1502 goto err; 1503 } 1504 1505 r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_USB0, 0, 56); 1506 if (r) 1507 goto err; 1508 1509 r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_TWSI2, 0, 59); 1510 if (r) 1511 goto err; 1512 1513 /* CIU_1 */ 1514 for (i = 0; i < 16; i++) { 1515 r = octeon_irq_set_ciu_mapping( 1516 i + OCTEON_IRQ_WDOG0, 1, i + 0, 0, chip_wd, 1517 handle_level_irq); 1518 if (r) 1519 goto err; 1520 } 1521 1522 r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_USB1, 1, 17); 1523 if (r) 1524 goto err; 1525 1526 /* Enable the CIU lines */ 1527 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 1528 if (octeon_irq_use_ip4) 1529 set_c0_status(STATUSF_IP4); 1530 else 1531 clear_c0_status(STATUSF_IP4); 1532 1533 return 0; 1534 err: 1535 return r; 1536 } 1537 1538 static int __init octeon_irq_init_gpio( 1539 struct device_node *gpio_node, struct device_node *parent) 1540 { 1541 struct octeon_irq_gpio_domain_data *gpiod; 1542 u32 interrupt_cells; 1543 unsigned int base_hwirq; 1544 int r; 1545 1546 r = of_property_read_u32(parent, "#interrupt-cells", &interrupt_cells); 1547 if (r) 1548 return r; 1549 1550 if (interrupt_cells == 1) { 1551 u32 v; 1552 1553 r = of_property_read_u32_index(gpio_node, "interrupts", 0, &v); 1554 if (r) { 1555 pr_warn("No \"interrupts\" property.\n"); 1556 return r; 1557 } 1558 base_hwirq = v; 1559 } else if (interrupt_cells == 2) { 1560 u32 v0, v1; 1561 1562 r = of_property_read_u32_index(gpio_node, "interrupts", 0, &v0); 1563 if (r) { 1564 pr_warn("No \"interrupts\" property.\n"); 1565 return r; 1566 } 1567 r = of_property_read_u32_index(gpio_node, "interrupts", 1, &v1); 1568 if (r) { 1569 pr_warn("No \"interrupts\" property.\n"); 1570 return r; 1571 } 1572 base_hwirq = (v0 << 6) | v1; 1573 } else { 1574 pr_warn("Bad \"#interrupt-cells\" property: %u\n", 1575 interrupt_cells); 1576 return -EINVAL; 1577 } 1578 1579 gpiod = kzalloc(sizeof(*gpiod), GFP_KERNEL); 1580 if (gpiod) { 1581 /* gpio domain host_data is the base hwirq number. */ 1582 gpiod->base_hwirq = base_hwirq; 1583 irq_domain_add_linear( 1584 gpio_node, 16, &octeon_irq_domain_gpio_ops, gpiod); 1585 } else { 1586 pr_warn("Cannot allocate memory for GPIO irq_domain.\n"); 1587 return -ENOMEM; 1588 } 1589 1590 return 0; 1591 } 1592 /* 1593 * Watchdog interrupts are special. They are associated with a single 1594 * core, so we hardwire the affinity to that core. 1595 */ 1596 static void octeon_irq_ciu2_wd_enable(struct irq_data *data) 1597 { 1598 u64 mask; 1599 u64 en_addr; 1600 int coreid = data->irq - OCTEON_IRQ_WDOG0; 1601 struct octeon_ciu_chip_data *cd; 1602 1603 cd = irq_data_get_irq_chip_data(data); 1604 mask = 1ull << (cd->bit); 1605 1606 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) + 1607 (0x1000ull * cd->line); 1608 cvmx_write_csr(en_addr, mask); 1609 1610 } 1611 1612 static void octeon_irq_ciu2_enable(struct irq_data *data) 1613 { 1614 u64 mask; 1615 u64 en_addr; 1616 int cpu = next_cpu_for_irq(data); 1617 int coreid = octeon_coreid_for_cpu(cpu); 1618 struct octeon_ciu_chip_data *cd; 1619 1620 cd = irq_data_get_irq_chip_data(data); 1621 mask = 1ull << (cd->bit); 1622 1623 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) + 1624 (0x1000ull * cd->line); 1625 cvmx_write_csr(en_addr, mask); 1626 } 1627 1628 static void octeon_irq_ciu2_enable_local(struct irq_data *data) 1629 { 1630 u64 mask; 1631 u64 en_addr; 1632 int coreid = cvmx_get_core_num(); 1633 struct octeon_ciu_chip_data *cd; 1634 1635 cd = irq_data_get_irq_chip_data(data); 1636 mask = 1ull << (cd->bit); 1637 1638 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) + 1639 (0x1000ull * cd->line); 1640 cvmx_write_csr(en_addr, mask); 1641 1642 } 1643 1644 static void octeon_irq_ciu2_disable_local(struct irq_data *data) 1645 { 1646 u64 mask; 1647 u64 en_addr; 1648 int coreid = cvmx_get_core_num(); 1649 struct octeon_ciu_chip_data *cd; 1650 1651 cd = irq_data_get_irq_chip_data(data); 1652 mask = 1ull << (cd->bit); 1653 1654 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C(coreid) + 1655 (0x1000ull * cd->line); 1656 cvmx_write_csr(en_addr, mask); 1657 1658 } 1659 1660 static void octeon_irq_ciu2_ack(struct irq_data *data) 1661 { 1662 u64 mask; 1663 u64 en_addr; 1664 int coreid = cvmx_get_core_num(); 1665 struct octeon_ciu_chip_data *cd; 1666 1667 cd = irq_data_get_irq_chip_data(data); 1668 mask = 1ull << (cd->bit); 1669 1670 en_addr = CVMX_CIU2_RAW_PPX_IP2_WRKQ(coreid) + (0x1000ull * cd->line); 1671 cvmx_write_csr(en_addr, mask); 1672 1673 } 1674 1675 static void octeon_irq_ciu2_disable_all(struct irq_data *data) 1676 { 1677 int cpu; 1678 u64 mask; 1679 struct octeon_ciu_chip_data *cd; 1680 1681 cd = irq_data_get_irq_chip_data(data); 1682 mask = 1ull << (cd->bit); 1683 1684 for_each_online_cpu(cpu) { 1685 u64 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C( 1686 octeon_coreid_for_cpu(cpu)) + (0x1000ull * cd->line); 1687 cvmx_write_csr(en_addr, mask); 1688 } 1689 } 1690 1691 static void octeon_irq_ciu2_mbox_enable_all(struct irq_data *data) 1692 { 1693 int cpu; 1694 u64 mask; 1695 1696 mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0); 1697 1698 for_each_online_cpu(cpu) { 1699 u64 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1S( 1700 octeon_coreid_for_cpu(cpu)); 1701 cvmx_write_csr(en_addr, mask); 1702 } 1703 } 1704 1705 static void octeon_irq_ciu2_mbox_disable_all(struct irq_data *data) 1706 { 1707 int cpu; 1708 u64 mask; 1709 1710 mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0); 1711 1712 for_each_online_cpu(cpu) { 1713 u64 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1C( 1714 octeon_coreid_for_cpu(cpu)); 1715 cvmx_write_csr(en_addr, mask); 1716 } 1717 } 1718 1719 static void octeon_irq_ciu2_mbox_enable_local(struct irq_data *data) 1720 { 1721 u64 mask; 1722 u64 en_addr; 1723 int coreid = cvmx_get_core_num(); 1724 1725 mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0); 1726 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1S(coreid); 1727 cvmx_write_csr(en_addr, mask); 1728 } 1729 1730 static void octeon_irq_ciu2_mbox_disable_local(struct irq_data *data) 1731 { 1732 u64 mask; 1733 u64 en_addr; 1734 int coreid = cvmx_get_core_num(); 1735 1736 mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0); 1737 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1C(coreid); 1738 cvmx_write_csr(en_addr, mask); 1739 } 1740 1741 #ifdef CONFIG_SMP 1742 static int octeon_irq_ciu2_set_affinity(struct irq_data *data, 1743 const struct cpumask *dest, bool force) 1744 { 1745 int cpu; 1746 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 1747 u64 mask; 1748 struct octeon_ciu_chip_data *cd; 1749 1750 if (!enable_one) 1751 return 0; 1752 1753 cd = irq_data_get_irq_chip_data(data); 1754 mask = 1ull << cd->bit; 1755 1756 for_each_online_cpu(cpu) { 1757 u64 en_addr; 1758 if (cpumask_test_cpu(cpu, dest) && enable_one) { 1759 enable_one = false; 1760 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S( 1761 octeon_coreid_for_cpu(cpu)) + 1762 (0x1000ull * cd->line); 1763 } else { 1764 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C( 1765 octeon_coreid_for_cpu(cpu)) + 1766 (0x1000ull * cd->line); 1767 } 1768 cvmx_write_csr(en_addr, mask); 1769 } 1770 1771 return 0; 1772 } 1773 #endif 1774 1775 static void octeon_irq_ciu2_enable_gpio(struct irq_data *data) 1776 { 1777 octeon_irq_gpio_setup(data); 1778 octeon_irq_ciu2_enable(data); 1779 } 1780 1781 static void octeon_irq_ciu2_disable_gpio(struct irq_data *data) 1782 { 1783 struct octeon_ciu_chip_data *cd; 1784 1785 cd = irq_data_get_irq_chip_data(data); 1786 1787 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0); 1788 1789 octeon_irq_ciu2_disable_all(data); 1790 } 1791 1792 static struct irq_chip octeon_irq_chip_ciu2 = { 1793 .name = "CIU2-E", 1794 .irq_enable = octeon_irq_ciu2_enable, 1795 .irq_disable = octeon_irq_ciu2_disable_all, 1796 .irq_mask = octeon_irq_ciu2_disable_local, 1797 .irq_unmask = octeon_irq_ciu2_enable, 1798 #ifdef CONFIG_SMP 1799 .irq_set_affinity = octeon_irq_ciu2_set_affinity, 1800 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1801 #endif 1802 }; 1803 1804 static struct irq_chip octeon_irq_chip_ciu2_edge = { 1805 .name = "CIU2-E", 1806 .irq_enable = octeon_irq_ciu2_enable, 1807 .irq_disable = octeon_irq_ciu2_disable_all, 1808 .irq_ack = octeon_irq_ciu2_ack, 1809 .irq_mask = octeon_irq_ciu2_disable_local, 1810 .irq_unmask = octeon_irq_ciu2_enable, 1811 #ifdef CONFIG_SMP 1812 .irq_set_affinity = octeon_irq_ciu2_set_affinity, 1813 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1814 #endif 1815 }; 1816 1817 static struct irq_chip octeon_irq_chip_ciu2_mbox = { 1818 .name = "CIU2-M", 1819 .irq_enable = octeon_irq_ciu2_mbox_enable_all, 1820 .irq_disable = octeon_irq_ciu2_mbox_disable_all, 1821 .irq_ack = octeon_irq_ciu2_mbox_disable_local, 1822 .irq_eoi = octeon_irq_ciu2_mbox_enable_local, 1823 1824 .irq_cpu_online = octeon_irq_ciu2_mbox_enable_local, 1825 .irq_cpu_offline = octeon_irq_ciu2_mbox_disable_local, 1826 .flags = IRQCHIP_ONOFFLINE_ENABLED, 1827 }; 1828 1829 static struct irq_chip octeon_irq_chip_ciu2_wd = { 1830 .name = "CIU2-W", 1831 .irq_enable = octeon_irq_ciu2_wd_enable, 1832 .irq_disable = octeon_irq_ciu2_disable_all, 1833 .irq_mask = octeon_irq_ciu2_disable_local, 1834 .irq_unmask = octeon_irq_ciu2_enable_local, 1835 }; 1836 1837 static struct irq_chip octeon_irq_chip_ciu2_gpio = { 1838 .name = "CIU-GPIO", 1839 .irq_enable = octeon_irq_ciu2_enable_gpio, 1840 .irq_disable = octeon_irq_ciu2_disable_gpio, 1841 .irq_ack = octeon_irq_ciu_gpio_ack, 1842 .irq_mask = octeon_irq_ciu2_disable_local, 1843 .irq_unmask = octeon_irq_ciu2_enable, 1844 .irq_set_type = octeon_irq_ciu_gpio_set_type, 1845 #ifdef CONFIG_SMP 1846 .irq_set_affinity = octeon_irq_ciu2_set_affinity, 1847 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1848 #endif 1849 .flags = IRQCHIP_SET_TYPE_MASKED, 1850 }; 1851 1852 static int octeon_irq_ciu2_xlat(struct irq_domain *d, 1853 struct device_node *node, 1854 const u32 *intspec, 1855 unsigned int intsize, 1856 unsigned long *out_hwirq, 1857 unsigned int *out_type) 1858 { 1859 unsigned int ciu, bit; 1860 1861 ciu = intspec[0]; 1862 bit = intspec[1]; 1863 1864 *out_hwirq = (ciu << 6) | bit; 1865 *out_type = 0; 1866 1867 return 0; 1868 } 1869 1870 static bool octeon_irq_ciu2_is_edge(unsigned int line, unsigned int bit) 1871 { 1872 bool edge = false; 1873 1874 if (line == 3) /* MIO */ 1875 switch (bit) { 1876 case 2: /* IPD_DRP */ 1877 case 8 ... 11: /* Timers */ 1878 case 48: /* PTP */ 1879 edge = true; 1880 break; 1881 default: 1882 break; 1883 } 1884 else if (line == 6) /* PKT */ 1885 switch (bit) { 1886 case 52 ... 53: /* ILK_DRP */ 1887 case 8 ... 12: /* GMX_DRP */ 1888 edge = true; 1889 break; 1890 default: 1891 break; 1892 } 1893 return edge; 1894 } 1895 1896 static int octeon_irq_ciu2_map(struct irq_domain *d, 1897 unsigned int virq, irq_hw_number_t hw) 1898 { 1899 unsigned int line = hw >> 6; 1900 unsigned int bit = hw & 63; 1901 1902 if (!octeon_irq_virq_in_range(virq)) 1903 return -EINVAL; 1904 1905 /* 1906 * Don't map irq if it is reserved for GPIO. 1907 * (Line 7 are the GPIO lines.) 1908 */ 1909 if (line == 7) 1910 return 0; 1911 1912 if (line > 7 || octeon_irq_ciu_to_irq[line][bit] != 0) 1913 return -EINVAL; 1914 1915 if (octeon_irq_ciu2_is_edge(line, bit)) 1916 octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1917 &octeon_irq_chip_ciu2_edge, 1918 handle_edge_irq); 1919 else 1920 octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1921 &octeon_irq_chip_ciu2, 1922 handle_level_irq); 1923 1924 return 0; 1925 } 1926 1927 static struct irq_domain_ops octeon_irq_domain_ciu2_ops = { 1928 .map = octeon_irq_ciu2_map, 1929 .unmap = octeon_irq_free_cd, 1930 .xlate = octeon_irq_ciu2_xlat, 1931 }; 1932 1933 static void octeon_irq_ciu2(void) 1934 { 1935 int line; 1936 int bit; 1937 int irq; 1938 u64 src_reg, src, sum; 1939 const unsigned long core_id = cvmx_get_core_num(); 1940 1941 sum = cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(core_id)) & 0xfful; 1942 1943 if (unlikely(!sum)) 1944 goto spurious; 1945 1946 line = fls64(sum) - 1; 1947 src_reg = CVMX_CIU2_SRC_PPX_IP2_WRKQ(core_id) + (0x1000 * line); 1948 src = cvmx_read_csr(src_reg); 1949 1950 if (unlikely(!src)) 1951 goto spurious; 1952 1953 bit = fls64(src) - 1; 1954 irq = octeon_irq_ciu_to_irq[line][bit]; 1955 if (unlikely(!irq)) 1956 goto spurious; 1957 1958 do_IRQ(irq); 1959 goto out; 1960 1961 spurious: 1962 spurious_interrupt(); 1963 out: 1964 /* CN68XX pass 1.x has an errata that accessing the ACK registers 1965 can stop interrupts from propagating */ 1966 if (OCTEON_IS_MODEL(OCTEON_CN68XX)) 1967 cvmx_read_csr(CVMX_CIU2_INTR_CIU_READY); 1968 else 1969 cvmx_read_csr(CVMX_CIU2_ACK_PPX_IP2(core_id)); 1970 return; 1971 } 1972 1973 static void octeon_irq_ciu2_mbox(void) 1974 { 1975 int line; 1976 1977 const unsigned long core_id = cvmx_get_core_num(); 1978 u64 sum = cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP3(core_id)) >> 60; 1979 1980 if (unlikely(!sum)) 1981 goto spurious; 1982 1983 line = fls64(sum) - 1; 1984 1985 do_IRQ(OCTEON_IRQ_MBOX0 + line); 1986 goto out; 1987 1988 spurious: 1989 spurious_interrupt(); 1990 out: 1991 /* CN68XX pass 1.x has an errata that accessing the ACK registers 1992 can stop interrupts from propagating */ 1993 if (OCTEON_IS_MODEL(OCTEON_CN68XX)) 1994 cvmx_read_csr(CVMX_CIU2_INTR_CIU_READY); 1995 else 1996 cvmx_read_csr(CVMX_CIU2_ACK_PPX_IP3(core_id)); 1997 return; 1998 } 1999 2000 static int __init octeon_irq_init_ciu2( 2001 struct device_node *ciu_node, struct device_node *parent) 2002 { 2003 unsigned int i, r; 2004 struct irq_domain *ciu_domain = NULL; 2005 2006 octeon_irq_init_ciu2_percpu(); 2007 octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu2; 2008 2009 octeon_irq_gpio_chip = &octeon_irq_chip_ciu2_gpio; 2010 octeon_irq_ip2 = octeon_irq_ciu2; 2011 octeon_irq_ip3 = octeon_irq_ciu2_mbox; 2012 octeon_irq_ip4 = octeon_irq_ip4_mask; 2013 2014 /* Mips internal */ 2015 octeon_irq_init_core(); 2016 2017 ciu_domain = irq_domain_add_tree( 2018 ciu_node, &octeon_irq_domain_ciu2_ops, NULL); 2019 irq_set_default_host(ciu_domain); 2020 2021 /* CUI2 */ 2022 for (i = 0; i < 64; i++) { 2023 r = octeon_irq_force_ciu_mapping( 2024 ciu_domain, i + OCTEON_IRQ_WORKQ0, 0, i); 2025 if (r) 2026 goto err; 2027 } 2028 2029 for (i = 0; i < 32; i++) { 2030 r = octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_WDOG0, 1, i, 0, 2031 &octeon_irq_chip_ciu2_wd, handle_level_irq); 2032 if (r) 2033 goto err; 2034 } 2035 2036 for (i = 0; i < 4; i++) { 2037 r = octeon_irq_force_ciu_mapping( 2038 ciu_domain, i + OCTEON_IRQ_TIMER0, 3, i + 8); 2039 if (r) 2040 goto err; 2041 } 2042 2043 r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_USB0, 3, 44); 2044 if (r) 2045 goto err; 2046 2047 for (i = 0; i < 4; i++) { 2048 r = octeon_irq_force_ciu_mapping( 2049 ciu_domain, i + OCTEON_IRQ_PCI_INT0, 4, i); 2050 if (r) 2051 goto err; 2052 } 2053 2054 for (i = 0; i < 4; i++) { 2055 r = octeon_irq_force_ciu_mapping( 2056 ciu_domain, i + OCTEON_IRQ_PCI_MSI0, 4, i + 8); 2057 if (r) 2058 goto err; 2059 } 2060 2061 irq_set_chip_and_handler(OCTEON_IRQ_MBOX0, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq); 2062 irq_set_chip_and_handler(OCTEON_IRQ_MBOX1, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq); 2063 irq_set_chip_and_handler(OCTEON_IRQ_MBOX2, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq); 2064 irq_set_chip_and_handler(OCTEON_IRQ_MBOX3, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq); 2065 2066 /* Enable the CIU lines */ 2067 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 2068 clear_c0_status(STATUSF_IP4); 2069 return 0; 2070 err: 2071 return r; 2072 } 2073 2074 struct octeon_irq_cib_host_data { 2075 raw_spinlock_t lock; 2076 u64 raw_reg; 2077 u64 en_reg; 2078 int max_bits; 2079 }; 2080 2081 struct octeon_irq_cib_chip_data { 2082 struct octeon_irq_cib_host_data *host_data; 2083 int bit; 2084 }; 2085 2086 static void octeon_irq_cib_enable(struct irq_data *data) 2087 { 2088 unsigned long flags; 2089 u64 en; 2090 struct octeon_irq_cib_chip_data *cd = irq_data_get_irq_chip_data(data); 2091 struct octeon_irq_cib_host_data *host_data = cd->host_data; 2092 2093 raw_spin_lock_irqsave(&host_data->lock, flags); 2094 en = cvmx_read_csr(host_data->en_reg); 2095 en |= 1ull << cd->bit; 2096 cvmx_write_csr(host_data->en_reg, en); 2097 raw_spin_unlock_irqrestore(&host_data->lock, flags); 2098 } 2099 2100 static void octeon_irq_cib_disable(struct irq_data *data) 2101 { 2102 unsigned long flags; 2103 u64 en; 2104 struct octeon_irq_cib_chip_data *cd = irq_data_get_irq_chip_data(data); 2105 struct octeon_irq_cib_host_data *host_data = cd->host_data; 2106 2107 raw_spin_lock_irqsave(&host_data->lock, flags); 2108 en = cvmx_read_csr(host_data->en_reg); 2109 en &= ~(1ull << cd->bit); 2110 cvmx_write_csr(host_data->en_reg, en); 2111 raw_spin_unlock_irqrestore(&host_data->lock, flags); 2112 } 2113 2114 static int octeon_irq_cib_set_type(struct irq_data *data, unsigned int t) 2115 { 2116 irqd_set_trigger_type(data, t); 2117 return IRQ_SET_MASK_OK; 2118 } 2119 2120 static struct irq_chip octeon_irq_chip_cib = { 2121 .name = "CIB", 2122 .irq_enable = octeon_irq_cib_enable, 2123 .irq_disable = octeon_irq_cib_disable, 2124 .irq_mask = octeon_irq_cib_disable, 2125 .irq_unmask = octeon_irq_cib_enable, 2126 .irq_set_type = octeon_irq_cib_set_type, 2127 }; 2128 2129 static int octeon_irq_cib_xlat(struct irq_domain *d, 2130 struct device_node *node, 2131 const u32 *intspec, 2132 unsigned int intsize, 2133 unsigned long *out_hwirq, 2134 unsigned int *out_type) 2135 { 2136 unsigned int type = 0; 2137 2138 if (intsize == 2) 2139 type = intspec[1]; 2140 2141 switch (type) { 2142 case 0: /* unofficial value, but we might as well let it work. */ 2143 case 4: /* official value for level triggering. */ 2144 *out_type = IRQ_TYPE_LEVEL_HIGH; 2145 break; 2146 case 1: /* official value for edge triggering. */ 2147 *out_type = IRQ_TYPE_EDGE_RISING; 2148 break; 2149 default: /* Nothing else is acceptable. */ 2150 return -EINVAL; 2151 } 2152 2153 *out_hwirq = intspec[0]; 2154 2155 return 0; 2156 } 2157 2158 static int octeon_irq_cib_map(struct irq_domain *d, 2159 unsigned int virq, irq_hw_number_t hw) 2160 { 2161 struct octeon_irq_cib_host_data *host_data = d->host_data; 2162 struct octeon_irq_cib_chip_data *cd; 2163 2164 if (hw >= host_data->max_bits) { 2165 pr_err("ERROR: %s mapping %u is to big!\n", 2166 irq_domain_get_of_node(d)->name, (unsigned)hw); 2167 return -EINVAL; 2168 } 2169 2170 cd = kzalloc(sizeof(*cd), GFP_KERNEL); 2171 cd->host_data = host_data; 2172 cd->bit = hw; 2173 2174 irq_set_chip_and_handler(virq, &octeon_irq_chip_cib, 2175 handle_simple_irq); 2176 irq_set_chip_data(virq, cd); 2177 return 0; 2178 } 2179 2180 static struct irq_domain_ops octeon_irq_domain_cib_ops = { 2181 .map = octeon_irq_cib_map, 2182 .unmap = octeon_irq_free_cd, 2183 .xlate = octeon_irq_cib_xlat, 2184 }; 2185 2186 /* Chain to real handler. */ 2187 static irqreturn_t octeon_irq_cib_handler(int my_irq, void *data) 2188 { 2189 u64 en; 2190 u64 raw; 2191 u64 bits; 2192 int i; 2193 int irq; 2194 struct irq_domain *cib_domain = data; 2195 struct octeon_irq_cib_host_data *host_data = cib_domain->host_data; 2196 2197 en = cvmx_read_csr(host_data->en_reg); 2198 raw = cvmx_read_csr(host_data->raw_reg); 2199 2200 bits = en & raw; 2201 2202 for (i = 0; i < host_data->max_bits; i++) { 2203 if ((bits & 1ull << i) == 0) 2204 continue; 2205 irq = irq_find_mapping(cib_domain, i); 2206 if (!irq) { 2207 unsigned long flags; 2208 2209 pr_err("ERROR: CIB bit %d@%llx IRQ unhandled, disabling\n", 2210 i, host_data->raw_reg); 2211 raw_spin_lock_irqsave(&host_data->lock, flags); 2212 en = cvmx_read_csr(host_data->en_reg); 2213 en &= ~(1ull << i); 2214 cvmx_write_csr(host_data->en_reg, en); 2215 cvmx_write_csr(host_data->raw_reg, 1ull << i); 2216 raw_spin_unlock_irqrestore(&host_data->lock, flags); 2217 } else { 2218 struct irq_desc *desc = irq_to_desc(irq); 2219 struct irq_data *irq_data = irq_desc_get_irq_data(desc); 2220 /* If edge, acknowledge the bit we will be sending. */ 2221 if (irqd_get_trigger_type(irq_data) & 2222 IRQ_TYPE_EDGE_BOTH) 2223 cvmx_write_csr(host_data->raw_reg, 1ull << i); 2224 generic_handle_irq_desc(desc); 2225 } 2226 } 2227 2228 return IRQ_HANDLED; 2229 } 2230 2231 static int __init octeon_irq_init_cib(struct device_node *ciu_node, 2232 struct device_node *parent) 2233 { 2234 const __be32 *addr; 2235 u32 val; 2236 struct octeon_irq_cib_host_data *host_data; 2237 int parent_irq; 2238 int r; 2239 struct irq_domain *cib_domain; 2240 2241 parent_irq = irq_of_parse_and_map(ciu_node, 0); 2242 if (!parent_irq) { 2243 pr_err("ERROR: Couldn't acquire parent_irq for %s\n.", 2244 ciu_node->name); 2245 return -EINVAL; 2246 } 2247 2248 host_data = kzalloc(sizeof(*host_data), GFP_KERNEL); 2249 raw_spin_lock_init(&host_data->lock); 2250 2251 addr = of_get_address(ciu_node, 0, NULL, NULL); 2252 if (!addr) { 2253 pr_err("ERROR: Couldn't acquire reg(0) %s\n.", ciu_node->name); 2254 return -EINVAL; 2255 } 2256 host_data->raw_reg = (u64)phys_to_virt( 2257 of_translate_address(ciu_node, addr)); 2258 2259 addr = of_get_address(ciu_node, 1, NULL, NULL); 2260 if (!addr) { 2261 pr_err("ERROR: Couldn't acquire reg(1) %s\n.", ciu_node->name); 2262 return -EINVAL; 2263 } 2264 host_data->en_reg = (u64)phys_to_virt( 2265 of_translate_address(ciu_node, addr)); 2266 2267 r = of_property_read_u32(ciu_node, "cavium,max-bits", &val); 2268 if (r) { 2269 pr_err("ERROR: Couldn't read cavium,max-bits from %s\n.", 2270 ciu_node->name); 2271 return r; 2272 } 2273 host_data->max_bits = val; 2274 2275 cib_domain = irq_domain_add_linear(ciu_node, host_data->max_bits, 2276 &octeon_irq_domain_cib_ops, 2277 host_data); 2278 if (!cib_domain) { 2279 pr_err("ERROR: Couldn't irq_domain_add_linear()\n."); 2280 return -ENOMEM; 2281 } 2282 2283 cvmx_write_csr(host_data->en_reg, 0); /* disable all IRQs */ 2284 cvmx_write_csr(host_data->raw_reg, ~0); /* ack any outstanding */ 2285 2286 r = request_irq(parent_irq, octeon_irq_cib_handler, 2287 IRQF_NO_THREAD, "cib", cib_domain); 2288 if (r) { 2289 pr_err("request_irq cib failed %d\n", r); 2290 return r; 2291 } 2292 pr_info("CIB interrupt controller probed: %llx %d\n", 2293 host_data->raw_reg, host_data->max_bits); 2294 return 0; 2295 } 2296 2297 static struct of_device_id ciu_types[] __initdata = { 2298 {.compatible = "cavium,octeon-3860-ciu", .data = octeon_irq_init_ciu}, 2299 {.compatible = "cavium,octeon-3860-gpio", .data = octeon_irq_init_gpio}, 2300 {.compatible = "cavium,octeon-6880-ciu2", .data = octeon_irq_init_ciu2}, 2301 {.compatible = "cavium,octeon-7130-cib", .data = octeon_irq_init_cib}, 2302 {} 2303 }; 2304 2305 void __init arch_init_irq(void) 2306 { 2307 #ifdef CONFIG_SMP 2308 /* Set the default affinity to the boot cpu. */ 2309 cpumask_clear(irq_default_affinity); 2310 cpumask_set_cpu(smp_processor_id(), irq_default_affinity); 2311 #endif 2312 of_irq_init(ciu_types); 2313 } 2314 2315 asmlinkage void plat_irq_dispatch(void) 2316 { 2317 unsigned long cop0_cause; 2318 unsigned long cop0_status; 2319 2320 while (1) { 2321 cop0_cause = read_c0_cause(); 2322 cop0_status = read_c0_status(); 2323 cop0_cause &= cop0_status; 2324 cop0_cause &= ST0_IM; 2325 2326 if (cop0_cause & STATUSF_IP2) 2327 octeon_irq_ip2(); 2328 else if (cop0_cause & STATUSF_IP3) 2329 octeon_irq_ip3(); 2330 else if (cop0_cause & STATUSF_IP4) 2331 octeon_irq_ip4(); 2332 else if (cop0_cause) 2333 do_IRQ(fls(cop0_cause) - 9 + MIPS_CPU_IRQ_BASE); 2334 else 2335 break; 2336 } 2337 } 2338 2339 #ifdef CONFIG_HOTPLUG_CPU 2340 2341 void octeon_fixup_irqs(void) 2342 { 2343 irq_cpu_offline(); 2344 } 2345 2346 #endif /* CONFIG_HOTPLUG_CPU */ 2347