1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 2004-2016 Cavium, Inc. 7 */ 8 9 #include <linux/of_address.h> 10 #include <linux/interrupt.h> 11 #include <linux/irqdomain.h> 12 #include <linux/bitops.h> 13 #include <linux/of_irq.h> 14 #include <linux/percpu.h> 15 #include <linux/slab.h> 16 #include <linux/irq.h> 17 #include <linux/smp.h> 18 #include <linux/of.h> 19 20 #include <asm/octeon/octeon.h> 21 #include <asm/octeon/cvmx-ciu2-defs.h> 22 #include <asm/octeon/cvmx-ciu3-defs.h> 23 24 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu0_en_mirror); 25 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu1_en_mirror); 26 static DEFINE_PER_CPU(raw_spinlock_t, octeon_irq_ciu_spinlock); 27 static DEFINE_PER_CPU(unsigned int, octeon_irq_ciu3_idt_ip2); 28 29 static DEFINE_PER_CPU(unsigned int, octeon_irq_ciu3_idt_ip3); 30 static DEFINE_PER_CPU(struct octeon_ciu3_info *, octeon_ciu3_info); 31 #define CIU3_MBOX_PER_CORE 10 32 33 /* 34 * The 8 most significant bits of the intsn identify the interrupt major block. 35 * Each major block might use its own interrupt domain. Thus 256 domains are 36 * needed. 37 */ 38 #define MAX_CIU3_DOMAINS 256 39 40 typedef irq_hw_number_t (*octeon_ciu3_intsn2hw_t)(struct irq_domain *, unsigned int); 41 42 /* Information for each ciu3 in the system */ 43 struct octeon_ciu3_info { 44 u64 ciu3_addr; 45 int node; 46 struct irq_domain *domain[MAX_CIU3_DOMAINS]; 47 octeon_ciu3_intsn2hw_t intsn2hw[MAX_CIU3_DOMAINS]; 48 }; 49 50 /* Each ciu3 in the system uses its own data (one ciu3 per node) */ 51 static struct octeon_ciu3_info *octeon_ciu3_info_per_node[4]; 52 53 struct octeon_irq_ciu_domain_data { 54 int num_sum; /* number of sum registers (2 or 3). */ 55 }; 56 57 /* Register offsets from ciu3_addr */ 58 #define CIU3_CONST 0x220 59 #define CIU3_IDT_CTL(_idt) ((_idt) * 8 + 0x110000) 60 #define CIU3_IDT_PP(_idt, _idx) ((_idt) * 32 + (_idx) * 8 + 0x120000) 61 #define CIU3_IDT_IO(_idt) ((_idt) * 8 + 0x130000) 62 #define CIU3_DEST_PP_INT(_pp_ip) ((_pp_ip) * 8 + 0x200000) 63 #define CIU3_DEST_IO_INT(_io) ((_io) * 8 + 0x210000) 64 #define CIU3_ISC_CTL(_intsn) ((_intsn) * 8 + 0x80000000) 65 #define CIU3_ISC_W1C(_intsn) ((_intsn) * 8 + 0x90000000) 66 #define CIU3_ISC_W1S(_intsn) ((_intsn) * 8 + 0xa0000000) 67 68 static __read_mostly int octeon_irq_ciu_to_irq[8][64]; 69 70 struct octeon_ciu_chip_data { 71 union { 72 struct { /* only used for ciu3 */ 73 u64 ciu3_addr; 74 unsigned int intsn; 75 }; 76 struct { /* only used for ciu/ciu2 */ 77 u8 line; 78 u8 bit; 79 }; 80 }; 81 int gpio_line; 82 int current_cpu; /* Next CPU expected to take this irq */ 83 int ciu_node; /* NUMA node number of the CIU */ 84 }; 85 86 struct octeon_core_chip_data { 87 struct mutex core_irq_mutex; 88 bool current_en; 89 bool desired_en; 90 u8 bit; 91 }; 92 93 #define MIPS_CORE_IRQ_LINES 8 94 95 static struct octeon_core_chip_data octeon_irq_core_chip_data[MIPS_CORE_IRQ_LINES]; 96 97 static int octeon_irq_set_ciu_mapping(int irq, int line, int bit, int gpio_line, 98 struct irq_chip *chip, 99 irq_flow_handler_t handler) 100 { 101 struct octeon_ciu_chip_data *cd; 102 103 cd = kzalloc(sizeof(*cd), GFP_KERNEL); 104 if (!cd) 105 return -ENOMEM; 106 107 irq_set_chip_and_handler(irq, chip, handler); 108 109 cd->line = line; 110 cd->bit = bit; 111 cd->gpio_line = gpio_line; 112 113 irq_set_chip_data(irq, cd); 114 octeon_irq_ciu_to_irq[line][bit] = irq; 115 return 0; 116 } 117 118 static void octeon_irq_free_cd(struct irq_domain *d, unsigned int irq) 119 { 120 struct irq_data *data = irq_get_irq_data(irq); 121 struct octeon_ciu_chip_data *cd = irq_data_get_irq_chip_data(data); 122 123 irq_set_chip_data(irq, NULL); 124 kfree(cd); 125 } 126 127 static int octeon_irq_force_ciu_mapping(struct irq_domain *domain, 128 int irq, int line, int bit) 129 { 130 return irq_domain_associate(domain, irq, line << 6 | bit); 131 } 132 133 static int octeon_coreid_for_cpu(int cpu) 134 { 135 #ifdef CONFIG_SMP 136 return cpu_logical_map(cpu); 137 #else 138 return cvmx_get_core_num(); 139 #endif 140 } 141 142 static int octeon_cpu_for_coreid(int coreid) 143 { 144 #ifdef CONFIG_SMP 145 return cpu_number_map(coreid); 146 #else 147 return smp_processor_id(); 148 #endif 149 } 150 151 static void octeon_irq_core_ack(struct irq_data *data) 152 { 153 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 154 unsigned int bit = cd->bit; 155 156 /* 157 * We don't need to disable IRQs to make these atomic since 158 * they are already disabled earlier in the low level 159 * interrupt code. 160 */ 161 clear_c0_status(0x100 << bit); 162 /* The two user interrupts must be cleared manually. */ 163 if (bit < 2) 164 clear_c0_cause(0x100 << bit); 165 } 166 167 static void octeon_irq_core_eoi(struct irq_data *data) 168 { 169 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 170 171 /* 172 * We don't need to disable IRQs to make these atomic since 173 * they are already disabled earlier in the low level 174 * interrupt code. 175 */ 176 set_c0_status(0x100 << cd->bit); 177 } 178 179 static void octeon_irq_core_set_enable_local(void *arg) 180 { 181 struct irq_data *data = arg; 182 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 183 unsigned int mask = 0x100 << cd->bit; 184 185 /* 186 * Interrupts are already disabled, so these are atomic. 187 */ 188 if (cd->desired_en) 189 set_c0_status(mask); 190 else 191 clear_c0_status(mask); 192 193 } 194 195 static void octeon_irq_core_disable(struct irq_data *data) 196 { 197 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 198 cd->desired_en = false; 199 } 200 201 static void octeon_irq_core_enable(struct irq_data *data) 202 { 203 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 204 cd->desired_en = true; 205 } 206 207 static void octeon_irq_core_bus_lock(struct irq_data *data) 208 { 209 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 210 211 mutex_lock(&cd->core_irq_mutex); 212 } 213 214 static void octeon_irq_core_bus_sync_unlock(struct irq_data *data) 215 { 216 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 217 218 if (cd->desired_en != cd->current_en) { 219 on_each_cpu(octeon_irq_core_set_enable_local, data, 1); 220 221 cd->current_en = cd->desired_en; 222 } 223 224 mutex_unlock(&cd->core_irq_mutex); 225 } 226 227 static struct irq_chip octeon_irq_chip_core = { 228 .name = "Core", 229 .irq_enable = octeon_irq_core_enable, 230 .irq_disable = octeon_irq_core_disable, 231 .irq_ack = octeon_irq_core_ack, 232 .irq_eoi = octeon_irq_core_eoi, 233 .irq_bus_lock = octeon_irq_core_bus_lock, 234 .irq_bus_sync_unlock = octeon_irq_core_bus_sync_unlock, 235 236 .irq_cpu_online = octeon_irq_core_eoi, 237 .irq_cpu_offline = octeon_irq_core_ack, 238 .flags = IRQCHIP_ONOFFLINE_ENABLED, 239 }; 240 241 static void __init octeon_irq_init_core(void) 242 { 243 int i; 244 int irq; 245 struct octeon_core_chip_data *cd; 246 247 for (i = 0; i < MIPS_CORE_IRQ_LINES; i++) { 248 cd = &octeon_irq_core_chip_data[i]; 249 cd->current_en = false; 250 cd->desired_en = false; 251 cd->bit = i; 252 mutex_init(&cd->core_irq_mutex); 253 254 irq = OCTEON_IRQ_SW0 + i; 255 irq_set_chip_data(irq, cd); 256 irq_set_chip_and_handler(irq, &octeon_irq_chip_core, 257 handle_percpu_irq); 258 } 259 } 260 261 static int next_cpu_for_irq(struct irq_data *data) 262 { 263 264 #ifdef CONFIG_SMP 265 int cpu; 266 struct cpumask *mask = irq_data_get_affinity_mask(data); 267 int weight = cpumask_weight(mask); 268 struct octeon_ciu_chip_data *cd = irq_data_get_irq_chip_data(data); 269 270 if (weight > 1) { 271 cpu = cd->current_cpu; 272 for (;;) { 273 cpu = cpumask_next(cpu, mask); 274 if (cpu >= nr_cpu_ids) { 275 cpu = -1; 276 continue; 277 } else if (cpumask_test_cpu(cpu, cpu_online_mask)) { 278 break; 279 } 280 } 281 } else if (weight == 1) { 282 cpu = cpumask_first(mask); 283 } else { 284 cpu = smp_processor_id(); 285 } 286 cd->current_cpu = cpu; 287 return cpu; 288 #else 289 return smp_processor_id(); 290 #endif 291 } 292 293 static void octeon_irq_ciu_enable(struct irq_data *data) 294 { 295 int cpu = next_cpu_for_irq(data); 296 int coreid = octeon_coreid_for_cpu(cpu); 297 unsigned long *pen; 298 unsigned long flags; 299 struct octeon_ciu_chip_data *cd; 300 raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 301 302 cd = irq_data_get_irq_chip_data(data); 303 304 raw_spin_lock_irqsave(lock, flags); 305 if (cd->line == 0) { 306 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 307 __set_bit(cd->bit, pen); 308 /* 309 * Must be visible to octeon_irq_ip{2,3}_ciu() before 310 * enabling the irq. 311 */ 312 wmb(); 313 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 314 } else { 315 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 316 __set_bit(cd->bit, pen); 317 /* 318 * Must be visible to octeon_irq_ip{2,3}_ciu() before 319 * enabling the irq. 320 */ 321 wmb(); 322 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 323 } 324 raw_spin_unlock_irqrestore(lock, flags); 325 } 326 327 static void octeon_irq_ciu_enable_local(struct irq_data *data) 328 { 329 unsigned long *pen; 330 unsigned long flags; 331 struct octeon_ciu_chip_data *cd; 332 raw_spinlock_t *lock = this_cpu_ptr(&octeon_irq_ciu_spinlock); 333 334 cd = irq_data_get_irq_chip_data(data); 335 336 raw_spin_lock_irqsave(lock, flags); 337 if (cd->line == 0) { 338 pen = this_cpu_ptr(&octeon_irq_ciu0_en_mirror); 339 __set_bit(cd->bit, pen); 340 /* 341 * Must be visible to octeon_irq_ip{2,3}_ciu() before 342 * enabling the irq. 343 */ 344 wmb(); 345 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen); 346 } else { 347 pen = this_cpu_ptr(&octeon_irq_ciu1_en_mirror); 348 __set_bit(cd->bit, pen); 349 /* 350 * Must be visible to octeon_irq_ip{2,3}_ciu() before 351 * enabling the irq. 352 */ 353 wmb(); 354 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen); 355 } 356 raw_spin_unlock_irqrestore(lock, flags); 357 } 358 359 static void octeon_irq_ciu_disable_local(struct irq_data *data) 360 { 361 unsigned long *pen; 362 unsigned long flags; 363 struct octeon_ciu_chip_data *cd; 364 raw_spinlock_t *lock = this_cpu_ptr(&octeon_irq_ciu_spinlock); 365 366 cd = irq_data_get_irq_chip_data(data); 367 368 raw_spin_lock_irqsave(lock, flags); 369 if (cd->line == 0) { 370 pen = this_cpu_ptr(&octeon_irq_ciu0_en_mirror); 371 __clear_bit(cd->bit, pen); 372 /* 373 * Must be visible to octeon_irq_ip{2,3}_ciu() before 374 * enabling the irq. 375 */ 376 wmb(); 377 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen); 378 } else { 379 pen = this_cpu_ptr(&octeon_irq_ciu1_en_mirror); 380 __clear_bit(cd->bit, pen); 381 /* 382 * Must be visible to octeon_irq_ip{2,3}_ciu() before 383 * enabling the irq. 384 */ 385 wmb(); 386 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen); 387 } 388 raw_spin_unlock_irqrestore(lock, flags); 389 } 390 391 static void octeon_irq_ciu_disable_all(struct irq_data *data) 392 { 393 unsigned long flags; 394 unsigned long *pen; 395 int cpu; 396 struct octeon_ciu_chip_data *cd; 397 raw_spinlock_t *lock; 398 399 cd = irq_data_get_irq_chip_data(data); 400 401 for_each_online_cpu(cpu) { 402 int coreid = octeon_coreid_for_cpu(cpu); 403 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 404 if (cd->line == 0) 405 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 406 else 407 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 408 409 raw_spin_lock_irqsave(lock, flags); 410 __clear_bit(cd->bit, pen); 411 /* 412 * Must be visible to octeon_irq_ip{2,3}_ciu() before 413 * enabling the irq. 414 */ 415 wmb(); 416 if (cd->line == 0) 417 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 418 else 419 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 420 raw_spin_unlock_irqrestore(lock, flags); 421 } 422 } 423 424 static void octeon_irq_ciu_enable_all(struct irq_data *data) 425 { 426 unsigned long flags; 427 unsigned long *pen; 428 int cpu; 429 struct octeon_ciu_chip_data *cd; 430 raw_spinlock_t *lock; 431 432 cd = irq_data_get_irq_chip_data(data); 433 434 for_each_online_cpu(cpu) { 435 int coreid = octeon_coreid_for_cpu(cpu); 436 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 437 if (cd->line == 0) 438 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 439 else 440 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 441 442 raw_spin_lock_irqsave(lock, flags); 443 __set_bit(cd->bit, pen); 444 /* 445 * Must be visible to octeon_irq_ip{2,3}_ciu() before 446 * enabling the irq. 447 */ 448 wmb(); 449 if (cd->line == 0) 450 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 451 else 452 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 453 raw_spin_unlock_irqrestore(lock, flags); 454 } 455 } 456 457 /* 458 * Enable the irq on the next core in the affinity set for chips that 459 * have the EN*_W1{S,C} registers. 460 */ 461 static void octeon_irq_ciu_enable_v2(struct irq_data *data) 462 { 463 u64 mask; 464 int cpu = next_cpu_for_irq(data); 465 struct octeon_ciu_chip_data *cd; 466 467 cd = irq_data_get_irq_chip_data(data); 468 mask = 1ull << (cd->bit); 469 470 /* 471 * Called under the desc lock, so these should never get out 472 * of sync. 473 */ 474 if (cd->line == 0) { 475 int index = octeon_coreid_for_cpu(cpu) * 2; 476 set_bit(cd->bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu)); 477 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 478 } else { 479 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 480 set_bit(cd->bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 481 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 482 } 483 } 484 485 /* 486 * Enable the irq in the sum2 registers. 487 */ 488 static void octeon_irq_ciu_enable_sum2(struct irq_data *data) 489 { 490 u64 mask; 491 int cpu = next_cpu_for_irq(data); 492 int index = octeon_coreid_for_cpu(cpu); 493 struct octeon_ciu_chip_data *cd; 494 495 cd = irq_data_get_irq_chip_data(data); 496 mask = 1ull << (cd->bit); 497 498 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1S(index), mask); 499 } 500 501 /* 502 * Disable the irq in the sum2 registers. 503 */ 504 static void octeon_irq_ciu_disable_local_sum2(struct irq_data *data) 505 { 506 u64 mask; 507 int cpu = next_cpu_for_irq(data); 508 int index = octeon_coreid_for_cpu(cpu); 509 struct octeon_ciu_chip_data *cd; 510 511 cd = irq_data_get_irq_chip_data(data); 512 mask = 1ull << (cd->bit); 513 514 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(index), mask); 515 } 516 517 static void octeon_irq_ciu_ack_sum2(struct irq_data *data) 518 { 519 u64 mask; 520 int cpu = next_cpu_for_irq(data); 521 int index = octeon_coreid_for_cpu(cpu); 522 struct octeon_ciu_chip_data *cd; 523 524 cd = irq_data_get_irq_chip_data(data); 525 mask = 1ull << (cd->bit); 526 527 cvmx_write_csr(CVMX_CIU_SUM2_PPX_IP4(index), mask); 528 } 529 530 static void octeon_irq_ciu_disable_all_sum2(struct irq_data *data) 531 { 532 int cpu; 533 struct octeon_ciu_chip_data *cd; 534 u64 mask; 535 536 cd = irq_data_get_irq_chip_data(data); 537 mask = 1ull << (cd->bit); 538 539 for_each_online_cpu(cpu) { 540 int coreid = octeon_coreid_for_cpu(cpu); 541 542 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(coreid), mask); 543 } 544 } 545 546 /* 547 * Enable the irq on the current CPU for chips that 548 * have the EN*_W1{S,C} registers. 549 */ 550 static void octeon_irq_ciu_enable_local_v2(struct irq_data *data) 551 { 552 u64 mask; 553 struct octeon_ciu_chip_data *cd; 554 555 cd = irq_data_get_irq_chip_data(data); 556 mask = 1ull << (cd->bit); 557 558 if (cd->line == 0) { 559 int index = cvmx_get_core_num() * 2; 560 set_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu0_en_mirror)); 561 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 562 } else { 563 int index = cvmx_get_core_num() * 2 + 1; 564 set_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu1_en_mirror)); 565 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 566 } 567 } 568 569 static void octeon_irq_ciu_disable_local_v2(struct irq_data *data) 570 { 571 u64 mask; 572 struct octeon_ciu_chip_data *cd; 573 574 cd = irq_data_get_irq_chip_data(data); 575 mask = 1ull << (cd->bit); 576 577 if (cd->line == 0) { 578 int index = cvmx_get_core_num() * 2; 579 clear_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu0_en_mirror)); 580 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); 581 } else { 582 int index = cvmx_get_core_num() * 2 + 1; 583 clear_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu1_en_mirror)); 584 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask); 585 } 586 } 587 588 /* 589 * Write to the W1C bit in CVMX_CIU_INTX_SUM0 to clear the irq. 590 */ 591 static void octeon_irq_ciu_ack(struct irq_data *data) 592 { 593 u64 mask; 594 struct octeon_ciu_chip_data *cd; 595 596 cd = irq_data_get_irq_chip_data(data); 597 mask = 1ull << (cd->bit); 598 599 if (cd->line == 0) { 600 int index = cvmx_get_core_num() * 2; 601 cvmx_write_csr(CVMX_CIU_INTX_SUM0(index), mask); 602 } else { 603 cvmx_write_csr(CVMX_CIU_INT_SUM1, mask); 604 } 605 } 606 607 /* 608 * Disable the irq on the all cores for chips that have the EN*_W1{S,C} 609 * registers. 610 */ 611 static void octeon_irq_ciu_disable_all_v2(struct irq_data *data) 612 { 613 int cpu; 614 u64 mask; 615 struct octeon_ciu_chip_data *cd; 616 617 cd = irq_data_get_irq_chip_data(data); 618 mask = 1ull << (cd->bit); 619 620 if (cd->line == 0) { 621 for_each_online_cpu(cpu) { 622 int index = octeon_coreid_for_cpu(cpu) * 2; 623 clear_bit(cd->bit, 624 &per_cpu(octeon_irq_ciu0_en_mirror, cpu)); 625 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); 626 } 627 } else { 628 for_each_online_cpu(cpu) { 629 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 630 clear_bit(cd->bit, 631 &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 632 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask); 633 } 634 } 635 } 636 637 /* 638 * Enable the irq on the all cores for chips that have the EN*_W1{S,C} 639 * registers. 640 */ 641 static void octeon_irq_ciu_enable_all_v2(struct irq_data *data) 642 { 643 int cpu; 644 u64 mask; 645 struct octeon_ciu_chip_data *cd; 646 647 cd = irq_data_get_irq_chip_data(data); 648 mask = 1ull << (cd->bit); 649 650 if (cd->line == 0) { 651 for_each_online_cpu(cpu) { 652 int index = octeon_coreid_for_cpu(cpu) * 2; 653 set_bit(cd->bit, 654 &per_cpu(octeon_irq_ciu0_en_mirror, cpu)); 655 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 656 } 657 } else { 658 for_each_online_cpu(cpu) { 659 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 660 set_bit(cd->bit, 661 &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 662 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 663 } 664 } 665 } 666 667 static int octeon_irq_ciu_set_type(struct irq_data *data, unsigned int t) 668 { 669 irqd_set_trigger_type(data, t); 670 671 if (t & IRQ_TYPE_EDGE_BOTH) 672 irq_set_handler_locked(data, handle_edge_irq); 673 else 674 irq_set_handler_locked(data, handle_level_irq); 675 676 return IRQ_SET_MASK_OK; 677 } 678 679 static void octeon_irq_gpio_setup(struct irq_data *data) 680 { 681 union cvmx_gpio_bit_cfgx cfg; 682 struct octeon_ciu_chip_data *cd; 683 u32 t = irqd_get_trigger_type(data); 684 685 cd = irq_data_get_irq_chip_data(data); 686 687 cfg.u64 = 0; 688 cfg.s.int_en = 1; 689 cfg.s.int_type = (t & IRQ_TYPE_EDGE_BOTH) != 0; 690 cfg.s.rx_xor = (t & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_EDGE_FALLING)) != 0; 691 692 /* 140 nS glitch filter*/ 693 cfg.s.fil_cnt = 7; 694 cfg.s.fil_sel = 3; 695 696 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), cfg.u64); 697 } 698 699 static void octeon_irq_ciu_enable_gpio_v2(struct irq_data *data) 700 { 701 octeon_irq_gpio_setup(data); 702 octeon_irq_ciu_enable_v2(data); 703 } 704 705 static void octeon_irq_ciu_enable_gpio(struct irq_data *data) 706 { 707 octeon_irq_gpio_setup(data); 708 octeon_irq_ciu_enable(data); 709 } 710 711 static int octeon_irq_ciu_gpio_set_type(struct irq_data *data, unsigned int t) 712 { 713 irqd_set_trigger_type(data, t); 714 octeon_irq_gpio_setup(data); 715 716 if (t & IRQ_TYPE_EDGE_BOTH) 717 irq_set_handler_locked(data, handle_edge_irq); 718 else 719 irq_set_handler_locked(data, handle_level_irq); 720 721 return IRQ_SET_MASK_OK; 722 } 723 724 static void octeon_irq_ciu_disable_gpio_v2(struct irq_data *data) 725 { 726 struct octeon_ciu_chip_data *cd; 727 728 cd = irq_data_get_irq_chip_data(data); 729 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0); 730 731 octeon_irq_ciu_disable_all_v2(data); 732 } 733 734 static void octeon_irq_ciu_disable_gpio(struct irq_data *data) 735 { 736 struct octeon_ciu_chip_data *cd; 737 738 cd = irq_data_get_irq_chip_data(data); 739 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0); 740 741 octeon_irq_ciu_disable_all(data); 742 } 743 744 static void octeon_irq_ciu_gpio_ack(struct irq_data *data) 745 { 746 struct octeon_ciu_chip_data *cd; 747 u64 mask; 748 749 cd = irq_data_get_irq_chip_data(data); 750 mask = 1ull << (cd->gpio_line); 751 752 cvmx_write_csr(CVMX_GPIO_INT_CLR, mask); 753 } 754 755 #ifdef CONFIG_SMP 756 757 static void octeon_irq_cpu_offline_ciu(struct irq_data *data) 758 { 759 int cpu = smp_processor_id(); 760 cpumask_t new_affinity; 761 struct cpumask *mask = irq_data_get_affinity_mask(data); 762 763 if (!cpumask_test_cpu(cpu, mask)) 764 return; 765 766 if (cpumask_weight(mask) > 1) { 767 /* 768 * It has multi CPU affinity, just remove this CPU 769 * from the affinity set. 770 */ 771 cpumask_copy(&new_affinity, mask); 772 cpumask_clear_cpu(cpu, &new_affinity); 773 } else { 774 /* Otherwise, put it on lowest numbered online CPU. */ 775 cpumask_clear(&new_affinity); 776 cpumask_set_cpu(cpumask_first(cpu_online_mask), &new_affinity); 777 } 778 irq_set_affinity_locked(data, &new_affinity, false); 779 } 780 781 static int octeon_irq_ciu_set_affinity(struct irq_data *data, 782 const struct cpumask *dest, bool force) 783 { 784 int cpu; 785 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 786 unsigned long flags; 787 struct octeon_ciu_chip_data *cd; 788 unsigned long *pen; 789 raw_spinlock_t *lock; 790 791 cd = irq_data_get_irq_chip_data(data); 792 793 /* 794 * For non-v2 CIU, we will allow only single CPU affinity. 795 * This removes the need to do locking in the .ack/.eoi 796 * functions. 797 */ 798 if (cpumask_weight(dest) != 1) 799 return -EINVAL; 800 801 if (!enable_one) 802 return 0; 803 804 805 for_each_online_cpu(cpu) { 806 int coreid = octeon_coreid_for_cpu(cpu); 807 808 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 809 raw_spin_lock_irqsave(lock, flags); 810 811 if (cd->line == 0) 812 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 813 else 814 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 815 816 if (cpumask_test_cpu(cpu, dest) && enable_one) { 817 enable_one = false; 818 __set_bit(cd->bit, pen); 819 } else { 820 __clear_bit(cd->bit, pen); 821 } 822 /* 823 * Must be visible to octeon_irq_ip{2,3}_ciu() before 824 * enabling the irq. 825 */ 826 wmb(); 827 828 if (cd->line == 0) 829 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 830 else 831 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 832 833 raw_spin_unlock_irqrestore(lock, flags); 834 } 835 return 0; 836 } 837 838 /* 839 * Set affinity for the irq for chips that have the EN*_W1{S,C} 840 * registers. 841 */ 842 static int octeon_irq_ciu_set_affinity_v2(struct irq_data *data, 843 const struct cpumask *dest, 844 bool force) 845 { 846 int cpu; 847 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 848 u64 mask; 849 struct octeon_ciu_chip_data *cd; 850 851 if (!enable_one) 852 return 0; 853 854 cd = irq_data_get_irq_chip_data(data); 855 mask = 1ull << cd->bit; 856 857 if (cd->line == 0) { 858 for_each_online_cpu(cpu) { 859 unsigned long *pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 860 int index = octeon_coreid_for_cpu(cpu) * 2; 861 if (cpumask_test_cpu(cpu, dest) && enable_one) { 862 enable_one = false; 863 set_bit(cd->bit, pen); 864 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 865 } else { 866 clear_bit(cd->bit, pen); 867 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); 868 } 869 } 870 } else { 871 for_each_online_cpu(cpu) { 872 unsigned long *pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 873 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 874 if (cpumask_test_cpu(cpu, dest) && enable_one) { 875 enable_one = false; 876 set_bit(cd->bit, pen); 877 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 878 } else { 879 clear_bit(cd->bit, pen); 880 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask); 881 } 882 } 883 } 884 return 0; 885 } 886 887 static int octeon_irq_ciu_set_affinity_sum2(struct irq_data *data, 888 const struct cpumask *dest, 889 bool force) 890 { 891 int cpu; 892 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 893 u64 mask; 894 struct octeon_ciu_chip_data *cd; 895 896 if (!enable_one) 897 return 0; 898 899 cd = irq_data_get_irq_chip_data(data); 900 mask = 1ull << cd->bit; 901 902 for_each_online_cpu(cpu) { 903 int index = octeon_coreid_for_cpu(cpu); 904 905 if (cpumask_test_cpu(cpu, dest) && enable_one) { 906 enable_one = false; 907 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1S(index), mask); 908 } else { 909 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(index), mask); 910 } 911 } 912 return 0; 913 } 914 #endif 915 916 static unsigned int edge_startup(struct irq_data *data) 917 { 918 /* ack any pending edge-irq at startup, so there is 919 * an _edge_ to fire on when the event reappears. 920 */ 921 data->chip->irq_ack(data); 922 data->chip->irq_enable(data); 923 return 0; 924 } 925 926 /* 927 * Newer octeon chips have support for lockless CIU operation. 928 */ 929 static struct irq_chip octeon_irq_chip_ciu_v2 = { 930 .name = "CIU", 931 .irq_enable = octeon_irq_ciu_enable_v2, 932 .irq_disable = octeon_irq_ciu_disable_all_v2, 933 .irq_mask = octeon_irq_ciu_disable_local_v2, 934 .irq_unmask = octeon_irq_ciu_enable_v2, 935 #ifdef CONFIG_SMP 936 .irq_set_affinity = octeon_irq_ciu_set_affinity_v2, 937 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 938 #endif 939 }; 940 941 static struct irq_chip octeon_irq_chip_ciu_v2_edge = { 942 .name = "CIU", 943 .irq_enable = octeon_irq_ciu_enable_v2, 944 .irq_disable = octeon_irq_ciu_disable_all_v2, 945 .irq_ack = octeon_irq_ciu_ack, 946 .irq_mask = octeon_irq_ciu_disable_local_v2, 947 .irq_unmask = octeon_irq_ciu_enable_v2, 948 #ifdef CONFIG_SMP 949 .irq_set_affinity = octeon_irq_ciu_set_affinity_v2, 950 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 951 #endif 952 }; 953 954 /* 955 * Newer octeon chips have support for lockless CIU operation. 956 */ 957 static struct irq_chip octeon_irq_chip_ciu_sum2 = { 958 .name = "CIU", 959 .irq_enable = octeon_irq_ciu_enable_sum2, 960 .irq_disable = octeon_irq_ciu_disable_all_sum2, 961 .irq_mask = octeon_irq_ciu_disable_local_sum2, 962 .irq_unmask = octeon_irq_ciu_enable_sum2, 963 #ifdef CONFIG_SMP 964 .irq_set_affinity = octeon_irq_ciu_set_affinity_sum2, 965 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 966 #endif 967 }; 968 969 static struct irq_chip octeon_irq_chip_ciu_sum2_edge = { 970 .name = "CIU", 971 .irq_enable = octeon_irq_ciu_enable_sum2, 972 .irq_disable = octeon_irq_ciu_disable_all_sum2, 973 .irq_ack = octeon_irq_ciu_ack_sum2, 974 .irq_mask = octeon_irq_ciu_disable_local_sum2, 975 .irq_unmask = octeon_irq_ciu_enable_sum2, 976 #ifdef CONFIG_SMP 977 .irq_set_affinity = octeon_irq_ciu_set_affinity_sum2, 978 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 979 #endif 980 }; 981 982 static struct irq_chip octeon_irq_chip_ciu = { 983 .name = "CIU", 984 .irq_enable = octeon_irq_ciu_enable, 985 .irq_disable = octeon_irq_ciu_disable_all, 986 .irq_mask = octeon_irq_ciu_disable_local, 987 .irq_unmask = octeon_irq_ciu_enable, 988 #ifdef CONFIG_SMP 989 .irq_set_affinity = octeon_irq_ciu_set_affinity, 990 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 991 #endif 992 }; 993 994 static struct irq_chip octeon_irq_chip_ciu_edge = { 995 .name = "CIU", 996 .irq_enable = octeon_irq_ciu_enable, 997 .irq_disable = octeon_irq_ciu_disable_all, 998 .irq_ack = octeon_irq_ciu_ack, 999 .irq_mask = octeon_irq_ciu_disable_local, 1000 .irq_unmask = octeon_irq_ciu_enable, 1001 #ifdef CONFIG_SMP 1002 .irq_set_affinity = octeon_irq_ciu_set_affinity, 1003 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1004 #endif 1005 }; 1006 1007 /* The mbox versions don't do any affinity or round-robin. */ 1008 static struct irq_chip octeon_irq_chip_ciu_mbox_v2 = { 1009 .name = "CIU-M", 1010 .irq_enable = octeon_irq_ciu_enable_all_v2, 1011 .irq_disable = octeon_irq_ciu_disable_all_v2, 1012 .irq_ack = octeon_irq_ciu_disable_local_v2, 1013 .irq_eoi = octeon_irq_ciu_enable_local_v2, 1014 1015 .irq_cpu_online = octeon_irq_ciu_enable_local_v2, 1016 .irq_cpu_offline = octeon_irq_ciu_disable_local_v2, 1017 .flags = IRQCHIP_ONOFFLINE_ENABLED, 1018 }; 1019 1020 static struct irq_chip octeon_irq_chip_ciu_mbox = { 1021 .name = "CIU-M", 1022 .irq_enable = octeon_irq_ciu_enable_all, 1023 .irq_disable = octeon_irq_ciu_disable_all, 1024 .irq_ack = octeon_irq_ciu_disable_local, 1025 .irq_eoi = octeon_irq_ciu_enable_local, 1026 1027 .irq_cpu_online = octeon_irq_ciu_enable_local, 1028 .irq_cpu_offline = octeon_irq_ciu_disable_local, 1029 .flags = IRQCHIP_ONOFFLINE_ENABLED, 1030 }; 1031 1032 static struct irq_chip octeon_irq_chip_ciu_gpio_v2 = { 1033 .name = "CIU-GPIO", 1034 .irq_enable = octeon_irq_ciu_enable_gpio_v2, 1035 .irq_disable = octeon_irq_ciu_disable_gpio_v2, 1036 .irq_ack = octeon_irq_ciu_gpio_ack, 1037 .irq_mask = octeon_irq_ciu_disable_local_v2, 1038 .irq_unmask = octeon_irq_ciu_enable_v2, 1039 .irq_set_type = octeon_irq_ciu_gpio_set_type, 1040 #ifdef CONFIG_SMP 1041 .irq_set_affinity = octeon_irq_ciu_set_affinity_v2, 1042 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1043 #endif 1044 .flags = IRQCHIP_SET_TYPE_MASKED, 1045 }; 1046 1047 static struct irq_chip octeon_irq_chip_ciu_gpio = { 1048 .name = "CIU-GPIO", 1049 .irq_enable = octeon_irq_ciu_enable_gpio, 1050 .irq_disable = octeon_irq_ciu_disable_gpio, 1051 .irq_mask = octeon_irq_ciu_disable_local, 1052 .irq_unmask = octeon_irq_ciu_enable, 1053 .irq_ack = octeon_irq_ciu_gpio_ack, 1054 .irq_set_type = octeon_irq_ciu_gpio_set_type, 1055 #ifdef CONFIG_SMP 1056 .irq_set_affinity = octeon_irq_ciu_set_affinity, 1057 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1058 #endif 1059 .flags = IRQCHIP_SET_TYPE_MASKED, 1060 }; 1061 1062 /* 1063 * Watchdog interrupts are special. They are associated with a single 1064 * core, so we hardwire the affinity to that core. 1065 */ 1066 static void octeon_irq_ciu_wd_enable(struct irq_data *data) 1067 { 1068 unsigned long flags; 1069 unsigned long *pen; 1070 int coreid = data->irq - OCTEON_IRQ_WDOG0; /* Bit 0-63 of EN1 */ 1071 int cpu = octeon_cpu_for_coreid(coreid); 1072 raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 1073 1074 raw_spin_lock_irqsave(lock, flags); 1075 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 1076 __set_bit(coreid, pen); 1077 /* 1078 * Must be visible to octeon_irq_ip{2,3}_ciu() before enabling 1079 * the irq. 1080 */ 1081 wmb(); 1082 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 1083 raw_spin_unlock_irqrestore(lock, flags); 1084 } 1085 1086 /* 1087 * Watchdog interrupts are special. They are associated with a single 1088 * core, so we hardwire the affinity to that core. 1089 */ 1090 static void octeon_irq_ciu1_wd_enable_v2(struct irq_data *data) 1091 { 1092 int coreid = data->irq - OCTEON_IRQ_WDOG0; 1093 int cpu = octeon_cpu_for_coreid(coreid); 1094 1095 set_bit(coreid, &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 1096 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(coreid * 2 + 1), 1ull << coreid); 1097 } 1098 1099 1100 static struct irq_chip octeon_irq_chip_ciu_wd_v2 = { 1101 .name = "CIU-W", 1102 .irq_enable = octeon_irq_ciu1_wd_enable_v2, 1103 .irq_disable = octeon_irq_ciu_disable_all_v2, 1104 .irq_mask = octeon_irq_ciu_disable_local_v2, 1105 .irq_unmask = octeon_irq_ciu_enable_local_v2, 1106 }; 1107 1108 static struct irq_chip octeon_irq_chip_ciu_wd = { 1109 .name = "CIU-W", 1110 .irq_enable = octeon_irq_ciu_wd_enable, 1111 .irq_disable = octeon_irq_ciu_disable_all, 1112 .irq_mask = octeon_irq_ciu_disable_local, 1113 .irq_unmask = octeon_irq_ciu_enable_local, 1114 }; 1115 1116 static bool octeon_irq_ciu_is_edge(unsigned int line, unsigned int bit) 1117 { 1118 bool edge = false; 1119 1120 if (line == 0) 1121 switch (bit) { 1122 case 48 ... 49: /* GMX DRP */ 1123 case 50: /* IPD_DRP */ 1124 case 52 ... 55: /* Timers */ 1125 case 58: /* MPI */ 1126 edge = true; 1127 break; 1128 default: 1129 break; 1130 } 1131 else /* line == 1 */ 1132 switch (bit) { 1133 case 47: /* PTP */ 1134 edge = true; 1135 break; 1136 default: 1137 break; 1138 } 1139 return edge; 1140 } 1141 1142 struct octeon_irq_gpio_domain_data { 1143 unsigned int base_hwirq; 1144 }; 1145 1146 static int octeon_irq_gpio_xlat(struct irq_domain *d, 1147 struct device_node *node, 1148 const u32 *intspec, 1149 unsigned int intsize, 1150 unsigned long *out_hwirq, 1151 unsigned int *out_type) 1152 { 1153 unsigned int type; 1154 unsigned int pin; 1155 unsigned int trigger; 1156 1157 if (irq_domain_get_of_node(d) != node) 1158 return -EINVAL; 1159 1160 if (intsize < 2) 1161 return -EINVAL; 1162 1163 pin = intspec[0]; 1164 if (pin >= 16) 1165 return -EINVAL; 1166 1167 trigger = intspec[1]; 1168 1169 switch (trigger) { 1170 case 1: 1171 type = IRQ_TYPE_EDGE_RISING; 1172 break; 1173 case 2: 1174 type = IRQ_TYPE_EDGE_FALLING; 1175 break; 1176 case 4: 1177 type = IRQ_TYPE_LEVEL_HIGH; 1178 break; 1179 case 8: 1180 type = IRQ_TYPE_LEVEL_LOW; 1181 break; 1182 default: 1183 pr_err("Error: (%pOFn) Invalid irq trigger specification: %x\n", 1184 node, 1185 trigger); 1186 type = IRQ_TYPE_LEVEL_LOW; 1187 break; 1188 } 1189 *out_type = type; 1190 *out_hwirq = pin; 1191 1192 return 0; 1193 } 1194 1195 static int octeon_irq_ciu_xlat(struct irq_domain *d, 1196 struct device_node *node, 1197 const u32 *intspec, 1198 unsigned int intsize, 1199 unsigned long *out_hwirq, 1200 unsigned int *out_type) 1201 { 1202 unsigned int ciu, bit; 1203 struct octeon_irq_ciu_domain_data *dd = d->host_data; 1204 1205 ciu = intspec[0]; 1206 bit = intspec[1]; 1207 1208 if (ciu >= dd->num_sum || bit > 63) 1209 return -EINVAL; 1210 1211 *out_hwirq = (ciu << 6) | bit; 1212 *out_type = 0; 1213 1214 return 0; 1215 } 1216 1217 static struct irq_chip *octeon_irq_ciu_chip; 1218 static struct irq_chip *octeon_irq_ciu_chip_edge; 1219 static struct irq_chip *octeon_irq_gpio_chip; 1220 1221 static int octeon_irq_ciu_map(struct irq_domain *d, 1222 unsigned int virq, irq_hw_number_t hw) 1223 { 1224 int rv; 1225 unsigned int line = hw >> 6; 1226 unsigned int bit = hw & 63; 1227 struct octeon_irq_ciu_domain_data *dd = d->host_data; 1228 1229 if (line >= dd->num_sum || octeon_irq_ciu_to_irq[line][bit] != 0) 1230 return -EINVAL; 1231 1232 if (line == 2) { 1233 if (octeon_irq_ciu_is_edge(line, bit)) 1234 rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1235 &octeon_irq_chip_ciu_sum2_edge, 1236 handle_edge_irq); 1237 else 1238 rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1239 &octeon_irq_chip_ciu_sum2, 1240 handle_level_irq); 1241 } else { 1242 if (octeon_irq_ciu_is_edge(line, bit)) 1243 rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1244 octeon_irq_ciu_chip_edge, 1245 handle_edge_irq); 1246 else 1247 rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1248 octeon_irq_ciu_chip, 1249 handle_level_irq); 1250 } 1251 return rv; 1252 } 1253 1254 static int octeon_irq_gpio_map(struct irq_domain *d, 1255 unsigned int virq, irq_hw_number_t hw) 1256 { 1257 struct octeon_irq_gpio_domain_data *gpiod = d->host_data; 1258 unsigned int line, bit; 1259 int r; 1260 1261 line = (hw + gpiod->base_hwirq) >> 6; 1262 bit = (hw + gpiod->base_hwirq) & 63; 1263 if (line >= ARRAY_SIZE(octeon_irq_ciu_to_irq) || 1264 octeon_irq_ciu_to_irq[line][bit] != 0) 1265 return -EINVAL; 1266 1267 /* 1268 * Default to handle_level_irq. If the DT contains a different 1269 * trigger type, it will call the irq_set_type callback and 1270 * the handler gets updated. 1271 */ 1272 r = octeon_irq_set_ciu_mapping(virq, line, bit, hw, 1273 octeon_irq_gpio_chip, handle_level_irq); 1274 return r; 1275 } 1276 1277 static struct irq_domain_ops octeon_irq_domain_ciu_ops = { 1278 .map = octeon_irq_ciu_map, 1279 .unmap = octeon_irq_free_cd, 1280 .xlate = octeon_irq_ciu_xlat, 1281 }; 1282 1283 static struct irq_domain_ops octeon_irq_domain_gpio_ops = { 1284 .map = octeon_irq_gpio_map, 1285 .unmap = octeon_irq_free_cd, 1286 .xlate = octeon_irq_gpio_xlat, 1287 }; 1288 1289 static void octeon_irq_ip2_ciu(void) 1290 { 1291 const unsigned long core_id = cvmx_get_core_num(); 1292 u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INTX_SUM0(core_id * 2)); 1293 1294 ciu_sum &= __this_cpu_read(octeon_irq_ciu0_en_mirror); 1295 if (likely(ciu_sum)) { 1296 int bit = fls64(ciu_sum) - 1; 1297 int irq = octeon_irq_ciu_to_irq[0][bit]; 1298 if (likely(irq)) 1299 do_IRQ(irq); 1300 else 1301 spurious_interrupt(); 1302 } else { 1303 spurious_interrupt(); 1304 } 1305 } 1306 1307 static void octeon_irq_ip3_ciu(void) 1308 { 1309 u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INT_SUM1); 1310 1311 ciu_sum &= __this_cpu_read(octeon_irq_ciu1_en_mirror); 1312 if (likely(ciu_sum)) { 1313 int bit = fls64(ciu_sum) - 1; 1314 int irq = octeon_irq_ciu_to_irq[1][bit]; 1315 if (likely(irq)) 1316 do_IRQ(irq); 1317 else 1318 spurious_interrupt(); 1319 } else { 1320 spurious_interrupt(); 1321 } 1322 } 1323 1324 static void octeon_irq_ip4_ciu(void) 1325 { 1326 int coreid = cvmx_get_core_num(); 1327 u64 ciu_sum = cvmx_read_csr(CVMX_CIU_SUM2_PPX_IP4(coreid)); 1328 u64 ciu_en = cvmx_read_csr(CVMX_CIU_EN2_PPX_IP4(coreid)); 1329 1330 ciu_sum &= ciu_en; 1331 if (likely(ciu_sum)) { 1332 int bit = fls64(ciu_sum) - 1; 1333 int irq = octeon_irq_ciu_to_irq[2][bit]; 1334 1335 if (likely(irq)) 1336 do_IRQ(irq); 1337 else 1338 spurious_interrupt(); 1339 } else { 1340 spurious_interrupt(); 1341 } 1342 } 1343 1344 static bool octeon_irq_use_ip4; 1345 1346 static void octeon_irq_local_enable_ip4(void *arg) 1347 { 1348 set_c0_status(STATUSF_IP4); 1349 } 1350 1351 static void octeon_irq_ip4_mask(void) 1352 { 1353 clear_c0_status(STATUSF_IP4); 1354 spurious_interrupt(); 1355 } 1356 1357 static void (*octeon_irq_ip2)(void); 1358 static void (*octeon_irq_ip3)(void); 1359 static void (*octeon_irq_ip4)(void); 1360 1361 void (*octeon_irq_setup_secondary)(void); 1362 1363 void octeon_irq_set_ip4_handler(octeon_irq_ip4_handler_t h) 1364 { 1365 octeon_irq_ip4 = h; 1366 octeon_irq_use_ip4 = true; 1367 on_each_cpu(octeon_irq_local_enable_ip4, NULL, 1); 1368 } 1369 1370 static void octeon_irq_percpu_enable(void) 1371 { 1372 irq_cpu_online(); 1373 } 1374 1375 static void octeon_irq_init_ciu_percpu(void) 1376 { 1377 int coreid = cvmx_get_core_num(); 1378 1379 1380 __this_cpu_write(octeon_irq_ciu0_en_mirror, 0); 1381 __this_cpu_write(octeon_irq_ciu1_en_mirror, 0); 1382 wmb(); 1383 raw_spin_lock_init(this_cpu_ptr(&octeon_irq_ciu_spinlock)); 1384 /* 1385 * Disable All CIU Interrupts. The ones we need will be 1386 * enabled later. Read the SUM register so we know the write 1387 * completed. 1388 */ 1389 cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2)), 0); 1390 cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2 + 1)), 0); 1391 cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2)), 0); 1392 cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2 + 1)), 0); 1393 cvmx_read_csr(CVMX_CIU_INTX_SUM0((coreid * 2))); 1394 } 1395 1396 static void octeon_irq_init_ciu2_percpu(void) 1397 { 1398 u64 regx, ipx; 1399 int coreid = cvmx_get_core_num(); 1400 u64 base = CVMX_CIU2_EN_PPX_IP2_WRKQ(coreid); 1401 1402 /* 1403 * Disable All CIU2 Interrupts. The ones we need will be 1404 * enabled later. Read the SUM register so we know the write 1405 * completed. 1406 * 1407 * There are 9 registers and 3 IPX levels with strides 0x1000 1408 * and 0x200 respectivly. Use loops to clear them. 1409 */ 1410 for (regx = 0; regx <= 0x8000; regx += 0x1000) { 1411 for (ipx = 0; ipx <= 0x400; ipx += 0x200) 1412 cvmx_write_csr(base + regx + ipx, 0); 1413 } 1414 1415 cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(coreid)); 1416 } 1417 1418 static void octeon_irq_setup_secondary_ciu(void) 1419 { 1420 octeon_irq_init_ciu_percpu(); 1421 octeon_irq_percpu_enable(); 1422 1423 /* Enable the CIU lines */ 1424 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 1425 if (octeon_irq_use_ip4) 1426 set_c0_status(STATUSF_IP4); 1427 else 1428 clear_c0_status(STATUSF_IP4); 1429 } 1430 1431 static void octeon_irq_setup_secondary_ciu2(void) 1432 { 1433 octeon_irq_init_ciu2_percpu(); 1434 octeon_irq_percpu_enable(); 1435 1436 /* Enable the CIU lines */ 1437 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 1438 if (octeon_irq_use_ip4) 1439 set_c0_status(STATUSF_IP4); 1440 else 1441 clear_c0_status(STATUSF_IP4); 1442 } 1443 1444 static int __init octeon_irq_init_ciu( 1445 struct device_node *ciu_node, struct device_node *parent) 1446 { 1447 unsigned int i, r; 1448 struct irq_chip *chip; 1449 struct irq_chip *chip_edge; 1450 struct irq_chip *chip_mbox; 1451 struct irq_chip *chip_wd; 1452 struct irq_domain *ciu_domain = NULL; 1453 struct octeon_irq_ciu_domain_data *dd; 1454 1455 dd = kzalloc(sizeof(*dd), GFP_KERNEL); 1456 if (!dd) 1457 return -ENOMEM; 1458 1459 octeon_irq_init_ciu_percpu(); 1460 octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu; 1461 1462 octeon_irq_ip2 = octeon_irq_ip2_ciu; 1463 octeon_irq_ip3 = octeon_irq_ip3_ciu; 1464 if ((OCTEON_IS_OCTEON2() || OCTEON_IS_OCTEON3()) 1465 && !OCTEON_IS_MODEL(OCTEON_CN63XX)) { 1466 octeon_irq_ip4 = octeon_irq_ip4_ciu; 1467 dd->num_sum = 3; 1468 octeon_irq_use_ip4 = true; 1469 } else { 1470 octeon_irq_ip4 = octeon_irq_ip4_mask; 1471 dd->num_sum = 2; 1472 octeon_irq_use_ip4 = false; 1473 } 1474 if (OCTEON_IS_MODEL(OCTEON_CN58XX_PASS2_X) || 1475 OCTEON_IS_MODEL(OCTEON_CN56XX_PASS2_X) || 1476 OCTEON_IS_MODEL(OCTEON_CN52XX_PASS2_X) || 1477 OCTEON_IS_OCTEON2() || OCTEON_IS_OCTEON3()) { 1478 chip = &octeon_irq_chip_ciu_v2; 1479 chip_edge = &octeon_irq_chip_ciu_v2_edge; 1480 chip_mbox = &octeon_irq_chip_ciu_mbox_v2; 1481 chip_wd = &octeon_irq_chip_ciu_wd_v2; 1482 octeon_irq_gpio_chip = &octeon_irq_chip_ciu_gpio_v2; 1483 } else { 1484 chip = &octeon_irq_chip_ciu; 1485 chip_edge = &octeon_irq_chip_ciu_edge; 1486 chip_mbox = &octeon_irq_chip_ciu_mbox; 1487 chip_wd = &octeon_irq_chip_ciu_wd; 1488 octeon_irq_gpio_chip = &octeon_irq_chip_ciu_gpio; 1489 } 1490 octeon_irq_ciu_chip = chip; 1491 octeon_irq_ciu_chip_edge = chip_edge; 1492 1493 /* Mips internal */ 1494 octeon_irq_init_core(); 1495 1496 ciu_domain = irq_domain_add_tree( 1497 ciu_node, &octeon_irq_domain_ciu_ops, dd); 1498 irq_set_default_host(ciu_domain); 1499 1500 /* CIU_0 */ 1501 for (i = 0; i < 16; i++) { 1502 r = octeon_irq_force_ciu_mapping( 1503 ciu_domain, i + OCTEON_IRQ_WORKQ0, 0, i + 0); 1504 if (r) 1505 goto err; 1506 } 1507 1508 r = octeon_irq_set_ciu_mapping( 1509 OCTEON_IRQ_MBOX0, 0, 32, 0, chip_mbox, handle_percpu_irq); 1510 if (r) 1511 goto err; 1512 r = octeon_irq_set_ciu_mapping( 1513 OCTEON_IRQ_MBOX1, 0, 33, 0, chip_mbox, handle_percpu_irq); 1514 if (r) 1515 goto err; 1516 1517 for (i = 0; i < 4; i++) { 1518 r = octeon_irq_force_ciu_mapping( 1519 ciu_domain, i + OCTEON_IRQ_PCI_INT0, 0, i + 36); 1520 if (r) 1521 goto err; 1522 } 1523 for (i = 0; i < 4; i++) { 1524 r = octeon_irq_force_ciu_mapping( 1525 ciu_domain, i + OCTEON_IRQ_PCI_MSI0, 0, i + 40); 1526 if (r) 1527 goto err; 1528 } 1529 1530 r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_TWSI, 0, 45); 1531 if (r) 1532 goto err; 1533 1534 r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_RML, 0, 46); 1535 if (r) 1536 goto err; 1537 1538 for (i = 0; i < 4; i++) { 1539 r = octeon_irq_force_ciu_mapping( 1540 ciu_domain, i + OCTEON_IRQ_TIMER0, 0, i + 52); 1541 if (r) 1542 goto err; 1543 } 1544 1545 r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_TWSI2, 0, 59); 1546 if (r) 1547 goto err; 1548 1549 /* CIU_1 */ 1550 for (i = 0; i < 16; i++) { 1551 r = octeon_irq_set_ciu_mapping( 1552 i + OCTEON_IRQ_WDOG0, 1, i + 0, 0, chip_wd, 1553 handle_level_irq); 1554 if (r) 1555 goto err; 1556 } 1557 1558 /* Enable the CIU lines */ 1559 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 1560 if (octeon_irq_use_ip4) 1561 set_c0_status(STATUSF_IP4); 1562 else 1563 clear_c0_status(STATUSF_IP4); 1564 1565 return 0; 1566 err: 1567 return r; 1568 } 1569 1570 static int __init octeon_irq_init_gpio( 1571 struct device_node *gpio_node, struct device_node *parent) 1572 { 1573 struct octeon_irq_gpio_domain_data *gpiod; 1574 u32 interrupt_cells; 1575 unsigned int base_hwirq; 1576 int r; 1577 1578 r = of_property_read_u32(parent, "#interrupt-cells", &interrupt_cells); 1579 if (r) 1580 return r; 1581 1582 if (interrupt_cells == 1) { 1583 u32 v; 1584 1585 r = of_property_read_u32_index(gpio_node, "interrupts", 0, &v); 1586 if (r) { 1587 pr_warn("No \"interrupts\" property.\n"); 1588 return r; 1589 } 1590 base_hwirq = v; 1591 } else if (interrupt_cells == 2) { 1592 u32 v0, v1; 1593 1594 r = of_property_read_u32_index(gpio_node, "interrupts", 0, &v0); 1595 if (r) { 1596 pr_warn("No \"interrupts\" property.\n"); 1597 return r; 1598 } 1599 r = of_property_read_u32_index(gpio_node, "interrupts", 1, &v1); 1600 if (r) { 1601 pr_warn("No \"interrupts\" property.\n"); 1602 return r; 1603 } 1604 base_hwirq = (v0 << 6) | v1; 1605 } else { 1606 pr_warn("Bad \"#interrupt-cells\" property: %u\n", 1607 interrupt_cells); 1608 return -EINVAL; 1609 } 1610 1611 gpiod = kzalloc(sizeof(*gpiod), GFP_KERNEL); 1612 if (gpiod) { 1613 /* gpio domain host_data is the base hwirq number. */ 1614 gpiod->base_hwirq = base_hwirq; 1615 irq_domain_add_linear( 1616 gpio_node, 16, &octeon_irq_domain_gpio_ops, gpiod); 1617 } else { 1618 pr_warn("Cannot allocate memory for GPIO irq_domain.\n"); 1619 return -ENOMEM; 1620 } 1621 1622 /* 1623 * Clear the OF_POPULATED flag that was set by of_irq_init() 1624 * so that all GPIO devices will be probed. 1625 */ 1626 of_node_clear_flag(gpio_node, OF_POPULATED); 1627 1628 return 0; 1629 } 1630 /* 1631 * Watchdog interrupts are special. They are associated with a single 1632 * core, so we hardwire the affinity to that core. 1633 */ 1634 static void octeon_irq_ciu2_wd_enable(struct irq_data *data) 1635 { 1636 u64 mask; 1637 u64 en_addr; 1638 int coreid = data->irq - OCTEON_IRQ_WDOG0; 1639 struct octeon_ciu_chip_data *cd; 1640 1641 cd = irq_data_get_irq_chip_data(data); 1642 mask = 1ull << (cd->bit); 1643 1644 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) + 1645 (0x1000ull * cd->line); 1646 cvmx_write_csr(en_addr, mask); 1647 1648 } 1649 1650 static void octeon_irq_ciu2_enable(struct irq_data *data) 1651 { 1652 u64 mask; 1653 u64 en_addr; 1654 int cpu = next_cpu_for_irq(data); 1655 int coreid = octeon_coreid_for_cpu(cpu); 1656 struct octeon_ciu_chip_data *cd; 1657 1658 cd = irq_data_get_irq_chip_data(data); 1659 mask = 1ull << (cd->bit); 1660 1661 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) + 1662 (0x1000ull * cd->line); 1663 cvmx_write_csr(en_addr, mask); 1664 } 1665 1666 static void octeon_irq_ciu2_enable_local(struct irq_data *data) 1667 { 1668 u64 mask; 1669 u64 en_addr; 1670 int coreid = cvmx_get_core_num(); 1671 struct octeon_ciu_chip_data *cd; 1672 1673 cd = irq_data_get_irq_chip_data(data); 1674 mask = 1ull << (cd->bit); 1675 1676 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) + 1677 (0x1000ull * cd->line); 1678 cvmx_write_csr(en_addr, mask); 1679 1680 } 1681 1682 static void octeon_irq_ciu2_disable_local(struct irq_data *data) 1683 { 1684 u64 mask; 1685 u64 en_addr; 1686 int coreid = cvmx_get_core_num(); 1687 struct octeon_ciu_chip_data *cd; 1688 1689 cd = irq_data_get_irq_chip_data(data); 1690 mask = 1ull << (cd->bit); 1691 1692 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C(coreid) + 1693 (0x1000ull * cd->line); 1694 cvmx_write_csr(en_addr, mask); 1695 1696 } 1697 1698 static void octeon_irq_ciu2_ack(struct irq_data *data) 1699 { 1700 u64 mask; 1701 u64 en_addr; 1702 int coreid = cvmx_get_core_num(); 1703 struct octeon_ciu_chip_data *cd; 1704 1705 cd = irq_data_get_irq_chip_data(data); 1706 mask = 1ull << (cd->bit); 1707 1708 en_addr = CVMX_CIU2_RAW_PPX_IP2_WRKQ(coreid) + (0x1000ull * cd->line); 1709 cvmx_write_csr(en_addr, mask); 1710 1711 } 1712 1713 static void octeon_irq_ciu2_disable_all(struct irq_data *data) 1714 { 1715 int cpu; 1716 u64 mask; 1717 struct octeon_ciu_chip_data *cd; 1718 1719 cd = irq_data_get_irq_chip_data(data); 1720 mask = 1ull << (cd->bit); 1721 1722 for_each_online_cpu(cpu) { 1723 u64 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C( 1724 octeon_coreid_for_cpu(cpu)) + (0x1000ull * cd->line); 1725 cvmx_write_csr(en_addr, mask); 1726 } 1727 } 1728 1729 static void octeon_irq_ciu2_mbox_enable_all(struct irq_data *data) 1730 { 1731 int cpu; 1732 u64 mask; 1733 1734 mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0); 1735 1736 for_each_online_cpu(cpu) { 1737 u64 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1S( 1738 octeon_coreid_for_cpu(cpu)); 1739 cvmx_write_csr(en_addr, mask); 1740 } 1741 } 1742 1743 static void octeon_irq_ciu2_mbox_disable_all(struct irq_data *data) 1744 { 1745 int cpu; 1746 u64 mask; 1747 1748 mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0); 1749 1750 for_each_online_cpu(cpu) { 1751 u64 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1C( 1752 octeon_coreid_for_cpu(cpu)); 1753 cvmx_write_csr(en_addr, mask); 1754 } 1755 } 1756 1757 static void octeon_irq_ciu2_mbox_enable_local(struct irq_data *data) 1758 { 1759 u64 mask; 1760 u64 en_addr; 1761 int coreid = cvmx_get_core_num(); 1762 1763 mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0); 1764 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1S(coreid); 1765 cvmx_write_csr(en_addr, mask); 1766 } 1767 1768 static void octeon_irq_ciu2_mbox_disable_local(struct irq_data *data) 1769 { 1770 u64 mask; 1771 u64 en_addr; 1772 int coreid = cvmx_get_core_num(); 1773 1774 mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0); 1775 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1C(coreid); 1776 cvmx_write_csr(en_addr, mask); 1777 } 1778 1779 #ifdef CONFIG_SMP 1780 static int octeon_irq_ciu2_set_affinity(struct irq_data *data, 1781 const struct cpumask *dest, bool force) 1782 { 1783 int cpu; 1784 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 1785 u64 mask; 1786 struct octeon_ciu_chip_data *cd; 1787 1788 if (!enable_one) 1789 return 0; 1790 1791 cd = irq_data_get_irq_chip_data(data); 1792 mask = 1ull << cd->bit; 1793 1794 for_each_online_cpu(cpu) { 1795 u64 en_addr; 1796 if (cpumask_test_cpu(cpu, dest) && enable_one) { 1797 enable_one = false; 1798 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S( 1799 octeon_coreid_for_cpu(cpu)) + 1800 (0x1000ull * cd->line); 1801 } else { 1802 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C( 1803 octeon_coreid_for_cpu(cpu)) + 1804 (0x1000ull * cd->line); 1805 } 1806 cvmx_write_csr(en_addr, mask); 1807 } 1808 1809 return 0; 1810 } 1811 #endif 1812 1813 static void octeon_irq_ciu2_enable_gpio(struct irq_data *data) 1814 { 1815 octeon_irq_gpio_setup(data); 1816 octeon_irq_ciu2_enable(data); 1817 } 1818 1819 static void octeon_irq_ciu2_disable_gpio(struct irq_data *data) 1820 { 1821 struct octeon_ciu_chip_data *cd; 1822 1823 cd = irq_data_get_irq_chip_data(data); 1824 1825 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0); 1826 1827 octeon_irq_ciu2_disable_all(data); 1828 } 1829 1830 static struct irq_chip octeon_irq_chip_ciu2 = { 1831 .name = "CIU2-E", 1832 .irq_enable = octeon_irq_ciu2_enable, 1833 .irq_disable = octeon_irq_ciu2_disable_all, 1834 .irq_mask = octeon_irq_ciu2_disable_local, 1835 .irq_unmask = octeon_irq_ciu2_enable, 1836 #ifdef CONFIG_SMP 1837 .irq_set_affinity = octeon_irq_ciu2_set_affinity, 1838 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1839 #endif 1840 }; 1841 1842 static struct irq_chip octeon_irq_chip_ciu2_edge = { 1843 .name = "CIU2-E", 1844 .irq_enable = octeon_irq_ciu2_enable, 1845 .irq_disable = octeon_irq_ciu2_disable_all, 1846 .irq_ack = octeon_irq_ciu2_ack, 1847 .irq_mask = octeon_irq_ciu2_disable_local, 1848 .irq_unmask = octeon_irq_ciu2_enable, 1849 #ifdef CONFIG_SMP 1850 .irq_set_affinity = octeon_irq_ciu2_set_affinity, 1851 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1852 #endif 1853 }; 1854 1855 static struct irq_chip octeon_irq_chip_ciu2_mbox = { 1856 .name = "CIU2-M", 1857 .irq_enable = octeon_irq_ciu2_mbox_enable_all, 1858 .irq_disable = octeon_irq_ciu2_mbox_disable_all, 1859 .irq_ack = octeon_irq_ciu2_mbox_disable_local, 1860 .irq_eoi = octeon_irq_ciu2_mbox_enable_local, 1861 1862 .irq_cpu_online = octeon_irq_ciu2_mbox_enable_local, 1863 .irq_cpu_offline = octeon_irq_ciu2_mbox_disable_local, 1864 .flags = IRQCHIP_ONOFFLINE_ENABLED, 1865 }; 1866 1867 static struct irq_chip octeon_irq_chip_ciu2_wd = { 1868 .name = "CIU2-W", 1869 .irq_enable = octeon_irq_ciu2_wd_enable, 1870 .irq_disable = octeon_irq_ciu2_disable_all, 1871 .irq_mask = octeon_irq_ciu2_disable_local, 1872 .irq_unmask = octeon_irq_ciu2_enable_local, 1873 }; 1874 1875 static struct irq_chip octeon_irq_chip_ciu2_gpio = { 1876 .name = "CIU-GPIO", 1877 .irq_enable = octeon_irq_ciu2_enable_gpio, 1878 .irq_disable = octeon_irq_ciu2_disable_gpio, 1879 .irq_ack = octeon_irq_ciu_gpio_ack, 1880 .irq_mask = octeon_irq_ciu2_disable_local, 1881 .irq_unmask = octeon_irq_ciu2_enable, 1882 .irq_set_type = octeon_irq_ciu_gpio_set_type, 1883 #ifdef CONFIG_SMP 1884 .irq_set_affinity = octeon_irq_ciu2_set_affinity, 1885 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1886 #endif 1887 .flags = IRQCHIP_SET_TYPE_MASKED, 1888 }; 1889 1890 static int octeon_irq_ciu2_xlat(struct irq_domain *d, 1891 struct device_node *node, 1892 const u32 *intspec, 1893 unsigned int intsize, 1894 unsigned long *out_hwirq, 1895 unsigned int *out_type) 1896 { 1897 unsigned int ciu, bit; 1898 1899 ciu = intspec[0]; 1900 bit = intspec[1]; 1901 1902 *out_hwirq = (ciu << 6) | bit; 1903 *out_type = 0; 1904 1905 return 0; 1906 } 1907 1908 static bool octeon_irq_ciu2_is_edge(unsigned int line, unsigned int bit) 1909 { 1910 bool edge = false; 1911 1912 if (line == 3) /* MIO */ 1913 switch (bit) { 1914 case 2: /* IPD_DRP */ 1915 case 8 ... 11: /* Timers */ 1916 case 48: /* PTP */ 1917 edge = true; 1918 break; 1919 default: 1920 break; 1921 } 1922 else if (line == 6) /* PKT */ 1923 switch (bit) { 1924 case 52 ... 53: /* ILK_DRP */ 1925 case 8 ... 12: /* GMX_DRP */ 1926 edge = true; 1927 break; 1928 default: 1929 break; 1930 } 1931 return edge; 1932 } 1933 1934 static int octeon_irq_ciu2_map(struct irq_domain *d, 1935 unsigned int virq, irq_hw_number_t hw) 1936 { 1937 unsigned int line = hw >> 6; 1938 unsigned int bit = hw & 63; 1939 1940 /* 1941 * Don't map irq if it is reserved for GPIO. 1942 * (Line 7 are the GPIO lines.) 1943 */ 1944 if (line == 7) 1945 return 0; 1946 1947 if (line > 7 || octeon_irq_ciu_to_irq[line][bit] != 0) 1948 return -EINVAL; 1949 1950 if (octeon_irq_ciu2_is_edge(line, bit)) 1951 octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1952 &octeon_irq_chip_ciu2_edge, 1953 handle_edge_irq); 1954 else 1955 octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1956 &octeon_irq_chip_ciu2, 1957 handle_level_irq); 1958 1959 return 0; 1960 } 1961 1962 static struct irq_domain_ops octeon_irq_domain_ciu2_ops = { 1963 .map = octeon_irq_ciu2_map, 1964 .unmap = octeon_irq_free_cd, 1965 .xlate = octeon_irq_ciu2_xlat, 1966 }; 1967 1968 static void octeon_irq_ciu2(void) 1969 { 1970 int line; 1971 int bit; 1972 int irq; 1973 u64 src_reg, src, sum; 1974 const unsigned long core_id = cvmx_get_core_num(); 1975 1976 sum = cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(core_id)) & 0xfful; 1977 1978 if (unlikely(!sum)) 1979 goto spurious; 1980 1981 line = fls64(sum) - 1; 1982 src_reg = CVMX_CIU2_SRC_PPX_IP2_WRKQ(core_id) + (0x1000 * line); 1983 src = cvmx_read_csr(src_reg); 1984 1985 if (unlikely(!src)) 1986 goto spurious; 1987 1988 bit = fls64(src) - 1; 1989 irq = octeon_irq_ciu_to_irq[line][bit]; 1990 if (unlikely(!irq)) 1991 goto spurious; 1992 1993 do_IRQ(irq); 1994 goto out; 1995 1996 spurious: 1997 spurious_interrupt(); 1998 out: 1999 /* CN68XX pass 1.x has an errata that accessing the ACK registers 2000 can stop interrupts from propagating */ 2001 if (OCTEON_IS_MODEL(OCTEON_CN68XX)) 2002 cvmx_read_csr(CVMX_CIU2_INTR_CIU_READY); 2003 else 2004 cvmx_read_csr(CVMX_CIU2_ACK_PPX_IP2(core_id)); 2005 return; 2006 } 2007 2008 static void octeon_irq_ciu2_mbox(void) 2009 { 2010 int line; 2011 2012 const unsigned long core_id = cvmx_get_core_num(); 2013 u64 sum = cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP3(core_id)) >> 60; 2014 2015 if (unlikely(!sum)) 2016 goto spurious; 2017 2018 line = fls64(sum) - 1; 2019 2020 do_IRQ(OCTEON_IRQ_MBOX0 + line); 2021 goto out; 2022 2023 spurious: 2024 spurious_interrupt(); 2025 out: 2026 /* CN68XX pass 1.x has an errata that accessing the ACK registers 2027 can stop interrupts from propagating */ 2028 if (OCTEON_IS_MODEL(OCTEON_CN68XX)) 2029 cvmx_read_csr(CVMX_CIU2_INTR_CIU_READY); 2030 else 2031 cvmx_read_csr(CVMX_CIU2_ACK_PPX_IP3(core_id)); 2032 return; 2033 } 2034 2035 static int __init octeon_irq_init_ciu2( 2036 struct device_node *ciu_node, struct device_node *parent) 2037 { 2038 unsigned int i, r; 2039 struct irq_domain *ciu_domain = NULL; 2040 2041 octeon_irq_init_ciu2_percpu(); 2042 octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu2; 2043 2044 octeon_irq_gpio_chip = &octeon_irq_chip_ciu2_gpio; 2045 octeon_irq_ip2 = octeon_irq_ciu2; 2046 octeon_irq_ip3 = octeon_irq_ciu2_mbox; 2047 octeon_irq_ip4 = octeon_irq_ip4_mask; 2048 2049 /* Mips internal */ 2050 octeon_irq_init_core(); 2051 2052 ciu_domain = irq_domain_add_tree( 2053 ciu_node, &octeon_irq_domain_ciu2_ops, NULL); 2054 irq_set_default_host(ciu_domain); 2055 2056 /* CUI2 */ 2057 for (i = 0; i < 64; i++) { 2058 r = octeon_irq_force_ciu_mapping( 2059 ciu_domain, i + OCTEON_IRQ_WORKQ0, 0, i); 2060 if (r) 2061 goto err; 2062 } 2063 2064 for (i = 0; i < 32; i++) { 2065 r = octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_WDOG0, 1, i, 0, 2066 &octeon_irq_chip_ciu2_wd, handle_level_irq); 2067 if (r) 2068 goto err; 2069 } 2070 2071 for (i = 0; i < 4; i++) { 2072 r = octeon_irq_force_ciu_mapping( 2073 ciu_domain, i + OCTEON_IRQ_TIMER0, 3, i + 8); 2074 if (r) 2075 goto err; 2076 } 2077 2078 for (i = 0; i < 4; i++) { 2079 r = octeon_irq_force_ciu_mapping( 2080 ciu_domain, i + OCTEON_IRQ_PCI_INT0, 4, i); 2081 if (r) 2082 goto err; 2083 } 2084 2085 for (i = 0; i < 4; i++) { 2086 r = octeon_irq_force_ciu_mapping( 2087 ciu_domain, i + OCTEON_IRQ_PCI_MSI0, 4, i + 8); 2088 if (r) 2089 goto err; 2090 } 2091 2092 irq_set_chip_and_handler(OCTEON_IRQ_MBOX0, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq); 2093 irq_set_chip_and_handler(OCTEON_IRQ_MBOX1, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq); 2094 irq_set_chip_and_handler(OCTEON_IRQ_MBOX2, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq); 2095 irq_set_chip_and_handler(OCTEON_IRQ_MBOX3, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq); 2096 2097 /* Enable the CIU lines */ 2098 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 2099 clear_c0_status(STATUSF_IP4); 2100 return 0; 2101 err: 2102 return r; 2103 } 2104 2105 struct octeon_irq_cib_host_data { 2106 raw_spinlock_t lock; 2107 u64 raw_reg; 2108 u64 en_reg; 2109 int max_bits; 2110 }; 2111 2112 struct octeon_irq_cib_chip_data { 2113 struct octeon_irq_cib_host_data *host_data; 2114 int bit; 2115 }; 2116 2117 static void octeon_irq_cib_enable(struct irq_data *data) 2118 { 2119 unsigned long flags; 2120 u64 en; 2121 struct octeon_irq_cib_chip_data *cd = irq_data_get_irq_chip_data(data); 2122 struct octeon_irq_cib_host_data *host_data = cd->host_data; 2123 2124 raw_spin_lock_irqsave(&host_data->lock, flags); 2125 en = cvmx_read_csr(host_data->en_reg); 2126 en |= 1ull << cd->bit; 2127 cvmx_write_csr(host_data->en_reg, en); 2128 raw_spin_unlock_irqrestore(&host_data->lock, flags); 2129 } 2130 2131 static void octeon_irq_cib_disable(struct irq_data *data) 2132 { 2133 unsigned long flags; 2134 u64 en; 2135 struct octeon_irq_cib_chip_data *cd = irq_data_get_irq_chip_data(data); 2136 struct octeon_irq_cib_host_data *host_data = cd->host_data; 2137 2138 raw_spin_lock_irqsave(&host_data->lock, flags); 2139 en = cvmx_read_csr(host_data->en_reg); 2140 en &= ~(1ull << cd->bit); 2141 cvmx_write_csr(host_data->en_reg, en); 2142 raw_spin_unlock_irqrestore(&host_data->lock, flags); 2143 } 2144 2145 static int octeon_irq_cib_set_type(struct irq_data *data, unsigned int t) 2146 { 2147 irqd_set_trigger_type(data, t); 2148 return IRQ_SET_MASK_OK; 2149 } 2150 2151 static struct irq_chip octeon_irq_chip_cib = { 2152 .name = "CIB", 2153 .irq_enable = octeon_irq_cib_enable, 2154 .irq_disable = octeon_irq_cib_disable, 2155 .irq_mask = octeon_irq_cib_disable, 2156 .irq_unmask = octeon_irq_cib_enable, 2157 .irq_set_type = octeon_irq_cib_set_type, 2158 }; 2159 2160 static int octeon_irq_cib_xlat(struct irq_domain *d, 2161 struct device_node *node, 2162 const u32 *intspec, 2163 unsigned int intsize, 2164 unsigned long *out_hwirq, 2165 unsigned int *out_type) 2166 { 2167 unsigned int type = 0; 2168 2169 if (intsize == 2) 2170 type = intspec[1]; 2171 2172 switch (type) { 2173 case 0: /* unofficial value, but we might as well let it work. */ 2174 case 4: /* official value for level triggering. */ 2175 *out_type = IRQ_TYPE_LEVEL_HIGH; 2176 break; 2177 case 1: /* official value for edge triggering. */ 2178 *out_type = IRQ_TYPE_EDGE_RISING; 2179 break; 2180 default: /* Nothing else is acceptable. */ 2181 return -EINVAL; 2182 } 2183 2184 *out_hwirq = intspec[0]; 2185 2186 return 0; 2187 } 2188 2189 static int octeon_irq_cib_map(struct irq_domain *d, 2190 unsigned int virq, irq_hw_number_t hw) 2191 { 2192 struct octeon_irq_cib_host_data *host_data = d->host_data; 2193 struct octeon_irq_cib_chip_data *cd; 2194 2195 if (hw >= host_data->max_bits) { 2196 pr_err("ERROR: %s mapping %u is to big!\n", 2197 irq_domain_get_of_node(d)->name, (unsigned)hw); 2198 return -EINVAL; 2199 } 2200 2201 cd = kzalloc(sizeof(*cd), GFP_KERNEL); 2202 cd->host_data = host_data; 2203 cd->bit = hw; 2204 2205 irq_set_chip_and_handler(virq, &octeon_irq_chip_cib, 2206 handle_simple_irq); 2207 irq_set_chip_data(virq, cd); 2208 return 0; 2209 } 2210 2211 static struct irq_domain_ops octeon_irq_domain_cib_ops = { 2212 .map = octeon_irq_cib_map, 2213 .unmap = octeon_irq_free_cd, 2214 .xlate = octeon_irq_cib_xlat, 2215 }; 2216 2217 /* Chain to real handler. */ 2218 static irqreturn_t octeon_irq_cib_handler(int my_irq, void *data) 2219 { 2220 u64 en; 2221 u64 raw; 2222 u64 bits; 2223 int i; 2224 int irq; 2225 struct irq_domain *cib_domain = data; 2226 struct octeon_irq_cib_host_data *host_data = cib_domain->host_data; 2227 2228 en = cvmx_read_csr(host_data->en_reg); 2229 raw = cvmx_read_csr(host_data->raw_reg); 2230 2231 bits = en & raw; 2232 2233 for (i = 0; i < host_data->max_bits; i++) { 2234 if ((bits & 1ull << i) == 0) 2235 continue; 2236 irq = irq_find_mapping(cib_domain, i); 2237 if (!irq) { 2238 unsigned long flags; 2239 2240 pr_err("ERROR: CIB bit %d@%llx IRQ unhandled, disabling\n", 2241 i, host_data->raw_reg); 2242 raw_spin_lock_irqsave(&host_data->lock, flags); 2243 en = cvmx_read_csr(host_data->en_reg); 2244 en &= ~(1ull << i); 2245 cvmx_write_csr(host_data->en_reg, en); 2246 cvmx_write_csr(host_data->raw_reg, 1ull << i); 2247 raw_spin_unlock_irqrestore(&host_data->lock, flags); 2248 } else { 2249 struct irq_desc *desc = irq_to_desc(irq); 2250 struct irq_data *irq_data = irq_desc_get_irq_data(desc); 2251 /* If edge, acknowledge the bit we will be sending. */ 2252 if (irqd_get_trigger_type(irq_data) & 2253 IRQ_TYPE_EDGE_BOTH) 2254 cvmx_write_csr(host_data->raw_reg, 1ull << i); 2255 generic_handle_irq_desc(desc); 2256 } 2257 } 2258 2259 return IRQ_HANDLED; 2260 } 2261 2262 static int __init octeon_irq_init_cib(struct device_node *ciu_node, 2263 struct device_node *parent) 2264 { 2265 const __be32 *addr; 2266 u32 val; 2267 struct octeon_irq_cib_host_data *host_data; 2268 int parent_irq; 2269 int r; 2270 struct irq_domain *cib_domain; 2271 2272 parent_irq = irq_of_parse_and_map(ciu_node, 0); 2273 if (!parent_irq) { 2274 pr_err("ERROR: Couldn't acquire parent_irq for %pOFn\n", 2275 ciu_node); 2276 return -EINVAL; 2277 } 2278 2279 host_data = kzalloc(sizeof(*host_data), GFP_KERNEL); 2280 if (!host_data) 2281 return -ENOMEM; 2282 raw_spin_lock_init(&host_data->lock); 2283 2284 addr = of_get_address(ciu_node, 0, NULL, NULL); 2285 if (!addr) { 2286 pr_err("ERROR: Couldn't acquire reg(0) %pOFn\n", ciu_node); 2287 return -EINVAL; 2288 } 2289 host_data->raw_reg = (u64)phys_to_virt( 2290 of_translate_address(ciu_node, addr)); 2291 2292 addr = of_get_address(ciu_node, 1, NULL, NULL); 2293 if (!addr) { 2294 pr_err("ERROR: Couldn't acquire reg(1) %pOFn\n", ciu_node); 2295 return -EINVAL; 2296 } 2297 host_data->en_reg = (u64)phys_to_virt( 2298 of_translate_address(ciu_node, addr)); 2299 2300 r = of_property_read_u32(ciu_node, "cavium,max-bits", &val); 2301 if (r) { 2302 pr_err("ERROR: Couldn't read cavium,max-bits from %pOFn\n", 2303 ciu_node); 2304 return r; 2305 } 2306 host_data->max_bits = val; 2307 2308 cib_domain = irq_domain_add_linear(ciu_node, host_data->max_bits, 2309 &octeon_irq_domain_cib_ops, 2310 host_data); 2311 if (!cib_domain) { 2312 pr_err("ERROR: Couldn't irq_domain_add_linear()\n"); 2313 return -ENOMEM; 2314 } 2315 2316 cvmx_write_csr(host_data->en_reg, 0); /* disable all IRQs */ 2317 cvmx_write_csr(host_data->raw_reg, ~0); /* ack any outstanding */ 2318 2319 r = request_irq(parent_irq, octeon_irq_cib_handler, 2320 IRQF_NO_THREAD, "cib", cib_domain); 2321 if (r) { 2322 pr_err("request_irq cib failed %d\n", r); 2323 return r; 2324 } 2325 pr_info("CIB interrupt controller probed: %llx %d\n", 2326 host_data->raw_reg, host_data->max_bits); 2327 return 0; 2328 } 2329 2330 int octeon_irq_ciu3_xlat(struct irq_domain *d, 2331 struct device_node *node, 2332 const u32 *intspec, 2333 unsigned int intsize, 2334 unsigned long *out_hwirq, 2335 unsigned int *out_type) 2336 { 2337 struct octeon_ciu3_info *ciu3_info = d->host_data; 2338 unsigned int hwirq, type, intsn_major; 2339 union cvmx_ciu3_iscx_ctl isc; 2340 2341 if (intsize < 2) 2342 return -EINVAL; 2343 hwirq = intspec[0]; 2344 type = intspec[1]; 2345 2346 if (hwirq >= (1 << 20)) 2347 return -EINVAL; 2348 2349 intsn_major = hwirq >> 12; 2350 switch (intsn_major) { 2351 case 0x04: /* Software handled separately. */ 2352 return -EINVAL; 2353 default: 2354 break; 2355 } 2356 2357 isc.u64 = cvmx_read_csr(ciu3_info->ciu3_addr + CIU3_ISC_CTL(hwirq)); 2358 if (!isc.s.imp) 2359 return -EINVAL; 2360 2361 switch (type) { 2362 case 4: /* official value for level triggering. */ 2363 *out_type = IRQ_TYPE_LEVEL_HIGH; 2364 break; 2365 case 0: /* unofficial value, but we might as well let it work. */ 2366 case 1: /* official value for edge triggering. */ 2367 *out_type = IRQ_TYPE_EDGE_RISING; 2368 break; 2369 default: /* Nothing else is acceptable. */ 2370 return -EINVAL; 2371 } 2372 2373 *out_hwirq = hwirq; 2374 2375 return 0; 2376 } 2377 2378 void octeon_irq_ciu3_enable(struct irq_data *data) 2379 { 2380 int cpu; 2381 union cvmx_ciu3_iscx_ctl isc_ctl; 2382 union cvmx_ciu3_iscx_w1c isc_w1c; 2383 u64 isc_ctl_addr; 2384 2385 struct octeon_ciu_chip_data *cd; 2386 2387 cpu = next_cpu_for_irq(data); 2388 2389 cd = irq_data_get_irq_chip_data(data); 2390 2391 isc_w1c.u64 = 0; 2392 isc_w1c.s.en = 1; 2393 cvmx_write_csr(cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn), isc_w1c.u64); 2394 2395 isc_ctl_addr = cd->ciu3_addr + CIU3_ISC_CTL(cd->intsn); 2396 isc_ctl.u64 = 0; 2397 isc_ctl.s.en = 1; 2398 isc_ctl.s.idt = per_cpu(octeon_irq_ciu3_idt_ip2, cpu); 2399 cvmx_write_csr(isc_ctl_addr, isc_ctl.u64); 2400 cvmx_read_csr(isc_ctl_addr); 2401 } 2402 2403 void octeon_irq_ciu3_disable(struct irq_data *data) 2404 { 2405 u64 isc_ctl_addr; 2406 union cvmx_ciu3_iscx_w1c isc_w1c; 2407 2408 struct octeon_ciu_chip_data *cd; 2409 2410 cd = irq_data_get_irq_chip_data(data); 2411 2412 isc_w1c.u64 = 0; 2413 isc_w1c.s.en = 1; 2414 2415 isc_ctl_addr = cd->ciu3_addr + CIU3_ISC_CTL(cd->intsn); 2416 cvmx_write_csr(cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn), isc_w1c.u64); 2417 cvmx_write_csr(isc_ctl_addr, 0); 2418 cvmx_read_csr(isc_ctl_addr); 2419 } 2420 2421 void octeon_irq_ciu3_ack(struct irq_data *data) 2422 { 2423 u64 isc_w1c_addr; 2424 union cvmx_ciu3_iscx_w1c isc_w1c; 2425 struct octeon_ciu_chip_data *cd; 2426 u32 trigger_type = irqd_get_trigger_type(data); 2427 2428 /* 2429 * We use a single irq_chip, so we have to do nothing to ack a 2430 * level interrupt. 2431 */ 2432 if (!(trigger_type & IRQ_TYPE_EDGE_BOTH)) 2433 return; 2434 2435 cd = irq_data_get_irq_chip_data(data); 2436 2437 isc_w1c.u64 = 0; 2438 isc_w1c.s.raw = 1; 2439 2440 isc_w1c_addr = cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn); 2441 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2442 cvmx_read_csr(isc_w1c_addr); 2443 } 2444 2445 void octeon_irq_ciu3_mask(struct irq_data *data) 2446 { 2447 union cvmx_ciu3_iscx_w1c isc_w1c; 2448 u64 isc_w1c_addr; 2449 struct octeon_ciu_chip_data *cd; 2450 2451 cd = irq_data_get_irq_chip_data(data); 2452 2453 isc_w1c.u64 = 0; 2454 isc_w1c.s.en = 1; 2455 2456 isc_w1c_addr = cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn); 2457 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2458 cvmx_read_csr(isc_w1c_addr); 2459 } 2460 2461 void octeon_irq_ciu3_mask_ack(struct irq_data *data) 2462 { 2463 union cvmx_ciu3_iscx_w1c isc_w1c; 2464 u64 isc_w1c_addr; 2465 struct octeon_ciu_chip_data *cd; 2466 u32 trigger_type = irqd_get_trigger_type(data); 2467 2468 cd = irq_data_get_irq_chip_data(data); 2469 2470 isc_w1c.u64 = 0; 2471 isc_w1c.s.en = 1; 2472 2473 /* 2474 * We use a single irq_chip, so only ack an edge (!level) 2475 * interrupt. 2476 */ 2477 if (trigger_type & IRQ_TYPE_EDGE_BOTH) 2478 isc_w1c.s.raw = 1; 2479 2480 isc_w1c_addr = cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn); 2481 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2482 cvmx_read_csr(isc_w1c_addr); 2483 } 2484 2485 #ifdef CONFIG_SMP 2486 static int octeon_irq_ciu3_set_affinity(struct irq_data *data, 2487 const struct cpumask *dest, bool force) 2488 { 2489 union cvmx_ciu3_iscx_ctl isc_ctl; 2490 union cvmx_ciu3_iscx_w1c isc_w1c; 2491 u64 isc_ctl_addr; 2492 int cpu; 2493 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 2494 struct octeon_ciu_chip_data *cd = irq_data_get_irq_chip_data(data); 2495 2496 if (!cpumask_subset(dest, cpumask_of_node(cd->ciu_node))) 2497 return -EINVAL; 2498 2499 if (!enable_one) 2500 return IRQ_SET_MASK_OK; 2501 2502 cd = irq_data_get_irq_chip_data(data); 2503 cpu = cpumask_first(dest); 2504 if (cpu >= nr_cpu_ids) 2505 cpu = smp_processor_id(); 2506 cd->current_cpu = cpu; 2507 2508 isc_w1c.u64 = 0; 2509 isc_w1c.s.en = 1; 2510 cvmx_write_csr(cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn), isc_w1c.u64); 2511 2512 isc_ctl_addr = cd->ciu3_addr + CIU3_ISC_CTL(cd->intsn); 2513 isc_ctl.u64 = 0; 2514 isc_ctl.s.en = 1; 2515 isc_ctl.s.idt = per_cpu(octeon_irq_ciu3_idt_ip2, cpu); 2516 cvmx_write_csr(isc_ctl_addr, isc_ctl.u64); 2517 cvmx_read_csr(isc_ctl_addr); 2518 2519 return IRQ_SET_MASK_OK; 2520 } 2521 #endif 2522 2523 static struct irq_chip octeon_irq_chip_ciu3 = { 2524 .name = "CIU3", 2525 .irq_startup = edge_startup, 2526 .irq_enable = octeon_irq_ciu3_enable, 2527 .irq_disable = octeon_irq_ciu3_disable, 2528 .irq_ack = octeon_irq_ciu3_ack, 2529 .irq_mask = octeon_irq_ciu3_mask, 2530 .irq_mask_ack = octeon_irq_ciu3_mask_ack, 2531 .irq_unmask = octeon_irq_ciu3_enable, 2532 .irq_set_type = octeon_irq_ciu_set_type, 2533 #ifdef CONFIG_SMP 2534 .irq_set_affinity = octeon_irq_ciu3_set_affinity, 2535 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 2536 #endif 2537 }; 2538 2539 int octeon_irq_ciu3_mapx(struct irq_domain *d, unsigned int virq, 2540 irq_hw_number_t hw, struct irq_chip *chip) 2541 { 2542 struct octeon_ciu3_info *ciu3_info = d->host_data; 2543 struct octeon_ciu_chip_data *cd = kzalloc_node(sizeof(*cd), GFP_KERNEL, 2544 ciu3_info->node); 2545 if (!cd) 2546 return -ENOMEM; 2547 cd->intsn = hw; 2548 cd->current_cpu = -1; 2549 cd->ciu3_addr = ciu3_info->ciu3_addr; 2550 cd->ciu_node = ciu3_info->node; 2551 irq_set_chip_and_handler(virq, chip, handle_edge_irq); 2552 irq_set_chip_data(virq, cd); 2553 2554 return 0; 2555 } 2556 2557 static int octeon_irq_ciu3_map(struct irq_domain *d, 2558 unsigned int virq, irq_hw_number_t hw) 2559 { 2560 return octeon_irq_ciu3_mapx(d, virq, hw, &octeon_irq_chip_ciu3); 2561 } 2562 2563 static struct irq_domain_ops octeon_dflt_domain_ciu3_ops = { 2564 .map = octeon_irq_ciu3_map, 2565 .unmap = octeon_irq_free_cd, 2566 .xlate = octeon_irq_ciu3_xlat, 2567 }; 2568 2569 static void octeon_irq_ciu3_ip2(void) 2570 { 2571 union cvmx_ciu3_destx_pp_int dest_pp_int; 2572 struct octeon_ciu3_info *ciu3_info; 2573 u64 ciu3_addr; 2574 2575 ciu3_info = __this_cpu_read(octeon_ciu3_info); 2576 ciu3_addr = ciu3_info->ciu3_addr; 2577 2578 dest_pp_int.u64 = cvmx_read_csr(ciu3_addr + CIU3_DEST_PP_INT(3 * cvmx_get_local_core_num())); 2579 2580 if (likely(dest_pp_int.s.intr)) { 2581 irq_hw_number_t intsn = dest_pp_int.s.intsn; 2582 irq_hw_number_t hw; 2583 struct irq_domain *domain; 2584 /* Get the domain to use from the major block */ 2585 int block = intsn >> 12; 2586 int ret; 2587 2588 domain = ciu3_info->domain[block]; 2589 if (ciu3_info->intsn2hw[block]) 2590 hw = ciu3_info->intsn2hw[block](domain, intsn); 2591 else 2592 hw = intsn; 2593 2594 ret = handle_domain_irq(domain, hw, NULL); 2595 if (ret < 0) { 2596 union cvmx_ciu3_iscx_w1c isc_w1c; 2597 u64 isc_w1c_addr = ciu3_addr + CIU3_ISC_W1C(intsn); 2598 2599 isc_w1c.u64 = 0; 2600 isc_w1c.s.en = 1; 2601 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2602 cvmx_read_csr(isc_w1c_addr); 2603 spurious_interrupt(); 2604 } 2605 } else { 2606 spurious_interrupt(); 2607 } 2608 } 2609 2610 /* 2611 * 10 mbox per core starting from zero. 2612 * Base mbox is core * 10 2613 */ 2614 static unsigned int octeon_irq_ciu3_base_mbox_intsn(int core) 2615 { 2616 /* SW (mbox) are 0x04 in bits 12..19 */ 2617 return 0x04000 + CIU3_MBOX_PER_CORE * core; 2618 } 2619 2620 static unsigned int octeon_irq_ciu3_mbox_intsn_for_core(int core, unsigned int mbox) 2621 { 2622 return octeon_irq_ciu3_base_mbox_intsn(core) + mbox; 2623 } 2624 2625 static unsigned int octeon_irq_ciu3_mbox_intsn_for_cpu(int cpu, unsigned int mbox) 2626 { 2627 int local_core = octeon_coreid_for_cpu(cpu) & 0x3f; 2628 2629 return octeon_irq_ciu3_mbox_intsn_for_core(local_core, mbox); 2630 } 2631 2632 static void octeon_irq_ciu3_mbox(void) 2633 { 2634 union cvmx_ciu3_destx_pp_int dest_pp_int; 2635 struct octeon_ciu3_info *ciu3_info; 2636 u64 ciu3_addr; 2637 int core = cvmx_get_local_core_num(); 2638 2639 ciu3_info = __this_cpu_read(octeon_ciu3_info); 2640 ciu3_addr = ciu3_info->ciu3_addr; 2641 2642 dest_pp_int.u64 = cvmx_read_csr(ciu3_addr + CIU3_DEST_PP_INT(1 + 3 * core)); 2643 2644 if (likely(dest_pp_int.s.intr)) { 2645 irq_hw_number_t intsn = dest_pp_int.s.intsn; 2646 int mbox = intsn - octeon_irq_ciu3_base_mbox_intsn(core); 2647 2648 if (likely(mbox >= 0 && mbox < CIU3_MBOX_PER_CORE)) { 2649 do_IRQ(mbox + OCTEON_IRQ_MBOX0); 2650 } else { 2651 union cvmx_ciu3_iscx_w1c isc_w1c; 2652 u64 isc_w1c_addr = ciu3_addr + CIU3_ISC_W1C(intsn); 2653 2654 isc_w1c.u64 = 0; 2655 isc_w1c.s.en = 1; 2656 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2657 cvmx_read_csr(isc_w1c_addr); 2658 spurious_interrupt(); 2659 } 2660 } else { 2661 spurious_interrupt(); 2662 } 2663 } 2664 2665 void octeon_ciu3_mbox_send(int cpu, unsigned int mbox) 2666 { 2667 struct octeon_ciu3_info *ciu3_info; 2668 unsigned int intsn; 2669 union cvmx_ciu3_iscx_w1s isc_w1s; 2670 u64 isc_w1s_addr; 2671 2672 if (WARN_ON_ONCE(mbox >= CIU3_MBOX_PER_CORE)) 2673 return; 2674 2675 intsn = octeon_irq_ciu3_mbox_intsn_for_cpu(cpu, mbox); 2676 ciu3_info = per_cpu(octeon_ciu3_info, cpu); 2677 isc_w1s_addr = ciu3_info->ciu3_addr + CIU3_ISC_W1S(intsn); 2678 2679 isc_w1s.u64 = 0; 2680 isc_w1s.s.raw = 1; 2681 2682 cvmx_write_csr(isc_w1s_addr, isc_w1s.u64); 2683 cvmx_read_csr(isc_w1s_addr); 2684 } 2685 2686 static void octeon_irq_ciu3_mbox_set_enable(struct irq_data *data, int cpu, bool en) 2687 { 2688 struct octeon_ciu3_info *ciu3_info; 2689 unsigned int intsn; 2690 u64 isc_ctl_addr, isc_w1c_addr; 2691 union cvmx_ciu3_iscx_ctl isc_ctl; 2692 unsigned int mbox = data->irq - OCTEON_IRQ_MBOX0; 2693 2694 intsn = octeon_irq_ciu3_mbox_intsn_for_cpu(cpu, mbox); 2695 ciu3_info = per_cpu(octeon_ciu3_info, cpu); 2696 isc_w1c_addr = ciu3_info->ciu3_addr + CIU3_ISC_W1C(intsn); 2697 isc_ctl_addr = ciu3_info->ciu3_addr + CIU3_ISC_CTL(intsn); 2698 2699 isc_ctl.u64 = 0; 2700 isc_ctl.s.en = 1; 2701 2702 cvmx_write_csr(isc_w1c_addr, isc_ctl.u64); 2703 cvmx_write_csr(isc_ctl_addr, 0); 2704 if (en) { 2705 unsigned int idt = per_cpu(octeon_irq_ciu3_idt_ip3, cpu); 2706 2707 isc_ctl.u64 = 0; 2708 isc_ctl.s.en = 1; 2709 isc_ctl.s.idt = idt; 2710 cvmx_write_csr(isc_ctl_addr, isc_ctl.u64); 2711 } 2712 cvmx_read_csr(isc_ctl_addr); 2713 } 2714 2715 static void octeon_irq_ciu3_mbox_enable(struct irq_data *data) 2716 { 2717 int cpu; 2718 unsigned int mbox = data->irq - OCTEON_IRQ_MBOX0; 2719 2720 WARN_ON(mbox >= CIU3_MBOX_PER_CORE); 2721 2722 for_each_online_cpu(cpu) 2723 octeon_irq_ciu3_mbox_set_enable(data, cpu, true); 2724 } 2725 2726 static void octeon_irq_ciu3_mbox_disable(struct irq_data *data) 2727 { 2728 int cpu; 2729 unsigned int mbox = data->irq - OCTEON_IRQ_MBOX0; 2730 2731 WARN_ON(mbox >= CIU3_MBOX_PER_CORE); 2732 2733 for_each_online_cpu(cpu) 2734 octeon_irq_ciu3_mbox_set_enable(data, cpu, false); 2735 } 2736 2737 static void octeon_irq_ciu3_mbox_ack(struct irq_data *data) 2738 { 2739 struct octeon_ciu3_info *ciu3_info; 2740 unsigned int intsn; 2741 u64 isc_w1c_addr; 2742 union cvmx_ciu3_iscx_w1c isc_w1c; 2743 unsigned int mbox = data->irq - OCTEON_IRQ_MBOX0; 2744 2745 intsn = octeon_irq_ciu3_mbox_intsn_for_core(cvmx_get_local_core_num(), mbox); 2746 2747 isc_w1c.u64 = 0; 2748 isc_w1c.s.raw = 1; 2749 2750 ciu3_info = __this_cpu_read(octeon_ciu3_info); 2751 isc_w1c_addr = ciu3_info->ciu3_addr + CIU3_ISC_W1C(intsn); 2752 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2753 cvmx_read_csr(isc_w1c_addr); 2754 } 2755 2756 static void octeon_irq_ciu3_mbox_cpu_online(struct irq_data *data) 2757 { 2758 octeon_irq_ciu3_mbox_set_enable(data, smp_processor_id(), true); 2759 } 2760 2761 static void octeon_irq_ciu3_mbox_cpu_offline(struct irq_data *data) 2762 { 2763 octeon_irq_ciu3_mbox_set_enable(data, smp_processor_id(), false); 2764 } 2765 2766 static int octeon_irq_ciu3_alloc_resources(struct octeon_ciu3_info *ciu3_info) 2767 { 2768 u64 b = ciu3_info->ciu3_addr; 2769 int idt_ip2, idt_ip3, idt_ip4; 2770 int unused_idt2; 2771 int core = cvmx_get_local_core_num(); 2772 int i; 2773 2774 __this_cpu_write(octeon_ciu3_info, ciu3_info); 2775 2776 /* 2777 * 4 idt per core starting from 1 because zero is reserved. 2778 * Base idt per core is 4 * core + 1 2779 */ 2780 idt_ip2 = core * 4 + 1; 2781 idt_ip3 = core * 4 + 2; 2782 idt_ip4 = core * 4 + 3; 2783 unused_idt2 = core * 4 + 4; 2784 __this_cpu_write(octeon_irq_ciu3_idt_ip2, idt_ip2); 2785 __this_cpu_write(octeon_irq_ciu3_idt_ip3, idt_ip3); 2786 2787 /* ip2 interrupts for this CPU */ 2788 cvmx_write_csr(b + CIU3_IDT_CTL(idt_ip2), 0); 2789 cvmx_write_csr(b + CIU3_IDT_PP(idt_ip2, 0), 1ull << core); 2790 cvmx_write_csr(b + CIU3_IDT_IO(idt_ip2), 0); 2791 2792 /* ip3 interrupts for this CPU */ 2793 cvmx_write_csr(b + CIU3_IDT_CTL(idt_ip3), 1); 2794 cvmx_write_csr(b + CIU3_IDT_PP(idt_ip3, 0), 1ull << core); 2795 cvmx_write_csr(b + CIU3_IDT_IO(idt_ip3), 0); 2796 2797 /* ip4 interrupts for this CPU */ 2798 cvmx_write_csr(b + CIU3_IDT_CTL(idt_ip4), 2); 2799 cvmx_write_csr(b + CIU3_IDT_PP(idt_ip4, 0), 0); 2800 cvmx_write_csr(b + CIU3_IDT_IO(idt_ip4), 0); 2801 2802 cvmx_write_csr(b + CIU3_IDT_CTL(unused_idt2), 0); 2803 cvmx_write_csr(b + CIU3_IDT_PP(unused_idt2, 0), 0); 2804 cvmx_write_csr(b + CIU3_IDT_IO(unused_idt2), 0); 2805 2806 for (i = 0; i < CIU3_MBOX_PER_CORE; i++) { 2807 unsigned int intsn = octeon_irq_ciu3_mbox_intsn_for_core(core, i); 2808 2809 cvmx_write_csr(b + CIU3_ISC_W1C(intsn), 2); 2810 cvmx_write_csr(b + CIU3_ISC_CTL(intsn), 0); 2811 } 2812 2813 return 0; 2814 } 2815 2816 static void octeon_irq_setup_secondary_ciu3(void) 2817 { 2818 struct octeon_ciu3_info *ciu3_info; 2819 2820 ciu3_info = octeon_ciu3_info_per_node[cvmx_get_node_num()]; 2821 octeon_irq_ciu3_alloc_resources(ciu3_info); 2822 irq_cpu_online(); 2823 2824 /* Enable the CIU lines */ 2825 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 2826 if (octeon_irq_use_ip4) 2827 set_c0_status(STATUSF_IP4); 2828 else 2829 clear_c0_status(STATUSF_IP4); 2830 } 2831 2832 static struct irq_chip octeon_irq_chip_ciu3_mbox = { 2833 .name = "CIU3-M", 2834 .irq_enable = octeon_irq_ciu3_mbox_enable, 2835 .irq_disable = octeon_irq_ciu3_mbox_disable, 2836 .irq_ack = octeon_irq_ciu3_mbox_ack, 2837 2838 .irq_cpu_online = octeon_irq_ciu3_mbox_cpu_online, 2839 .irq_cpu_offline = octeon_irq_ciu3_mbox_cpu_offline, 2840 .flags = IRQCHIP_ONOFFLINE_ENABLED, 2841 }; 2842 2843 static int __init octeon_irq_init_ciu3(struct device_node *ciu_node, 2844 struct device_node *parent) 2845 { 2846 int i; 2847 int node; 2848 struct irq_domain *domain; 2849 struct octeon_ciu3_info *ciu3_info; 2850 const __be32 *zero_addr; 2851 u64 base_addr; 2852 union cvmx_ciu3_const consts; 2853 2854 node = 0; /* of_node_to_nid(ciu_node); */ 2855 ciu3_info = kzalloc_node(sizeof(*ciu3_info), GFP_KERNEL, node); 2856 2857 if (!ciu3_info) 2858 return -ENOMEM; 2859 2860 zero_addr = of_get_address(ciu_node, 0, NULL, NULL); 2861 if (WARN_ON(!zero_addr)) 2862 return -EINVAL; 2863 2864 base_addr = of_translate_address(ciu_node, zero_addr); 2865 base_addr = (u64)phys_to_virt(base_addr); 2866 2867 ciu3_info->ciu3_addr = base_addr; 2868 ciu3_info->node = node; 2869 2870 consts.u64 = cvmx_read_csr(base_addr + CIU3_CONST); 2871 2872 octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu3; 2873 2874 octeon_irq_ip2 = octeon_irq_ciu3_ip2; 2875 octeon_irq_ip3 = octeon_irq_ciu3_mbox; 2876 octeon_irq_ip4 = octeon_irq_ip4_mask; 2877 2878 if (node == cvmx_get_node_num()) { 2879 /* Mips internal */ 2880 octeon_irq_init_core(); 2881 2882 /* Only do per CPU things if it is the CIU of the boot node. */ 2883 i = irq_alloc_descs_from(OCTEON_IRQ_MBOX0, 8, node); 2884 WARN_ON(i < 0); 2885 2886 for (i = 0; i < 8; i++) 2887 irq_set_chip_and_handler(i + OCTEON_IRQ_MBOX0, 2888 &octeon_irq_chip_ciu3_mbox, handle_percpu_irq); 2889 } 2890 2891 /* 2892 * Initialize all domains to use the default domain. Specific major 2893 * blocks will overwrite the default domain as needed. 2894 */ 2895 domain = irq_domain_add_tree(ciu_node, &octeon_dflt_domain_ciu3_ops, 2896 ciu3_info); 2897 for (i = 0; i < MAX_CIU3_DOMAINS; i++) 2898 ciu3_info->domain[i] = domain; 2899 2900 octeon_ciu3_info_per_node[node] = ciu3_info; 2901 2902 if (node == cvmx_get_node_num()) { 2903 /* Only do per CPU things if it is the CIU of the boot node. */ 2904 octeon_irq_ciu3_alloc_resources(ciu3_info); 2905 if (node == 0) 2906 irq_set_default_host(domain); 2907 2908 octeon_irq_use_ip4 = false; 2909 /* Enable the CIU lines */ 2910 set_c0_status(STATUSF_IP2 | STATUSF_IP3); 2911 clear_c0_status(STATUSF_IP4); 2912 } 2913 2914 return 0; 2915 } 2916 2917 static struct of_device_id ciu_types[] __initdata = { 2918 {.compatible = "cavium,octeon-3860-ciu", .data = octeon_irq_init_ciu}, 2919 {.compatible = "cavium,octeon-3860-gpio", .data = octeon_irq_init_gpio}, 2920 {.compatible = "cavium,octeon-6880-ciu2", .data = octeon_irq_init_ciu2}, 2921 {.compatible = "cavium,octeon-7890-ciu3", .data = octeon_irq_init_ciu3}, 2922 {.compatible = "cavium,octeon-7130-cib", .data = octeon_irq_init_cib}, 2923 {} 2924 }; 2925 2926 void __init arch_init_irq(void) 2927 { 2928 #ifdef CONFIG_SMP 2929 /* Set the default affinity to the boot cpu. */ 2930 cpumask_clear(irq_default_affinity); 2931 cpumask_set_cpu(smp_processor_id(), irq_default_affinity); 2932 #endif 2933 of_irq_init(ciu_types); 2934 } 2935 2936 asmlinkage void plat_irq_dispatch(void) 2937 { 2938 unsigned long cop0_cause; 2939 unsigned long cop0_status; 2940 2941 while (1) { 2942 cop0_cause = read_c0_cause(); 2943 cop0_status = read_c0_status(); 2944 cop0_cause &= cop0_status; 2945 cop0_cause &= ST0_IM; 2946 2947 if (cop0_cause & STATUSF_IP2) 2948 octeon_irq_ip2(); 2949 else if (cop0_cause & STATUSF_IP3) 2950 octeon_irq_ip3(); 2951 else if (cop0_cause & STATUSF_IP4) 2952 octeon_irq_ip4(); 2953 else if (cop0_cause) 2954 do_IRQ(fls(cop0_cause) - 9 + MIPS_CPU_IRQ_BASE); 2955 else 2956 break; 2957 } 2958 } 2959 2960 #ifdef CONFIG_HOTPLUG_CPU 2961 2962 void octeon_fixup_irqs(void) 2963 { 2964 irq_cpu_offline(); 2965 } 2966 2967 #endif /* CONFIG_HOTPLUG_CPU */ 2968 2969 struct irq_domain *octeon_irq_get_block_domain(int node, uint8_t block) 2970 { 2971 struct octeon_ciu3_info *ciu3_info; 2972 2973 ciu3_info = octeon_ciu3_info_per_node[node & CVMX_NODE_MASK]; 2974 return ciu3_info->domain[block]; 2975 } 2976 EXPORT_SYMBOL(octeon_irq_get_block_domain); 2977