1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 2004-2016 Cavium, Inc. 7 */ 8 9 #include <linux/of_address.h> 10 #include <linux/interrupt.h> 11 #include <linux/irqdomain.h> 12 #include <linux/bitops.h> 13 #include <linux/of_irq.h> 14 #include <linux/percpu.h> 15 #include <linux/slab.h> 16 #include <linux/irq.h> 17 #include <linux/smp.h> 18 #include <linux/of.h> 19 20 #include <asm/octeon/octeon.h> 21 #include <asm/octeon/cvmx-ciu2-defs.h> 22 #include <asm/octeon/cvmx-ciu3-defs.h> 23 24 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu0_en_mirror); 25 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu1_en_mirror); 26 static DEFINE_PER_CPU(raw_spinlock_t, octeon_irq_ciu_spinlock); 27 static DEFINE_PER_CPU(unsigned int, octeon_irq_ciu3_idt_ip2); 28 29 static DEFINE_PER_CPU(unsigned int, octeon_irq_ciu3_idt_ip3); 30 static DEFINE_PER_CPU(struct octeon_ciu3_info *, octeon_ciu3_info); 31 #define CIU3_MBOX_PER_CORE 10 32 33 /* 34 * The 8 most significant bits of the intsn identify the interrupt major block. 35 * Each major block might use its own interrupt domain. Thus 256 domains are 36 * needed. 37 */ 38 #define MAX_CIU3_DOMAINS 256 39 40 typedef irq_hw_number_t (*octeon_ciu3_intsn2hw_t)(struct irq_domain *, unsigned int); 41 42 /* Information for each ciu3 in the system */ 43 struct octeon_ciu3_info { 44 u64 ciu3_addr; 45 int node; 46 struct irq_domain *domain[MAX_CIU3_DOMAINS]; 47 octeon_ciu3_intsn2hw_t intsn2hw[MAX_CIU3_DOMAINS]; 48 }; 49 50 /* Each ciu3 in the system uses its own data (one ciu3 per node) */ 51 static struct octeon_ciu3_info *octeon_ciu3_info_per_node[4]; 52 53 struct octeon_irq_ciu_domain_data { 54 int num_sum; /* number of sum registers (2 or 3). */ 55 }; 56 57 /* Register offsets from ciu3_addr */ 58 #define CIU3_CONST 0x220 59 #define CIU3_IDT_CTL(_idt) ((_idt) * 8 + 0x110000) 60 #define CIU3_IDT_PP(_idt, _idx) ((_idt) * 32 + (_idx) * 8 + 0x120000) 61 #define CIU3_IDT_IO(_idt) ((_idt) * 8 + 0x130000) 62 #define CIU3_DEST_PP_INT(_pp_ip) ((_pp_ip) * 8 + 0x200000) 63 #define CIU3_DEST_IO_INT(_io) ((_io) * 8 + 0x210000) 64 #define CIU3_ISC_CTL(_intsn) ((_intsn) * 8 + 0x80000000) 65 #define CIU3_ISC_W1C(_intsn) ((_intsn) * 8 + 0x90000000) 66 #define CIU3_ISC_W1S(_intsn) ((_intsn) * 8 + 0xa0000000) 67 68 static __read_mostly int octeon_irq_ciu_to_irq[8][64]; 69 70 struct octeon_ciu_chip_data { 71 union { 72 struct { /* only used for ciu3 */ 73 u64 ciu3_addr; 74 unsigned int intsn; 75 }; 76 struct { /* only used for ciu/ciu2 */ 77 u8 line; 78 u8 bit; 79 }; 80 }; 81 int gpio_line; 82 int current_cpu; /* Next CPU expected to take this irq */ 83 int ciu_node; /* NUMA node number of the CIU */ 84 }; 85 86 struct octeon_core_chip_data { 87 struct mutex core_irq_mutex; 88 bool current_en; 89 bool desired_en; 90 u8 bit; 91 }; 92 93 #define MIPS_CORE_IRQ_LINES 8 94 95 static struct octeon_core_chip_data octeon_irq_core_chip_data[MIPS_CORE_IRQ_LINES]; 96 97 static int octeon_irq_set_ciu_mapping(int irq, int line, int bit, int gpio_line, 98 struct irq_chip *chip, 99 irq_flow_handler_t handler) 100 { 101 struct octeon_ciu_chip_data *cd; 102 103 cd = kzalloc(sizeof(*cd), GFP_KERNEL); 104 if (!cd) 105 return -ENOMEM; 106 107 irq_set_chip_and_handler(irq, chip, handler); 108 109 cd->line = line; 110 cd->bit = bit; 111 cd->gpio_line = gpio_line; 112 113 irq_set_chip_data(irq, cd); 114 octeon_irq_ciu_to_irq[line][bit] = irq; 115 return 0; 116 } 117 118 static void octeon_irq_free_cd(struct irq_domain *d, unsigned int irq) 119 { 120 struct irq_data *data = irq_get_irq_data(irq); 121 struct octeon_ciu_chip_data *cd = irq_data_get_irq_chip_data(data); 122 123 irq_set_chip_data(irq, NULL); 124 kfree(cd); 125 } 126 127 static int octeon_irq_force_ciu_mapping(struct irq_domain *domain, 128 int irq, int line, int bit) 129 { 130 return irq_domain_associate(domain, irq, line << 6 | bit); 131 } 132 133 static int octeon_coreid_for_cpu(int cpu) 134 { 135 #ifdef CONFIG_SMP 136 return cpu_logical_map(cpu); 137 #else 138 return cvmx_get_core_num(); 139 #endif 140 } 141 142 static int octeon_cpu_for_coreid(int coreid) 143 { 144 #ifdef CONFIG_SMP 145 return cpu_number_map(coreid); 146 #else 147 return smp_processor_id(); 148 #endif 149 } 150 151 static void octeon_irq_core_ack(struct irq_data *data) 152 { 153 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 154 unsigned int bit = cd->bit; 155 156 /* 157 * We don't need to disable IRQs to make these atomic since 158 * they are already disabled earlier in the low level 159 * interrupt code. 160 */ 161 clear_c0_status(0x100 << bit); 162 /* The two user interrupts must be cleared manually. */ 163 if (bit < 2) 164 clear_c0_cause(0x100 << bit); 165 } 166 167 static void octeon_irq_core_eoi(struct irq_data *data) 168 { 169 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 170 171 /* 172 * We don't need to disable IRQs to make these atomic since 173 * they are already disabled earlier in the low level 174 * interrupt code. 175 */ 176 set_c0_status(0x100 << cd->bit); 177 } 178 179 static void octeon_irq_core_set_enable_local(void *arg) 180 { 181 struct irq_data *data = arg; 182 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 183 unsigned int mask = 0x100 << cd->bit; 184 185 /* 186 * Interrupts are already disabled, so these are atomic. 187 */ 188 if (cd->desired_en) 189 set_c0_status(mask); 190 else 191 clear_c0_status(mask); 192 193 } 194 195 static void octeon_irq_core_disable(struct irq_data *data) 196 { 197 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 198 cd->desired_en = false; 199 } 200 201 static void octeon_irq_core_enable(struct irq_data *data) 202 { 203 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 204 cd->desired_en = true; 205 } 206 207 static void octeon_irq_core_bus_lock(struct irq_data *data) 208 { 209 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 210 211 mutex_lock(&cd->core_irq_mutex); 212 } 213 214 static void octeon_irq_core_bus_sync_unlock(struct irq_data *data) 215 { 216 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 217 218 if (cd->desired_en != cd->current_en) { 219 on_each_cpu(octeon_irq_core_set_enable_local, data, 1); 220 221 cd->current_en = cd->desired_en; 222 } 223 224 mutex_unlock(&cd->core_irq_mutex); 225 } 226 227 static struct irq_chip octeon_irq_chip_core = { 228 .name = "Core", 229 .irq_enable = octeon_irq_core_enable, 230 .irq_disable = octeon_irq_core_disable, 231 .irq_ack = octeon_irq_core_ack, 232 .irq_eoi = octeon_irq_core_eoi, 233 .irq_bus_lock = octeon_irq_core_bus_lock, 234 .irq_bus_sync_unlock = octeon_irq_core_bus_sync_unlock, 235 236 .irq_cpu_online = octeon_irq_core_eoi, 237 .irq_cpu_offline = octeon_irq_core_ack, 238 .flags = IRQCHIP_ONOFFLINE_ENABLED, 239 }; 240 241 static void __init octeon_irq_init_core(void) 242 { 243 int i; 244 int irq; 245 struct octeon_core_chip_data *cd; 246 247 for (i = 0; i < MIPS_CORE_IRQ_LINES; i++) { 248 cd = &octeon_irq_core_chip_data[i]; 249 cd->current_en = false; 250 cd->desired_en = false; 251 cd->bit = i; 252 mutex_init(&cd->core_irq_mutex); 253 254 irq = OCTEON_IRQ_SW0 + i; 255 irq_set_chip_data(irq, cd); 256 irq_set_chip_and_handler(irq, &octeon_irq_chip_core, 257 handle_percpu_irq); 258 } 259 } 260 261 static int next_cpu_for_irq(struct irq_data *data) 262 { 263 264 #ifdef CONFIG_SMP 265 int cpu; 266 struct cpumask *mask = irq_data_get_affinity_mask(data); 267 int weight = cpumask_weight(mask); 268 struct octeon_ciu_chip_data *cd = irq_data_get_irq_chip_data(data); 269 270 if (weight > 1) { 271 cpu = cd->current_cpu; 272 for (;;) { 273 cpu = cpumask_next(cpu, mask); 274 if (cpu >= nr_cpu_ids) { 275 cpu = -1; 276 continue; 277 } else if (cpumask_test_cpu(cpu, cpu_online_mask)) { 278 break; 279 } 280 } 281 } else if (weight == 1) { 282 cpu = cpumask_first(mask); 283 } else { 284 cpu = smp_processor_id(); 285 } 286 cd->current_cpu = cpu; 287 return cpu; 288 #else 289 return smp_processor_id(); 290 #endif 291 } 292 293 static void octeon_irq_ciu_enable(struct irq_data *data) 294 { 295 int cpu = next_cpu_for_irq(data); 296 int coreid = octeon_coreid_for_cpu(cpu); 297 unsigned long *pen; 298 unsigned long flags; 299 struct octeon_ciu_chip_data *cd; 300 raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 301 302 cd = irq_data_get_irq_chip_data(data); 303 304 raw_spin_lock_irqsave(lock, flags); 305 if (cd->line == 0) { 306 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 307 __set_bit(cd->bit, pen); 308 /* 309 * Must be visible to octeon_irq_ip{2,3}_ciu() before 310 * enabling the irq. 311 */ 312 wmb(); 313 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 314 } else { 315 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 316 __set_bit(cd->bit, pen); 317 /* 318 * Must be visible to octeon_irq_ip{2,3}_ciu() before 319 * enabling the irq. 320 */ 321 wmb(); 322 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 323 } 324 raw_spin_unlock_irqrestore(lock, flags); 325 } 326 327 static void octeon_irq_ciu_enable_local(struct irq_data *data) 328 { 329 unsigned long *pen; 330 unsigned long flags; 331 struct octeon_ciu_chip_data *cd; 332 raw_spinlock_t *lock = this_cpu_ptr(&octeon_irq_ciu_spinlock); 333 334 cd = irq_data_get_irq_chip_data(data); 335 336 raw_spin_lock_irqsave(lock, flags); 337 if (cd->line == 0) { 338 pen = this_cpu_ptr(&octeon_irq_ciu0_en_mirror); 339 __set_bit(cd->bit, pen); 340 /* 341 * Must be visible to octeon_irq_ip{2,3}_ciu() before 342 * enabling the irq. 343 */ 344 wmb(); 345 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen); 346 } else { 347 pen = this_cpu_ptr(&octeon_irq_ciu1_en_mirror); 348 __set_bit(cd->bit, pen); 349 /* 350 * Must be visible to octeon_irq_ip{2,3}_ciu() before 351 * enabling the irq. 352 */ 353 wmb(); 354 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen); 355 } 356 raw_spin_unlock_irqrestore(lock, flags); 357 } 358 359 static void octeon_irq_ciu_disable_local(struct irq_data *data) 360 { 361 unsigned long *pen; 362 unsigned long flags; 363 struct octeon_ciu_chip_data *cd; 364 raw_spinlock_t *lock = this_cpu_ptr(&octeon_irq_ciu_spinlock); 365 366 cd = irq_data_get_irq_chip_data(data); 367 368 raw_spin_lock_irqsave(lock, flags); 369 if (cd->line == 0) { 370 pen = this_cpu_ptr(&octeon_irq_ciu0_en_mirror); 371 __clear_bit(cd->bit, pen); 372 /* 373 * Must be visible to octeon_irq_ip{2,3}_ciu() before 374 * enabling the irq. 375 */ 376 wmb(); 377 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen); 378 } else { 379 pen = this_cpu_ptr(&octeon_irq_ciu1_en_mirror); 380 __clear_bit(cd->bit, pen); 381 /* 382 * Must be visible to octeon_irq_ip{2,3}_ciu() before 383 * enabling the irq. 384 */ 385 wmb(); 386 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen); 387 } 388 raw_spin_unlock_irqrestore(lock, flags); 389 } 390 391 static void octeon_irq_ciu_disable_all(struct irq_data *data) 392 { 393 unsigned long flags; 394 unsigned long *pen; 395 int cpu; 396 struct octeon_ciu_chip_data *cd; 397 raw_spinlock_t *lock; 398 399 cd = irq_data_get_irq_chip_data(data); 400 401 for_each_online_cpu(cpu) { 402 int coreid = octeon_coreid_for_cpu(cpu); 403 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 404 if (cd->line == 0) 405 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 406 else 407 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 408 409 raw_spin_lock_irqsave(lock, flags); 410 __clear_bit(cd->bit, pen); 411 /* 412 * Must be visible to octeon_irq_ip{2,3}_ciu() before 413 * enabling the irq. 414 */ 415 wmb(); 416 if (cd->line == 0) 417 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 418 else 419 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 420 raw_spin_unlock_irqrestore(lock, flags); 421 } 422 } 423 424 static void octeon_irq_ciu_enable_all(struct irq_data *data) 425 { 426 unsigned long flags; 427 unsigned long *pen; 428 int cpu; 429 struct octeon_ciu_chip_data *cd; 430 raw_spinlock_t *lock; 431 432 cd = irq_data_get_irq_chip_data(data); 433 434 for_each_online_cpu(cpu) { 435 int coreid = octeon_coreid_for_cpu(cpu); 436 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 437 if (cd->line == 0) 438 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 439 else 440 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 441 442 raw_spin_lock_irqsave(lock, flags); 443 __set_bit(cd->bit, pen); 444 /* 445 * Must be visible to octeon_irq_ip{2,3}_ciu() before 446 * enabling the irq. 447 */ 448 wmb(); 449 if (cd->line == 0) 450 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 451 else 452 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 453 raw_spin_unlock_irqrestore(lock, flags); 454 } 455 } 456 457 /* 458 * Enable the irq on the next core in the affinity set for chips that 459 * have the EN*_W1{S,C} registers. 460 */ 461 static void octeon_irq_ciu_enable_v2(struct irq_data *data) 462 { 463 u64 mask; 464 int cpu = next_cpu_for_irq(data); 465 struct octeon_ciu_chip_data *cd; 466 467 cd = irq_data_get_irq_chip_data(data); 468 mask = 1ull << (cd->bit); 469 470 /* 471 * Called under the desc lock, so these should never get out 472 * of sync. 473 */ 474 if (cd->line == 0) { 475 int index = octeon_coreid_for_cpu(cpu) * 2; 476 set_bit(cd->bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu)); 477 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 478 } else { 479 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 480 set_bit(cd->bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 481 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 482 } 483 } 484 485 /* 486 * Enable the irq in the sum2 registers. 487 */ 488 static void octeon_irq_ciu_enable_sum2(struct irq_data *data) 489 { 490 u64 mask; 491 int cpu = next_cpu_for_irq(data); 492 int index = octeon_coreid_for_cpu(cpu); 493 struct octeon_ciu_chip_data *cd; 494 495 cd = irq_data_get_irq_chip_data(data); 496 mask = 1ull << (cd->bit); 497 498 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1S(index), mask); 499 } 500 501 /* 502 * Disable the irq in the sum2 registers. 503 */ 504 static void octeon_irq_ciu_disable_local_sum2(struct irq_data *data) 505 { 506 u64 mask; 507 int cpu = next_cpu_for_irq(data); 508 int index = octeon_coreid_for_cpu(cpu); 509 struct octeon_ciu_chip_data *cd; 510 511 cd = irq_data_get_irq_chip_data(data); 512 mask = 1ull << (cd->bit); 513 514 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(index), mask); 515 } 516 517 static void octeon_irq_ciu_ack_sum2(struct irq_data *data) 518 { 519 u64 mask; 520 int cpu = next_cpu_for_irq(data); 521 int index = octeon_coreid_for_cpu(cpu); 522 struct octeon_ciu_chip_data *cd; 523 524 cd = irq_data_get_irq_chip_data(data); 525 mask = 1ull << (cd->bit); 526 527 cvmx_write_csr(CVMX_CIU_SUM2_PPX_IP4(index), mask); 528 } 529 530 static void octeon_irq_ciu_disable_all_sum2(struct irq_data *data) 531 { 532 int cpu; 533 struct octeon_ciu_chip_data *cd; 534 u64 mask; 535 536 cd = irq_data_get_irq_chip_data(data); 537 mask = 1ull << (cd->bit); 538 539 for_each_online_cpu(cpu) { 540 int coreid = octeon_coreid_for_cpu(cpu); 541 542 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(coreid), mask); 543 } 544 } 545 546 /* 547 * Enable the irq on the current CPU for chips that 548 * have the EN*_W1{S,C} registers. 549 */ 550 static void octeon_irq_ciu_enable_local_v2(struct irq_data *data) 551 { 552 u64 mask; 553 struct octeon_ciu_chip_data *cd; 554 555 cd = irq_data_get_irq_chip_data(data); 556 mask = 1ull << (cd->bit); 557 558 if (cd->line == 0) { 559 int index = cvmx_get_core_num() * 2; 560 set_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu0_en_mirror)); 561 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 562 } else { 563 int index = cvmx_get_core_num() * 2 + 1; 564 set_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu1_en_mirror)); 565 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 566 } 567 } 568 569 static void octeon_irq_ciu_disable_local_v2(struct irq_data *data) 570 { 571 u64 mask; 572 struct octeon_ciu_chip_data *cd; 573 574 cd = irq_data_get_irq_chip_data(data); 575 mask = 1ull << (cd->bit); 576 577 if (cd->line == 0) { 578 int index = cvmx_get_core_num() * 2; 579 clear_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu0_en_mirror)); 580 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); 581 } else { 582 int index = cvmx_get_core_num() * 2 + 1; 583 clear_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu1_en_mirror)); 584 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask); 585 } 586 } 587 588 /* 589 * Write to the W1C bit in CVMX_CIU_INTX_SUM0 to clear the irq. 590 */ 591 static void octeon_irq_ciu_ack(struct irq_data *data) 592 { 593 u64 mask; 594 struct octeon_ciu_chip_data *cd; 595 596 cd = irq_data_get_irq_chip_data(data); 597 mask = 1ull << (cd->bit); 598 599 if (cd->line == 0) { 600 int index = cvmx_get_core_num() * 2; 601 cvmx_write_csr(CVMX_CIU_INTX_SUM0(index), mask); 602 } else { 603 cvmx_write_csr(CVMX_CIU_INT_SUM1, mask); 604 } 605 } 606 607 /* 608 * Disable the irq on the all cores for chips that have the EN*_W1{S,C} 609 * registers. 610 */ 611 static void octeon_irq_ciu_disable_all_v2(struct irq_data *data) 612 { 613 int cpu; 614 u64 mask; 615 struct octeon_ciu_chip_data *cd; 616 617 cd = irq_data_get_irq_chip_data(data); 618 mask = 1ull << (cd->bit); 619 620 if (cd->line == 0) { 621 for_each_online_cpu(cpu) { 622 int index = octeon_coreid_for_cpu(cpu) * 2; 623 clear_bit(cd->bit, 624 &per_cpu(octeon_irq_ciu0_en_mirror, cpu)); 625 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); 626 } 627 } else { 628 for_each_online_cpu(cpu) { 629 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 630 clear_bit(cd->bit, 631 &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 632 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask); 633 } 634 } 635 } 636 637 /* 638 * Enable the irq on the all cores for chips that have the EN*_W1{S,C} 639 * registers. 640 */ 641 static void octeon_irq_ciu_enable_all_v2(struct irq_data *data) 642 { 643 int cpu; 644 u64 mask; 645 struct octeon_ciu_chip_data *cd; 646 647 cd = irq_data_get_irq_chip_data(data); 648 mask = 1ull << (cd->bit); 649 650 if (cd->line == 0) { 651 for_each_online_cpu(cpu) { 652 int index = octeon_coreid_for_cpu(cpu) * 2; 653 set_bit(cd->bit, 654 &per_cpu(octeon_irq_ciu0_en_mirror, cpu)); 655 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 656 } 657 } else { 658 for_each_online_cpu(cpu) { 659 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 660 set_bit(cd->bit, 661 &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 662 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 663 } 664 } 665 } 666 667 static int octeon_irq_ciu_set_type(struct irq_data *data, unsigned int t) 668 { 669 irqd_set_trigger_type(data, t); 670 671 if (t & IRQ_TYPE_EDGE_BOTH) 672 irq_set_handler_locked(data, handle_edge_irq); 673 else 674 irq_set_handler_locked(data, handle_level_irq); 675 676 return IRQ_SET_MASK_OK; 677 } 678 679 static void octeon_irq_gpio_setup(struct irq_data *data) 680 { 681 union cvmx_gpio_bit_cfgx cfg; 682 struct octeon_ciu_chip_data *cd; 683 u32 t = irqd_get_trigger_type(data); 684 685 cd = irq_data_get_irq_chip_data(data); 686 687 cfg.u64 = 0; 688 cfg.s.int_en = 1; 689 cfg.s.int_type = (t & IRQ_TYPE_EDGE_BOTH) != 0; 690 cfg.s.rx_xor = (t & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_EDGE_FALLING)) != 0; 691 692 /* 140 nS glitch filter*/ 693 cfg.s.fil_cnt = 7; 694 cfg.s.fil_sel = 3; 695 696 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), cfg.u64); 697 } 698 699 static void octeon_irq_ciu_enable_gpio_v2(struct irq_data *data) 700 { 701 octeon_irq_gpio_setup(data); 702 octeon_irq_ciu_enable_v2(data); 703 } 704 705 static void octeon_irq_ciu_enable_gpio(struct irq_data *data) 706 { 707 octeon_irq_gpio_setup(data); 708 octeon_irq_ciu_enable(data); 709 } 710 711 static int octeon_irq_ciu_gpio_set_type(struct irq_data *data, unsigned int t) 712 { 713 irqd_set_trigger_type(data, t); 714 octeon_irq_gpio_setup(data); 715 716 if (t & IRQ_TYPE_EDGE_BOTH) 717 irq_set_handler_locked(data, handle_edge_irq); 718 else 719 irq_set_handler_locked(data, handle_level_irq); 720 721 return IRQ_SET_MASK_OK; 722 } 723 724 static void octeon_irq_ciu_disable_gpio_v2(struct irq_data *data) 725 { 726 struct octeon_ciu_chip_data *cd; 727 728 cd = irq_data_get_irq_chip_data(data); 729 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0); 730 731 octeon_irq_ciu_disable_all_v2(data); 732 } 733 734 static void octeon_irq_ciu_disable_gpio(struct irq_data *data) 735 { 736 struct octeon_ciu_chip_data *cd; 737 738 cd = irq_data_get_irq_chip_data(data); 739 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0); 740 741 octeon_irq_ciu_disable_all(data); 742 } 743 744 static void octeon_irq_ciu_gpio_ack(struct irq_data *data) 745 { 746 struct octeon_ciu_chip_data *cd; 747 u64 mask; 748 749 cd = irq_data_get_irq_chip_data(data); 750 mask = 1ull << (cd->gpio_line); 751 752 cvmx_write_csr(CVMX_GPIO_INT_CLR, mask); 753 } 754 755 #ifdef CONFIG_SMP 756 757 static void octeon_irq_cpu_offline_ciu(struct irq_data *data) 758 { 759 int cpu = smp_processor_id(); 760 cpumask_t new_affinity; 761 struct cpumask *mask = irq_data_get_affinity_mask(data); 762 763 if (!cpumask_test_cpu(cpu, mask)) 764 return; 765 766 if (cpumask_weight(mask) > 1) { 767 /* 768 * It has multi CPU affinity, just remove this CPU 769 * from the affinity set. 770 */ 771 cpumask_copy(&new_affinity, mask); 772 cpumask_clear_cpu(cpu, &new_affinity); 773 } else { 774 /* Otherwise, put it on lowest numbered online CPU. */ 775 cpumask_clear(&new_affinity); 776 cpumask_set_cpu(cpumask_first(cpu_online_mask), &new_affinity); 777 } 778 irq_set_affinity_locked(data, &new_affinity, false); 779 } 780 781 static int octeon_irq_ciu_set_affinity(struct irq_data *data, 782 const struct cpumask *dest, bool force) 783 { 784 int cpu; 785 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 786 unsigned long flags; 787 struct octeon_ciu_chip_data *cd; 788 unsigned long *pen; 789 raw_spinlock_t *lock; 790 791 cd = irq_data_get_irq_chip_data(data); 792 793 /* 794 * For non-v2 CIU, we will allow only single CPU affinity. 795 * This removes the need to do locking in the .ack/.eoi 796 * functions. 797 */ 798 if (cpumask_weight(dest) != 1) 799 return -EINVAL; 800 801 if (!enable_one) 802 return 0; 803 804 805 for_each_online_cpu(cpu) { 806 int coreid = octeon_coreid_for_cpu(cpu); 807 808 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 809 raw_spin_lock_irqsave(lock, flags); 810 811 if (cd->line == 0) 812 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 813 else 814 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 815 816 if (cpumask_test_cpu(cpu, dest) && enable_one) { 817 enable_one = 0; 818 __set_bit(cd->bit, pen); 819 } else { 820 __clear_bit(cd->bit, pen); 821 } 822 /* 823 * Must be visible to octeon_irq_ip{2,3}_ciu() before 824 * enabling the irq. 825 */ 826 wmb(); 827 828 if (cd->line == 0) 829 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 830 else 831 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 832 833 raw_spin_unlock_irqrestore(lock, flags); 834 } 835 return 0; 836 } 837 838 /* 839 * Set affinity for the irq for chips that have the EN*_W1{S,C} 840 * registers. 841 */ 842 static int octeon_irq_ciu_set_affinity_v2(struct irq_data *data, 843 const struct cpumask *dest, 844 bool force) 845 { 846 int cpu; 847 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 848 u64 mask; 849 struct octeon_ciu_chip_data *cd; 850 851 if (!enable_one) 852 return 0; 853 854 cd = irq_data_get_irq_chip_data(data); 855 mask = 1ull << cd->bit; 856 857 if (cd->line == 0) { 858 for_each_online_cpu(cpu) { 859 unsigned long *pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 860 int index = octeon_coreid_for_cpu(cpu) * 2; 861 if (cpumask_test_cpu(cpu, dest) && enable_one) { 862 enable_one = false; 863 set_bit(cd->bit, pen); 864 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 865 } else { 866 clear_bit(cd->bit, pen); 867 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); 868 } 869 } 870 } else { 871 for_each_online_cpu(cpu) { 872 unsigned long *pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 873 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 874 if (cpumask_test_cpu(cpu, dest) && enable_one) { 875 enable_one = false; 876 set_bit(cd->bit, pen); 877 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 878 } else { 879 clear_bit(cd->bit, pen); 880 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask); 881 } 882 } 883 } 884 return 0; 885 } 886 887 static int octeon_irq_ciu_set_affinity_sum2(struct irq_data *data, 888 const struct cpumask *dest, 889 bool force) 890 { 891 int cpu; 892 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 893 u64 mask; 894 struct octeon_ciu_chip_data *cd; 895 896 if (!enable_one) 897 return 0; 898 899 cd = irq_data_get_irq_chip_data(data); 900 mask = 1ull << cd->bit; 901 902 for_each_online_cpu(cpu) { 903 int index = octeon_coreid_for_cpu(cpu); 904 905 if (cpumask_test_cpu(cpu, dest) && enable_one) { 906 enable_one = false; 907 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1S(index), mask); 908 } else { 909 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(index), mask); 910 } 911 } 912 return 0; 913 } 914 #endif 915 916 static unsigned int edge_startup(struct irq_data *data) 917 { 918 /* ack any pending edge-irq at startup, so there is 919 * an _edge_ to fire on when the event reappears. 920 */ 921 data->chip->irq_ack(data); 922 data->chip->irq_enable(data); 923 return 0; 924 } 925 926 /* 927 * Newer octeon chips have support for lockless CIU operation. 928 */ 929 static struct irq_chip octeon_irq_chip_ciu_v2 = { 930 .name = "CIU", 931 .irq_enable = octeon_irq_ciu_enable_v2, 932 .irq_disable = octeon_irq_ciu_disable_all_v2, 933 .irq_mask = octeon_irq_ciu_disable_local_v2, 934 .irq_unmask = octeon_irq_ciu_enable_v2, 935 #ifdef CONFIG_SMP 936 .irq_set_affinity = octeon_irq_ciu_set_affinity_v2, 937 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 938 #endif 939 }; 940 941 static struct irq_chip octeon_irq_chip_ciu_v2_edge = { 942 .name = "CIU", 943 .irq_enable = octeon_irq_ciu_enable_v2, 944 .irq_disable = octeon_irq_ciu_disable_all_v2, 945 .irq_ack = octeon_irq_ciu_ack, 946 .irq_mask = octeon_irq_ciu_disable_local_v2, 947 .irq_unmask = octeon_irq_ciu_enable_v2, 948 #ifdef CONFIG_SMP 949 .irq_set_affinity = octeon_irq_ciu_set_affinity_v2, 950 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 951 #endif 952 }; 953 954 /* 955 * Newer octeon chips have support for lockless CIU operation. 956 */ 957 static struct irq_chip octeon_irq_chip_ciu_sum2 = { 958 .name = "CIU", 959 .irq_enable = octeon_irq_ciu_enable_sum2, 960 .irq_disable = octeon_irq_ciu_disable_all_sum2, 961 .irq_mask = octeon_irq_ciu_disable_local_sum2, 962 .irq_unmask = octeon_irq_ciu_enable_sum2, 963 #ifdef CONFIG_SMP 964 .irq_set_affinity = octeon_irq_ciu_set_affinity_sum2, 965 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 966 #endif 967 }; 968 969 static struct irq_chip octeon_irq_chip_ciu_sum2_edge = { 970 .name = "CIU", 971 .irq_enable = octeon_irq_ciu_enable_sum2, 972 .irq_disable = octeon_irq_ciu_disable_all_sum2, 973 .irq_ack = octeon_irq_ciu_ack_sum2, 974 .irq_mask = octeon_irq_ciu_disable_local_sum2, 975 .irq_unmask = octeon_irq_ciu_enable_sum2, 976 #ifdef CONFIG_SMP 977 .irq_set_affinity = octeon_irq_ciu_set_affinity_sum2, 978 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 979 #endif 980 }; 981 982 static struct irq_chip octeon_irq_chip_ciu = { 983 .name = "CIU", 984 .irq_enable = octeon_irq_ciu_enable, 985 .irq_disable = octeon_irq_ciu_disable_all, 986 .irq_mask = octeon_irq_ciu_disable_local, 987 .irq_unmask = octeon_irq_ciu_enable, 988 #ifdef CONFIG_SMP 989 .irq_set_affinity = octeon_irq_ciu_set_affinity, 990 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 991 #endif 992 }; 993 994 static struct irq_chip octeon_irq_chip_ciu_edge = { 995 .name = "CIU", 996 .irq_enable = octeon_irq_ciu_enable, 997 .irq_disable = octeon_irq_ciu_disable_all, 998 .irq_ack = octeon_irq_ciu_ack, 999 .irq_mask = octeon_irq_ciu_disable_local, 1000 .irq_unmask = octeon_irq_ciu_enable, 1001 #ifdef CONFIG_SMP 1002 .irq_set_affinity = octeon_irq_ciu_set_affinity, 1003 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1004 #endif 1005 }; 1006 1007 /* The mbox versions don't do any affinity or round-robin. */ 1008 static struct irq_chip octeon_irq_chip_ciu_mbox_v2 = { 1009 .name = "CIU-M", 1010 .irq_enable = octeon_irq_ciu_enable_all_v2, 1011 .irq_disable = octeon_irq_ciu_disable_all_v2, 1012 .irq_ack = octeon_irq_ciu_disable_local_v2, 1013 .irq_eoi = octeon_irq_ciu_enable_local_v2, 1014 1015 .irq_cpu_online = octeon_irq_ciu_enable_local_v2, 1016 .irq_cpu_offline = octeon_irq_ciu_disable_local_v2, 1017 .flags = IRQCHIP_ONOFFLINE_ENABLED, 1018 }; 1019 1020 static struct irq_chip octeon_irq_chip_ciu_mbox = { 1021 .name = "CIU-M", 1022 .irq_enable = octeon_irq_ciu_enable_all, 1023 .irq_disable = octeon_irq_ciu_disable_all, 1024 .irq_ack = octeon_irq_ciu_disable_local, 1025 .irq_eoi = octeon_irq_ciu_enable_local, 1026 1027 .irq_cpu_online = octeon_irq_ciu_enable_local, 1028 .irq_cpu_offline = octeon_irq_ciu_disable_local, 1029 .flags = IRQCHIP_ONOFFLINE_ENABLED, 1030 }; 1031 1032 static struct irq_chip octeon_irq_chip_ciu_gpio_v2 = { 1033 .name = "CIU-GPIO", 1034 .irq_enable = octeon_irq_ciu_enable_gpio_v2, 1035 .irq_disable = octeon_irq_ciu_disable_gpio_v2, 1036 .irq_ack = octeon_irq_ciu_gpio_ack, 1037 .irq_mask = octeon_irq_ciu_disable_local_v2, 1038 .irq_unmask = octeon_irq_ciu_enable_v2, 1039 .irq_set_type = octeon_irq_ciu_gpio_set_type, 1040 #ifdef CONFIG_SMP 1041 .irq_set_affinity = octeon_irq_ciu_set_affinity_v2, 1042 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1043 #endif 1044 .flags = IRQCHIP_SET_TYPE_MASKED, 1045 }; 1046 1047 static struct irq_chip octeon_irq_chip_ciu_gpio = { 1048 .name = "CIU-GPIO", 1049 .irq_enable = octeon_irq_ciu_enable_gpio, 1050 .irq_disable = octeon_irq_ciu_disable_gpio, 1051 .irq_mask = octeon_irq_ciu_disable_local, 1052 .irq_unmask = octeon_irq_ciu_enable, 1053 .irq_ack = octeon_irq_ciu_gpio_ack, 1054 .irq_set_type = octeon_irq_ciu_gpio_set_type, 1055 #ifdef CONFIG_SMP 1056 .irq_set_affinity = octeon_irq_ciu_set_affinity, 1057 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1058 #endif 1059 .flags = IRQCHIP_SET_TYPE_MASKED, 1060 }; 1061 1062 /* 1063 * Watchdog interrupts are special. They are associated with a single 1064 * core, so we hardwire the affinity to that core. 1065 */ 1066 static void octeon_irq_ciu_wd_enable(struct irq_data *data) 1067 { 1068 unsigned long flags; 1069 unsigned long *pen; 1070 int coreid = data->irq - OCTEON_IRQ_WDOG0; /* Bit 0-63 of EN1 */ 1071 int cpu = octeon_cpu_for_coreid(coreid); 1072 raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 1073 1074 raw_spin_lock_irqsave(lock, flags); 1075 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 1076 __set_bit(coreid, pen); 1077 /* 1078 * Must be visible to octeon_irq_ip{2,3}_ciu() before enabling 1079 * the irq. 1080 */ 1081 wmb(); 1082 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 1083 raw_spin_unlock_irqrestore(lock, flags); 1084 } 1085 1086 /* 1087 * Watchdog interrupts are special. They are associated with a single 1088 * core, so we hardwire the affinity to that core. 1089 */ 1090 static void octeon_irq_ciu1_wd_enable_v2(struct irq_data *data) 1091 { 1092 int coreid = data->irq - OCTEON_IRQ_WDOG0; 1093 int cpu = octeon_cpu_for_coreid(coreid); 1094 1095 set_bit(coreid, &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 1096 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(coreid * 2 + 1), 1ull << coreid); 1097 } 1098 1099 1100 static struct irq_chip octeon_irq_chip_ciu_wd_v2 = { 1101 .name = "CIU-W", 1102 .irq_enable = octeon_irq_ciu1_wd_enable_v2, 1103 .irq_disable = octeon_irq_ciu_disable_all_v2, 1104 .irq_mask = octeon_irq_ciu_disable_local_v2, 1105 .irq_unmask = octeon_irq_ciu_enable_local_v2, 1106 }; 1107 1108 static struct irq_chip octeon_irq_chip_ciu_wd = { 1109 .name = "CIU-W", 1110 .irq_enable = octeon_irq_ciu_wd_enable, 1111 .irq_disable = octeon_irq_ciu_disable_all, 1112 .irq_mask = octeon_irq_ciu_disable_local, 1113 .irq_unmask = octeon_irq_ciu_enable_local, 1114 }; 1115 1116 static bool octeon_irq_ciu_is_edge(unsigned int line, unsigned int bit) 1117 { 1118 bool edge = false; 1119 1120 if (line == 0) 1121 switch (bit) { 1122 case 48 ... 49: /* GMX DRP */ 1123 case 50: /* IPD_DRP */ 1124 case 52 ... 55: /* Timers */ 1125 case 58: /* MPI */ 1126 edge = true; 1127 break; 1128 default: 1129 break; 1130 } 1131 else /* line == 1 */ 1132 switch (bit) { 1133 case 47: /* PTP */ 1134 edge = true; 1135 break; 1136 default: 1137 break; 1138 } 1139 return edge; 1140 } 1141 1142 struct octeon_irq_gpio_domain_data { 1143 unsigned int base_hwirq; 1144 }; 1145 1146 static int octeon_irq_gpio_xlat(struct irq_domain *d, 1147 struct device_node *node, 1148 const u32 *intspec, 1149 unsigned int intsize, 1150 unsigned long *out_hwirq, 1151 unsigned int *out_type) 1152 { 1153 unsigned int type; 1154 unsigned int pin; 1155 unsigned int trigger; 1156 1157 if (irq_domain_get_of_node(d) != node) 1158 return -EINVAL; 1159 1160 if (intsize < 2) 1161 return -EINVAL; 1162 1163 pin = intspec[0]; 1164 if (pin >= 16) 1165 return -EINVAL; 1166 1167 trigger = intspec[1]; 1168 1169 switch (trigger) { 1170 case 1: 1171 type = IRQ_TYPE_EDGE_RISING; 1172 break; 1173 case 2: 1174 type = IRQ_TYPE_EDGE_FALLING; 1175 break; 1176 case 4: 1177 type = IRQ_TYPE_LEVEL_HIGH; 1178 break; 1179 case 8: 1180 type = IRQ_TYPE_LEVEL_LOW; 1181 break; 1182 default: 1183 pr_err("Error: (%s) Invalid irq trigger specification: %x\n", 1184 node->name, 1185 trigger); 1186 type = IRQ_TYPE_LEVEL_LOW; 1187 break; 1188 } 1189 *out_type = type; 1190 *out_hwirq = pin; 1191 1192 return 0; 1193 } 1194 1195 static int octeon_irq_ciu_xlat(struct irq_domain *d, 1196 struct device_node *node, 1197 const u32 *intspec, 1198 unsigned int intsize, 1199 unsigned long *out_hwirq, 1200 unsigned int *out_type) 1201 { 1202 unsigned int ciu, bit; 1203 struct octeon_irq_ciu_domain_data *dd = d->host_data; 1204 1205 ciu = intspec[0]; 1206 bit = intspec[1]; 1207 1208 if (ciu >= dd->num_sum || bit > 63) 1209 return -EINVAL; 1210 1211 *out_hwirq = (ciu << 6) | bit; 1212 *out_type = 0; 1213 1214 return 0; 1215 } 1216 1217 static struct irq_chip *octeon_irq_ciu_chip; 1218 static struct irq_chip *octeon_irq_ciu_chip_edge; 1219 static struct irq_chip *octeon_irq_gpio_chip; 1220 1221 static int octeon_irq_ciu_map(struct irq_domain *d, 1222 unsigned int virq, irq_hw_number_t hw) 1223 { 1224 int rv; 1225 unsigned int line = hw >> 6; 1226 unsigned int bit = hw & 63; 1227 struct octeon_irq_ciu_domain_data *dd = d->host_data; 1228 1229 if (line >= dd->num_sum || octeon_irq_ciu_to_irq[line][bit] != 0) 1230 return -EINVAL; 1231 1232 if (line == 2) { 1233 if (octeon_irq_ciu_is_edge(line, bit)) 1234 rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1235 &octeon_irq_chip_ciu_sum2_edge, 1236 handle_edge_irq); 1237 else 1238 rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1239 &octeon_irq_chip_ciu_sum2, 1240 handle_level_irq); 1241 } else { 1242 if (octeon_irq_ciu_is_edge(line, bit)) 1243 rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1244 octeon_irq_ciu_chip_edge, 1245 handle_edge_irq); 1246 else 1247 rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1248 octeon_irq_ciu_chip, 1249 handle_level_irq); 1250 } 1251 return rv; 1252 } 1253 1254 static int octeon_irq_gpio_map(struct irq_domain *d, 1255 unsigned int virq, irq_hw_number_t hw) 1256 { 1257 struct octeon_irq_gpio_domain_data *gpiod = d->host_data; 1258 unsigned int line, bit; 1259 int r; 1260 1261 line = (hw + gpiod->base_hwirq) >> 6; 1262 bit = (hw + gpiod->base_hwirq) & 63; 1263 if (line >= ARRAY_SIZE(octeon_irq_ciu_to_irq) || 1264 octeon_irq_ciu_to_irq[line][bit] != 0) 1265 return -EINVAL; 1266 1267 /* 1268 * Default to handle_level_irq. If the DT contains a different 1269 * trigger type, it will call the irq_set_type callback and 1270 * the handler gets updated. 1271 */ 1272 r = octeon_irq_set_ciu_mapping(virq, line, bit, hw, 1273 octeon_irq_gpio_chip, handle_level_irq); 1274 return r; 1275 } 1276 1277 static struct irq_domain_ops octeon_irq_domain_ciu_ops = { 1278 .map = octeon_irq_ciu_map, 1279 .unmap = octeon_irq_free_cd, 1280 .xlate = octeon_irq_ciu_xlat, 1281 }; 1282 1283 static struct irq_domain_ops octeon_irq_domain_gpio_ops = { 1284 .map = octeon_irq_gpio_map, 1285 .unmap = octeon_irq_free_cd, 1286 .xlate = octeon_irq_gpio_xlat, 1287 }; 1288 1289 static void octeon_irq_ip2_ciu(void) 1290 { 1291 const unsigned long core_id = cvmx_get_core_num(); 1292 u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INTX_SUM0(core_id * 2)); 1293 1294 ciu_sum &= __this_cpu_read(octeon_irq_ciu0_en_mirror); 1295 if (likely(ciu_sum)) { 1296 int bit = fls64(ciu_sum) - 1; 1297 int irq = octeon_irq_ciu_to_irq[0][bit]; 1298 if (likely(irq)) 1299 do_IRQ(irq); 1300 else 1301 spurious_interrupt(); 1302 } else { 1303 spurious_interrupt(); 1304 } 1305 } 1306 1307 static void octeon_irq_ip3_ciu(void) 1308 { 1309 u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INT_SUM1); 1310 1311 ciu_sum &= __this_cpu_read(octeon_irq_ciu1_en_mirror); 1312 if (likely(ciu_sum)) { 1313 int bit = fls64(ciu_sum) - 1; 1314 int irq = octeon_irq_ciu_to_irq[1][bit]; 1315 if (likely(irq)) 1316 do_IRQ(irq); 1317 else 1318 spurious_interrupt(); 1319 } else { 1320 spurious_interrupt(); 1321 } 1322 } 1323 1324 static void octeon_irq_ip4_ciu(void) 1325 { 1326 int coreid = cvmx_get_core_num(); 1327 u64 ciu_sum = cvmx_read_csr(CVMX_CIU_SUM2_PPX_IP4(coreid)); 1328 u64 ciu_en = cvmx_read_csr(CVMX_CIU_EN2_PPX_IP4(coreid)); 1329 1330 ciu_sum &= ciu_en; 1331 if (likely(ciu_sum)) { 1332 int bit = fls64(ciu_sum) - 1; 1333 int irq = octeon_irq_ciu_to_irq[2][bit]; 1334 1335 if (likely(irq)) 1336 do_IRQ(irq); 1337 else 1338 spurious_interrupt(); 1339 } else { 1340 spurious_interrupt(); 1341 } 1342 } 1343 1344 static bool octeon_irq_use_ip4; 1345 1346 static void octeon_irq_local_enable_ip4(void *arg) 1347 { 1348 set_c0_status(STATUSF_IP4); 1349 } 1350 1351 static void octeon_irq_ip4_mask(void) 1352 { 1353 clear_c0_status(STATUSF_IP4); 1354 spurious_interrupt(); 1355 } 1356 1357 static void (*octeon_irq_ip2)(void); 1358 static void (*octeon_irq_ip3)(void); 1359 static void (*octeon_irq_ip4)(void); 1360 1361 void (*octeon_irq_setup_secondary)(void); 1362 1363 void octeon_irq_set_ip4_handler(octeon_irq_ip4_handler_t h) 1364 { 1365 octeon_irq_ip4 = h; 1366 octeon_irq_use_ip4 = true; 1367 on_each_cpu(octeon_irq_local_enable_ip4, NULL, 1); 1368 } 1369 1370 static void octeon_irq_percpu_enable(void) 1371 { 1372 irq_cpu_online(); 1373 } 1374 1375 static void octeon_irq_init_ciu_percpu(void) 1376 { 1377 int coreid = cvmx_get_core_num(); 1378 1379 1380 __this_cpu_write(octeon_irq_ciu0_en_mirror, 0); 1381 __this_cpu_write(octeon_irq_ciu1_en_mirror, 0); 1382 wmb(); 1383 raw_spin_lock_init(this_cpu_ptr(&octeon_irq_ciu_spinlock)); 1384 /* 1385 * Disable All CIU Interrupts. The ones we need will be 1386 * enabled later. Read the SUM register so we know the write 1387 * completed. 1388 */ 1389 cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2)), 0); 1390 cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2 + 1)), 0); 1391 cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2)), 0); 1392 cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2 + 1)), 0); 1393 cvmx_read_csr(CVMX_CIU_INTX_SUM0((coreid * 2))); 1394 } 1395 1396 static void octeon_irq_init_ciu2_percpu(void) 1397 { 1398 u64 regx, ipx; 1399 int coreid = cvmx_get_core_num(); 1400 u64 base = CVMX_CIU2_EN_PPX_IP2_WRKQ(coreid); 1401 1402 /* 1403 * Disable All CIU2 Interrupts. The ones we need will be 1404 * enabled later. Read the SUM register so we know the write 1405 * completed. 1406 * 1407 * There are 9 registers and 3 IPX levels with strides 0x1000 1408 * and 0x200 respectivly. Use loops to clear them. 1409 */ 1410 for (regx = 0; regx <= 0x8000; regx += 0x1000) { 1411 for (ipx = 0; ipx <= 0x400; ipx += 0x200) 1412 cvmx_write_csr(base + regx + ipx, 0); 1413 } 1414 1415 cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(coreid)); 1416 } 1417 1418 static void octeon_irq_setup_secondary_ciu(void) 1419 { 1420 octeon_irq_init_ciu_percpu(); 1421 octeon_irq_percpu_enable(); 1422 1423 /* Enable the CIU lines */ 1424 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 1425 if (octeon_irq_use_ip4) 1426 set_c0_status(STATUSF_IP4); 1427 else 1428 clear_c0_status(STATUSF_IP4); 1429 } 1430 1431 static void octeon_irq_setup_secondary_ciu2(void) 1432 { 1433 octeon_irq_init_ciu2_percpu(); 1434 octeon_irq_percpu_enable(); 1435 1436 /* Enable the CIU lines */ 1437 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 1438 if (octeon_irq_use_ip4) 1439 set_c0_status(STATUSF_IP4); 1440 else 1441 clear_c0_status(STATUSF_IP4); 1442 } 1443 1444 static int __init octeon_irq_init_ciu( 1445 struct device_node *ciu_node, struct device_node *parent) 1446 { 1447 unsigned int i, r; 1448 struct irq_chip *chip; 1449 struct irq_chip *chip_edge; 1450 struct irq_chip *chip_mbox; 1451 struct irq_chip *chip_wd; 1452 struct irq_domain *ciu_domain = NULL; 1453 struct octeon_irq_ciu_domain_data *dd; 1454 1455 dd = kzalloc(sizeof(*dd), GFP_KERNEL); 1456 if (!dd) 1457 return -ENOMEM; 1458 1459 octeon_irq_init_ciu_percpu(); 1460 octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu; 1461 1462 octeon_irq_ip2 = octeon_irq_ip2_ciu; 1463 octeon_irq_ip3 = octeon_irq_ip3_ciu; 1464 if ((OCTEON_IS_OCTEON2() || OCTEON_IS_OCTEON3()) 1465 && !OCTEON_IS_MODEL(OCTEON_CN63XX)) { 1466 octeon_irq_ip4 = octeon_irq_ip4_ciu; 1467 dd->num_sum = 3; 1468 octeon_irq_use_ip4 = true; 1469 } else { 1470 octeon_irq_ip4 = octeon_irq_ip4_mask; 1471 dd->num_sum = 2; 1472 octeon_irq_use_ip4 = false; 1473 } 1474 if (OCTEON_IS_MODEL(OCTEON_CN58XX_PASS2_X) || 1475 OCTEON_IS_MODEL(OCTEON_CN56XX_PASS2_X) || 1476 OCTEON_IS_MODEL(OCTEON_CN52XX_PASS2_X) || 1477 OCTEON_IS_OCTEON2() || OCTEON_IS_OCTEON3()) { 1478 chip = &octeon_irq_chip_ciu_v2; 1479 chip_edge = &octeon_irq_chip_ciu_v2_edge; 1480 chip_mbox = &octeon_irq_chip_ciu_mbox_v2; 1481 chip_wd = &octeon_irq_chip_ciu_wd_v2; 1482 octeon_irq_gpio_chip = &octeon_irq_chip_ciu_gpio_v2; 1483 } else { 1484 chip = &octeon_irq_chip_ciu; 1485 chip_edge = &octeon_irq_chip_ciu_edge; 1486 chip_mbox = &octeon_irq_chip_ciu_mbox; 1487 chip_wd = &octeon_irq_chip_ciu_wd; 1488 octeon_irq_gpio_chip = &octeon_irq_chip_ciu_gpio; 1489 } 1490 octeon_irq_ciu_chip = chip; 1491 octeon_irq_ciu_chip_edge = chip_edge; 1492 1493 /* Mips internal */ 1494 octeon_irq_init_core(); 1495 1496 ciu_domain = irq_domain_add_tree( 1497 ciu_node, &octeon_irq_domain_ciu_ops, dd); 1498 irq_set_default_host(ciu_domain); 1499 1500 /* CIU_0 */ 1501 for (i = 0; i < 16; i++) { 1502 r = octeon_irq_force_ciu_mapping( 1503 ciu_domain, i + OCTEON_IRQ_WORKQ0, 0, i + 0); 1504 if (r) 1505 goto err; 1506 } 1507 1508 r = octeon_irq_set_ciu_mapping( 1509 OCTEON_IRQ_MBOX0, 0, 32, 0, chip_mbox, handle_percpu_irq); 1510 if (r) 1511 goto err; 1512 r = octeon_irq_set_ciu_mapping( 1513 OCTEON_IRQ_MBOX1, 0, 33, 0, chip_mbox, handle_percpu_irq); 1514 if (r) 1515 goto err; 1516 1517 for (i = 0; i < 4; i++) { 1518 r = octeon_irq_force_ciu_mapping( 1519 ciu_domain, i + OCTEON_IRQ_PCI_INT0, 0, i + 36); 1520 if (r) 1521 goto err; 1522 } 1523 for (i = 0; i < 4; i++) { 1524 r = octeon_irq_force_ciu_mapping( 1525 ciu_domain, i + OCTEON_IRQ_PCI_MSI0, 0, i + 40); 1526 if (r) 1527 goto err; 1528 } 1529 1530 r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_TWSI, 0, 45); 1531 if (r) 1532 goto err; 1533 1534 r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_RML, 0, 46); 1535 if (r) 1536 goto err; 1537 1538 for (i = 0; i < 4; i++) { 1539 r = octeon_irq_force_ciu_mapping( 1540 ciu_domain, i + OCTEON_IRQ_TIMER0, 0, i + 52); 1541 if (r) 1542 goto err; 1543 } 1544 1545 r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_TWSI2, 0, 59); 1546 if (r) 1547 goto err; 1548 1549 /* CIU_1 */ 1550 for (i = 0; i < 16; i++) { 1551 r = octeon_irq_set_ciu_mapping( 1552 i + OCTEON_IRQ_WDOG0, 1, i + 0, 0, chip_wd, 1553 handle_level_irq); 1554 if (r) 1555 goto err; 1556 } 1557 1558 /* Enable the CIU lines */ 1559 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 1560 if (octeon_irq_use_ip4) 1561 set_c0_status(STATUSF_IP4); 1562 else 1563 clear_c0_status(STATUSF_IP4); 1564 1565 return 0; 1566 err: 1567 return r; 1568 } 1569 1570 static int __init octeon_irq_init_gpio( 1571 struct device_node *gpio_node, struct device_node *parent) 1572 { 1573 struct octeon_irq_gpio_domain_data *gpiod; 1574 u32 interrupt_cells; 1575 unsigned int base_hwirq; 1576 int r; 1577 1578 r = of_property_read_u32(parent, "#interrupt-cells", &interrupt_cells); 1579 if (r) 1580 return r; 1581 1582 if (interrupt_cells == 1) { 1583 u32 v; 1584 1585 r = of_property_read_u32_index(gpio_node, "interrupts", 0, &v); 1586 if (r) { 1587 pr_warn("No \"interrupts\" property.\n"); 1588 return r; 1589 } 1590 base_hwirq = v; 1591 } else if (interrupt_cells == 2) { 1592 u32 v0, v1; 1593 1594 r = of_property_read_u32_index(gpio_node, "interrupts", 0, &v0); 1595 if (r) { 1596 pr_warn("No \"interrupts\" property.\n"); 1597 return r; 1598 } 1599 r = of_property_read_u32_index(gpio_node, "interrupts", 1, &v1); 1600 if (r) { 1601 pr_warn("No \"interrupts\" property.\n"); 1602 return r; 1603 } 1604 base_hwirq = (v0 << 6) | v1; 1605 } else { 1606 pr_warn("Bad \"#interrupt-cells\" property: %u\n", 1607 interrupt_cells); 1608 return -EINVAL; 1609 } 1610 1611 gpiod = kzalloc(sizeof(*gpiod), GFP_KERNEL); 1612 if (gpiod) { 1613 /* gpio domain host_data is the base hwirq number. */ 1614 gpiod->base_hwirq = base_hwirq; 1615 irq_domain_add_linear( 1616 gpio_node, 16, &octeon_irq_domain_gpio_ops, gpiod); 1617 } else { 1618 pr_warn("Cannot allocate memory for GPIO irq_domain.\n"); 1619 return -ENOMEM; 1620 } 1621 1622 return 0; 1623 } 1624 /* 1625 * Watchdog interrupts are special. They are associated with a single 1626 * core, so we hardwire the affinity to that core. 1627 */ 1628 static void octeon_irq_ciu2_wd_enable(struct irq_data *data) 1629 { 1630 u64 mask; 1631 u64 en_addr; 1632 int coreid = data->irq - OCTEON_IRQ_WDOG0; 1633 struct octeon_ciu_chip_data *cd; 1634 1635 cd = irq_data_get_irq_chip_data(data); 1636 mask = 1ull << (cd->bit); 1637 1638 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) + 1639 (0x1000ull * cd->line); 1640 cvmx_write_csr(en_addr, mask); 1641 1642 } 1643 1644 static void octeon_irq_ciu2_enable(struct irq_data *data) 1645 { 1646 u64 mask; 1647 u64 en_addr; 1648 int cpu = next_cpu_for_irq(data); 1649 int coreid = octeon_coreid_for_cpu(cpu); 1650 struct octeon_ciu_chip_data *cd; 1651 1652 cd = irq_data_get_irq_chip_data(data); 1653 mask = 1ull << (cd->bit); 1654 1655 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) + 1656 (0x1000ull * cd->line); 1657 cvmx_write_csr(en_addr, mask); 1658 } 1659 1660 static void octeon_irq_ciu2_enable_local(struct irq_data *data) 1661 { 1662 u64 mask; 1663 u64 en_addr; 1664 int coreid = cvmx_get_core_num(); 1665 struct octeon_ciu_chip_data *cd; 1666 1667 cd = irq_data_get_irq_chip_data(data); 1668 mask = 1ull << (cd->bit); 1669 1670 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) + 1671 (0x1000ull * cd->line); 1672 cvmx_write_csr(en_addr, mask); 1673 1674 } 1675 1676 static void octeon_irq_ciu2_disable_local(struct irq_data *data) 1677 { 1678 u64 mask; 1679 u64 en_addr; 1680 int coreid = cvmx_get_core_num(); 1681 struct octeon_ciu_chip_data *cd; 1682 1683 cd = irq_data_get_irq_chip_data(data); 1684 mask = 1ull << (cd->bit); 1685 1686 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C(coreid) + 1687 (0x1000ull * cd->line); 1688 cvmx_write_csr(en_addr, mask); 1689 1690 } 1691 1692 static void octeon_irq_ciu2_ack(struct irq_data *data) 1693 { 1694 u64 mask; 1695 u64 en_addr; 1696 int coreid = cvmx_get_core_num(); 1697 struct octeon_ciu_chip_data *cd; 1698 1699 cd = irq_data_get_irq_chip_data(data); 1700 mask = 1ull << (cd->bit); 1701 1702 en_addr = CVMX_CIU2_RAW_PPX_IP2_WRKQ(coreid) + (0x1000ull * cd->line); 1703 cvmx_write_csr(en_addr, mask); 1704 1705 } 1706 1707 static void octeon_irq_ciu2_disable_all(struct irq_data *data) 1708 { 1709 int cpu; 1710 u64 mask; 1711 struct octeon_ciu_chip_data *cd; 1712 1713 cd = irq_data_get_irq_chip_data(data); 1714 mask = 1ull << (cd->bit); 1715 1716 for_each_online_cpu(cpu) { 1717 u64 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C( 1718 octeon_coreid_for_cpu(cpu)) + (0x1000ull * cd->line); 1719 cvmx_write_csr(en_addr, mask); 1720 } 1721 } 1722 1723 static void octeon_irq_ciu2_mbox_enable_all(struct irq_data *data) 1724 { 1725 int cpu; 1726 u64 mask; 1727 1728 mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0); 1729 1730 for_each_online_cpu(cpu) { 1731 u64 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1S( 1732 octeon_coreid_for_cpu(cpu)); 1733 cvmx_write_csr(en_addr, mask); 1734 } 1735 } 1736 1737 static void octeon_irq_ciu2_mbox_disable_all(struct irq_data *data) 1738 { 1739 int cpu; 1740 u64 mask; 1741 1742 mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0); 1743 1744 for_each_online_cpu(cpu) { 1745 u64 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1C( 1746 octeon_coreid_for_cpu(cpu)); 1747 cvmx_write_csr(en_addr, mask); 1748 } 1749 } 1750 1751 static void octeon_irq_ciu2_mbox_enable_local(struct irq_data *data) 1752 { 1753 u64 mask; 1754 u64 en_addr; 1755 int coreid = cvmx_get_core_num(); 1756 1757 mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0); 1758 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1S(coreid); 1759 cvmx_write_csr(en_addr, mask); 1760 } 1761 1762 static void octeon_irq_ciu2_mbox_disable_local(struct irq_data *data) 1763 { 1764 u64 mask; 1765 u64 en_addr; 1766 int coreid = cvmx_get_core_num(); 1767 1768 mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0); 1769 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1C(coreid); 1770 cvmx_write_csr(en_addr, mask); 1771 } 1772 1773 #ifdef CONFIG_SMP 1774 static int octeon_irq_ciu2_set_affinity(struct irq_data *data, 1775 const struct cpumask *dest, bool force) 1776 { 1777 int cpu; 1778 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 1779 u64 mask; 1780 struct octeon_ciu_chip_data *cd; 1781 1782 if (!enable_one) 1783 return 0; 1784 1785 cd = irq_data_get_irq_chip_data(data); 1786 mask = 1ull << cd->bit; 1787 1788 for_each_online_cpu(cpu) { 1789 u64 en_addr; 1790 if (cpumask_test_cpu(cpu, dest) && enable_one) { 1791 enable_one = false; 1792 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S( 1793 octeon_coreid_for_cpu(cpu)) + 1794 (0x1000ull * cd->line); 1795 } else { 1796 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C( 1797 octeon_coreid_for_cpu(cpu)) + 1798 (0x1000ull * cd->line); 1799 } 1800 cvmx_write_csr(en_addr, mask); 1801 } 1802 1803 return 0; 1804 } 1805 #endif 1806 1807 static void octeon_irq_ciu2_enable_gpio(struct irq_data *data) 1808 { 1809 octeon_irq_gpio_setup(data); 1810 octeon_irq_ciu2_enable(data); 1811 } 1812 1813 static void octeon_irq_ciu2_disable_gpio(struct irq_data *data) 1814 { 1815 struct octeon_ciu_chip_data *cd; 1816 1817 cd = irq_data_get_irq_chip_data(data); 1818 1819 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0); 1820 1821 octeon_irq_ciu2_disable_all(data); 1822 } 1823 1824 static struct irq_chip octeon_irq_chip_ciu2 = { 1825 .name = "CIU2-E", 1826 .irq_enable = octeon_irq_ciu2_enable, 1827 .irq_disable = octeon_irq_ciu2_disable_all, 1828 .irq_mask = octeon_irq_ciu2_disable_local, 1829 .irq_unmask = octeon_irq_ciu2_enable, 1830 #ifdef CONFIG_SMP 1831 .irq_set_affinity = octeon_irq_ciu2_set_affinity, 1832 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1833 #endif 1834 }; 1835 1836 static struct irq_chip octeon_irq_chip_ciu2_edge = { 1837 .name = "CIU2-E", 1838 .irq_enable = octeon_irq_ciu2_enable, 1839 .irq_disable = octeon_irq_ciu2_disable_all, 1840 .irq_ack = octeon_irq_ciu2_ack, 1841 .irq_mask = octeon_irq_ciu2_disable_local, 1842 .irq_unmask = octeon_irq_ciu2_enable, 1843 #ifdef CONFIG_SMP 1844 .irq_set_affinity = octeon_irq_ciu2_set_affinity, 1845 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1846 #endif 1847 }; 1848 1849 static struct irq_chip octeon_irq_chip_ciu2_mbox = { 1850 .name = "CIU2-M", 1851 .irq_enable = octeon_irq_ciu2_mbox_enable_all, 1852 .irq_disable = octeon_irq_ciu2_mbox_disable_all, 1853 .irq_ack = octeon_irq_ciu2_mbox_disable_local, 1854 .irq_eoi = octeon_irq_ciu2_mbox_enable_local, 1855 1856 .irq_cpu_online = octeon_irq_ciu2_mbox_enable_local, 1857 .irq_cpu_offline = octeon_irq_ciu2_mbox_disable_local, 1858 .flags = IRQCHIP_ONOFFLINE_ENABLED, 1859 }; 1860 1861 static struct irq_chip octeon_irq_chip_ciu2_wd = { 1862 .name = "CIU2-W", 1863 .irq_enable = octeon_irq_ciu2_wd_enable, 1864 .irq_disable = octeon_irq_ciu2_disable_all, 1865 .irq_mask = octeon_irq_ciu2_disable_local, 1866 .irq_unmask = octeon_irq_ciu2_enable_local, 1867 }; 1868 1869 static struct irq_chip octeon_irq_chip_ciu2_gpio = { 1870 .name = "CIU-GPIO", 1871 .irq_enable = octeon_irq_ciu2_enable_gpio, 1872 .irq_disable = octeon_irq_ciu2_disable_gpio, 1873 .irq_ack = octeon_irq_ciu_gpio_ack, 1874 .irq_mask = octeon_irq_ciu2_disable_local, 1875 .irq_unmask = octeon_irq_ciu2_enable, 1876 .irq_set_type = octeon_irq_ciu_gpio_set_type, 1877 #ifdef CONFIG_SMP 1878 .irq_set_affinity = octeon_irq_ciu2_set_affinity, 1879 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1880 #endif 1881 .flags = IRQCHIP_SET_TYPE_MASKED, 1882 }; 1883 1884 static int octeon_irq_ciu2_xlat(struct irq_domain *d, 1885 struct device_node *node, 1886 const u32 *intspec, 1887 unsigned int intsize, 1888 unsigned long *out_hwirq, 1889 unsigned int *out_type) 1890 { 1891 unsigned int ciu, bit; 1892 1893 ciu = intspec[0]; 1894 bit = intspec[1]; 1895 1896 *out_hwirq = (ciu << 6) | bit; 1897 *out_type = 0; 1898 1899 return 0; 1900 } 1901 1902 static bool octeon_irq_ciu2_is_edge(unsigned int line, unsigned int bit) 1903 { 1904 bool edge = false; 1905 1906 if (line == 3) /* MIO */ 1907 switch (bit) { 1908 case 2: /* IPD_DRP */ 1909 case 8 ... 11: /* Timers */ 1910 case 48: /* PTP */ 1911 edge = true; 1912 break; 1913 default: 1914 break; 1915 } 1916 else if (line == 6) /* PKT */ 1917 switch (bit) { 1918 case 52 ... 53: /* ILK_DRP */ 1919 case 8 ... 12: /* GMX_DRP */ 1920 edge = true; 1921 break; 1922 default: 1923 break; 1924 } 1925 return edge; 1926 } 1927 1928 static int octeon_irq_ciu2_map(struct irq_domain *d, 1929 unsigned int virq, irq_hw_number_t hw) 1930 { 1931 unsigned int line = hw >> 6; 1932 unsigned int bit = hw & 63; 1933 1934 /* 1935 * Don't map irq if it is reserved for GPIO. 1936 * (Line 7 are the GPIO lines.) 1937 */ 1938 if (line == 7) 1939 return 0; 1940 1941 if (line > 7 || octeon_irq_ciu_to_irq[line][bit] != 0) 1942 return -EINVAL; 1943 1944 if (octeon_irq_ciu2_is_edge(line, bit)) 1945 octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1946 &octeon_irq_chip_ciu2_edge, 1947 handle_edge_irq); 1948 else 1949 octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1950 &octeon_irq_chip_ciu2, 1951 handle_level_irq); 1952 1953 return 0; 1954 } 1955 1956 static struct irq_domain_ops octeon_irq_domain_ciu2_ops = { 1957 .map = octeon_irq_ciu2_map, 1958 .unmap = octeon_irq_free_cd, 1959 .xlate = octeon_irq_ciu2_xlat, 1960 }; 1961 1962 static void octeon_irq_ciu2(void) 1963 { 1964 int line; 1965 int bit; 1966 int irq; 1967 u64 src_reg, src, sum; 1968 const unsigned long core_id = cvmx_get_core_num(); 1969 1970 sum = cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(core_id)) & 0xfful; 1971 1972 if (unlikely(!sum)) 1973 goto spurious; 1974 1975 line = fls64(sum) - 1; 1976 src_reg = CVMX_CIU2_SRC_PPX_IP2_WRKQ(core_id) + (0x1000 * line); 1977 src = cvmx_read_csr(src_reg); 1978 1979 if (unlikely(!src)) 1980 goto spurious; 1981 1982 bit = fls64(src) - 1; 1983 irq = octeon_irq_ciu_to_irq[line][bit]; 1984 if (unlikely(!irq)) 1985 goto spurious; 1986 1987 do_IRQ(irq); 1988 goto out; 1989 1990 spurious: 1991 spurious_interrupt(); 1992 out: 1993 /* CN68XX pass 1.x has an errata that accessing the ACK registers 1994 can stop interrupts from propagating */ 1995 if (OCTEON_IS_MODEL(OCTEON_CN68XX)) 1996 cvmx_read_csr(CVMX_CIU2_INTR_CIU_READY); 1997 else 1998 cvmx_read_csr(CVMX_CIU2_ACK_PPX_IP2(core_id)); 1999 return; 2000 } 2001 2002 static void octeon_irq_ciu2_mbox(void) 2003 { 2004 int line; 2005 2006 const unsigned long core_id = cvmx_get_core_num(); 2007 u64 sum = cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP3(core_id)) >> 60; 2008 2009 if (unlikely(!sum)) 2010 goto spurious; 2011 2012 line = fls64(sum) - 1; 2013 2014 do_IRQ(OCTEON_IRQ_MBOX0 + line); 2015 goto out; 2016 2017 spurious: 2018 spurious_interrupt(); 2019 out: 2020 /* CN68XX pass 1.x has an errata that accessing the ACK registers 2021 can stop interrupts from propagating */ 2022 if (OCTEON_IS_MODEL(OCTEON_CN68XX)) 2023 cvmx_read_csr(CVMX_CIU2_INTR_CIU_READY); 2024 else 2025 cvmx_read_csr(CVMX_CIU2_ACK_PPX_IP3(core_id)); 2026 return; 2027 } 2028 2029 static int __init octeon_irq_init_ciu2( 2030 struct device_node *ciu_node, struct device_node *parent) 2031 { 2032 unsigned int i, r; 2033 struct irq_domain *ciu_domain = NULL; 2034 2035 octeon_irq_init_ciu2_percpu(); 2036 octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu2; 2037 2038 octeon_irq_gpio_chip = &octeon_irq_chip_ciu2_gpio; 2039 octeon_irq_ip2 = octeon_irq_ciu2; 2040 octeon_irq_ip3 = octeon_irq_ciu2_mbox; 2041 octeon_irq_ip4 = octeon_irq_ip4_mask; 2042 2043 /* Mips internal */ 2044 octeon_irq_init_core(); 2045 2046 ciu_domain = irq_domain_add_tree( 2047 ciu_node, &octeon_irq_domain_ciu2_ops, NULL); 2048 irq_set_default_host(ciu_domain); 2049 2050 /* CUI2 */ 2051 for (i = 0; i < 64; i++) { 2052 r = octeon_irq_force_ciu_mapping( 2053 ciu_domain, i + OCTEON_IRQ_WORKQ0, 0, i); 2054 if (r) 2055 goto err; 2056 } 2057 2058 for (i = 0; i < 32; i++) { 2059 r = octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_WDOG0, 1, i, 0, 2060 &octeon_irq_chip_ciu2_wd, handle_level_irq); 2061 if (r) 2062 goto err; 2063 } 2064 2065 for (i = 0; i < 4; i++) { 2066 r = octeon_irq_force_ciu_mapping( 2067 ciu_domain, i + OCTEON_IRQ_TIMER0, 3, i + 8); 2068 if (r) 2069 goto err; 2070 } 2071 2072 for (i = 0; i < 4; i++) { 2073 r = octeon_irq_force_ciu_mapping( 2074 ciu_domain, i + OCTEON_IRQ_PCI_INT0, 4, i); 2075 if (r) 2076 goto err; 2077 } 2078 2079 for (i = 0; i < 4; i++) { 2080 r = octeon_irq_force_ciu_mapping( 2081 ciu_domain, i + OCTEON_IRQ_PCI_MSI0, 4, i + 8); 2082 if (r) 2083 goto err; 2084 } 2085 2086 irq_set_chip_and_handler(OCTEON_IRQ_MBOX0, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq); 2087 irq_set_chip_and_handler(OCTEON_IRQ_MBOX1, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq); 2088 irq_set_chip_and_handler(OCTEON_IRQ_MBOX2, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq); 2089 irq_set_chip_and_handler(OCTEON_IRQ_MBOX3, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq); 2090 2091 /* Enable the CIU lines */ 2092 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 2093 clear_c0_status(STATUSF_IP4); 2094 return 0; 2095 err: 2096 return r; 2097 } 2098 2099 struct octeon_irq_cib_host_data { 2100 raw_spinlock_t lock; 2101 u64 raw_reg; 2102 u64 en_reg; 2103 int max_bits; 2104 }; 2105 2106 struct octeon_irq_cib_chip_data { 2107 struct octeon_irq_cib_host_data *host_data; 2108 int bit; 2109 }; 2110 2111 static void octeon_irq_cib_enable(struct irq_data *data) 2112 { 2113 unsigned long flags; 2114 u64 en; 2115 struct octeon_irq_cib_chip_data *cd = irq_data_get_irq_chip_data(data); 2116 struct octeon_irq_cib_host_data *host_data = cd->host_data; 2117 2118 raw_spin_lock_irqsave(&host_data->lock, flags); 2119 en = cvmx_read_csr(host_data->en_reg); 2120 en |= 1ull << cd->bit; 2121 cvmx_write_csr(host_data->en_reg, en); 2122 raw_spin_unlock_irqrestore(&host_data->lock, flags); 2123 } 2124 2125 static void octeon_irq_cib_disable(struct irq_data *data) 2126 { 2127 unsigned long flags; 2128 u64 en; 2129 struct octeon_irq_cib_chip_data *cd = irq_data_get_irq_chip_data(data); 2130 struct octeon_irq_cib_host_data *host_data = cd->host_data; 2131 2132 raw_spin_lock_irqsave(&host_data->lock, flags); 2133 en = cvmx_read_csr(host_data->en_reg); 2134 en &= ~(1ull << cd->bit); 2135 cvmx_write_csr(host_data->en_reg, en); 2136 raw_spin_unlock_irqrestore(&host_data->lock, flags); 2137 } 2138 2139 static int octeon_irq_cib_set_type(struct irq_data *data, unsigned int t) 2140 { 2141 irqd_set_trigger_type(data, t); 2142 return IRQ_SET_MASK_OK; 2143 } 2144 2145 static struct irq_chip octeon_irq_chip_cib = { 2146 .name = "CIB", 2147 .irq_enable = octeon_irq_cib_enable, 2148 .irq_disable = octeon_irq_cib_disable, 2149 .irq_mask = octeon_irq_cib_disable, 2150 .irq_unmask = octeon_irq_cib_enable, 2151 .irq_set_type = octeon_irq_cib_set_type, 2152 }; 2153 2154 static int octeon_irq_cib_xlat(struct irq_domain *d, 2155 struct device_node *node, 2156 const u32 *intspec, 2157 unsigned int intsize, 2158 unsigned long *out_hwirq, 2159 unsigned int *out_type) 2160 { 2161 unsigned int type = 0; 2162 2163 if (intsize == 2) 2164 type = intspec[1]; 2165 2166 switch (type) { 2167 case 0: /* unofficial value, but we might as well let it work. */ 2168 case 4: /* official value for level triggering. */ 2169 *out_type = IRQ_TYPE_LEVEL_HIGH; 2170 break; 2171 case 1: /* official value for edge triggering. */ 2172 *out_type = IRQ_TYPE_EDGE_RISING; 2173 break; 2174 default: /* Nothing else is acceptable. */ 2175 return -EINVAL; 2176 } 2177 2178 *out_hwirq = intspec[0]; 2179 2180 return 0; 2181 } 2182 2183 static int octeon_irq_cib_map(struct irq_domain *d, 2184 unsigned int virq, irq_hw_number_t hw) 2185 { 2186 struct octeon_irq_cib_host_data *host_data = d->host_data; 2187 struct octeon_irq_cib_chip_data *cd; 2188 2189 if (hw >= host_data->max_bits) { 2190 pr_err("ERROR: %s mapping %u is to big!\n", 2191 irq_domain_get_of_node(d)->name, (unsigned)hw); 2192 return -EINVAL; 2193 } 2194 2195 cd = kzalloc(sizeof(*cd), GFP_KERNEL); 2196 cd->host_data = host_data; 2197 cd->bit = hw; 2198 2199 irq_set_chip_and_handler(virq, &octeon_irq_chip_cib, 2200 handle_simple_irq); 2201 irq_set_chip_data(virq, cd); 2202 return 0; 2203 } 2204 2205 static struct irq_domain_ops octeon_irq_domain_cib_ops = { 2206 .map = octeon_irq_cib_map, 2207 .unmap = octeon_irq_free_cd, 2208 .xlate = octeon_irq_cib_xlat, 2209 }; 2210 2211 /* Chain to real handler. */ 2212 static irqreturn_t octeon_irq_cib_handler(int my_irq, void *data) 2213 { 2214 u64 en; 2215 u64 raw; 2216 u64 bits; 2217 int i; 2218 int irq; 2219 struct irq_domain *cib_domain = data; 2220 struct octeon_irq_cib_host_data *host_data = cib_domain->host_data; 2221 2222 en = cvmx_read_csr(host_data->en_reg); 2223 raw = cvmx_read_csr(host_data->raw_reg); 2224 2225 bits = en & raw; 2226 2227 for (i = 0; i < host_data->max_bits; i++) { 2228 if ((bits & 1ull << i) == 0) 2229 continue; 2230 irq = irq_find_mapping(cib_domain, i); 2231 if (!irq) { 2232 unsigned long flags; 2233 2234 pr_err("ERROR: CIB bit %d@%llx IRQ unhandled, disabling\n", 2235 i, host_data->raw_reg); 2236 raw_spin_lock_irqsave(&host_data->lock, flags); 2237 en = cvmx_read_csr(host_data->en_reg); 2238 en &= ~(1ull << i); 2239 cvmx_write_csr(host_data->en_reg, en); 2240 cvmx_write_csr(host_data->raw_reg, 1ull << i); 2241 raw_spin_unlock_irqrestore(&host_data->lock, flags); 2242 } else { 2243 struct irq_desc *desc = irq_to_desc(irq); 2244 struct irq_data *irq_data = irq_desc_get_irq_data(desc); 2245 /* If edge, acknowledge the bit we will be sending. */ 2246 if (irqd_get_trigger_type(irq_data) & 2247 IRQ_TYPE_EDGE_BOTH) 2248 cvmx_write_csr(host_data->raw_reg, 1ull << i); 2249 generic_handle_irq_desc(desc); 2250 } 2251 } 2252 2253 return IRQ_HANDLED; 2254 } 2255 2256 static int __init octeon_irq_init_cib(struct device_node *ciu_node, 2257 struct device_node *parent) 2258 { 2259 const __be32 *addr; 2260 u32 val; 2261 struct octeon_irq_cib_host_data *host_data; 2262 int parent_irq; 2263 int r; 2264 struct irq_domain *cib_domain; 2265 2266 parent_irq = irq_of_parse_and_map(ciu_node, 0); 2267 if (!parent_irq) { 2268 pr_err("ERROR: Couldn't acquire parent_irq for %s\n.", 2269 ciu_node->name); 2270 return -EINVAL; 2271 } 2272 2273 host_data = kzalloc(sizeof(*host_data), GFP_KERNEL); 2274 raw_spin_lock_init(&host_data->lock); 2275 2276 addr = of_get_address(ciu_node, 0, NULL, NULL); 2277 if (!addr) { 2278 pr_err("ERROR: Couldn't acquire reg(0) %s\n.", ciu_node->name); 2279 return -EINVAL; 2280 } 2281 host_data->raw_reg = (u64)phys_to_virt( 2282 of_translate_address(ciu_node, addr)); 2283 2284 addr = of_get_address(ciu_node, 1, NULL, NULL); 2285 if (!addr) { 2286 pr_err("ERROR: Couldn't acquire reg(1) %s\n.", ciu_node->name); 2287 return -EINVAL; 2288 } 2289 host_data->en_reg = (u64)phys_to_virt( 2290 of_translate_address(ciu_node, addr)); 2291 2292 r = of_property_read_u32(ciu_node, "cavium,max-bits", &val); 2293 if (r) { 2294 pr_err("ERROR: Couldn't read cavium,max-bits from %s\n.", 2295 ciu_node->name); 2296 return r; 2297 } 2298 host_data->max_bits = val; 2299 2300 cib_domain = irq_domain_add_linear(ciu_node, host_data->max_bits, 2301 &octeon_irq_domain_cib_ops, 2302 host_data); 2303 if (!cib_domain) { 2304 pr_err("ERROR: Couldn't irq_domain_add_linear()\n."); 2305 return -ENOMEM; 2306 } 2307 2308 cvmx_write_csr(host_data->en_reg, 0); /* disable all IRQs */ 2309 cvmx_write_csr(host_data->raw_reg, ~0); /* ack any outstanding */ 2310 2311 r = request_irq(parent_irq, octeon_irq_cib_handler, 2312 IRQF_NO_THREAD, "cib", cib_domain); 2313 if (r) { 2314 pr_err("request_irq cib failed %d\n", r); 2315 return r; 2316 } 2317 pr_info("CIB interrupt controller probed: %llx %d\n", 2318 host_data->raw_reg, host_data->max_bits); 2319 return 0; 2320 } 2321 2322 int octeon_irq_ciu3_xlat(struct irq_domain *d, 2323 struct device_node *node, 2324 const u32 *intspec, 2325 unsigned int intsize, 2326 unsigned long *out_hwirq, 2327 unsigned int *out_type) 2328 { 2329 struct octeon_ciu3_info *ciu3_info = d->host_data; 2330 unsigned int hwirq, type, intsn_major; 2331 union cvmx_ciu3_iscx_ctl isc; 2332 2333 if (intsize < 2) 2334 return -EINVAL; 2335 hwirq = intspec[0]; 2336 type = intspec[1]; 2337 2338 if (hwirq >= (1 << 20)) 2339 return -EINVAL; 2340 2341 intsn_major = hwirq >> 12; 2342 switch (intsn_major) { 2343 case 0x04: /* Software handled separately. */ 2344 return -EINVAL; 2345 default: 2346 break; 2347 } 2348 2349 isc.u64 = cvmx_read_csr(ciu3_info->ciu3_addr + CIU3_ISC_CTL(hwirq)); 2350 if (!isc.s.imp) 2351 return -EINVAL; 2352 2353 switch (type) { 2354 case 4: /* official value for level triggering. */ 2355 *out_type = IRQ_TYPE_LEVEL_HIGH; 2356 break; 2357 case 0: /* unofficial value, but we might as well let it work. */ 2358 case 1: /* official value for edge triggering. */ 2359 *out_type = IRQ_TYPE_EDGE_RISING; 2360 break; 2361 default: /* Nothing else is acceptable. */ 2362 return -EINVAL; 2363 } 2364 2365 *out_hwirq = hwirq; 2366 2367 return 0; 2368 } 2369 2370 void octeon_irq_ciu3_enable(struct irq_data *data) 2371 { 2372 int cpu; 2373 union cvmx_ciu3_iscx_ctl isc_ctl; 2374 union cvmx_ciu3_iscx_w1c isc_w1c; 2375 u64 isc_ctl_addr; 2376 2377 struct octeon_ciu_chip_data *cd; 2378 2379 cpu = next_cpu_for_irq(data); 2380 2381 cd = irq_data_get_irq_chip_data(data); 2382 2383 isc_w1c.u64 = 0; 2384 isc_w1c.s.en = 1; 2385 cvmx_write_csr(cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn), isc_w1c.u64); 2386 2387 isc_ctl_addr = cd->ciu3_addr + CIU3_ISC_CTL(cd->intsn); 2388 isc_ctl.u64 = 0; 2389 isc_ctl.s.en = 1; 2390 isc_ctl.s.idt = per_cpu(octeon_irq_ciu3_idt_ip2, cpu); 2391 cvmx_write_csr(isc_ctl_addr, isc_ctl.u64); 2392 cvmx_read_csr(isc_ctl_addr); 2393 } 2394 2395 void octeon_irq_ciu3_disable(struct irq_data *data) 2396 { 2397 u64 isc_ctl_addr; 2398 union cvmx_ciu3_iscx_w1c isc_w1c; 2399 2400 struct octeon_ciu_chip_data *cd; 2401 2402 cd = irq_data_get_irq_chip_data(data); 2403 2404 isc_w1c.u64 = 0; 2405 isc_w1c.s.en = 1; 2406 2407 isc_ctl_addr = cd->ciu3_addr + CIU3_ISC_CTL(cd->intsn); 2408 cvmx_write_csr(cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn), isc_w1c.u64); 2409 cvmx_write_csr(isc_ctl_addr, 0); 2410 cvmx_read_csr(isc_ctl_addr); 2411 } 2412 2413 void octeon_irq_ciu3_ack(struct irq_data *data) 2414 { 2415 u64 isc_w1c_addr; 2416 union cvmx_ciu3_iscx_w1c isc_w1c; 2417 struct octeon_ciu_chip_data *cd; 2418 u32 trigger_type = irqd_get_trigger_type(data); 2419 2420 /* 2421 * We use a single irq_chip, so we have to do nothing to ack a 2422 * level interrupt. 2423 */ 2424 if (!(trigger_type & IRQ_TYPE_EDGE_BOTH)) 2425 return; 2426 2427 cd = irq_data_get_irq_chip_data(data); 2428 2429 isc_w1c.u64 = 0; 2430 isc_w1c.s.raw = 1; 2431 2432 isc_w1c_addr = cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn); 2433 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2434 cvmx_read_csr(isc_w1c_addr); 2435 } 2436 2437 void octeon_irq_ciu3_mask(struct irq_data *data) 2438 { 2439 union cvmx_ciu3_iscx_w1c isc_w1c; 2440 u64 isc_w1c_addr; 2441 struct octeon_ciu_chip_data *cd; 2442 2443 cd = irq_data_get_irq_chip_data(data); 2444 2445 isc_w1c.u64 = 0; 2446 isc_w1c.s.en = 1; 2447 2448 isc_w1c_addr = cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn); 2449 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2450 cvmx_read_csr(isc_w1c_addr); 2451 } 2452 2453 void octeon_irq_ciu3_mask_ack(struct irq_data *data) 2454 { 2455 union cvmx_ciu3_iscx_w1c isc_w1c; 2456 u64 isc_w1c_addr; 2457 struct octeon_ciu_chip_data *cd; 2458 u32 trigger_type = irqd_get_trigger_type(data); 2459 2460 cd = irq_data_get_irq_chip_data(data); 2461 2462 isc_w1c.u64 = 0; 2463 isc_w1c.s.en = 1; 2464 2465 /* 2466 * We use a single irq_chip, so only ack an edge (!level) 2467 * interrupt. 2468 */ 2469 if (trigger_type & IRQ_TYPE_EDGE_BOTH) 2470 isc_w1c.s.raw = 1; 2471 2472 isc_w1c_addr = cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn); 2473 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2474 cvmx_read_csr(isc_w1c_addr); 2475 } 2476 2477 #ifdef CONFIG_SMP 2478 int octeon_irq_ciu3_set_affinity(struct irq_data *data, 2479 const struct cpumask *dest, bool force) 2480 { 2481 union cvmx_ciu3_iscx_ctl isc_ctl; 2482 union cvmx_ciu3_iscx_w1c isc_w1c; 2483 u64 isc_ctl_addr; 2484 int cpu; 2485 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 2486 struct octeon_ciu_chip_data *cd = irq_data_get_irq_chip_data(data); 2487 2488 if (!cpumask_subset(dest, cpumask_of_node(cd->ciu_node))) 2489 return -EINVAL; 2490 2491 if (!enable_one) 2492 return IRQ_SET_MASK_OK; 2493 2494 cd = irq_data_get_irq_chip_data(data); 2495 cpu = cpumask_first(dest); 2496 if (cpu >= nr_cpu_ids) 2497 cpu = smp_processor_id(); 2498 cd->current_cpu = cpu; 2499 2500 isc_w1c.u64 = 0; 2501 isc_w1c.s.en = 1; 2502 cvmx_write_csr(cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn), isc_w1c.u64); 2503 2504 isc_ctl_addr = cd->ciu3_addr + CIU3_ISC_CTL(cd->intsn); 2505 isc_ctl.u64 = 0; 2506 isc_ctl.s.en = 1; 2507 isc_ctl.s.idt = per_cpu(octeon_irq_ciu3_idt_ip2, cpu); 2508 cvmx_write_csr(isc_ctl_addr, isc_ctl.u64); 2509 cvmx_read_csr(isc_ctl_addr); 2510 2511 return IRQ_SET_MASK_OK; 2512 } 2513 #endif 2514 2515 static struct irq_chip octeon_irq_chip_ciu3 = { 2516 .name = "CIU3", 2517 .irq_startup = edge_startup, 2518 .irq_enable = octeon_irq_ciu3_enable, 2519 .irq_disable = octeon_irq_ciu3_disable, 2520 .irq_ack = octeon_irq_ciu3_ack, 2521 .irq_mask = octeon_irq_ciu3_mask, 2522 .irq_mask_ack = octeon_irq_ciu3_mask_ack, 2523 .irq_unmask = octeon_irq_ciu3_enable, 2524 .irq_set_type = octeon_irq_ciu_set_type, 2525 #ifdef CONFIG_SMP 2526 .irq_set_affinity = octeon_irq_ciu3_set_affinity, 2527 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 2528 #endif 2529 }; 2530 2531 int octeon_irq_ciu3_mapx(struct irq_domain *d, unsigned int virq, 2532 irq_hw_number_t hw, struct irq_chip *chip) 2533 { 2534 struct octeon_ciu3_info *ciu3_info = d->host_data; 2535 struct octeon_ciu_chip_data *cd = kzalloc_node(sizeof(*cd), GFP_KERNEL, 2536 ciu3_info->node); 2537 if (!cd) 2538 return -ENOMEM; 2539 cd->intsn = hw; 2540 cd->current_cpu = -1; 2541 cd->ciu3_addr = ciu3_info->ciu3_addr; 2542 cd->ciu_node = ciu3_info->node; 2543 irq_set_chip_and_handler(virq, chip, handle_edge_irq); 2544 irq_set_chip_data(virq, cd); 2545 2546 return 0; 2547 } 2548 2549 static int octeon_irq_ciu3_map(struct irq_domain *d, 2550 unsigned int virq, irq_hw_number_t hw) 2551 { 2552 return octeon_irq_ciu3_mapx(d, virq, hw, &octeon_irq_chip_ciu3); 2553 } 2554 2555 static struct irq_domain_ops octeon_dflt_domain_ciu3_ops = { 2556 .map = octeon_irq_ciu3_map, 2557 .unmap = octeon_irq_free_cd, 2558 .xlate = octeon_irq_ciu3_xlat, 2559 }; 2560 2561 static void octeon_irq_ciu3_ip2(void) 2562 { 2563 union cvmx_ciu3_destx_pp_int dest_pp_int; 2564 struct octeon_ciu3_info *ciu3_info; 2565 u64 ciu3_addr; 2566 2567 ciu3_info = __this_cpu_read(octeon_ciu3_info); 2568 ciu3_addr = ciu3_info->ciu3_addr; 2569 2570 dest_pp_int.u64 = cvmx_read_csr(ciu3_addr + CIU3_DEST_PP_INT(3 * cvmx_get_local_core_num())); 2571 2572 if (likely(dest_pp_int.s.intr)) { 2573 irq_hw_number_t intsn = dest_pp_int.s.intsn; 2574 irq_hw_number_t hw; 2575 struct irq_domain *domain; 2576 /* Get the domain to use from the major block */ 2577 int block = intsn >> 12; 2578 int ret; 2579 2580 domain = ciu3_info->domain[block]; 2581 if (ciu3_info->intsn2hw[block]) 2582 hw = ciu3_info->intsn2hw[block](domain, intsn); 2583 else 2584 hw = intsn; 2585 2586 ret = handle_domain_irq(domain, hw, NULL); 2587 if (ret < 0) { 2588 union cvmx_ciu3_iscx_w1c isc_w1c; 2589 u64 isc_w1c_addr = ciu3_addr + CIU3_ISC_W1C(intsn); 2590 2591 isc_w1c.u64 = 0; 2592 isc_w1c.s.en = 1; 2593 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2594 cvmx_read_csr(isc_w1c_addr); 2595 spurious_interrupt(); 2596 } 2597 } else { 2598 spurious_interrupt(); 2599 } 2600 } 2601 2602 /* 2603 * 10 mbox per core starting from zero. 2604 * Base mbox is core * 10 2605 */ 2606 static unsigned int octeon_irq_ciu3_base_mbox_intsn(int core) 2607 { 2608 /* SW (mbox) are 0x04 in bits 12..19 */ 2609 return 0x04000 + CIU3_MBOX_PER_CORE * core; 2610 } 2611 2612 static unsigned int octeon_irq_ciu3_mbox_intsn_for_core(int core, unsigned int mbox) 2613 { 2614 return octeon_irq_ciu3_base_mbox_intsn(core) + mbox; 2615 } 2616 2617 static unsigned int octeon_irq_ciu3_mbox_intsn_for_cpu(int cpu, unsigned int mbox) 2618 { 2619 int local_core = octeon_coreid_for_cpu(cpu) & 0x3f; 2620 2621 return octeon_irq_ciu3_mbox_intsn_for_core(local_core, mbox); 2622 } 2623 2624 static void octeon_irq_ciu3_mbox(void) 2625 { 2626 union cvmx_ciu3_destx_pp_int dest_pp_int; 2627 struct octeon_ciu3_info *ciu3_info; 2628 u64 ciu3_addr; 2629 int core = cvmx_get_local_core_num(); 2630 2631 ciu3_info = __this_cpu_read(octeon_ciu3_info); 2632 ciu3_addr = ciu3_info->ciu3_addr; 2633 2634 dest_pp_int.u64 = cvmx_read_csr(ciu3_addr + CIU3_DEST_PP_INT(1 + 3 * core)); 2635 2636 if (likely(dest_pp_int.s.intr)) { 2637 irq_hw_number_t intsn = dest_pp_int.s.intsn; 2638 int mbox = intsn - octeon_irq_ciu3_base_mbox_intsn(core); 2639 2640 if (likely(mbox >= 0 && mbox < CIU3_MBOX_PER_CORE)) { 2641 do_IRQ(mbox + OCTEON_IRQ_MBOX0); 2642 } else { 2643 union cvmx_ciu3_iscx_w1c isc_w1c; 2644 u64 isc_w1c_addr = ciu3_addr + CIU3_ISC_W1C(intsn); 2645 2646 isc_w1c.u64 = 0; 2647 isc_w1c.s.en = 1; 2648 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2649 cvmx_read_csr(isc_w1c_addr); 2650 spurious_interrupt(); 2651 } 2652 } else { 2653 spurious_interrupt(); 2654 } 2655 } 2656 2657 void octeon_ciu3_mbox_send(int cpu, unsigned int mbox) 2658 { 2659 struct octeon_ciu3_info *ciu3_info; 2660 unsigned int intsn; 2661 union cvmx_ciu3_iscx_w1s isc_w1s; 2662 u64 isc_w1s_addr; 2663 2664 if (WARN_ON_ONCE(mbox >= CIU3_MBOX_PER_CORE)) 2665 return; 2666 2667 intsn = octeon_irq_ciu3_mbox_intsn_for_cpu(cpu, mbox); 2668 ciu3_info = per_cpu(octeon_ciu3_info, cpu); 2669 isc_w1s_addr = ciu3_info->ciu3_addr + CIU3_ISC_W1S(intsn); 2670 2671 isc_w1s.u64 = 0; 2672 isc_w1s.s.raw = 1; 2673 2674 cvmx_write_csr(isc_w1s_addr, isc_w1s.u64); 2675 cvmx_read_csr(isc_w1s_addr); 2676 } 2677 2678 static void octeon_irq_ciu3_mbox_set_enable(struct irq_data *data, int cpu, bool en) 2679 { 2680 struct octeon_ciu3_info *ciu3_info; 2681 unsigned int intsn; 2682 u64 isc_ctl_addr, isc_w1c_addr; 2683 union cvmx_ciu3_iscx_ctl isc_ctl; 2684 unsigned int mbox = data->irq - OCTEON_IRQ_MBOX0; 2685 2686 intsn = octeon_irq_ciu3_mbox_intsn_for_cpu(cpu, mbox); 2687 ciu3_info = per_cpu(octeon_ciu3_info, cpu); 2688 isc_w1c_addr = ciu3_info->ciu3_addr + CIU3_ISC_W1C(intsn); 2689 isc_ctl_addr = ciu3_info->ciu3_addr + CIU3_ISC_CTL(intsn); 2690 2691 isc_ctl.u64 = 0; 2692 isc_ctl.s.en = 1; 2693 2694 cvmx_write_csr(isc_w1c_addr, isc_ctl.u64); 2695 cvmx_write_csr(isc_ctl_addr, 0); 2696 if (en) { 2697 unsigned int idt = per_cpu(octeon_irq_ciu3_idt_ip3, cpu); 2698 2699 isc_ctl.u64 = 0; 2700 isc_ctl.s.en = 1; 2701 isc_ctl.s.idt = idt; 2702 cvmx_write_csr(isc_ctl_addr, isc_ctl.u64); 2703 } 2704 cvmx_read_csr(isc_ctl_addr); 2705 } 2706 2707 static void octeon_irq_ciu3_mbox_enable(struct irq_data *data) 2708 { 2709 int cpu; 2710 unsigned int mbox = data->irq - OCTEON_IRQ_MBOX0; 2711 2712 WARN_ON(mbox >= CIU3_MBOX_PER_CORE); 2713 2714 for_each_online_cpu(cpu) 2715 octeon_irq_ciu3_mbox_set_enable(data, cpu, true); 2716 } 2717 2718 static void octeon_irq_ciu3_mbox_disable(struct irq_data *data) 2719 { 2720 int cpu; 2721 unsigned int mbox = data->irq - OCTEON_IRQ_MBOX0; 2722 2723 WARN_ON(mbox >= CIU3_MBOX_PER_CORE); 2724 2725 for_each_online_cpu(cpu) 2726 octeon_irq_ciu3_mbox_set_enable(data, cpu, false); 2727 } 2728 2729 static void octeon_irq_ciu3_mbox_ack(struct irq_data *data) 2730 { 2731 struct octeon_ciu3_info *ciu3_info; 2732 unsigned int intsn; 2733 u64 isc_w1c_addr; 2734 union cvmx_ciu3_iscx_w1c isc_w1c; 2735 unsigned int mbox = data->irq - OCTEON_IRQ_MBOX0; 2736 2737 intsn = octeon_irq_ciu3_mbox_intsn_for_core(cvmx_get_local_core_num(), mbox); 2738 2739 isc_w1c.u64 = 0; 2740 isc_w1c.s.raw = 1; 2741 2742 ciu3_info = __this_cpu_read(octeon_ciu3_info); 2743 isc_w1c_addr = ciu3_info->ciu3_addr + CIU3_ISC_W1C(intsn); 2744 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2745 cvmx_read_csr(isc_w1c_addr); 2746 } 2747 2748 static void octeon_irq_ciu3_mbox_cpu_online(struct irq_data *data) 2749 { 2750 octeon_irq_ciu3_mbox_set_enable(data, smp_processor_id(), true); 2751 } 2752 2753 static void octeon_irq_ciu3_mbox_cpu_offline(struct irq_data *data) 2754 { 2755 octeon_irq_ciu3_mbox_set_enable(data, smp_processor_id(), false); 2756 } 2757 2758 static int octeon_irq_ciu3_alloc_resources(struct octeon_ciu3_info *ciu3_info) 2759 { 2760 u64 b = ciu3_info->ciu3_addr; 2761 int idt_ip2, idt_ip3, idt_ip4; 2762 int unused_idt2; 2763 int core = cvmx_get_local_core_num(); 2764 int i; 2765 2766 __this_cpu_write(octeon_ciu3_info, ciu3_info); 2767 2768 /* 2769 * 4 idt per core starting from 1 because zero is reserved. 2770 * Base idt per core is 4 * core + 1 2771 */ 2772 idt_ip2 = core * 4 + 1; 2773 idt_ip3 = core * 4 + 2; 2774 idt_ip4 = core * 4 + 3; 2775 unused_idt2 = core * 4 + 4; 2776 __this_cpu_write(octeon_irq_ciu3_idt_ip2, idt_ip2); 2777 __this_cpu_write(octeon_irq_ciu3_idt_ip3, idt_ip3); 2778 2779 /* ip2 interrupts for this CPU */ 2780 cvmx_write_csr(b + CIU3_IDT_CTL(idt_ip2), 0); 2781 cvmx_write_csr(b + CIU3_IDT_PP(idt_ip2, 0), 1ull << core); 2782 cvmx_write_csr(b + CIU3_IDT_IO(idt_ip2), 0); 2783 2784 /* ip3 interrupts for this CPU */ 2785 cvmx_write_csr(b + CIU3_IDT_CTL(idt_ip3), 1); 2786 cvmx_write_csr(b + CIU3_IDT_PP(idt_ip3, 0), 1ull << core); 2787 cvmx_write_csr(b + CIU3_IDT_IO(idt_ip3), 0); 2788 2789 /* ip4 interrupts for this CPU */ 2790 cvmx_write_csr(b + CIU3_IDT_CTL(idt_ip4), 2); 2791 cvmx_write_csr(b + CIU3_IDT_PP(idt_ip4, 0), 0); 2792 cvmx_write_csr(b + CIU3_IDT_IO(idt_ip4), 0); 2793 2794 cvmx_write_csr(b + CIU3_IDT_CTL(unused_idt2), 0); 2795 cvmx_write_csr(b + CIU3_IDT_PP(unused_idt2, 0), 0); 2796 cvmx_write_csr(b + CIU3_IDT_IO(unused_idt2), 0); 2797 2798 for (i = 0; i < CIU3_MBOX_PER_CORE; i++) { 2799 unsigned int intsn = octeon_irq_ciu3_mbox_intsn_for_core(core, i); 2800 2801 cvmx_write_csr(b + CIU3_ISC_W1C(intsn), 2); 2802 cvmx_write_csr(b + CIU3_ISC_CTL(intsn), 0); 2803 } 2804 2805 return 0; 2806 } 2807 2808 static void octeon_irq_setup_secondary_ciu3(void) 2809 { 2810 struct octeon_ciu3_info *ciu3_info; 2811 2812 ciu3_info = octeon_ciu3_info_per_node[cvmx_get_node_num()]; 2813 octeon_irq_ciu3_alloc_resources(ciu3_info); 2814 irq_cpu_online(); 2815 2816 /* Enable the CIU lines */ 2817 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 2818 if (octeon_irq_use_ip4) 2819 set_c0_status(STATUSF_IP4); 2820 else 2821 clear_c0_status(STATUSF_IP4); 2822 } 2823 2824 static struct irq_chip octeon_irq_chip_ciu3_mbox = { 2825 .name = "CIU3-M", 2826 .irq_enable = octeon_irq_ciu3_mbox_enable, 2827 .irq_disable = octeon_irq_ciu3_mbox_disable, 2828 .irq_ack = octeon_irq_ciu3_mbox_ack, 2829 2830 .irq_cpu_online = octeon_irq_ciu3_mbox_cpu_online, 2831 .irq_cpu_offline = octeon_irq_ciu3_mbox_cpu_offline, 2832 .flags = IRQCHIP_ONOFFLINE_ENABLED, 2833 }; 2834 2835 static int __init octeon_irq_init_ciu3(struct device_node *ciu_node, 2836 struct device_node *parent) 2837 { 2838 int i; 2839 int node; 2840 struct irq_domain *domain; 2841 struct octeon_ciu3_info *ciu3_info; 2842 const __be32 *zero_addr; 2843 u64 base_addr; 2844 union cvmx_ciu3_const consts; 2845 2846 node = 0; /* of_node_to_nid(ciu_node); */ 2847 ciu3_info = kzalloc_node(sizeof(*ciu3_info), GFP_KERNEL, node); 2848 2849 if (!ciu3_info) 2850 return -ENOMEM; 2851 2852 zero_addr = of_get_address(ciu_node, 0, NULL, NULL); 2853 if (WARN_ON(!zero_addr)) 2854 return -EINVAL; 2855 2856 base_addr = of_translate_address(ciu_node, zero_addr); 2857 base_addr = (u64)phys_to_virt(base_addr); 2858 2859 ciu3_info->ciu3_addr = base_addr; 2860 ciu3_info->node = node; 2861 2862 consts.u64 = cvmx_read_csr(base_addr + CIU3_CONST); 2863 2864 octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu3; 2865 2866 octeon_irq_ip2 = octeon_irq_ciu3_ip2; 2867 octeon_irq_ip3 = octeon_irq_ciu3_mbox; 2868 octeon_irq_ip4 = octeon_irq_ip4_mask; 2869 2870 if (node == cvmx_get_node_num()) { 2871 /* Mips internal */ 2872 octeon_irq_init_core(); 2873 2874 /* Only do per CPU things if it is the CIU of the boot node. */ 2875 i = irq_alloc_descs_from(OCTEON_IRQ_MBOX0, 8, node); 2876 WARN_ON(i < 0); 2877 2878 for (i = 0; i < 8; i++) 2879 irq_set_chip_and_handler(i + OCTEON_IRQ_MBOX0, 2880 &octeon_irq_chip_ciu3_mbox, handle_percpu_irq); 2881 } 2882 2883 /* 2884 * Initialize all domains to use the default domain. Specific major 2885 * blocks will overwrite the default domain as needed. 2886 */ 2887 domain = irq_domain_add_tree(ciu_node, &octeon_dflt_domain_ciu3_ops, 2888 ciu3_info); 2889 for (i = 0; i < MAX_CIU3_DOMAINS; i++) 2890 ciu3_info->domain[i] = domain; 2891 2892 octeon_ciu3_info_per_node[node] = ciu3_info; 2893 2894 if (node == cvmx_get_node_num()) { 2895 /* Only do per CPU things if it is the CIU of the boot node. */ 2896 octeon_irq_ciu3_alloc_resources(ciu3_info); 2897 if (node == 0) 2898 irq_set_default_host(domain); 2899 2900 octeon_irq_use_ip4 = false; 2901 /* Enable the CIU lines */ 2902 set_c0_status(STATUSF_IP2 | STATUSF_IP3); 2903 clear_c0_status(STATUSF_IP4); 2904 } 2905 2906 return 0; 2907 } 2908 2909 static struct of_device_id ciu_types[] __initdata = { 2910 {.compatible = "cavium,octeon-3860-ciu", .data = octeon_irq_init_ciu}, 2911 {.compatible = "cavium,octeon-3860-gpio", .data = octeon_irq_init_gpio}, 2912 {.compatible = "cavium,octeon-6880-ciu2", .data = octeon_irq_init_ciu2}, 2913 {.compatible = "cavium,octeon-7890-ciu3", .data = octeon_irq_init_ciu3}, 2914 {.compatible = "cavium,octeon-7130-cib", .data = octeon_irq_init_cib}, 2915 {} 2916 }; 2917 2918 void __init arch_init_irq(void) 2919 { 2920 #ifdef CONFIG_SMP 2921 /* Set the default affinity to the boot cpu. */ 2922 cpumask_clear(irq_default_affinity); 2923 cpumask_set_cpu(smp_processor_id(), irq_default_affinity); 2924 #endif 2925 of_irq_init(ciu_types); 2926 } 2927 2928 asmlinkage void plat_irq_dispatch(void) 2929 { 2930 unsigned long cop0_cause; 2931 unsigned long cop0_status; 2932 2933 while (1) { 2934 cop0_cause = read_c0_cause(); 2935 cop0_status = read_c0_status(); 2936 cop0_cause &= cop0_status; 2937 cop0_cause &= ST0_IM; 2938 2939 if (cop0_cause & STATUSF_IP2) 2940 octeon_irq_ip2(); 2941 else if (cop0_cause & STATUSF_IP3) 2942 octeon_irq_ip3(); 2943 else if (cop0_cause & STATUSF_IP4) 2944 octeon_irq_ip4(); 2945 else if (cop0_cause) 2946 do_IRQ(fls(cop0_cause) - 9 + MIPS_CPU_IRQ_BASE); 2947 else 2948 break; 2949 } 2950 } 2951 2952 #ifdef CONFIG_HOTPLUG_CPU 2953 2954 void octeon_fixup_irqs(void) 2955 { 2956 irq_cpu_offline(); 2957 } 2958 2959 #endif /* CONFIG_HOTPLUG_CPU */ 2960