1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 2004-2016 Cavium, Inc. 7 */ 8 9 #include <linux/of_address.h> 10 #include <linux/interrupt.h> 11 #include <linux/irqdomain.h> 12 #include <linux/bitops.h> 13 #include <linux/of_irq.h> 14 #include <linux/percpu.h> 15 #include <linux/slab.h> 16 #include <linux/irq.h> 17 #include <linux/smp.h> 18 #include <linux/of.h> 19 20 #include <asm/octeon/octeon.h> 21 #include <asm/octeon/cvmx-ciu2-defs.h> 22 #include <asm/octeon/cvmx-ciu3-defs.h> 23 24 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu0_en_mirror); 25 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu1_en_mirror); 26 static DEFINE_PER_CPU(raw_spinlock_t, octeon_irq_ciu_spinlock); 27 static DEFINE_PER_CPU(unsigned int, octeon_irq_ciu3_idt_ip2); 28 29 static DEFINE_PER_CPU(unsigned int, octeon_irq_ciu3_idt_ip3); 30 static DEFINE_PER_CPU(struct octeon_ciu3_info *, octeon_ciu3_info); 31 #define CIU3_MBOX_PER_CORE 10 32 33 /* 34 * The 8 most significant bits of the intsn identify the interrupt major block. 35 * Each major block might use its own interrupt domain. Thus 256 domains are 36 * needed. 37 */ 38 #define MAX_CIU3_DOMAINS 256 39 40 typedef irq_hw_number_t (*octeon_ciu3_intsn2hw_t)(struct irq_domain *, unsigned int); 41 42 /* Information for each ciu3 in the system */ 43 struct octeon_ciu3_info { 44 u64 ciu3_addr; 45 int node; 46 struct irq_domain *domain[MAX_CIU3_DOMAINS]; 47 octeon_ciu3_intsn2hw_t intsn2hw[MAX_CIU3_DOMAINS]; 48 }; 49 50 /* Each ciu3 in the system uses its own data (one ciu3 per node) */ 51 static struct octeon_ciu3_info *octeon_ciu3_info_per_node[4]; 52 53 struct octeon_irq_ciu_domain_data { 54 int num_sum; /* number of sum registers (2 or 3). */ 55 }; 56 57 /* Register offsets from ciu3_addr */ 58 #define CIU3_CONST 0x220 59 #define CIU3_IDT_CTL(_idt) ((_idt) * 8 + 0x110000) 60 #define CIU3_IDT_PP(_idt, _idx) ((_idt) * 32 + (_idx) * 8 + 0x120000) 61 #define CIU3_IDT_IO(_idt) ((_idt) * 8 + 0x130000) 62 #define CIU3_DEST_PP_INT(_pp_ip) ((_pp_ip) * 8 + 0x200000) 63 #define CIU3_DEST_IO_INT(_io) ((_io) * 8 + 0x210000) 64 #define CIU3_ISC_CTL(_intsn) ((_intsn) * 8 + 0x80000000) 65 #define CIU3_ISC_W1C(_intsn) ((_intsn) * 8 + 0x90000000) 66 #define CIU3_ISC_W1S(_intsn) ((_intsn) * 8 + 0xa0000000) 67 68 static __read_mostly int octeon_irq_ciu_to_irq[8][64]; 69 70 struct octeon_ciu_chip_data { 71 union { 72 struct { /* only used for ciu3 */ 73 u64 ciu3_addr; 74 unsigned int intsn; 75 }; 76 struct { /* only used for ciu/ciu2 */ 77 u8 line; 78 u8 bit; 79 }; 80 }; 81 int gpio_line; 82 int current_cpu; /* Next CPU expected to take this irq */ 83 int ciu_node; /* NUMA node number of the CIU */ 84 }; 85 86 struct octeon_core_chip_data { 87 struct mutex core_irq_mutex; 88 bool current_en; 89 bool desired_en; 90 u8 bit; 91 }; 92 93 #define MIPS_CORE_IRQ_LINES 8 94 95 static struct octeon_core_chip_data octeon_irq_core_chip_data[MIPS_CORE_IRQ_LINES]; 96 97 static int octeon_irq_set_ciu_mapping(int irq, int line, int bit, int gpio_line, 98 struct irq_chip *chip, 99 irq_flow_handler_t handler) 100 { 101 struct octeon_ciu_chip_data *cd; 102 103 cd = kzalloc(sizeof(*cd), GFP_KERNEL); 104 if (!cd) 105 return -ENOMEM; 106 107 irq_set_chip_and_handler(irq, chip, handler); 108 109 cd->line = line; 110 cd->bit = bit; 111 cd->gpio_line = gpio_line; 112 113 irq_set_chip_data(irq, cd); 114 octeon_irq_ciu_to_irq[line][bit] = irq; 115 return 0; 116 } 117 118 static void octeon_irq_free_cd(struct irq_domain *d, unsigned int irq) 119 { 120 struct irq_data *data = irq_get_irq_data(irq); 121 struct octeon_ciu_chip_data *cd = irq_data_get_irq_chip_data(data); 122 123 irq_set_chip_data(irq, NULL); 124 kfree(cd); 125 } 126 127 static int octeon_irq_force_ciu_mapping(struct irq_domain *domain, 128 int irq, int line, int bit) 129 { 130 return irq_domain_associate(domain, irq, line << 6 | bit); 131 } 132 133 static int octeon_coreid_for_cpu(int cpu) 134 { 135 #ifdef CONFIG_SMP 136 return cpu_logical_map(cpu); 137 #else 138 return cvmx_get_core_num(); 139 #endif 140 } 141 142 static int octeon_cpu_for_coreid(int coreid) 143 { 144 #ifdef CONFIG_SMP 145 return cpu_number_map(coreid); 146 #else 147 return smp_processor_id(); 148 #endif 149 } 150 151 static void octeon_irq_core_ack(struct irq_data *data) 152 { 153 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 154 unsigned int bit = cd->bit; 155 156 /* 157 * We don't need to disable IRQs to make these atomic since 158 * they are already disabled earlier in the low level 159 * interrupt code. 160 */ 161 clear_c0_status(0x100 << bit); 162 /* The two user interrupts must be cleared manually. */ 163 if (bit < 2) 164 clear_c0_cause(0x100 << bit); 165 } 166 167 static void octeon_irq_core_eoi(struct irq_data *data) 168 { 169 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 170 171 /* 172 * We don't need to disable IRQs to make these atomic since 173 * they are already disabled earlier in the low level 174 * interrupt code. 175 */ 176 set_c0_status(0x100 << cd->bit); 177 } 178 179 static void octeon_irq_core_set_enable_local(void *arg) 180 { 181 struct irq_data *data = arg; 182 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 183 unsigned int mask = 0x100 << cd->bit; 184 185 /* 186 * Interrupts are already disabled, so these are atomic. 187 */ 188 if (cd->desired_en) 189 set_c0_status(mask); 190 else 191 clear_c0_status(mask); 192 193 } 194 195 static void octeon_irq_core_disable(struct irq_data *data) 196 { 197 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 198 cd->desired_en = false; 199 } 200 201 static void octeon_irq_core_enable(struct irq_data *data) 202 { 203 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 204 cd->desired_en = true; 205 } 206 207 static void octeon_irq_core_bus_lock(struct irq_data *data) 208 { 209 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 210 211 mutex_lock(&cd->core_irq_mutex); 212 } 213 214 static void octeon_irq_core_bus_sync_unlock(struct irq_data *data) 215 { 216 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 217 218 if (cd->desired_en != cd->current_en) { 219 on_each_cpu(octeon_irq_core_set_enable_local, data, 1); 220 221 cd->current_en = cd->desired_en; 222 } 223 224 mutex_unlock(&cd->core_irq_mutex); 225 } 226 227 static struct irq_chip octeon_irq_chip_core = { 228 .name = "Core", 229 .irq_enable = octeon_irq_core_enable, 230 .irq_disable = octeon_irq_core_disable, 231 .irq_ack = octeon_irq_core_ack, 232 .irq_eoi = octeon_irq_core_eoi, 233 .irq_bus_lock = octeon_irq_core_bus_lock, 234 .irq_bus_sync_unlock = octeon_irq_core_bus_sync_unlock, 235 236 .irq_cpu_online = octeon_irq_core_eoi, 237 .irq_cpu_offline = octeon_irq_core_ack, 238 .flags = IRQCHIP_ONOFFLINE_ENABLED, 239 }; 240 241 static void __init octeon_irq_init_core(void) 242 { 243 int i; 244 int irq; 245 struct octeon_core_chip_data *cd; 246 247 for (i = 0; i < MIPS_CORE_IRQ_LINES; i++) { 248 cd = &octeon_irq_core_chip_data[i]; 249 cd->current_en = false; 250 cd->desired_en = false; 251 cd->bit = i; 252 mutex_init(&cd->core_irq_mutex); 253 254 irq = OCTEON_IRQ_SW0 + i; 255 irq_set_chip_data(irq, cd); 256 irq_set_chip_and_handler(irq, &octeon_irq_chip_core, 257 handle_percpu_irq); 258 } 259 } 260 261 static int next_cpu_for_irq(struct irq_data *data) 262 { 263 264 #ifdef CONFIG_SMP 265 int cpu; 266 struct cpumask *mask = irq_data_get_affinity_mask(data); 267 int weight = cpumask_weight(mask); 268 struct octeon_ciu_chip_data *cd = irq_data_get_irq_chip_data(data); 269 270 if (weight > 1) { 271 cpu = cd->current_cpu; 272 for (;;) { 273 cpu = cpumask_next(cpu, mask); 274 if (cpu >= nr_cpu_ids) { 275 cpu = -1; 276 continue; 277 } else if (cpumask_test_cpu(cpu, cpu_online_mask)) { 278 break; 279 } 280 } 281 } else if (weight == 1) { 282 cpu = cpumask_first(mask); 283 } else { 284 cpu = smp_processor_id(); 285 } 286 cd->current_cpu = cpu; 287 return cpu; 288 #else 289 return smp_processor_id(); 290 #endif 291 } 292 293 static void octeon_irq_ciu_enable(struct irq_data *data) 294 { 295 int cpu = next_cpu_for_irq(data); 296 int coreid = octeon_coreid_for_cpu(cpu); 297 unsigned long *pen; 298 unsigned long flags; 299 struct octeon_ciu_chip_data *cd; 300 raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 301 302 cd = irq_data_get_irq_chip_data(data); 303 304 raw_spin_lock_irqsave(lock, flags); 305 if (cd->line == 0) { 306 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 307 __set_bit(cd->bit, pen); 308 /* 309 * Must be visible to octeon_irq_ip{2,3}_ciu() before 310 * enabling the irq. 311 */ 312 wmb(); 313 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 314 } else { 315 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 316 __set_bit(cd->bit, pen); 317 /* 318 * Must be visible to octeon_irq_ip{2,3}_ciu() before 319 * enabling the irq. 320 */ 321 wmb(); 322 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 323 } 324 raw_spin_unlock_irqrestore(lock, flags); 325 } 326 327 static void octeon_irq_ciu_enable_local(struct irq_data *data) 328 { 329 unsigned long *pen; 330 unsigned long flags; 331 struct octeon_ciu_chip_data *cd; 332 raw_spinlock_t *lock = this_cpu_ptr(&octeon_irq_ciu_spinlock); 333 334 cd = irq_data_get_irq_chip_data(data); 335 336 raw_spin_lock_irqsave(lock, flags); 337 if (cd->line == 0) { 338 pen = this_cpu_ptr(&octeon_irq_ciu0_en_mirror); 339 __set_bit(cd->bit, pen); 340 /* 341 * Must be visible to octeon_irq_ip{2,3}_ciu() before 342 * enabling the irq. 343 */ 344 wmb(); 345 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen); 346 } else { 347 pen = this_cpu_ptr(&octeon_irq_ciu1_en_mirror); 348 __set_bit(cd->bit, pen); 349 /* 350 * Must be visible to octeon_irq_ip{2,3}_ciu() before 351 * enabling the irq. 352 */ 353 wmb(); 354 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen); 355 } 356 raw_spin_unlock_irqrestore(lock, flags); 357 } 358 359 static void octeon_irq_ciu_disable_local(struct irq_data *data) 360 { 361 unsigned long *pen; 362 unsigned long flags; 363 struct octeon_ciu_chip_data *cd; 364 raw_spinlock_t *lock = this_cpu_ptr(&octeon_irq_ciu_spinlock); 365 366 cd = irq_data_get_irq_chip_data(data); 367 368 raw_spin_lock_irqsave(lock, flags); 369 if (cd->line == 0) { 370 pen = this_cpu_ptr(&octeon_irq_ciu0_en_mirror); 371 __clear_bit(cd->bit, pen); 372 /* 373 * Must be visible to octeon_irq_ip{2,3}_ciu() before 374 * enabling the irq. 375 */ 376 wmb(); 377 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen); 378 } else { 379 pen = this_cpu_ptr(&octeon_irq_ciu1_en_mirror); 380 __clear_bit(cd->bit, pen); 381 /* 382 * Must be visible to octeon_irq_ip{2,3}_ciu() before 383 * enabling the irq. 384 */ 385 wmb(); 386 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen); 387 } 388 raw_spin_unlock_irqrestore(lock, flags); 389 } 390 391 static void octeon_irq_ciu_disable_all(struct irq_data *data) 392 { 393 unsigned long flags; 394 unsigned long *pen; 395 int cpu; 396 struct octeon_ciu_chip_data *cd; 397 raw_spinlock_t *lock; 398 399 cd = irq_data_get_irq_chip_data(data); 400 401 for_each_online_cpu(cpu) { 402 int coreid = octeon_coreid_for_cpu(cpu); 403 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 404 if (cd->line == 0) 405 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 406 else 407 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 408 409 raw_spin_lock_irqsave(lock, flags); 410 __clear_bit(cd->bit, pen); 411 /* 412 * Must be visible to octeon_irq_ip{2,3}_ciu() before 413 * enabling the irq. 414 */ 415 wmb(); 416 if (cd->line == 0) 417 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 418 else 419 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 420 raw_spin_unlock_irqrestore(lock, flags); 421 } 422 } 423 424 static void octeon_irq_ciu_enable_all(struct irq_data *data) 425 { 426 unsigned long flags; 427 unsigned long *pen; 428 int cpu; 429 struct octeon_ciu_chip_data *cd; 430 raw_spinlock_t *lock; 431 432 cd = irq_data_get_irq_chip_data(data); 433 434 for_each_online_cpu(cpu) { 435 int coreid = octeon_coreid_for_cpu(cpu); 436 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 437 if (cd->line == 0) 438 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 439 else 440 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 441 442 raw_spin_lock_irqsave(lock, flags); 443 __set_bit(cd->bit, pen); 444 /* 445 * Must be visible to octeon_irq_ip{2,3}_ciu() before 446 * enabling the irq. 447 */ 448 wmb(); 449 if (cd->line == 0) 450 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 451 else 452 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 453 raw_spin_unlock_irqrestore(lock, flags); 454 } 455 } 456 457 /* 458 * Enable the irq on the next core in the affinity set for chips that 459 * have the EN*_W1{S,C} registers. 460 */ 461 static void octeon_irq_ciu_enable_v2(struct irq_data *data) 462 { 463 u64 mask; 464 int cpu = next_cpu_for_irq(data); 465 struct octeon_ciu_chip_data *cd; 466 467 cd = irq_data_get_irq_chip_data(data); 468 mask = 1ull << (cd->bit); 469 470 /* 471 * Called under the desc lock, so these should never get out 472 * of sync. 473 */ 474 if (cd->line == 0) { 475 int index = octeon_coreid_for_cpu(cpu) * 2; 476 set_bit(cd->bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu)); 477 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 478 } else { 479 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 480 set_bit(cd->bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 481 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 482 } 483 } 484 485 /* 486 * Enable the irq in the sum2 registers. 487 */ 488 static void octeon_irq_ciu_enable_sum2(struct irq_data *data) 489 { 490 u64 mask; 491 int cpu = next_cpu_for_irq(data); 492 int index = octeon_coreid_for_cpu(cpu); 493 struct octeon_ciu_chip_data *cd; 494 495 cd = irq_data_get_irq_chip_data(data); 496 mask = 1ull << (cd->bit); 497 498 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1S(index), mask); 499 } 500 501 /* 502 * Disable the irq in the sum2 registers. 503 */ 504 static void octeon_irq_ciu_disable_local_sum2(struct irq_data *data) 505 { 506 u64 mask; 507 int cpu = next_cpu_for_irq(data); 508 int index = octeon_coreid_for_cpu(cpu); 509 struct octeon_ciu_chip_data *cd; 510 511 cd = irq_data_get_irq_chip_data(data); 512 mask = 1ull << (cd->bit); 513 514 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(index), mask); 515 } 516 517 static void octeon_irq_ciu_ack_sum2(struct irq_data *data) 518 { 519 u64 mask; 520 int cpu = next_cpu_for_irq(data); 521 int index = octeon_coreid_for_cpu(cpu); 522 struct octeon_ciu_chip_data *cd; 523 524 cd = irq_data_get_irq_chip_data(data); 525 mask = 1ull << (cd->bit); 526 527 cvmx_write_csr(CVMX_CIU_SUM2_PPX_IP4(index), mask); 528 } 529 530 static void octeon_irq_ciu_disable_all_sum2(struct irq_data *data) 531 { 532 int cpu; 533 struct octeon_ciu_chip_data *cd; 534 u64 mask; 535 536 cd = irq_data_get_irq_chip_data(data); 537 mask = 1ull << (cd->bit); 538 539 for_each_online_cpu(cpu) { 540 int coreid = octeon_coreid_for_cpu(cpu); 541 542 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(coreid), mask); 543 } 544 } 545 546 /* 547 * Enable the irq on the current CPU for chips that 548 * have the EN*_W1{S,C} registers. 549 */ 550 static void octeon_irq_ciu_enable_local_v2(struct irq_data *data) 551 { 552 u64 mask; 553 struct octeon_ciu_chip_data *cd; 554 555 cd = irq_data_get_irq_chip_data(data); 556 mask = 1ull << (cd->bit); 557 558 if (cd->line == 0) { 559 int index = cvmx_get_core_num() * 2; 560 set_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu0_en_mirror)); 561 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 562 } else { 563 int index = cvmx_get_core_num() * 2 + 1; 564 set_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu1_en_mirror)); 565 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 566 } 567 } 568 569 static void octeon_irq_ciu_disable_local_v2(struct irq_data *data) 570 { 571 u64 mask; 572 struct octeon_ciu_chip_data *cd; 573 574 cd = irq_data_get_irq_chip_data(data); 575 mask = 1ull << (cd->bit); 576 577 if (cd->line == 0) { 578 int index = cvmx_get_core_num() * 2; 579 clear_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu0_en_mirror)); 580 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); 581 } else { 582 int index = cvmx_get_core_num() * 2 + 1; 583 clear_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu1_en_mirror)); 584 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask); 585 } 586 } 587 588 /* 589 * Write to the W1C bit in CVMX_CIU_INTX_SUM0 to clear the irq. 590 */ 591 static void octeon_irq_ciu_ack(struct irq_data *data) 592 { 593 u64 mask; 594 struct octeon_ciu_chip_data *cd; 595 596 cd = irq_data_get_irq_chip_data(data); 597 mask = 1ull << (cd->bit); 598 599 if (cd->line == 0) { 600 int index = cvmx_get_core_num() * 2; 601 cvmx_write_csr(CVMX_CIU_INTX_SUM0(index), mask); 602 } else { 603 cvmx_write_csr(CVMX_CIU_INT_SUM1, mask); 604 } 605 } 606 607 /* 608 * Disable the irq on the all cores for chips that have the EN*_W1{S,C} 609 * registers. 610 */ 611 static void octeon_irq_ciu_disable_all_v2(struct irq_data *data) 612 { 613 int cpu; 614 u64 mask; 615 struct octeon_ciu_chip_data *cd; 616 617 cd = irq_data_get_irq_chip_data(data); 618 mask = 1ull << (cd->bit); 619 620 if (cd->line == 0) { 621 for_each_online_cpu(cpu) { 622 int index = octeon_coreid_for_cpu(cpu) * 2; 623 clear_bit(cd->bit, 624 &per_cpu(octeon_irq_ciu0_en_mirror, cpu)); 625 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); 626 } 627 } else { 628 for_each_online_cpu(cpu) { 629 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 630 clear_bit(cd->bit, 631 &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 632 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask); 633 } 634 } 635 } 636 637 /* 638 * Enable the irq on the all cores for chips that have the EN*_W1{S,C} 639 * registers. 640 */ 641 static void octeon_irq_ciu_enable_all_v2(struct irq_data *data) 642 { 643 int cpu; 644 u64 mask; 645 struct octeon_ciu_chip_data *cd; 646 647 cd = irq_data_get_irq_chip_data(data); 648 mask = 1ull << (cd->bit); 649 650 if (cd->line == 0) { 651 for_each_online_cpu(cpu) { 652 int index = octeon_coreid_for_cpu(cpu) * 2; 653 set_bit(cd->bit, 654 &per_cpu(octeon_irq_ciu0_en_mirror, cpu)); 655 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 656 } 657 } else { 658 for_each_online_cpu(cpu) { 659 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 660 set_bit(cd->bit, 661 &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 662 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 663 } 664 } 665 } 666 667 static int octeon_irq_ciu_set_type(struct irq_data *data, unsigned int t) 668 { 669 irqd_set_trigger_type(data, t); 670 671 if (t & IRQ_TYPE_EDGE_BOTH) 672 irq_set_handler_locked(data, handle_edge_irq); 673 else 674 irq_set_handler_locked(data, handle_level_irq); 675 676 return IRQ_SET_MASK_OK; 677 } 678 679 static void octeon_irq_gpio_setup(struct irq_data *data) 680 { 681 union cvmx_gpio_bit_cfgx cfg; 682 struct octeon_ciu_chip_data *cd; 683 u32 t = irqd_get_trigger_type(data); 684 685 cd = irq_data_get_irq_chip_data(data); 686 687 cfg.u64 = 0; 688 cfg.s.int_en = 1; 689 cfg.s.int_type = (t & IRQ_TYPE_EDGE_BOTH) != 0; 690 cfg.s.rx_xor = (t & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_EDGE_FALLING)) != 0; 691 692 /* 140 nS glitch filter*/ 693 cfg.s.fil_cnt = 7; 694 cfg.s.fil_sel = 3; 695 696 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), cfg.u64); 697 } 698 699 static void octeon_irq_ciu_enable_gpio_v2(struct irq_data *data) 700 { 701 octeon_irq_gpio_setup(data); 702 octeon_irq_ciu_enable_v2(data); 703 } 704 705 static void octeon_irq_ciu_enable_gpio(struct irq_data *data) 706 { 707 octeon_irq_gpio_setup(data); 708 octeon_irq_ciu_enable(data); 709 } 710 711 static int octeon_irq_ciu_gpio_set_type(struct irq_data *data, unsigned int t) 712 { 713 irqd_set_trigger_type(data, t); 714 octeon_irq_gpio_setup(data); 715 716 if (t & IRQ_TYPE_EDGE_BOTH) 717 irq_set_handler_locked(data, handle_edge_irq); 718 else 719 irq_set_handler_locked(data, handle_level_irq); 720 721 return IRQ_SET_MASK_OK; 722 } 723 724 static void octeon_irq_ciu_disable_gpio_v2(struct irq_data *data) 725 { 726 struct octeon_ciu_chip_data *cd; 727 728 cd = irq_data_get_irq_chip_data(data); 729 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0); 730 731 octeon_irq_ciu_disable_all_v2(data); 732 } 733 734 static void octeon_irq_ciu_disable_gpio(struct irq_data *data) 735 { 736 struct octeon_ciu_chip_data *cd; 737 738 cd = irq_data_get_irq_chip_data(data); 739 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0); 740 741 octeon_irq_ciu_disable_all(data); 742 } 743 744 static void octeon_irq_ciu_gpio_ack(struct irq_data *data) 745 { 746 struct octeon_ciu_chip_data *cd; 747 u64 mask; 748 749 cd = irq_data_get_irq_chip_data(data); 750 mask = 1ull << (cd->gpio_line); 751 752 cvmx_write_csr(CVMX_GPIO_INT_CLR, mask); 753 } 754 755 #ifdef CONFIG_SMP 756 757 static void octeon_irq_cpu_offline_ciu(struct irq_data *data) 758 { 759 int cpu = smp_processor_id(); 760 cpumask_t new_affinity; 761 struct cpumask *mask = irq_data_get_affinity_mask(data); 762 763 if (!cpumask_test_cpu(cpu, mask)) 764 return; 765 766 if (cpumask_weight(mask) > 1) { 767 /* 768 * It has multi CPU affinity, just remove this CPU 769 * from the affinity set. 770 */ 771 cpumask_copy(&new_affinity, mask); 772 cpumask_clear_cpu(cpu, &new_affinity); 773 } else { 774 /* Otherwise, put it on lowest numbered online CPU. */ 775 cpumask_clear(&new_affinity); 776 cpumask_set_cpu(cpumask_first(cpu_online_mask), &new_affinity); 777 } 778 irq_set_affinity_locked(data, &new_affinity, false); 779 } 780 781 static int octeon_irq_ciu_set_affinity(struct irq_data *data, 782 const struct cpumask *dest, bool force) 783 { 784 int cpu; 785 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 786 unsigned long flags; 787 struct octeon_ciu_chip_data *cd; 788 unsigned long *pen; 789 raw_spinlock_t *lock; 790 791 cd = irq_data_get_irq_chip_data(data); 792 793 /* 794 * For non-v2 CIU, we will allow only single CPU affinity. 795 * This removes the need to do locking in the .ack/.eoi 796 * functions. 797 */ 798 if (cpumask_weight(dest) != 1) 799 return -EINVAL; 800 801 if (!enable_one) 802 return 0; 803 804 805 for_each_online_cpu(cpu) { 806 int coreid = octeon_coreid_for_cpu(cpu); 807 808 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 809 raw_spin_lock_irqsave(lock, flags); 810 811 if (cd->line == 0) 812 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 813 else 814 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 815 816 if (cpumask_test_cpu(cpu, dest) && enable_one) { 817 enable_one = 0; 818 __set_bit(cd->bit, pen); 819 } else { 820 __clear_bit(cd->bit, pen); 821 } 822 /* 823 * Must be visible to octeon_irq_ip{2,3}_ciu() before 824 * enabling the irq. 825 */ 826 wmb(); 827 828 if (cd->line == 0) 829 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 830 else 831 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 832 833 raw_spin_unlock_irqrestore(lock, flags); 834 } 835 return 0; 836 } 837 838 /* 839 * Set affinity for the irq for chips that have the EN*_W1{S,C} 840 * registers. 841 */ 842 static int octeon_irq_ciu_set_affinity_v2(struct irq_data *data, 843 const struct cpumask *dest, 844 bool force) 845 { 846 int cpu; 847 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 848 u64 mask; 849 struct octeon_ciu_chip_data *cd; 850 851 if (!enable_one) 852 return 0; 853 854 cd = irq_data_get_irq_chip_data(data); 855 mask = 1ull << cd->bit; 856 857 if (cd->line == 0) { 858 for_each_online_cpu(cpu) { 859 unsigned long *pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 860 int index = octeon_coreid_for_cpu(cpu) * 2; 861 if (cpumask_test_cpu(cpu, dest) && enable_one) { 862 enable_one = false; 863 set_bit(cd->bit, pen); 864 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 865 } else { 866 clear_bit(cd->bit, pen); 867 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); 868 } 869 } 870 } else { 871 for_each_online_cpu(cpu) { 872 unsigned long *pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 873 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 874 if (cpumask_test_cpu(cpu, dest) && enable_one) { 875 enable_one = false; 876 set_bit(cd->bit, pen); 877 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 878 } else { 879 clear_bit(cd->bit, pen); 880 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask); 881 } 882 } 883 } 884 return 0; 885 } 886 887 static int octeon_irq_ciu_set_affinity_sum2(struct irq_data *data, 888 const struct cpumask *dest, 889 bool force) 890 { 891 int cpu; 892 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 893 u64 mask; 894 struct octeon_ciu_chip_data *cd; 895 896 if (!enable_one) 897 return 0; 898 899 cd = irq_data_get_irq_chip_data(data); 900 mask = 1ull << cd->bit; 901 902 for_each_online_cpu(cpu) { 903 int index = octeon_coreid_for_cpu(cpu); 904 905 if (cpumask_test_cpu(cpu, dest) && enable_one) { 906 enable_one = false; 907 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1S(index), mask); 908 } else { 909 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(index), mask); 910 } 911 } 912 return 0; 913 } 914 #endif 915 916 static unsigned int edge_startup(struct irq_data *data) 917 { 918 /* ack any pending edge-irq at startup, so there is 919 * an _edge_ to fire on when the event reappears. 920 */ 921 data->chip->irq_ack(data); 922 data->chip->irq_enable(data); 923 return 0; 924 } 925 926 /* 927 * Newer octeon chips have support for lockless CIU operation. 928 */ 929 static struct irq_chip octeon_irq_chip_ciu_v2 = { 930 .name = "CIU", 931 .irq_enable = octeon_irq_ciu_enable_v2, 932 .irq_disable = octeon_irq_ciu_disable_all_v2, 933 .irq_mask = octeon_irq_ciu_disable_local_v2, 934 .irq_unmask = octeon_irq_ciu_enable_v2, 935 #ifdef CONFIG_SMP 936 .irq_set_affinity = octeon_irq_ciu_set_affinity_v2, 937 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 938 #endif 939 }; 940 941 static struct irq_chip octeon_irq_chip_ciu_v2_edge = { 942 .name = "CIU", 943 .irq_enable = octeon_irq_ciu_enable_v2, 944 .irq_disable = octeon_irq_ciu_disable_all_v2, 945 .irq_ack = octeon_irq_ciu_ack, 946 .irq_mask = octeon_irq_ciu_disable_local_v2, 947 .irq_unmask = octeon_irq_ciu_enable_v2, 948 #ifdef CONFIG_SMP 949 .irq_set_affinity = octeon_irq_ciu_set_affinity_v2, 950 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 951 #endif 952 }; 953 954 /* 955 * Newer octeon chips have support for lockless CIU operation. 956 */ 957 static struct irq_chip octeon_irq_chip_ciu_sum2 = { 958 .name = "CIU", 959 .irq_enable = octeon_irq_ciu_enable_sum2, 960 .irq_disable = octeon_irq_ciu_disable_all_sum2, 961 .irq_mask = octeon_irq_ciu_disable_local_sum2, 962 .irq_unmask = octeon_irq_ciu_enable_sum2, 963 #ifdef CONFIG_SMP 964 .irq_set_affinity = octeon_irq_ciu_set_affinity_sum2, 965 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 966 #endif 967 }; 968 969 static struct irq_chip octeon_irq_chip_ciu_sum2_edge = { 970 .name = "CIU", 971 .irq_enable = octeon_irq_ciu_enable_sum2, 972 .irq_disable = octeon_irq_ciu_disable_all_sum2, 973 .irq_ack = octeon_irq_ciu_ack_sum2, 974 .irq_mask = octeon_irq_ciu_disable_local_sum2, 975 .irq_unmask = octeon_irq_ciu_enable_sum2, 976 #ifdef CONFIG_SMP 977 .irq_set_affinity = octeon_irq_ciu_set_affinity_sum2, 978 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 979 #endif 980 }; 981 982 static struct irq_chip octeon_irq_chip_ciu = { 983 .name = "CIU", 984 .irq_enable = octeon_irq_ciu_enable, 985 .irq_disable = octeon_irq_ciu_disable_all, 986 .irq_mask = octeon_irq_ciu_disable_local, 987 .irq_unmask = octeon_irq_ciu_enable, 988 #ifdef CONFIG_SMP 989 .irq_set_affinity = octeon_irq_ciu_set_affinity, 990 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 991 #endif 992 }; 993 994 static struct irq_chip octeon_irq_chip_ciu_edge = { 995 .name = "CIU", 996 .irq_enable = octeon_irq_ciu_enable, 997 .irq_disable = octeon_irq_ciu_disable_all, 998 .irq_ack = octeon_irq_ciu_ack, 999 .irq_mask = octeon_irq_ciu_disable_local, 1000 .irq_unmask = octeon_irq_ciu_enable, 1001 #ifdef CONFIG_SMP 1002 .irq_set_affinity = octeon_irq_ciu_set_affinity, 1003 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1004 #endif 1005 }; 1006 1007 /* The mbox versions don't do any affinity or round-robin. */ 1008 static struct irq_chip octeon_irq_chip_ciu_mbox_v2 = { 1009 .name = "CIU-M", 1010 .irq_enable = octeon_irq_ciu_enable_all_v2, 1011 .irq_disable = octeon_irq_ciu_disable_all_v2, 1012 .irq_ack = octeon_irq_ciu_disable_local_v2, 1013 .irq_eoi = octeon_irq_ciu_enable_local_v2, 1014 1015 .irq_cpu_online = octeon_irq_ciu_enable_local_v2, 1016 .irq_cpu_offline = octeon_irq_ciu_disable_local_v2, 1017 .flags = IRQCHIP_ONOFFLINE_ENABLED, 1018 }; 1019 1020 static struct irq_chip octeon_irq_chip_ciu_mbox = { 1021 .name = "CIU-M", 1022 .irq_enable = octeon_irq_ciu_enable_all, 1023 .irq_disable = octeon_irq_ciu_disable_all, 1024 .irq_ack = octeon_irq_ciu_disable_local, 1025 .irq_eoi = octeon_irq_ciu_enable_local, 1026 1027 .irq_cpu_online = octeon_irq_ciu_enable_local, 1028 .irq_cpu_offline = octeon_irq_ciu_disable_local, 1029 .flags = IRQCHIP_ONOFFLINE_ENABLED, 1030 }; 1031 1032 static struct irq_chip octeon_irq_chip_ciu_gpio_v2 = { 1033 .name = "CIU-GPIO", 1034 .irq_enable = octeon_irq_ciu_enable_gpio_v2, 1035 .irq_disable = octeon_irq_ciu_disable_gpio_v2, 1036 .irq_ack = octeon_irq_ciu_gpio_ack, 1037 .irq_mask = octeon_irq_ciu_disable_local_v2, 1038 .irq_unmask = octeon_irq_ciu_enable_v2, 1039 .irq_set_type = octeon_irq_ciu_gpio_set_type, 1040 #ifdef CONFIG_SMP 1041 .irq_set_affinity = octeon_irq_ciu_set_affinity_v2, 1042 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1043 #endif 1044 .flags = IRQCHIP_SET_TYPE_MASKED, 1045 }; 1046 1047 static struct irq_chip octeon_irq_chip_ciu_gpio = { 1048 .name = "CIU-GPIO", 1049 .irq_enable = octeon_irq_ciu_enable_gpio, 1050 .irq_disable = octeon_irq_ciu_disable_gpio, 1051 .irq_mask = octeon_irq_ciu_disable_local, 1052 .irq_unmask = octeon_irq_ciu_enable, 1053 .irq_ack = octeon_irq_ciu_gpio_ack, 1054 .irq_set_type = octeon_irq_ciu_gpio_set_type, 1055 #ifdef CONFIG_SMP 1056 .irq_set_affinity = octeon_irq_ciu_set_affinity, 1057 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1058 #endif 1059 .flags = IRQCHIP_SET_TYPE_MASKED, 1060 }; 1061 1062 /* 1063 * Watchdog interrupts are special. They are associated with a single 1064 * core, so we hardwire the affinity to that core. 1065 */ 1066 static void octeon_irq_ciu_wd_enable(struct irq_data *data) 1067 { 1068 unsigned long flags; 1069 unsigned long *pen; 1070 int coreid = data->irq - OCTEON_IRQ_WDOG0; /* Bit 0-63 of EN1 */ 1071 int cpu = octeon_cpu_for_coreid(coreid); 1072 raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 1073 1074 raw_spin_lock_irqsave(lock, flags); 1075 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 1076 __set_bit(coreid, pen); 1077 /* 1078 * Must be visible to octeon_irq_ip{2,3}_ciu() before enabling 1079 * the irq. 1080 */ 1081 wmb(); 1082 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 1083 raw_spin_unlock_irqrestore(lock, flags); 1084 } 1085 1086 /* 1087 * Watchdog interrupts are special. They are associated with a single 1088 * core, so we hardwire the affinity to that core. 1089 */ 1090 static void octeon_irq_ciu1_wd_enable_v2(struct irq_data *data) 1091 { 1092 int coreid = data->irq - OCTEON_IRQ_WDOG0; 1093 int cpu = octeon_cpu_for_coreid(coreid); 1094 1095 set_bit(coreid, &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 1096 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(coreid * 2 + 1), 1ull << coreid); 1097 } 1098 1099 1100 static struct irq_chip octeon_irq_chip_ciu_wd_v2 = { 1101 .name = "CIU-W", 1102 .irq_enable = octeon_irq_ciu1_wd_enable_v2, 1103 .irq_disable = octeon_irq_ciu_disable_all_v2, 1104 .irq_mask = octeon_irq_ciu_disable_local_v2, 1105 .irq_unmask = octeon_irq_ciu_enable_local_v2, 1106 }; 1107 1108 static struct irq_chip octeon_irq_chip_ciu_wd = { 1109 .name = "CIU-W", 1110 .irq_enable = octeon_irq_ciu_wd_enable, 1111 .irq_disable = octeon_irq_ciu_disable_all, 1112 .irq_mask = octeon_irq_ciu_disable_local, 1113 .irq_unmask = octeon_irq_ciu_enable_local, 1114 }; 1115 1116 static bool octeon_irq_ciu_is_edge(unsigned int line, unsigned int bit) 1117 { 1118 bool edge = false; 1119 1120 if (line == 0) 1121 switch (bit) { 1122 case 48 ... 49: /* GMX DRP */ 1123 case 50: /* IPD_DRP */ 1124 case 52 ... 55: /* Timers */ 1125 case 58: /* MPI */ 1126 edge = true; 1127 break; 1128 default: 1129 break; 1130 } 1131 else /* line == 1 */ 1132 switch (bit) { 1133 case 47: /* PTP */ 1134 edge = true; 1135 break; 1136 default: 1137 break; 1138 } 1139 return edge; 1140 } 1141 1142 struct octeon_irq_gpio_domain_data { 1143 unsigned int base_hwirq; 1144 }; 1145 1146 static int octeon_irq_gpio_xlat(struct irq_domain *d, 1147 struct device_node *node, 1148 const u32 *intspec, 1149 unsigned int intsize, 1150 unsigned long *out_hwirq, 1151 unsigned int *out_type) 1152 { 1153 unsigned int type; 1154 unsigned int pin; 1155 unsigned int trigger; 1156 1157 if (irq_domain_get_of_node(d) != node) 1158 return -EINVAL; 1159 1160 if (intsize < 2) 1161 return -EINVAL; 1162 1163 pin = intspec[0]; 1164 if (pin >= 16) 1165 return -EINVAL; 1166 1167 trigger = intspec[1]; 1168 1169 switch (trigger) { 1170 case 1: 1171 type = IRQ_TYPE_EDGE_RISING; 1172 break; 1173 case 2: 1174 type = IRQ_TYPE_EDGE_FALLING; 1175 break; 1176 case 4: 1177 type = IRQ_TYPE_LEVEL_HIGH; 1178 break; 1179 case 8: 1180 type = IRQ_TYPE_LEVEL_LOW; 1181 break; 1182 default: 1183 pr_err("Error: (%s) Invalid irq trigger specification: %x\n", 1184 node->name, 1185 trigger); 1186 type = IRQ_TYPE_LEVEL_LOW; 1187 break; 1188 } 1189 *out_type = type; 1190 *out_hwirq = pin; 1191 1192 return 0; 1193 } 1194 1195 static int octeon_irq_ciu_xlat(struct irq_domain *d, 1196 struct device_node *node, 1197 const u32 *intspec, 1198 unsigned int intsize, 1199 unsigned long *out_hwirq, 1200 unsigned int *out_type) 1201 { 1202 unsigned int ciu, bit; 1203 struct octeon_irq_ciu_domain_data *dd = d->host_data; 1204 1205 ciu = intspec[0]; 1206 bit = intspec[1]; 1207 1208 if (ciu >= dd->num_sum || bit > 63) 1209 return -EINVAL; 1210 1211 *out_hwirq = (ciu << 6) | bit; 1212 *out_type = 0; 1213 1214 return 0; 1215 } 1216 1217 static struct irq_chip *octeon_irq_ciu_chip; 1218 static struct irq_chip *octeon_irq_ciu_chip_edge; 1219 static struct irq_chip *octeon_irq_gpio_chip; 1220 1221 static int octeon_irq_ciu_map(struct irq_domain *d, 1222 unsigned int virq, irq_hw_number_t hw) 1223 { 1224 int rv; 1225 unsigned int line = hw >> 6; 1226 unsigned int bit = hw & 63; 1227 struct octeon_irq_ciu_domain_data *dd = d->host_data; 1228 1229 if (line >= dd->num_sum || octeon_irq_ciu_to_irq[line][bit] != 0) 1230 return -EINVAL; 1231 1232 if (line == 2) { 1233 if (octeon_irq_ciu_is_edge(line, bit)) 1234 rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1235 &octeon_irq_chip_ciu_sum2_edge, 1236 handle_edge_irq); 1237 else 1238 rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1239 &octeon_irq_chip_ciu_sum2, 1240 handle_level_irq); 1241 } else { 1242 if (octeon_irq_ciu_is_edge(line, bit)) 1243 rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1244 octeon_irq_ciu_chip_edge, 1245 handle_edge_irq); 1246 else 1247 rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1248 octeon_irq_ciu_chip, 1249 handle_level_irq); 1250 } 1251 return rv; 1252 } 1253 1254 static int octeon_irq_gpio_map(struct irq_domain *d, 1255 unsigned int virq, irq_hw_number_t hw) 1256 { 1257 struct octeon_irq_gpio_domain_data *gpiod = d->host_data; 1258 unsigned int line, bit; 1259 int r; 1260 1261 line = (hw + gpiod->base_hwirq) >> 6; 1262 bit = (hw + gpiod->base_hwirq) & 63; 1263 if (line >= ARRAY_SIZE(octeon_irq_ciu_to_irq) || 1264 octeon_irq_ciu_to_irq[line][bit] != 0) 1265 return -EINVAL; 1266 1267 /* 1268 * Default to handle_level_irq. If the DT contains a different 1269 * trigger type, it will call the irq_set_type callback and 1270 * the handler gets updated. 1271 */ 1272 r = octeon_irq_set_ciu_mapping(virq, line, bit, hw, 1273 octeon_irq_gpio_chip, handle_level_irq); 1274 return r; 1275 } 1276 1277 static struct irq_domain_ops octeon_irq_domain_ciu_ops = { 1278 .map = octeon_irq_ciu_map, 1279 .unmap = octeon_irq_free_cd, 1280 .xlate = octeon_irq_ciu_xlat, 1281 }; 1282 1283 static struct irq_domain_ops octeon_irq_domain_gpio_ops = { 1284 .map = octeon_irq_gpio_map, 1285 .unmap = octeon_irq_free_cd, 1286 .xlate = octeon_irq_gpio_xlat, 1287 }; 1288 1289 static void octeon_irq_ip2_ciu(void) 1290 { 1291 const unsigned long core_id = cvmx_get_core_num(); 1292 u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INTX_SUM0(core_id * 2)); 1293 1294 ciu_sum &= __this_cpu_read(octeon_irq_ciu0_en_mirror); 1295 if (likely(ciu_sum)) { 1296 int bit = fls64(ciu_sum) - 1; 1297 int irq = octeon_irq_ciu_to_irq[0][bit]; 1298 if (likely(irq)) 1299 do_IRQ(irq); 1300 else 1301 spurious_interrupt(); 1302 } else { 1303 spurious_interrupt(); 1304 } 1305 } 1306 1307 static void octeon_irq_ip3_ciu(void) 1308 { 1309 u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INT_SUM1); 1310 1311 ciu_sum &= __this_cpu_read(octeon_irq_ciu1_en_mirror); 1312 if (likely(ciu_sum)) { 1313 int bit = fls64(ciu_sum) - 1; 1314 int irq = octeon_irq_ciu_to_irq[1][bit]; 1315 if (likely(irq)) 1316 do_IRQ(irq); 1317 else 1318 spurious_interrupt(); 1319 } else { 1320 spurious_interrupt(); 1321 } 1322 } 1323 1324 static void octeon_irq_ip4_ciu(void) 1325 { 1326 int coreid = cvmx_get_core_num(); 1327 u64 ciu_sum = cvmx_read_csr(CVMX_CIU_SUM2_PPX_IP4(coreid)); 1328 u64 ciu_en = cvmx_read_csr(CVMX_CIU_EN2_PPX_IP4(coreid)); 1329 1330 ciu_sum &= ciu_en; 1331 if (likely(ciu_sum)) { 1332 int bit = fls64(ciu_sum) - 1; 1333 int irq = octeon_irq_ciu_to_irq[2][bit]; 1334 1335 if (likely(irq)) 1336 do_IRQ(irq); 1337 else 1338 spurious_interrupt(); 1339 } else { 1340 spurious_interrupt(); 1341 } 1342 } 1343 1344 static bool octeon_irq_use_ip4; 1345 1346 static void octeon_irq_local_enable_ip4(void *arg) 1347 { 1348 set_c0_status(STATUSF_IP4); 1349 } 1350 1351 static void octeon_irq_ip4_mask(void) 1352 { 1353 clear_c0_status(STATUSF_IP4); 1354 spurious_interrupt(); 1355 } 1356 1357 static void (*octeon_irq_ip2)(void); 1358 static void (*octeon_irq_ip3)(void); 1359 static void (*octeon_irq_ip4)(void); 1360 1361 void (*octeon_irq_setup_secondary)(void); 1362 1363 void octeon_irq_set_ip4_handler(octeon_irq_ip4_handler_t h) 1364 { 1365 octeon_irq_ip4 = h; 1366 octeon_irq_use_ip4 = true; 1367 on_each_cpu(octeon_irq_local_enable_ip4, NULL, 1); 1368 } 1369 1370 static void octeon_irq_percpu_enable(void) 1371 { 1372 irq_cpu_online(); 1373 } 1374 1375 static void octeon_irq_init_ciu_percpu(void) 1376 { 1377 int coreid = cvmx_get_core_num(); 1378 1379 1380 __this_cpu_write(octeon_irq_ciu0_en_mirror, 0); 1381 __this_cpu_write(octeon_irq_ciu1_en_mirror, 0); 1382 wmb(); 1383 raw_spin_lock_init(this_cpu_ptr(&octeon_irq_ciu_spinlock)); 1384 /* 1385 * Disable All CIU Interrupts. The ones we need will be 1386 * enabled later. Read the SUM register so we know the write 1387 * completed. 1388 */ 1389 cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2)), 0); 1390 cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2 + 1)), 0); 1391 cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2)), 0); 1392 cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2 + 1)), 0); 1393 cvmx_read_csr(CVMX_CIU_INTX_SUM0((coreid * 2))); 1394 } 1395 1396 static void octeon_irq_init_ciu2_percpu(void) 1397 { 1398 u64 regx, ipx; 1399 int coreid = cvmx_get_core_num(); 1400 u64 base = CVMX_CIU2_EN_PPX_IP2_WRKQ(coreid); 1401 1402 /* 1403 * Disable All CIU2 Interrupts. The ones we need will be 1404 * enabled later. Read the SUM register so we know the write 1405 * completed. 1406 * 1407 * There are 9 registers and 3 IPX levels with strides 0x1000 1408 * and 0x200 respectivly. Use loops to clear them. 1409 */ 1410 for (regx = 0; regx <= 0x8000; regx += 0x1000) { 1411 for (ipx = 0; ipx <= 0x400; ipx += 0x200) 1412 cvmx_write_csr(base + regx + ipx, 0); 1413 } 1414 1415 cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(coreid)); 1416 } 1417 1418 static void octeon_irq_setup_secondary_ciu(void) 1419 { 1420 octeon_irq_init_ciu_percpu(); 1421 octeon_irq_percpu_enable(); 1422 1423 /* Enable the CIU lines */ 1424 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 1425 if (octeon_irq_use_ip4) 1426 set_c0_status(STATUSF_IP4); 1427 else 1428 clear_c0_status(STATUSF_IP4); 1429 } 1430 1431 static void octeon_irq_setup_secondary_ciu2(void) 1432 { 1433 octeon_irq_init_ciu2_percpu(); 1434 octeon_irq_percpu_enable(); 1435 1436 /* Enable the CIU lines */ 1437 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 1438 if (octeon_irq_use_ip4) 1439 set_c0_status(STATUSF_IP4); 1440 else 1441 clear_c0_status(STATUSF_IP4); 1442 } 1443 1444 static int __init octeon_irq_init_ciu( 1445 struct device_node *ciu_node, struct device_node *parent) 1446 { 1447 unsigned int i, r; 1448 struct irq_chip *chip; 1449 struct irq_chip *chip_edge; 1450 struct irq_chip *chip_mbox; 1451 struct irq_chip *chip_wd; 1452 struct irq_domain *ciu_domain = NULL; 1453 struct octeon_irq_ciu_domain_data *dd; 1454 1455 dd = kzalloc(sizeof(*dd), GFP_KERNEL); 1456 if (!dd) 1457 return -ENOMEM; 1458 1459 octeon_irq_init_ciu_percpu(); 1460 octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu; 1461 1462 octeon_irq_ip2 = octeon_irq_ip2_ciu; 1463 octeon_irq_ip3 = octeon_irq_ip3_ciu; 1464 if ((OCTEON_IS_OCTEON2() || OCTEON_IS_OCTEON3()) 1465 && !OCTEON_IS_MODEL(OCTEON_CN63XX)) { 1466 octeon_irq_ip4 = octeon_irq_ip4_ciu; 1467 dd->num_sum = 3; 1468 octeon_irq_use_ip4 = true; 1469 } else { 1470 octeon_irq_ip4 = octeon_irq_ip4_mask; 1471 dd->num_sum = 2; 1472 octeon_irq_use_ip4 = false; 1473 } 1474 if (OCTEON_IS_MODEL(OCTEON_CN58XX_PASS2_X) || 1475 OCTEON_IS_MODEL(OCTEON_CN56XX_PASS2_X) || 1476 OCTEON_IS_MODEL(OCTEON_CN52XX_PASS2_X) || 1477 OCTEON_IS_OCTEON2() || OCTEON_IS_OCTEON3()) { 1478 chip = &octeon_irq_chip_ciu_v2; 1479 chip_edge = &octeon_irq_chip_ciu_v2_edge; 1480 chip_mbox = &octeon_irq_chip_ciu_mbox_v2; 1481 chip_wd = &octeon_irq_chip_ciu_wd_v2; 1482 octeon_irq_gpio_chip = &octeon_irq_chip_ciu_gpio_v2; 1483 } else { 1484 chip = &octeon_irq_chip_ciu; 1485 chip_edge = &octeon_irq_chip_ciu_edge; 1486 chip_mbox = &octeon_irq_chip_ciu_mbox; 1487 chip_wd = &octeon_irq_chip_ciu_wd; 1488 octeon_irq_gpio_chip = &octeon_irq_chip_ciu_gpio; 1489 } 1490 octeon_irq_ciu_chip = chip; 1491 octeon_irq_ciu_chip_edge = chip_edge; 1492 1493 /* Mips internal */ 1494 octeon_irq_init_core(); 1495 1496 ciu_domain = irq_domain_add_tree( 1497 ciu_node, &octeon_irq_domain_ciu_ops, dd); 1498 irq_set_default_host(ciu_domain); 1499 1500 /* CIU_0 */ 1501 for (i = 0; i < 16; i++) { 1502 r = octeon_irq_force_ciu_mapping( 1503 ciu_domain, i + OCTEON_IRQ_WORKQ0, 0, i + 0); 1504 if (r) 1505 goto err; 1506 } 1507 1508 r = octeon_irq_set_ciu_mapping( 1509 OCTEON_IRQ_MBOX0, 0, 32, 0, chip_mbox, handle_percpu_irq); 1510 if (r) 1511 goto err; 1512 r = octeon_irq_set_ciu_mapping( 1513 OCTEON_IRQ_MBOX1, 0, 33, 0, chip_mbox, handle_percpu_irq); 1514 if (r) 1515 goto err; 1516 1517 for (i = 0; i < 4; i++) { 1518 r = octeon_irq_force_ciu_mapping( 1519 ciu_domain, i + OCTEON_IRQ_PCI_INT0, 0, i + 36); 1520 if (r) 1521 goto err; 1522 } 1523 for (i = 0; i < 4; i++) { 1524 r = octeon_irq_force_ciu_mapping( 1525 ciu_domain, i + OCTEON_IRQ_PCI_MSI0, 0, i + 40); 1526 if (r) 1527 goto err; 1528 } 1529 1530 r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_TWSI, 0, 45); 1531 if (r) 1532 goto err; 1533 1534 r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_RML, 0, 46); 1535 if (r) 1536 goto err; 1537 1538 for (i = 0; i < 4; i++) { 1539 r = octeon_irq_force_ciu_mapping( 1540 ciu_domain, i + OCTEON_IRQ_TIMER0, 0, i + 52); 1541 if (r) 1542 goto err; 1543 } 1544 1545 r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_TWSI2, 0, 59); 1546 if (r) 1547 goto err; 1548 1549 /* CIU_1 */ 1550 for (i = 0; i < 16; i++) { 1551 r = octeon_irq_set_ciu_mapping( 1552 i + OCTEON_IRQ_WDOG0, 1, i + 0, 0, chip_wd, 1553 handle_level_irq); 1554 if (r) 1555 goto err; 1556 } 1557 1558 /* Enable the CIU lines */ 1559 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 1560 if (octeon_irq_use_ip4) 1561 set_c0_status(STATUSF_IP4); 1562 else 1563 clear_c0_status(STATUSF_IP4); 1564 1565 return 0; 1566 err: 1567 return r; 1568 } 1569 1570 static int __init octeon_irq_init_gpio( 1571 struct device_node *gpio_node, struct device_node *parent) 1572 { 1573 struct octeon_irq_gpio_domain_data *gpiod; 1574 u32 interrupt_cells; 1575 unsigned int base_hwirq; 1576 int r; 1577 1578 r = of_property_read_u32(parent, "#interrupt-cells", &interrupt_cells); 1579 if (r) 1580 return r; 1581 1582 if (interrupt_cells == 1) { 1583 u32 v; 1584 1585 r = of_property_read_u32_index(gpio_node, "interrupts", 0, &v); 1586 if (r) { 1587 pr_warn("No \"interrupts\" property.\n"); 1588 return r; 1589 } 1590 base_hwirq = v; 1591 } else if (interrupt_cells == 2) { 1592 u32 v0, v1; 1593 1594 r = of_property_read_u32_index(gpio_node, "interrupts", 0, &v0); 1595 if (r) { 1596 pr_warn("No \"interrupts\" property.\n"); 1597 return r; 1598 } 1599 r = of_property_read_u32_index(gpio_node, "interrupts", 1, &v1); 1600 if (r) { 1601 pr_warn("No \"interrupts\" property.\n"); 1602 return r; 1603 } 1604 base_hwirq = (v0 << 6) | v1; 1605 } else { 1606 pr_warn("Bad \"#interrupt-cells\" property: %u\n", 1607 interrupt_cells); 1608 return -EINVAL; 1609 } 1610 1611 gpiod = kzalloc(sizeof(*gpiod), GFP_KERNEL); 1612 if (gpiod) { 1613 /* gpio domain host_data is the base hwirq number. */ 1614 gpiod->base_hwirq = base_hwirq; 1615 irq_domain_add_linear( 1616 gpio_node, 16, &octeon_irq_domain_gpio_ops, gpiod); 1617 } else { 1618 pr_warn("Cannot allocate memory for GPIO irq_domain.\n"); 1619 return -ENOMEM; 1620 } 1621 1622 /* 1623 * Clear the OF_POPULATED flag that was set by of_irq_init() 1624 * so that all GPIO devices will be probed. 1625 */ 1626 of_node_clear_flag(gpio_node, OF_POPULATED); 1627 1628 return 0; 1629 } 1630 /* 1631 * Watchdog interrupts are special. They are associated with a single 1632 * core, so we hardwire the affinity to that core. 1633 */ 1634 static void octeon_irq_ciu2_wd_enable(struct irq_data *data) 1635 { 1636 u64 mask; 1637 u64 en_addr; 1638 int coreid = data->irq - OCTEON_IRQ_WDOG0; 1639 struct octeon_ciu_chip_data *cd; 1640 1641 cd = irq_data_get_irq_chip_data(data); 1642 mask = 1ull << (cd->bit); 1643 1644 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) + 1645 (0x1000ull * cd->line); 1646 cvmx_write_csr(en_addr, mask); 1647 1648 } 1649 1650 static void octeon_irq_ciu2_enable(struct irq_data *data) 1651 { 1652 u64 mask; 1653 u64 en_addr; 1654 int cpu = next_cpu_for_irq(data); 1655 int coreid = octeon_coreid_for_cpu(cpu); 1656 struct octeon_ciu_chip_data *cd; 1657 1658 cd = irq_data_get_irq_chip_data(data); 1659 mask = 1ull << (cd->bit); 1660 1661 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) + 1662 (0x1000ull * cd->line); 1663 cvmx_write_csr(en_addr, mask); 1664 } 1665 1666 static void octeon_irq_ciu2_enable_local(struct irq_data *data) 1667 { 1668 u64 mask; 1669 u64 en_addr; 1670 int coreid = cvmx_get_core_num(); 1671 struct octeon_ciu_chip_data *cd; 1672 1673 cd = irq_data_get_irq_chip_data(data); 1674 mask = 1ull << (cd->bit); 1675 1676 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) + 1677 (0x1000ull * cd->line); 1678 cvmx_write_csr(en_addr, mask); 1679 1680 } 1681 1682 static void octeon_irq_ciu2_disable_local(struct irq_data *data) 1683 { 1684 u64 mask; 1685 u64 en_addr; 1686 int coreid = cvmx_get_core_num(); 1687 struct octeon_ciu_chip_data *cd; 1688 1689 cd = irq_data_get_irq_chip_data(data); 1690 mask = 1ull << (cd->bit); 1691 1692 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C(coreid) + 1693 (0x1000ull * cd->line); 1694 cvmx_write_csr(en_addr, mask); 1695 1696 } 1697 1698 static void octeon_irq_ciu2_ack(struct irq_data *data) 1699 { 1700 u64 mask; 1701 u64 en_addr; 1702 int coreid = cvmx_get_core_num(); 1703 struct octeon_ciu_chip_data *cd; 1704 1705 cd = irq_data_get_irq_chip_data(data); 1706 mask = 1ull << (cd->bit); 1707 1708 en_addr = CVMX_CIU2_RAW_PPX_IP2_WRKQ(coreid) + (0x1000ull * cd->line); 1709 cvmx_write_csr(en_addr, mask); 1710 1711 } 1712 1713 static void octeon_irq_ciu2_disable_all(struct irq_data *data) 1714 { 1715 int cpu; 1716 u64 mask; 1717 struct octeon_ciu_chip_data *cd; 1718 1719 cd = irq_data_get_irq_chip_data(data); 1720 mask = 1ull << (cd->bit); 1721 1722 for_each_online_cpu(cpu) { 1723 u64 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C( 1724 octeon_coreid_for_cpu(cpu)) + (0x1000ull * cd->line); 1725 cvmx_write_csr(en_addr, mask); 1726 } 1727 } 1728 1729 static void octeon_irq_ciu2_mbox_enable_all(struct irq_data *data) 1730 { 1731 int cpu; 1732 u64 mask; 1733 1734 mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0); 1735 1736 for_each_online_cpu(cpu) { 1737 u64 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1S( 1738 octeon_coreid_for_cpu(cpu)); 1739 cvmx_write_csr(en_addr, mask); 1740 } 1741 } 1742 1743 static void octeon_irq_ciu2_mbox_disable_all(struct irq_data *data) 1744 { 1745 int cpu; 1746 u64 mask; 1747 1748 mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0); 1749 1750 for_each_online_cpu(cpu) { 1751 u64 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1C( 1752 octeon_coreid_for_cpu(cpu)); 1753 cvmx_write_csr(en_addr, mask); 1754 } 1755 } 1756 1757 static void octeon_irq_ciu2_mbox_enable_local(struct irq_data *data) 1758 { 1759 u64 mask; 1760 u64 en_addr; 1761 int coreid = cvmx_get_core_num(); 1762 1763 mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0); 1764 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1S(coreid); 1765 cvmx_write_csr(en_addr, mask); 1766 } 1767 1768 static void octeon_irq_ciu2_mbox_disable_local(struct irq_data *data) 1769 { 1770 u64 mask; 1771 u64 en_addr; 1772 int coreid = cvmx_get_core_num(); 1773 1774 mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0); 1775 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1C(coreid); 1776 cvmx_write_csr(en_addr, mask); 1777 } 1778 1779 #ifdef CONFIG_SMP 1780 static int octeon_irq_ciu2_set_affinity(struct irq_data *data, 1781 const struct cpumask *dest, bool force) 1782 { 1783 int cpu; 1784 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 1785 u64 mask; 1786 struct octeon_ciu_chip_data *cd; 1787 1788 if (!enable_one) 1789 return 0; 1790 1791 cd = irq_data_get_irq_chip_data(data); 1792 mask = 1ull << cd->bit; 1793 1794 for_each_online_cpu(cpu) { 1795 u64 en_addr; 1796 if (cpumask_test_cpu(cpu, dest) && enable_one) { 1797 enable_one = false; 1798 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S( 1799 octeon_coreid_for_cpu(cpu)) + 1800 (0x1000ull * cd->line); 1801 } else { 1802 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C( 1803 octeon_coreid_for_cpu(cpu)) + 1804 (0x1000ull * cd->line); 1805 } 1806 cvmx_write_csr(en_addr, mask); 1807 } 1808 1809 return 0; 1810 } 1811 #endif 1812 1813 static void octeon_irq_ciu2_enable_gpio(struct irq_data *data) 1814 { 1815 octeon_irq_gpio_setup(data); 1816 octeon_irq_ciu2_enable(data); 1817 } 1818 1819 static void octeon_irq_ciu2_disable_gpio(struct irq_data *data) 1820 { 1821 struct octeon_ciu_chip_data *cd; 1822 1823 cd = irq_data_get_irq_chip_data(data); 1824 1825 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0); 1826 1827 octeon_irq_ciu2_disable_all(data); 1828 } 1829 1830 static struct irq_chip octeon_irq_chip_ciu2 = { 1831 .name = "CIU2-E", 1832 .irq_enable = octeon_irq_ciu2_enable, 1833 .irq_disable = octeon_irq_ciu2_disable_all, 1834 .irq_mask = octeon_irq_ciu2_disable_local, 1835 .irq_unmask = octeon_irq_ciu2_enable, 1836 #ifdef CONFIG_SMP 1837 .irq_set_affinity = octeon_irq_ciu2_set_affinity, 1838 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1839 #endif 1840 }; 1841 1842 static struct irq_chip octeon_irq_chip_ciu2_edge = { 1843 .name = "CIU2-E", 1844 .irq_enable = octeon_irq_ciu2_enable, 1845 .irq_disable = octeon_irq_ciu2_disable_all, 1846 .irq_ack = octeon_irq_ciu2_ack, 1847 .irq_mask = octeon_irq_ciu2_disable_local, 1848 .irq_unmask = octeon_irq_ciu2_enable, 1849 #ifdef CONFIG_SMP 1850 .irq_set_affinity = octeon_irq_ciu2_set_affinity, 1851 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1852 #endif 1853 }; 1854 1855 static struct irq_chip octeon_irq_chip_ciu2_mbox = { 1856 .name = "CIU2-M", 1857 .irq_enable = octeon_irq_ciu2_mbox_enable_all, 1858 .irq_disable = octeon_irq_ciu2_mbox_disable_all, 1859 .irq_ack = octeon_irq_ciu2_mbox_disable_local, 1860 .irq_eoi = octeon_irq_ciu2_mbox_enable_local, 1861 1862 .irq_cpu_online = octeon_irq_ciu2_mbox_enable_local, 1863 .irq_cpu_offline = octeon_irq_ciu2_mbox_disable_local, 1864 .flags = IRQCHIP_ONOFFLINE_ENABLED, 1865 }; 1866 1867 static struct irq_chip octeon_irq_chip_ciu2_wd = { 1868 .name = "CIU2-W", 1869 .irq_enable = octeon_irq_ciu2_wd_enable, 1870 .irq_disable = octeon_irq_ciu2_disable_all, 1871 .irq_mask = octeon_irq_ciu2_disable_local, 1872 .irq_unmask = octeon_irq_ciu2_enable_local, 1873 }; 1874 1875 static struct irq_chip octeon_irq_chip_ciu2_gpio = { 1876 .name = "CIU-GPIO", 1877 .irq_enable = octeon_irq_ciu2_enable_gpio, 1878 .irq_disable = octeon_irq_ciu2_disable_gpio, 1879 .irq_ack = octeon_irq_ciu_gpio_ack, 1880 .irq_mask = octeon_irq_ciu2_disable_local, 1881 .irq_unmask = octeon_irq_ciu2_enable, 1882 .irq_set_type = octeon_irq_ciu_gpio_set_type, 1883 #ifdef CONFIG_SMP 1884 .irq_set_affinity = octeon_irq_ciu2_set_affinity, 1885 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1886 #endif 1887 .flags = IRQCHIP_SET_TYPE_MASKED, 1888 }; 1889 1890 static int octeon_irq_ciu2_xlat(struct irq_domain *d, 1891 struct device_node *node, 1892 const u32 *intspec, 1893 unsigned int intsize, 1894 unsigned long *out_hwirq, 1895 unsigned int *out_type) 1896 { 1897 unsigned int ciu, bit; 1898 1899 ciu = intspec[0]; 1900 bit = intspec[1]; 1901 1902 *out_hwirq = (ciu << 6) | bit; 1903 *out_type = 0; 1904 1905 return 0; 1906 } 1907 1908 static bool octeon_irq_ciu2_is_edge(unsigned int line, unsigned int bit) 1909 { 1910 bool edge = false; 1911 1912 if (line == 3) /* MIO */ 1913 switch (bit) { 1914 case 2: /* IPD_DRP */ 1915 case 8 ... 11: /* Timers */ 1916 case 48: /* PTP */ 1917 edge = true; 1918 break; 1919 default: 1920 break; 1921 } 1922 else if (line == 6) /* PKT */ 1923 switch (bit) { 1924 case 52 ... 53: /* ILK_DRP */ 1925 case 8 ... 12: /* GMX_DRP */ 1926 edge = true; 1927 break; 1928 default: 1929 break; 1930 } 1931 return edge; 1932 } 1933 1934 static int octeon_irq_ciu2_map(struct irq_domain *d, 1935 unsigned int virq, irq_hw_number_t hw) 1936 { 1937 unsigned int line = hw >> 6; 1938 unsigned int bit = hw & 63; 1939 1940 /* 1941 * Don't map irq if it is reserved for GPIO. 1942 * (Line 7 are the GPIO lines.) 1943 */ 1944 if (line == 7) 1945 return 0; 1946 1947 if (line > 7 || octeon_irq_ciu_to_irq[line][bit] != 0) 1948 return -EINVAL; 1949 1950 if (octeon_irq_ciu2_is_edge(line, bit)) 1951 octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1952 &octeon_irq_chip_ciu2_edge, 1953 handle_edge_irq); 1954 else 1955 octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1956 &octeon_irq_chip_ciu2, 1957 handle_level_irq); 1958 1959 return 0; 1960 } 1961 1962 static struct irq_domain_ops octeon_irq_domain_ciu2_ops = { 1963 .map = octeon_irq_ciu2_map, 1964 .unmap = octeon_irq_free_cd, 1965 .xlate = octeon_irq_ciu2_xlat, 1966 }; 1967 1968 static void octeon_irq_ciu2(void) 1969 { 1970 int line; 1971 int bit; 1972 int irq; 1973 u64 src_reg, src, sum; 1974 const unsigned long core_id = cvmx_get_core_num(); 1975 1976 sum = cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(core_id)) & 0xfful; 1977 1978 if (unlikely(!sum)) 1979 goto spurious; 1980 1981 line = fls64(sum) - 1; 1982 src_reg = CVMX_CIU2_SRC_PPX_IP2_WRKQ(core_id) + (0x1000 * line); 1983 src = cvmx_read_csr(src_reg); 1984 1985 if (unlikely(!src)) 1986 goto spurious; 1987 1988 bit = fls64(src) - 1; 1989 irq = octeon_irq_ciu_to_irq[line][bit]; 1990 if (unlikely(!irq)) 1991 goto spurious; 1992 1993 do_IRQ(irq); 1994 goto out; 1995 1996 spurious: 1997 spurious_interrupt(); 1998 out: 1999 /* CN68XX pass 1.x has an errata that accessing the ACK registers 2000 can stop interrupts from propagating */ 2001 if (OCTEON_IS_MODEL(OCTEON_CN68XX)) 2002 cvmx_read_csr(CVMX_CIU2_INTR_CIU_READY); 2003 else 2004 cvmx_read_csr(CVMX_CIU2_ACK_PPX_IP2(core_id)); 2005 return; 2006 } 2007 2008 static void octeon_irq_ciu2_mbox(void) 2009 { 2010 int line; 2011 2012 const unsigned long core_id = cvmx_get_core_num(); 2013 u64 sum = cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP3(core_id)) >> 60; 2014 2015 if (unlikely(!sum)) 2016 goto spurious; 2017 2018 line = fls64(sum) - 1; 2019 2020 do_IRQ(OCTEON_IRQ_MBOX0 + line); 2021 goto out; 2022 2023 spurious: 2024 spurious_interrupt(); 2025 out: 2026 /* CN68XX pass 1.x has an errata that accessing the ACK registers 2027 can stop interrupts from propagating */ 2028 if (OCTEON_IS_MODEL(OCTEON_CN68XX)) 2029 cvmx_read_csr(CVMX_CIU2_INTR_CIU_READY); 2030 else 2031 cvmx_read_csr(CVMX_CIU2_ACK_PPX_IP3(core_id)); 2032 return; 2033 } 2034 2035 static int __init octeon_irq_init_ciu2( 2036 struct device_node *ciu_node, struct device_node *parent) 2037 { 2038 unsigned int i, r; 2039 struct irq_domain *ciu_domain = NULL; 2040 2041 octeon_irq_init_ciu2_percpu(); 2042 octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu2; 2043 2044 octeon_irq_gpio_chip = &octeon_irq_chip_ciu2_gpio; 2045 octeon_irq_ip2 = octeon_irq_ciu2; 2046 octeon_irq_ip3 = octeon_irq_ciu2_mbox; 2047 octeon_irq_ip4 = octeon_irq_ip4_mask; 2048 2049 /* Mips internal */ 2050 octeon_irq_init_core(); 2051 2052 ciu_domain = irq_domain_add_tree( 2053 ciu_node, &octeon_irq_domain_ciu2_ops, NULL); 2054 irq_set_default_host(ciu_domain); 2055 2056 /* CUI2 */ 2057 for (i = 0; i < 64; i++) { 2058 r = octeon_irq_force_ciu_mapping( 2059 ciu_domain, i + OCTEON_IRQ_WORKQ0, 0, i); 2060 if (r) 2061 goto err; 2062 } 2063 2064 for (i = 0; i < 32; i++) { 2065 r = octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_WDOG0, 1, i, 0, 2066 &octeon_irq_chip_ciu2_wd, handle_level_irq); 2067 if (r) 2068 goto err; 2069 } 2070 2071 for (i = 0; i < 4; i++) { 2072 r = octeon_irq_force_ciu_mapping( 2073 ciu_domain, i + OCTEON_IRQ_TIMER0, 3, i + 8); 2074 if (r) 2075 goto err; 2076 } 2077 2078 for (i = 0; i < 4; i++) { 2079 r = octeon_irq_force_ciu_mapping( 2080 ciu_domain, i + OCTEON_IRQ_PCI_INT0, 4, i); 2081 if (r) 2082 goto err; 2083 } 2084 2085 for (i = 0; i < 4; i++) { 2086 r = octeon_irq_force_ciu_mapping( 2087 ciu_domain, i + OCTEON_IRQ_PCI_MSI0, 4, i + 8); 2088 if (r) 2089 goto err; 2090 } 2091 2092 irq_set_chip_and_handler(OCTEON_IRQ_MBOX0, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq); 2093 irq_set_chip_and_handler(OCTEON_IRQ_MBOX1, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq); 2094 irq_set_chip_and_handler(OCTEON_IRQ_MBOX2, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq); 2095 irq_set_chip_and_handler(OCTEON_IRQ_MBOX3, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq); 2096 2097 /* Enable the CIU lines */ 2098 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 2099 clear_c0_status(STATUSF_IP4); 2100 return 0; 2101 err: 2102 return r; 2103 } 2104 2105 struct octeon_irq_cib_host_data { 2106 raw_spinlock_t lock; 2107 u64 raw_reg; 2108 u64 en_reg; 2109 int max_bits; 2110 }; 2111 2112 struct octeon_irq_cib_chip_data { 2113 struct octeon_irq_cib_host_data *host_data; 2114 int bit; 2115 }; 2116 2117 static void octeon_irq_cib_enable(struct irq_data *data) 2118 { 2119 unsigned long flags; 2120 u64 en; 2121 struct octeon_irq_cib_chip_data *cd = irq_data_get_irq_chip_data(data); 2122 struct octeon_irq_cib_host_data *host_data = cd->host_data; 2123 2124 raw_spin_lock_irqsave(&host_data->lock, flags); 2125 en = cvmx_read_csr(host_data->en_reg); 2126 en |= 1ull << cd->bit; 2127 cvmx_write_csr(host_data->en_reg, en); 2128 raw_spin_unlock_irqrestore(&host_data->lock, flags); 2129 } 2130 2131 static void octeon_irq_cib_disable(struct irq_data *data) 2132 { 2133 unsigned long flags; 2134 u64 en; 2135 struct octeon_irq_cib_chip_data *cd = irq_data_get_irq_chip_data(data); 2136 struct octeon_irq_cib_host_data *host_data = cd->host_data; 2137 2138 raw_spin_lock_irqsave(&host_data->lock, flags); 2139 en = cvmx_read_csr(host_data->en_reg); 2140 en &= ~(1ull << cd->bit); 2141 cvmx_write_csr(host_data->en_reg, en); 2142 raw_spin_unlock_irqrestore(&host_data->lock, flags); 2143 } 2144 2145 static int octeon_irq_cib_set_type(struct irq_data *data, unsigned int t) 2146 { 2147 irqd_set_trigger_type(data, t); 2148 return IRQ_SET_MASK_OK; 2149 } 2150 2151 static struct irq_chip octeon_irq_chip_cib = { 2152 .name = "CIB", 2153 .irq_enable = octeon_irq_cib_enable, 2154 .irq_disable = octeon_irq_cib_disable, 2155 .irq_mask = octeon_irq_cib_disable, 2156 .irq_unmask = octeon_irq_cib_enable, 2157 .irq_set_type = octeon_irq_cib_set_type, 2158 }; 2159 2160 static int octeon_irq_cib_xlat(struct irq_domain *d, 2161 struct device_node *node, 2162 const u32 *intspec, 2163 unsigned int intsize, 2164 unsigned long *out_hwirq, 2165 unsigned int *out_type) 2166 { 2167 unsigned int type = 0; 2168 2169 if (intsize == 2) 2170 type = intspec[1]; 2171 2172 switch (type) { 2173 case 0: /* unofficial value, but we might as well let it work. */ 2174 case 4: /* official value for level triggering. */ 2175 *out_type = IRQ_TYPE_LEVEL_HIGH; 2176 break; 2177 case 1: /* official value for edge triggering. */ 2178 *out_type = IRQ_TYPE_EDGE_RISING; 2179 break; 2180 default: /* Nothing else is acceptable. */ 2181 return -EINVAL; 2182 } 2183 2184 *out_hwirq = intspec[0]; 2185 2186 return 0; 2187 } 2188 2189 static int octeon_irq_cib_map(struct irq_domain *d, 2190 unsigned int virq, irq_hw_number_t hw) 2191 { 2192 struct octeon_irq_cib_host_data *host_data = d->host_data; 2193 struct octeon_irq_cib_chip_data *cd; 2194 2195 if (hw >= host_data->max_bits) { 2196 pr_err("ERROR: %s mapping %u is to big!\n", 2197 irq_domain_get_of_node(d)->name, (unsigned)hw); 2198 return -EINVAL; 2199 } 2200 2201 cd = kzalloc(sizeof(*cd), GFP_KERNEL); 2202 cd->host_data = host_data; 2203 cd->bit = hw; 2204 2205 irq_set_chip_and_handler(virq, &octeon_irq_chip_cib, 2206 handle_simple_irq); 2207 irq_set_chip_data(virq, cd); 2208 return 0; 2209 } 2210 2211 static struct irq_domain_ops octeon_irq_domain_cib_ops = { 2212 .map = octeon_irq_cib_map, 2213 .unmap = octeon_irq_free_cd, 2214 .xlate = octeon_irq_cib_xlat, 2215 }; 2216 2217 /* Chain to real handler. */ 2218 static irqreturn_t octeon_irq_cib_handler(int my_irq, void *data) 2219 { 2220 u64 en; 2221 u64 raw; 2222 u64 bits; 2223 int i; 2224 int irq; 2225 struct irq_domain *cib_domain = data; 2226 struct octeon_irq_cib_host_data *host_data = cib_domain->host_data; 2227 2228 en = cvmx_read_csr(host_data->en_reg); 2229 raw = cvmx_read_csr(host_data->raw_reg); 2230 2231 bits = en & raw; 2232 2233 for (i = 0; i < host_data->max_bits; i++) { 2234 if ((bits & 1ull << i) == 0) 2235 continue; 2236 irq = irq_find_mapping(cib_domain, i); 2237 if (!irq) { 2238 unsigned long flags; 2239 2240 pr_err("ERROR: CIB bit %d@%llx IRQ unhandled, disabling\n", 2241 i, host_data->raw_reg); 2242 raw_spin_lock_irqsave(&host_data->lock, flags); 2243 en = cvmx_read_csr(host_data->en_reg); 2244 en &= ~(1ull << i); 2245 cvmx_write_csr(host_data->en_reg, en); 2246 cvmx_write_csr(host_data->raw_reg, 1ull << i); 2247 raw_spin_unlock_irqrestore(&host_data->lock, flags); 2248 } else { 2249 struct irq_desc *desc = irq_to_desc(irq); 2250 struct irq_data *irq_data = irq_desc_get_irq_data(desc); 2251 /* If edge, acknowledge the bit we will be sending. */ 2252 if (irqd_get_trigger_type(irq_data) & 2253 IRQ_TYPE_EDGE_BOTH) 2254 cvmx_write_csr(host_data->raw_reg, 1ull << i); 2255 generic_handle_irq_desc(desc); 2256 } 2257 } 2258 2259 return IRQ_HANDLED; 2260 } 2261 2262 static int __init octeon_irq_init_cib(struct device_node *ciu_node, 2263 struct device_node *parent) 2264 { 2265 const __be32 *addr; 2266 u32 val; 2267 struct octeon_irq_cib_host_data *host_data; 2268 int parent_irq; 2269 int r; 2270 struct irq_domain *cib_domain; 2271 2272 parent_irq = irq_of_parse_and_map(ciu_node, 0); 2273 if (!parent_irq) { 2274 pr_err("ERROR: Couldn't acquire parent_irq for %s\n.", 2275 ciu_node->name); 2276 return -EINVAL; 2277 } 2278 2279 host_data = kzalloc(sizeof(*host_data), GFP_KERNEL); 2280 raw_spin_lock_init(&host_data->lock); 2281 2282 addr = of_get_address(ciu_node, 0, NULL, NULL); 2283 if (!addr) { 2284 pr_err("ERROR: Couldn't acquire reg(0) %s\n.", ciu_node->name); 2285 return -EINVAL; 2286 } 2287 host_data->raw_reg = (u64)phys_to_virt( 2288 of_translate_address(ciu_node, addr)); 2289 2290 addr = of_get_address(ciu_node, 1, NULL, NULL); 2291 if (!addr) { 2292 pr_err("ERROR: Couldn't acquire reg(1) %s\n.", ciu_node->name); 2293 return -EINVAL; 2294 } 2295 host_data->en_reg = (u64)phys_to_virt( 2296 of_translate_address(ciu_node, addr)); 2297 2298 r = of_property_read_u32(ciu_node, "cavium,max-bits", &val); 2299 if (r) { 2300 pr_err("ERROR: Couldn't read cavium,max-bits from %s\n.", 2301 ciu_node->name); 2302 return r; 2303 } 2304 host_data->max_bits = val; 2305 2306 cib_domain = irq_domain_add_linear(ciu_node, host_data->max_bits, 2307 &octeon_irq_domain_cib_ops, 2308 host_data); 2309 if (!cib_domain) { 2310 pr_err("ERROR: Couldn't irq_domain_add_linear()\n."); 2311 return -ENOMEM; 2312 } 2313 2314 cvmx_write_csr(host_data->en_reg, 0); /* disable all IRQs */ 2315 cvmx_write_csr(host_data->raw_reg, ~0); /* ack any outstanding */ 2316 2317 r = request_irq(parent_irq, octeon_irq_cib_handler, 2318 IRQF_NO_THREAD, "cib", cib_domain); 2319 if (r) { 2320 pr_err("request_irq cib failed %d\n", r); 2321 return r; 2322 } 2323 pr_info("CIB interrupt controller probed: %llx %d\n", 2324 host_data->raw_reg, host_data->max_bits); 2325 return 0; 2326 } 2327 2328 int octeon_irq_ciu3_xlat(struct irq_domain *d, 2329 struct device_node *node, 2330 const u32 *intspec, 2331 unsigned int intsize, 2332 unsigned long *out_hwirq, 2333 unsigned int *out_type) 2334 { 2335 struct octeon_ciu3_info *ciu3_info = d->host_data; 2336 unsigned int hwirq, type, intsn_major; 2337 union cvmx_ciu3_iscx_ctl isc; 2338 2339 if (intsize < 2) 2340 return -EINVAL; 2341 hwirq = intspec[0]; 2342 type = intspec[1]; 2343 2344 if (hwirq >= (1 << 20)) 2345 return -EINVAL; 2346 2347 intsn_major = hwirq >> 12; 2348 switch (intsn_major) { 2349 case 0x04: /* Software handled separately. */ 2350 return -EINVAL; 2351 default: 2352 break; 2353 } 2354 2355 isc.u64 = cvmx_read_csr(ciu3_info->ciu3_addr + CIU3_ISC_CTL(hwirq)); 2356 if (!isc.s.imp) 2357 return -EINVAL; 2358 2359 switch (type) { 2360 case 4: /* official value for level triggering. */ 2361 *out_type = IRQ_TYPE_LEVEL_HIGH; 2362 break; 2363 case 0: /* unofficial value, but we might as well let it work. */ 2364 case 1: /* official value for edge triggering. */ 2365 *out_type = IRQ_TYPE_EDGE_RISING; 2366 break; 2367 default: /* Nothing else is acceptable. */ 2368 return -EINVAL; 2369 } 2370 2371 *out_hwirq = hwirq; 2372 2373 return 0; 2374 } 2375 2376 void octeon_irq_ciu3_enable(struct irq_data *data) 2377 { 2378 int cpu; 2379 union cvmx_ciu3_iscx_ctl isc_ctl; 2380 union cvmx_ciu3_iscx_w1c isc_w1c; 2381 u64 isc_ctl_addr; 2382 2383 struct octeon_ciu_chip_data *cd; 2384 2385 cpu = next_cpu_for_irq(data); 2386 2387 cd = irq_data_get_irq_chip_data(data); 2388 2389 isc_w1c.u64 = 0; 2390 isc_w1c.s.en = 1; 2391 cvmx_write_csr(cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn), isc_w1c.u64); 2392 2393 isc_ctl_addr = cd->ciu3_addr + CIU3_ISC_CTL(cd->intsn); 2394 isc_ctl.u64 = 0; 2395 isc_ctl.s.en = 1; 2396 isc_ctl.s.idt = per_cpu(octeon_irq_ciu3_idt_ip2, cpu); 2397 cvmx_write_csr(isc_ctl_addr, isc_ctl.u64); 2398 cvmx_read_csr(isc_ctl_addr); 2399 } 2400 2401 void octeon_irq_ciu3_disable(struct irq_data *data) 2402 { 2403 u64 isc_ctl_addr; 2404 union cvmx_ciu3_iscx_w1c isc_w1c; 2405 2406 struct octeon_ciu_chip_data *cd; 2407 2408 cd = irq_data_get_irq_chip_data(data); 2409 2410 isc_w1c.u64 = 0; 2411 isc_w1c.s.en = 1; 2412 2413 isc_ctl_addr = cd->ciu3_addr + CIU3_ISC_CTL(cd->intsn); 2414 cvmx_write_csr(cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn), isc_w1c.u64); 2415 cvmx_write_csr(isc_ctl_addr, 0); 2416 cvmx_read_csr(isc_ctl_addr); 2417 } 2418 2419 void octeon_irq_ciu3_ack(struct irq_data *data) 2420 { 2421 u64 isc_w1c_addr; 2422 union cvmx_ciu3_iscx_w1c isc_w1c; 2423 struct octeon_ciu_chip_data *cd; 2424 u32 trigger_type = irqd_get_trigger_type(data); 2425 2426 /* 2427 * We use a single irq_chip, so we have to do nothing to ack a 2428 * level interrupt. 2429 */ 2430 if (!(trigger_type & IRQ_TYPE_EDGE_BOTH)) 2431 return; 2432 2433 cd = irq_data_get_irq_chip_data(data); 2434 2435 isc_w1c.u64 = 0; 2436 isc_w1c.s.raw = 1; 2437 2438 isc_w1c_addr = cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn); 2439 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2440 cvmx_read_csr(isc_w1c_addr); 2441 } 2442 2443 void octeon_irq_ciu3_mask(struct irq_data *data) 2444 { 2445 union cvmx_ciu3_iscx_w1c isc_w1c; 2446 u64 isc_w1c_addr; 2447 struct octeon_ciu_chip_data *cd; 2448 2449 cd = irq_data_get_irq_chip_data(data); 2450 2451 isc_w1c.u64 = 0; 2452 isc_w1c.s.en = 1; 2453 2454 isc_w1c_addr = cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn); 2455 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2456 cvmx_read_csr(isc_w1c_addr); 2457 } 2458 2459 void octeon_irq_ciu3_mask_ack(struct irq_data *data) 2460 { 2461 union cvmx_ciu3_iscx_w1c isc_w1c; 2462 u64 isc_w1c_addr; 2463 struct octeon_ciu_chip_data *cd; 2464 u32 trigger_type = irqd_get_trigger_type(data); 2465 2466 cd = irq_data_get_irq_chip_data(data); 2467 2468 isc_w1c.u64 = 0; 2469 isc_w1c.s.en = 1; 2470 2471 /* 2472 * We use a single irq_chip, so only ack an edge (!level) 2473 * interrupt. 2474 */ 2475 if (trigger_type & IRQ_TYPE_EDGE_BOTH) 2476 isc_w1c.s.raw = 1; 2477 2478 isc_w1c_addr = cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn); 2479 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2480 cvmx_read_csr(isc_w1c_addr); 2481 } 2482 2483 #ifdef CONFIG_SMP 2484 int octeon_irq_ciu3_set_affinity(struct irq_data *data, 2485 const struct cpumask *dest, bool force) 2486 { 2487 union cvmx_ciu3_iscx_ctl isc_ctl; 2488 union cvmx_ciu3_iscx_w1c isc_w1c; 2489 u64 isc_ctl_addr; 2490 int cpu; 2491 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 2492 struct octeon_ciu_chip_data *cd = irq_data_get_irq_chip_data(data); 2493 2494 if (!cpumask_subset(dest, cpumask_of_node(cd->ciu_node))) 2495 return -EINVAL; 2496 2497 if (!enable_one) 2498 return IRQ_SET_MASK_OK; 2499 2500 cd = irq_data_get_irq_chip_data(data); 2501 cpu = cpumask_first(dest); 2502 if (cpu >= nr_cpu_ids) 2503 cpu = smp_processor_id(); 2504 cd->current_cpu = cpu; 2505 2506 isc_w1c.u64 = 0; 2507 isc_w1c.s.en = 1; 2508 cvmx_write_csr(cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn), isc_w1c.u64); 2509 2510 isc_ctl_addr = cd->ciu3_addr + CIU3_ISC_CTL(cd->intsn); 2511 isc_ctl.u64 = 0; 2512 isc_ctl.s.en = 1; 2513 isc_ctl.s.idt = per_cpu(octeon_irq_ciu3_idt_ip2, cpu); 2514 cvmx_write_csr(isc_ctl_addr, isc_ctl.u64); 2515 cvmx_read_csr(isc_ctl_addr); 2516 2517 return IRQ_SET_MASK_OK; 2518 } 2519 #endif 2520 2521 static struct irq_chip octeon_irq_chip_ciu3 = { 2522 .name = "CIU3", 2523 .irq_startup = edge_startup, 2524 .irq_enable = octeon_irq_ciu3_enable, 2525 .irq_disable = octeon_irq_ciu3_disable, 2526 .irq_ack = octeon_irq_ciu3_ack, 2527 .irq_mask = octeon_irq_ciu3_mask, 2528 .irq_mask_ack = octeon_irq_ciu3_mask_ack, 2529 .irq_unmask = octeon_irq_ciu3_enable, 2530 .irq_set_type = octeon_irq_ciu_set_type, 2531 #ifdef CONFIG_SMP 2532 .irq_set_affinity = octeon_irq_ciu3_set_affinity, 2533 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 2534 #endif 2535 }; 2536 2537 int octeon_irq_ciu3_mapx(struct irq_domain *d, unsigned int virq, 2538 irq_hw_number_t hw, struct irq_chip *chip) 2539 { 2540 struct octeon_ciu3_info *ciu3_info = d->host_data; 2541 struct octeon_ciu_chip_data *cd = kzalloc_node(sizeof(*cd), GFP_KERNEL, 2542 ciu3_info->node); 2543 if (!cd) 2544 return -ENOMEM; 2545 cd->intsn = hw; 2546 cd->current_cpu = -1; 2547 cd->ciu3_addr = ciu3_info->ciu3_addr; 2548 cd->ciu_node = ciu3_info->node; 2549 irq_set_chip_and_handler(virq, chip, handle_edge_irq); 2550 irq_set_chip_data(virq, cd); 2551 2552 return 0; 2553 } 2554 2555 static int octeon_irq_ciu3_map(struct irq_domain *d, 2556 unsigned int virq, irq_hw_number_t hw) 2557 { 2558 return octeon_irq_ciu3_mapx(d, virq, hw, &octeon_irq_chip_ciu3); 2559 } 2560 2561 static struct irq_domain_ops octeon_dflt_domain_ciu3_ops = { 2562 .map = octeon_irq_ciu3_map, 2563 .unmap = octeon_irq_free_cd, 2564 .xlate = octeon_irq_ciu3_xlat, 2565 }; 2566 2567 static void octeon_irq_ciu3_ip2(void) 2568 { 2569 union cvmx_ciu3_destx_pp_int dest_pp_int; 2570 struct octeon_ciu3_info *ciu3_info; 2571 u64 ciu3_addr; 2572 2573 ciu3_info = __this_cpu_read(octeon_ciu3_info); 2574 ciu3_addr = ciu3_info->ciu3_addr; 2575 2576 dest_pp_int.u64 = cvmx_read_csr(ciu3_addr + CIU3_DEST_PP_INT(3 * cvmx_get_local_core_num())); 2577 2578 if (likely(dest_pp_int.s.intr)) { 2579 irq_hw_number_t intsn = dest_pp_int.s.intsn; 2580 irq_hw_number_t hw; 2581 struct irq_domain *domain; 2582 /* Get the domain to use from the major block */ 2583 int block = intsn >> 12; 2584 int ret; 2585 2586 domain = ciu3_info->domain[block]; 2587 if (ciu3_info->intsn2hw[block]) 2588 hw = ciu3_info->intsn2hw[block](domain, intsn); 2589 else 2590 hw = intsn; 2591 2592 ret = handle_domain_irq(domain, hw, NULL); 2593 if (ret < 0) { 2594 union cvmx_ciu3_iscx_w1c isc_w1c; 2595 u64 isc_w1c_addr = ciu3_addr + CIU3_ISC_W1C(intsn); 2596 2597 isc_w1c.u64 = 0; 2598 isc_w1c.s.en = 1; 2599 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2600 cvmx_read_csr(isc_w1c_addr); 2601 spurious_interrupt(); 2602 } 2603 } else { 2604 spurious_interrupt(); 2605 } 2606 } 2607 2608 /* 2609 * 10 mbox per core starting from zero. 2610 * Base mbox is core * 10 2611 */ 2612 static unsigned int octeon_irq_ciu3_base_mbox_intsn(int core) 2613 { 2614 /* SW (mbox) are 0x04 in bits 12..19 */ 2615 return 0x04000 + CIU3_MBOX_PER_CORE * core; 2616 } 2617 2618 static unsigned int octeon_irq_ciu3_mbox_intsn_for_core(int core, unsigned int mbox) 2619 { 2620 return octeon_irq_ciu3_base_mbox_intsn(core) + mbox; 2621 } 2622 2623 static unsigned int octeon_irq_ciu3_mbox_intsn_for_cpu(int cpu, unsigned int mbox) 2624 { 2625 int local_core = octeon_coreid_for_cpu(cpu) & 0x3f; 2626 2627 return octeon_irq_ciu3_mbox_intsn_for_core(local_core, mbox); 2628 } 2629 2630 static void octeon_irq_ciu3_mbox(void) 2631 { 2632 union cvmx_ciu3_destx_pp_int dest_pp_int; 2633 struct octeon_ciu3_info *ciu3_info; 2634 u64 ciu3_addr; 2635 int core = cvmx_get_local_core_num(); 2636 2637 ciu3_info = __this_cpu_read(octeon_ciu3_info); 2638 ciu3_addr = ciu3_info->ciu3_addr; 2639 2640 dest_pp_int.u64 = cvmx_read_csr(ciu3_addr + CIU3_DEST_PP_INT(1 + 3 * core)); 2641 2642 if (likely(dest_pp_int.s.intr)) { 2643 irq_hw_number_t intsn = dest_pp_int.s.intsn; 2644 int mbox = intsn - octeon_irq_ciu3_base_mbox_intsn(core); 2645 2646 if (likely(mbox >= 0 && mbox < CIU3_MBOX_PER_CORE)) { 2647 do_IRQ(mbox + OCTEON_IRQ_MBOX0); 2648 } else { 2649 union cvmx_ciu3_iscx_w1c isc_w1c; 2650 u64 isc_w1c_addr = ciu3_addr + CIU3_ISC_W1C(intsn); 2651 2652 isc_w1c.u64 = 0; 2653 isc_w1c.s.en = 1; 2654 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2655 cvmx_read_csr(isc_w1c_addr); 2656 spurious_interrupt(); 2657 } 2658 } else { 2659 spurious_interrupt(); 2660 } 2661 } 2662 2663 void octeon_ciu3_mbox_send(int cpu, unsigned int mbox) 2664 { 2665 struct octeon_ciu3_info *ciu3_info; 2666 unsigned int intsn; 2667 union cvmx_ciu3_iscx_w1s isc_w1s; 2668 u64 isc_w1s_addr; 2669 2670 if (WARN_ON_ONCE(mbox >= CIU3_MBOX_PER_CORE)) 2671 return; 2672 2673 intsn = octeon_irq_ciu3_mbox_intsn_for_cpu(cpu, mbox); 2674 ciu3_info = per_cpu(octeon_ciu3_info, cpu); 2675 isc_w1s_addr = ciu3_info->ciu3_addr + CIU3_ISC_W1S(intsn); 2676 2677 isc_w1s.u64 = 0; 2678 isc_w1s.s.raw = 1; 2679 2680 cvmx_write_csr(isc_w1s_addr, isc_w1s.u64); 2681 cvmx_read_csr(isc_w1s_addr); 2682 } 2683 2684 static void octeon_irq_ciu3_mbox_set_enable(struct irq_data *data, int cpu, bool en) 2685 { 2686 struct octeon_ciu3_info *ciu3_info; 2687 unsigned int intsn; 2688 u64 isc_ctl_addr, isc_w1c_addr; 2689 union cvmx_ciu3_iscx_ctl isc_ctl; 2690 unsigned int mbox = data->irq - OCTEON_IRQ_MBOX0; 2691 2692 intsn = octeon_irq_ciu3_mbox_intsn_for_cpu(cpu, mbox); 2693 ciu3_info = per_cpu(octeon_ciu3_info, cpu); 2694 isc_w1c_addr = ciu3_info->ciu3_addr + CIU3_ISC_W1C(intsn); 2695 isc_ctl_addr = ciu3_info->ciu3_addr + CIU3_ISC_CTL(intsn); 2696 2697 isc_ctl.u64 = 0; 2698 isc_ctl.s.en = 1; 2699 2700 cvmx_write_csr(isc_w1c_addr, isc_ctl.u64); 2701 cvmx_write_csr(isc_ctl_addr, 0); 2702 if (en) { 2703 unsigned int idt = per_cpu(octeon_irq_ciu3_idt_ip3, cpu); 2704 2705 isc_ctl.u64 = 0; 2706 isc_ctl.s.en = 1; 2707 isc_ctl.s.idt = idt; 2708 cvmx_write_csr(isc_ctl_addr, isc_ctl.u64); 2709 } 2710 cvmx_read_csr(isc_ctl_addr); 2711 } 2712 2713 static void octeon_irq_ciu3_mbox_enable(struct irq_data *data) 2714 { 2715 int cpu; 2716 unsigned int mbox = data->irq - OCTEON_IRQ_MBOX0; 2717 2718 WARN_ON(mbox >= CIU3_MBOX_PER_CORE); 2719 2720 for_each_online_cpu(cpu) 2721 octeon_irq_ciu3_mbox_set_enable(data, cpu, true); 2722 } 2723 2724 static void octeon_irq_ciu3_mbox_disable(struct irq_data *data) 2725 { 2726 int cpu; 2727 unsigned int mbox = data->irq - OCTEON_IRQ_MBOX0; 2728 2729 WARN_ON(mbox >= CIU3_MBOX_PER_CORE); 2730 2731 for_each_online_cpu(cpu) 2732 octeon_irq_ciu3_mbox_set_enable(data, cpu, false); 2733 } 2734 2735 static void octeon_irq_ciu3_mbox_ack(struct irq_data *data) 2736 { 2737 struct octeon_ciu3_info *ciu3_info; 2738 unsigned int intsn; 2739 u64 isc_w1c_addr; 2740 union cvmx_ciu3_iscx_w1c isc_w1c; 2741 unsigned int mbox = data->irq - OCTEON_IRQ_MBOX0; 2742 2743 intsn = octeon_irq_ciu3_mbox_intsn_for_core(cvmx_get_local_core_num(), mbox); 2744 2745 isc_w1c.u64 = 0; 2746 isc_w1c.s.raw = 1; 2747 2748 ciu3_info = __this_cpu_read(octeon_ciu3_info); 2749 isc_w1c_addr = ciu3_info->ciu3_addr + CIU3_ISC_W1C(intsn); 2750 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2751 cvmx_read_csr(isc_w1c_addr); 2752 } 2753 2754 static void octeon_irq_ciu3_mbox_cpu_online(struct irq_data *data) 2755 { 2756 octeon_irq_ciu3_mbox_set_enable(data, smp_processor_id(), true); 2757 } 2758 2759 static void octeon_irq_ciu3_mbox_cpu_offline(struct irq_data *data) 2760 { 2761 octeon_irq_ciu3_mbox_set_enable(data, smp_processor_id(), false); 2762 } 2763 2764 static int octeon_irq_ciu3_alloc_resources(struct octeon_ciu3_info *ciu3_info) 2765 { 2766 u64 b = ciu3_info->ciu3_addr; 2767 int idt_ip2, idt_ip3, idt_ip4; 2768 int unused_idt2; 2769 int core = cvmx_get_local_core_num(); 2770 int i; 2771 2772 __this_cpu_write(octeon_ciu3_info, ciu3_info); 2773 2774 /* 2775 * 4 idt per core starting from 1 because zero is reserved. 2776 * Base idt per core is 4 * core + 1 2777 */ 2778 idt_ip2 = core * 4 + 1; 2779 idt_ip3 = core * 4 + 2; 2780 idt_ip4 = core * 4 + 3; 2781 unused_idt2 = core * 4 + 4; 2782 __this_cpu_write(octeon_irq_ciu3_idt_ip2, idt_ip2); 2783 __this_cpu_write(octeon_irq_ciu3_idt_ip3, idt_ip3); 2784 2785 /* ip2 interrupts for this CPU */ 2786 cvmx_write_csr(b + CIU3_IDT_CTL(idt_ip2), 0); 2787 cvmx_write_csr(b + CIU3_IDT_PP(idt_ip2, 0), 1ull << core); 2788 cvmx_write_csr(b + CIU3_IDT_IO(idt_ip2), 0); 2789 2790 /* ip3 interrupts for this CPU */ 2791 cvmx_write_csr(b + CIU3_IDT_CTL(idt_ip3), 1); 2792 cvmx_write_csr(b + CIU3_IDT_PP(idt_ip3, 0), 1ull << core); 2793 cvmx_write_csr(b + CIU3_IDT_IO(idt_ip3), 0); 2794 2795 /* ip4 interrupts for this CPU */ 2796 cvmx_write_csr(b + CIU3_IDT_CTL(idt_ip4), 2); 2797 cvmx_write_csr(b + CIU3_IDT_PP(idt_ip4, 0), 0); 2798 cvmx_write_csr(b + CIU3_IDT_IO(idt_ip4), 0); 2799 2800 cvmx_write_csr(b + CIU3_IDT_CTL(unused_idt2), 0); 2801 cvmx_write_csr(b + CIU3_IDT_PP(unused_idt2, 0), 0); 2802 cvmx_write_csr(b + CIU3_IDT_IO(unused_idt2), 0); 2803 2804 for (i = 0; i < CIU3_MBOX_PER_CORE; i++) { 2805 unsigned int intsn = octeon_irq_ciu3_mbox_intsn_for_core(core, i); 2806 2807 cvmx_write_csr(b + CIU3_ISC_W1C(intsn), 2); 2808 cvmx_write_csr(b + CIU3_ISC_CTL(intsn), 0); 2809 } 2810 2811 return 0; 2812 } 2813 2814 static void octeon_irq_setup_secondary_ciu3(void) 2815 { 2816 struct octeon_ciu3_info *ciu3_info; 2817 2818 ciu3_info = octeon_ciu3_info_per_node[cvmx_get_node_num()]; 2819 octeon_irq_ciu3_alloc_resources(ciu3_info); 2820 irq_cpu_online(); 2821 2822 /* Enable the CIU lines */ 2823 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 2824 if (octeon_irq_use_ip4) 2825 set_c0_status(STATUSF_IP4); 2826 else 2827 clear_c0_status(STATUSF_IP4); 2828 } 2829 2830 static struct irq_chip octeon_irq_chip_ciu3_mbox = { 2831 .name = "CIU3-M", 2832 .irq_enable = octeon_irq_ciu3_mbox_enable, 2833 .irq_disable = octeon_irq_ciu3_mbox_disable, 2834 .irq_ack = octeon_irq_ciu3_mbox_ack, 2835 2836 .irq_cpu_online = octeon_irq_ciu3_mbox_cpu_online, 2837 .irq_cpu_offline = octeon_irq_ciu3_mbox_cpu_offline, 2838 .flags = IRQCHIP_ONOFFLINE_ENABLED, 2839 }; 2840 2841 static int __init octeon_irq_init_ciu3(struct device_node *ciu_node, 2842 struct device_node *parent) 2843 { 2844 int i; 2845 int node; 2846 struct irq_domain *domain; 2847 struct octeon_ciu3_info *ciu3_info; 2848 const __be32 *zero_addr; 2849 u64 base_addr; 2850 union cvmx_ciu3_const consts; 2851 2852 node = 0; /* of_node_to_nid(ciu_node); */ 2853 ciu3_info = kzalloc_node(sizeof(*ciu3_info), GFP_KERNEL, node); 2854 2855 if (!ciu3_info) 2856 return -ENOMEM; 2857 2858 zero_addr = of_get_address(ciu_node, 0, NULL, NULL); 2859 if (WARN_ON(!zero_addr)) 2860 return -EINVAL; 2861 2862 base_addr = of_translate_address(ciu_node, zero_addr); 2863 base_addr = (u64)phys_to_virt(base_addr); 2864 2865 ciu3_info->ciu3_addr = base_addr; 2866 ciu3_info->node = node; 2867 2868 consts.u64 = cvmx_read_csr(base_addr + CIU3_CONST); 2869 2870 octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu3; 2871 2872 octeon_irq_ip2 = octeon_irq_ciu3_ip2; 2873 octeon_irq_ip3 = octeon_irq_ciu3_mbox; 2874 octeon_irq_ip4 = octeon_irq_ip4_mask; 2875 2876 if (node == cvmx_get_node_num()) { 2877 /* Mips internal */ 2878 octeon_irq_init_core(); 2879 2880 /* Only do per CPU things if it is the CIU of the boot node. */ 2881 i = irq_alloc_descs_from(OCTEON_IRQ_MBOX0, 8, node); 2882 WARN_ON(i < 0); 2883 2884 for (i = 0; i < 8; i++) 2885 irq_set_chip_and_handler(i + OCTEON_IRQ_MBOX0, 2886 &octeon_irq_chip_ciu3_mbox, handle_percpu_irq); 2887 } 2888 2889 /* 2890 * Initialize all domains to use the default domain. Specific major 2891 * blocks will overwrite the default domain as needed. 2892 */ 2893 domain = irq_domain_add_tree(ciu_node, &octeon_dflt_domain_ciu3_ops, 2894 ciu3_info); 2895 for (i = 0; i < MAX_CIU3_DOMAINS; i++) 2896 ciu3_info->domain[i] = domain; 2897 2898 octeon_ciu3_info_per_node[node] = ciu3_info; 2899 2900 if (node == cvmx_get_node_num()) { 2901 /* Only do per CPU things if it is the CIU of the boot node. */ 2902 octeon_irq_ciu3_alloc_resources(ciu3_info); 2903 if (node == 0) 2904 irq_set_default_host(domain); 2905 2906 octeon_irq_use_ip4 = false; 2907 /* Enable the CIU lines */ 2908 set_c0_status(STATUSF_IP2 | STATUSF_IP3); 2909 clear_c0_status(STATUSF_IP4); 2910 } 2911 2912 return 0; 2913 } 2914 2915 static struct of_device_id ciu_types[] __initdata = { 2916 {.compatible = "cavium,octeon-3860-ciu", .data = octeon_irq_init_ciu}, 2917 {.compatible = "cavium,octeon-3860-gpio", .data = octeon_irq_init_gpio}, 2918 {.compatible = "cavium,octeon-6880-ciu2", .data = octeon_irq_init_ciu2}, 2919 {.compatible = "cavium,octeon-7890-ciu3", .data = octeon_irq_init_ciu3}, 2920 {.compatible = "cavium,octeon-7130-cib", .data = octeon_irq_init_cib}, 2921 {} 2922 }; 2923 2924 void __init arch_init_irq(void) 2925 { 2926 #ifdef CONFIG_SMP 2927 /* Set the default affinity to the boot cpu. */ 2928 cpumask_clear(irq_default_affinity); 2929 cpumask_set_cpu(smp_processor_id(), irq_default_affinity); 2930 #endif 2931 of_irq_init(ciu_types); 2932 } 2933 2934 asmlinkage void plat_irq_dispatch(void) 2935 { 2936 unsigned long cop0_cause; 2937 unsigned long cop0_status; 2938 2939 while (1) { 2940 cop0_cause = read_c0_cause(); 2941 cop0_status = read_c0_status(); 2942 cop0_cause &= cop0_status; 2943 cop0_cause &= ST0_IM; 2944 2945 if (cop0_cause & STATUSF_IP2) 2946 octeon_irq_ip2(); 2947 else if (cop0_cause & STATUSF_IP3) 2948 octeon_irq_ip3(); 2949 else if (cop0_cause & STATUSF_IP4) 2950 octeon_irq_ip4(); 2951 else if (cop0_cause) 2952 do_IRQ(fls(cop0_cause) - 9 + MIPS_CPU_IRQ_BASE); 2953 else 2954 break; 2955 } 2956 } 2957 2958 #ifdef CONFIG_HOTPLUG_CPU 2959 2960 void octeon_fixup_irqs(void) 2961 { 2962 irq_cpu_offline(); 2963 } 2964 2965 #endif /* CONFIG_HOTPLUG_CPU */ 2966 2967 struct irq_domain *octeon_irq_get_block_domain(int node, uint8_t block) 2968 { 2969 struct octeon_ciu3_info *ciu3_info; 2970 2971 ciu3_info = octeon_ciu3_info_per_node[node & CVMX_NODE_MASK]; 2972 return ciu3_info->domain[block]; 2973 } 2974 EXPORT_SYMBOL(octeon_irq_get_block_domain); 2975