1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 2004-2016 Cavium, Inc. 7 */ 8 9 #include <linux/of_address.h> 10 #include <linux/interrupt.h> 11 #include <linux/irqdomain.h> 12 #include <linux/bitops.h> 13 #include <linux/of_irq.h> 14 #include <linux/percpu.h> 15 #include <linux/slab.h> 16 #include <linux/irq.h> 17 #include <linux/smp.h> 18 #include <linux/of.h> 19 20 #include <asm/octeon/octeon.h> 21 #include <asm/octeon/cvmx-ciu2-defs.h> 22 #include <asm/octeon/cvmx-ciu3-defs.h> 23 24 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu0_en_mirror); 25 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu1_en_mirror); 26 static DEFINE_PER_CPU(raw_spinlock_t, octeon_irq_ciu_spinlock); 27 static DEFINE_PER_CPU(unsigned int, octeon_irq_ciu3_idt_ip2); 28 29 static DEFINE_PER_CPU(unsigned int, octeon_irq_ciu3_idt_ip3); 30 static DEFINE_PER_CPU(struct octeon_ciu3_info *, octeon_ciu3_info); 31 #define CIU3_MBOX_PER_CORE 10 32 33 /* 34 * The 8 most significant bits of the intsn identify the interrupt major block. 35 * Each major block might use its own interrupt domain. Thus 256 domains are 36 * needed. 37 */ 38 #define MAX_CIU3_DOMAINS 256 39 40 typedef irq_hw_number_t (*octeon_ciu3_intsn2hw_t)(struct irq_domain *, unsigned int); 41 42 /* Information for each ciu3 in the system */ 43 struct octeon_ciu3_info { 44 u64 ciu3_addr; 45 int node; 46 struct irq_domain *domain[MAX_CIU3_DOMAINS]; 47 octeon_ciu3_intsn2hw_t intsn2hw[MAX_CIU3_DOMAINS]; 48 }; 49 50 /* Each ciu3 in the system uses its own data (one ciu3 per node) */ 51 static struct octeon_ciu3_info *octeon_ciu3_info_per_node[4]; 52 53 struct octeon_irq_ciu_domain_data { 54 int num_sum; /* number of sum registers (2 or 3). */ 55 }; 56 57 /* Register offsets from ciu3_addr */ 58 #define CIU3_CONST 0x220 59 #define CIU3_IDT_CTL(_idt) ((_idt) * 8 + 0x110000) 60 #define CIU3_IDT_PP(_idt, _idx) ((_idt) * 32 + (_idx) * 8 + 0x120000) 61 #define CIU3_IDT_IO(_idt) ((_idt) * 8 + 0x130000) 62 #define CIU3_DEST_PP_INT(_pp_ip) ((_pp_ip) * 8 + 0x200000) 63 #define CIU3_DEST_IO_INT(_io) ((_io) * 8 + 0x210000) 64 #define CIU3_ISC_CTL(_intsn) ((_intsn) * 8 + 0x80000000) 65 #define CIU3_ISC_W1C(_intsn) ((_intsn) * 8 + 0x90000000) 66 #define CIU3_ISC_W1S(_intsn) ((_intsn) * 8 + 0xa0000000) 67 68 static __read_mostly int octeon_irq_ciu_to_irq[8][64]; 69 70 struct octeon_ciu_chip_data { 71 union { 72 struct { /* only used for ciu3 */ 73 u64 ciu3_addr; 74 unsigned int intsn; 75 }; 76 struct { /* only used for ciu/ciu2 */ 77 u8 line; 78 u8 bit; 79 }; 80 }; 81 int gpio_line; 82 int current_cpu; /* Next CPU expected to take this irq */ 83 int ciu_node; /* NUMA node number of the CIU */ 84 }; 85 86 struct octeon_core_chip_data { 87 struct mutex core_irq_mutex; 88 bool current_en; 89 bool desired_en; 90 u8 bit; 91 }; 92 93 #define MIPS_CORE_IRQ_LINES 8 94 95 static struct octeon_core_chip_data octeon_irq_core_chip_data[MIPS_CORE_IRQ_LINES]; 96 97 static int octeon_irq_set_ciu_mapping(int irq, int line, int bit, int gpio_line, 98 struct irq_chip *chip, 99 irq_flow_handler_t handler) 100 { 101 struct octeon_ciu_chip_data *cd; 102 103 cd = kzalloc(sizeof(*cd), GFP_KERNEL); 104 if (!cd) 105 return -ENOMEM; 106 107 irq_set_chip_and_handler(irq, chip, handler); 108 109 cd->line = line; 110 cd->bit = bit; 111 cd->gpio_line = gpio_line; 112 113 irq_set_chip_data(irq, cd); 114 octeon_irq_ciu_to_irq[line][bit] = irq; 115 return 0; 116 } 117 118 static void octeon_irq_free_cd(struct irq_domain *d, unsigned int irq) 119 { 120 struct irq_data *data = irq_get_irq_data(irq); 121 struct octeon_ciu_chip_data *cd = irq_data_get_irq_chip_data(data); 122 123 irq_set_chip_data(irq, NULL); 124 kfree(cd); 125 } 126 127 static int octeon_irq_force_ciu_mapping(struct irq_domain *domain, 128 int irq, int line, int bit) 129 { 130 return irq_domain_associate(domain, irq, line << 6 | bit); 131 } 132 133 static int octeon_coreid_for_cpu(int cpu) 134 { 135 #ifdef CONFIG_SMP 136 return cpu_logical_map(cpu); 137 #else 138 return cvmx_get_core_num(); 139 #endif 140 } 141 142 static int octeon_cpu_for_coreid(int coreid) 143 { 144 #ifdef CONFIG_SMP 145 return cpu_number_map(coreid); 146 #else 147 return smp_processor_id(); 148 #endif 149 } 150 151 static void octeon_irq_core_ack(struct irq_data *data) 152 { 153 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 154 unsigned int bit = cd->bit; 155 156 /* 157 * We don't need to disable IRQs to make these atomic since 158 * they are already disabled earlier in the low level 159 * interrupt code. 160 */ 161 clear_c0_status(0x100 << bit); 162 /* The two user interrupts must be cleared manually. */ 163 if (bit < 2) 164 clear_c0_cause(0x100 << bit); 165 } 166 167 static void octeon_irq_core_eoi(struct irq_data *data) 168 { 169 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 170 171 /* 172 * We don't need to disable IRQs to make these atomic since 173 * they are already disabled earlier in the low level 174 * interrupt code. 175 */ 176 set_c0_status(0x100 << cd->bit); 177 } 178 179 static void octeon_irq_core_set_enable_local(void *arg) 180 { 181 struct irq_data *data = arg; 182 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 183 unsigned int mask = 0x100 << cd->bit; 184 185 /* 186 * Interrupts are already disabled, so these are atomic. 187 */ 188 if (cd->desired_en) 189 set_c0_status(mask); 190 else 191 clear_c0_status(mask); 192 193 } 194 195 static void octeon_irq_core_disable(struct irq_data *data) 196 { 197 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 198 cd->desired_en = false; 199 } 200 201 static void octeon_irq_core_enable(struct irq_data *data) 202 { 203 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 204 cd->desired_en = true; 205 } 206 207 static void octeon_irq_core_bus_lock(struct irq_data *data) 208 { 209 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 210 211 mutex_lock(&cd->core_irq_mutex); 212 } 213 214 static void octeon_irq_core_bus_sync_unlock(struct irq_data *data) 215 { 216 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 217 218 if (cd->desired_en != cd->current_en) { 219 on_each_cpu(octeon_irq_core_set_enable_local, data, 1); 220 221 cd->current_en = cd->desired_en; 222 } 223 224 mutex_unlock(&cd->core_irq_mutex); 225 } 226 227 static struct irq_chip octeon_irq_chip_core = { 228 .name = "Core", 229 .irq_enable = octeon_irq_core_enable, 230 .irq_disable = octeon_irq_core_disable, 231 .irq_ack = octeon_irq_core_ack, 232 .irq_eoi = octeon_irq_core_eoi, 233 .irq_bus_lock = octeon_irq_core_bus_lock, 234 .irq_bus_sync_unlock = octeon_irq_core_bus_sync_unlock, 235 236 .irq_cpu_online = octeon_irq_core_eoi, 237 .irq_cpu_offline = octeon_irq_core_ack, 238 .flags = IRQCHIP_ONOFFLINE_ENABLED, 239 }; 240 241 static void __init octeon_irq_init_core(void) 242 { 243 int i; 244 int irq; 245 struct octeon_core_chip_data *cd; 246 247 for (i = 0; i < MIPS_CORE_IRQ_LINES; i++) { 248 cd = &octeon_irq_core_chip_data[i]; 249 cd->current_en = false; 250 cd->desired_en = false; 251 cd->bit = i; 252 mutex_init(&cd->core_irq_mutex); 253 254 irq = OCTEON_IRQ_SW0 + i; 255 irq_set_chip_data(irq, cd); 256 irq_set_chip_and_handler(irq, &octeon_irq_chip_core, 257 handle_percpu_irq); 258 } 259 } 260 261 static int next_cpu_for_irq(struct irq_data *data) 262 { 263 264 #ifdef CONFIG_SMP 265 int cpu; 266 struct cpumask *mask = irq_data_get_affinity_mask(data); 267 int weight = cpumask_weight(mask); 268 struct octeon_ciu_chip_data *cd = irq_data_get_irq_chip_data(data); 269 270 if (weight > 1) { 271 cpu = cd->current_cpu; 272 for (;;) { 273 cpu = cpumask_next(cpu, mask); 274 if (cpu >= nr_cpu_ids) { 275 cpu = -1; 276 continue; 277 } else if (cpumask_test_cpu(cpu, cpu_online_mask)) { 278 break; 279 } 280 } 281 } else if (weight == 1) { 282 cpu = cpumask_first(mask); 283 } else { 284 cpu = smp_processor_id(); 285 } 286 cd->current_cpu = cpu; 287 return cpu; 288 #else 289 return smp_processor_id(); 290 #endif 291 } 292 293 static void octeon_irq_ciu_enable(struct irq_data *data) 294 { 295 int cpu = next_cpu_for_irq(data); 296 int coreid = octeon_coreid_for_cpu(cpu); 297 unsigned long *pen; 298 unsigned long flags; 299 struct octeon_ciu_chip_data *cd; 300 raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 301 302 cd = irq_data_get_irq_chip_data(data); 303 304 raw_spin_lock_irqsave(lock, flags); 305 if (cd->line == 0) { 306 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 307 __set_bit(cd->bit, pen); 308 /* 309 * Must be visible to octeon_irq_ip{2,3}_ciu() before 310 * enabling the irq. 311 */ 312 wmb(); 313 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 314 } else { 315 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 316 __set_bit(cd->bit, pen); 317 /* 318 * Must be visible to octeon_irq_ip{2,3}_ciu() before 319 * enabling the irq. 320 */ 321 wmb(); 322 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 323 } 324 raw_spin_unlock_irqrestore(lock, flags); 325 } 326 327 static void octeon_irq_ciu_enable_local(struct irq_data *data) 328 { 329 unsigned long *pen; 330 unsigned long flags; 331 struct octeon_ciu_chip_data *cd; 332 raw_spinlock_t *lock = this_cpu_ptr(&octeon_irq_ciu_spinlock); 333 334 cd = irq_data_get_irq_chip_data(data); 335 336 raw_spin_lock_irqsave(lock, flags); 337 if (cd->line == 0) { 338 pen = this_cpu_ptr(&octeon_irq_ciu0_en_mirror); 339 __set_bit(cd->bit, pen); 340 /* 341 * Must be visible to octeon_irq_ip{2,3}_ciu() before 342 * enabling the irq. 343 */ 344 wmb(); 345 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen); 346 } else { 347 pen = this_cpu_ptr(&octeon_irq_ciu1_en_mirror); 348 __set_bit(cd->bit, pen); 349 /* 350 * Must be visible to octeon_irq_ip{2,3}_ciu() before 351 * enabling the irq. 352 */ 353 wmb(); 354 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen); 355 } 356 raw_spin_unlock_irqrestore(lock, flags); 357 } 358 359 static void octeon_irq_ciu_disable_local(struct irq_data *data) 360 { 361 unsigned long *pen; 362 unsigned long flags; 363 struct octeon_ciu_chip_data *cd; 364 raw_spinlock_t *lock = this_cpu_ptr(&octeon_irq_ciu_spinlock); 365 366 cd = irq_data_get_irq_chip_data(data); 367 368 raw_spin_lock_irqsave(lock, flags); 369 if (cd->line == 0) { 370 pen = this_cpu_ptr(&octeon_irq_ciu0_en_mirror); 371 __clear_bit(cd->bit, pen); 372 /* 373 * Must be visible to octeon_irq_ip{2,3}_ciu() before 374 * enabling the irq. 375 */ 376 wmb(); 377 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen); 378 } else { 379 pen = this_cpu_ptr(&octeon_irq_ciu1_en_mirror); 380 __clear_bit(cd->bit, pen); 381 /* 382 * Must be visible to octeon_irq_ip{2,3}_ciu() before 383 * enabling the irq. 384 */ 385 wmb(); 386 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen); 387 } 388 raw_spin_unlock_irqrestore(lock, flags); 389 } 390 391 static void octeon_irq_ciu_disable_all(struct irq_data *data) 392 { 393 unsigned long flags; 394 unsigned long *pen; 395 int cpu; 396 struct octeon_ciu_chip_data *cd; 397 raw_spinlock_t *lock; 398 399 cd = irq_data_get_irq_chip_data(data); 400 401 for_each_online_cpu(cpu) { 402 int coreid = octeon_coreid_for_cpu(cpu); 403 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 404 if (cd->line == 0) 405 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 406 else 407 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 408 409 raw_spin_lock_irqsave(lock, flags); 410 __clear_bit(cd->bit, pen); 411 /* 412 * Must be visible to octeon_irq_ip{2,3}_ciu() before 413 * enabling the irq. 414 */ 415 wmb(); 416 if (cd->line == 0) 417 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 418 else 419 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 420 raw_spin_unlock_irqrestore(lock, flags); 421 } 422 } 423 424 static void octeon_irq_ciu_enable_all(struct irq_data *data) 425 { 426 unsigned long flags; 427 unsigned long *pen; 428 int cpu; 429 struct octeon_ciu_chip_data *cd; 430 raw_spinlock_t *lock; 431 432 cd = irq_data_get_irq_chip_data(data); 433 434 for_each_online_cpu(cpu) { 435 int coreid = octeon_coreid_for_cpu(cpu); 436 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 437 if (cd->line == 0) 438 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 439 else 440 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 441 442 raw_spin_lock_irqsave(lock, flags); 443 __set_bit(cd->bit, pen); 444 /* 445 * Must be visible to octeon_irq_ip{2,3}_ciu() before 446 * enabling the irq. 447 */ 448 wmb(); 449 if (cd->line == 0) 450 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 451 else 452 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 453 raw_spin_unlock_irqrestore(lock, flags); 454 } 455 } 456 457 /* 458 * Enable the irq on the next core in the affinity set for chips that 459 * have the EN*_W1{S,C} registers. 460 */ 461 static void octeon_irq_ciu_enable_v2(struct irq_data *data) 462 { 463 u64 mask; 464 int cpu = next_cpu_for_irq(data); 465 struct octeon_ciu_chip_data *cd; 466 467 cd = irq_data_get_irq_chip_data(data); 468 mask = 1ull << (cd->bit); 469 470 /* 471 * Called under the desc lock, so these should never get out 472 * of sync. 473 */ 474 if (cd->line == 0) { 475 int index = octeon_coreid_for_cpu(cpu) * 2; 476 set_bit(cd->bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu)); 477 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 478 } else { 479 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 480 set_bit(cd->bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 481 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 482 } 483 } 484 485 /* 486 * Enable the irq in the sum2 registers. 487 */ 488 static void octeon_irq_ciu_enable_sum2(struct irq_data *data) 489 { 490 u64 mask; 491 int cpu = next_cpu_for_irq(data); 492 int index = octeon_coreid_for_cpu(cpu); 493 struct octeon_ciu_chip_data *cd; 494 495 cd = irq_data_get_irq_chip_data(data); 496 mask = 1ull << (cd->bit); 497 498 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1S(index), mask); 499 } 500 501 /* 502 * Disable the irq in the sum2 registers. 503 */ 504 static void octeon_irq_ciu_disable_local_sum2(struct irq_data *data) 505 { 506 u64 mask; 507 int cpu = next_cpu_for_irq(data); 508 int index = octeon_coreid_for_cpu(cpu); 509 struct octeon_ciu_chip_data *cd; 510 511 cd = irq_data_get_irq_chip_data(data); 512 mask = 1ull << (cd->bit); 513 514 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(index), mask); 515 } 516 517 static void octeon_irq_ciu_ack_sum2(struct irq_data *data) 518 { 519 u64 mask; 520 int cpu = next_cpu_for_irq(data); 521 int index = octeon_coreid_for_cpu(cpu); 522 struct octeon_ciu_chip_data *cd; 523 524 cd = irq_data_get_irq_chip_data(data); 525 mask = 1ull << (cd->bit); 526 527 cvmx_write_csr(CVMX_CIU_SUM2_PPX_IP4(index), mask); 528 } 529 530 static void octeon_irq_ciu_disable_all_sum2(struct irq_data *data) 531 { 532 int cpu; 533 struct octeon_ciu_chip_data *cd; 534 u64 mask; 535 536 cd = irq_data_get_irq_chip_data(data); 537 mask = 1ull << (cd->bit); 538 539 for_each_online_cpu(cpu) { 540 int coreid = octeon_coreid_for_cpu(cpu); 541 542 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(coreid), mask); 543 } 544 } 545 546 /* 547 * Enable the irq on the current CPU for chips that 548 * have the EN*_W1{S,C} registers. 549 */ 550 static void octeon_irq_ciu_enable_local_v2(struct irq_data *data) 551 { 552 u64 mask; 553 struct octeon_ciu_chip_data *cd; 554 555 cd = irq_data_get_irq_chip_data(data); 556 mask = 1ull << (cd->bit); 557 558 if (cd->line == 0) { 559 int index = cvmx_get_core_num() * 2; 560 set_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu0_en_mirror)); 561 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 562 } else { 563 int index = cvmx_get_core_num() * 2 + 1; 564 set_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu1_en_mirror)); 565 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 566 } 567 } 568 569 static void octeon_irq_ciu_disable_local_v2(struct irq_data *data) 570 { 571 u64 mask; 572 struct octeon_ciu_chip_data *cd; 573 574 cd = irq_data_get_irq_chip_data(data); 575 mask = 1ull << (cd->bit); 576 577 if (cd->line == 0) { 578 int index = cvmx_get_core_num() * 2; 579 clear_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu0_en_mirror)); 580 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); 581 } else { 582 int index = cvmx_get_core_num() * 2 + 1; 583 clear_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu1_en_mirror)); 584 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask); 585 } 586 } 587 588 /* 589 * Write to the W1C bit in CVMX_CIU_INTX_SUM0 to clear the irq. 590 */ 591 static void octeon_irq_ciu_ack(struct irq_data *data) 592 { 593 u64 mask; 594 struct octeon_ciu_chip_data *cd; 595 596 cd = irq_data_get_irq_chip_data(data); 597 mask = 1ull << (cd->bit); 598 599 if (cd->line == 0) { 600 int index = cvmx_get_core_num() * 2; 601 cvmx_write_csr(CVMX_CIU_INTX_SUM0(index), mask); 602 } else { 603 cvmx_write_csr(CVMX_CIU_INT_SUM1, mask); 604 } 605 } 606 607 /* 608 * Disable the irq on the all cores for chips that have the EN*_W1{S,C} 609 * registers. 610 */ 611 static void octeon_irq_ciu_disable_all_v2(struct irq_data *data) 612 { 613 int cpu; 614 u64 mask; 615 struct octeon_ciu_chip_data *cd; 616 617 cd = irq_data_get_irq_chip_data(data); 618 mask = 1ull << (cd->bit); 619 620 if (cd->line == 0) { 621 for_each_online_cpu(cpu) { 622 int index = octeon_coreid_for_cpu(cpu) * 2; 623 clear_bit(cd->bit, 624 &per_cpu(octeon_irq_ciu0_en_mirror, cpu)); 625 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); 626 } 627 } else { 628 for_each_online_cpu(cpu) { 629 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 630 clear_bit(cd->bit, 631 &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 632 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask); 633 } 634 } 635 } 636 637 /* 638 * Enable the irq on the all cores for chips that have the EN*_W1{S,C} 639 * registers. 640 */ 641 static void octeon_irq_ciu_enable_all_v2(struct irq_data *data) 642 { 643 int cpu; 644 u64 mask; 645 struct octeon_ciu_chip_data *cd; 646 647 cd = irq_data_get_irq_chip_data(data); 648 mask = 1ull << (cd->bit); 649 650 if (cd->line == 0) { 651 for_each_online_cpu(cpu) { 652 int index = octeon_coreid_for_cpu(cpu) * 2; 653 set_bit(cd->bit, 654 &per_cpu(octeon_irq_ciu0_en_mirror, cpu)); 655 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 656 } 657 } else { 658 for_each_online_cpu(cpu) { 659 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 660 set_bit(cd->bit, 661 &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 662 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 663 } 664 } 665 } 666 667 static int octeon_irq_ciu_set_type(struct irq_data *data, unsigned int t) 668 { 669 irqd_set_trigger_type(data, t); 670 671 if (t & IRQ_TYPE_EDGE_BOTH) 672 irq_set_handler_locked(data, handle_edge_irq); 673 else 674 irq_set_handler_locked(data, handle_level_irq); 675 676 return IRQ_SET_MASK_OK; 677 } 678 679 static void octeon_irq_gpio_setup(struct irq_data *data) 680 { 681 union cvmx_gpio_bit_cfgx cfg; 682 struct octeon_ciu_chip_data *cd; 683 u32 t = irqd_get_trigger_type(data); 684 685 cd = irq_data_get_irq_chip_data(data); 686 687 cfg.u64 = 0; 688 cfg.s.int_en = 1; 689 cfg.s.int_type = (t & IRQ_TYPE_EDGE_BOTH) != 0; 690 cfg.s.rx_xor = (t & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_EDGE_FALLING)) != 0; 691 692 /* 140 nS glitch filter*/ 693 cfg.s.fil_cnt = 7; 694 cfg.s.fil_sel = 3; 695 696 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), cfg.u64); 697 } 698 699 static void octeon_irq_ciu_enable_gpio_v2(struct irq_data *data) 700 { 701 octeon_irq_gpio_setup(data); 702 octeon_irq_ciu_enable_v2(data); 703 } 704 705 static void octeon_irq_ciu_enable_gpio(struct irq_data *data) 706 { 707 octeon_irq_gpio_setup(data); 708 octeon_irq_ciu_enable(data); 709 } 710 711 static int octeon_irq_ciu_gpio_set_type(struct irq_data *data, unsigned int t) 712 { 713 irqd_set_trigger_type(data, t); 714 octeon_irq_gpio_setup(data); 715 716 if (t & IRQ_TYPE_EDGE_BOTH) 717 irq_set_handler_locked(data, handle_edge_irq); 718 else 719 irq_set_handler_locked(data, handle_level_irq); 720 721 return IRQ_SET_MASK_OK; 722 } 723 724 static void octeon_irq_ciu_disable_gpio_v2(struct irq_data *data) 725 { 726 struct octeon_ciu_chip_data *cd; 727 728 cd = irq_data_get_irq_chip_data(data); 729 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0); 730 731 octeon_irq_ciu_disable_all_v2(data); 732 } 733 734 static void octeon_irq_ciu_disable_gpio(struct irq_data *data) 735 { 736 struct octeon_ciu_chip_data *cd; 737 738 cd = irq_data_get_irq_chip_data(data); 739 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0); 740 741 octeon_irq_ciu_disable_all(data); 742 } 743 744 static void octeon_irq_ciu_gpio_ack(struct irq_data *data) 745 { 746 struct octeon_ciu_chip_data *cd; 747 u64 mask; 748 749 cd = irq_data_get_irq_chip_data(data); 750 mask = 1ull << (cd->gpio_line); 751 752 cvmx_write_csr(CVMX_GPIO_INT_CLR, mask); 753 } 754 755 #ifdef CONFIG_SMP 756 757 static void octeon_irq_cpu_offline_ciu(struct irq_data *data) 758 { 759 int cpu = smp_processor_id(); 760 cpumask_t new_affinity; 761 struct cpumask *mask = irq_data_get_affinity_mask(data); 762 763 if (!cpumask_test_cpu(cpu, mask)) 764 return; 765 766 if (cpumask_weight(mask) > 1) { 767 /* 768 * It has multi CPU affinity, just remove this CPU 769 * from the affinity set. 770 */ 771 cpumask_copy(&new_affinity, mask); 772 cpumask_clear_cpu(cpu, &new_affinity); 773 } else { 774 /* Otherwise, put it on lowest numbered online CPU. */ 775 cpumask_clear(&new_affinity); 776 cpumask_set_cpu(cpumask_first(cpu_online_mask), &new_affinity); 777 } 778 irq_set_affinity_locked(data, &new_affinity, false); 779 } 780 781 static int octeon_irq_ciu_set_affinity(struct irq_data *data, 782 const struct cpumask *dest, bool force) 783 { 784 int cpu; 785 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 786 unsigned long flags; 787 struct octeon_ciu_chip_data *cd; 788 unsigned long *pen; 789 raw_spinlock_t *lock; 790 791 cd = irq_data_get_irq_chip_data(data); 792 793 /* 794 * For non-v2 CIU, we will allow only single CPU affinity. 795 * This removes the need to do locking in the .ack/.eoi 796 * functions. 797 */ 798 if (cpumask_weight(dest) != 1) 799 return -EINVAL; 800 801 if (!enable_one) 802 return 0; 803 804 805 for_each_online_cpu(cpu) { 806 int coreid = octeon_coreid_for_cpu(cpu); 807 808 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 809 raw_spin_lock_irqsave(lock, flags); 810 811 if (cd->line == 0) 812 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 813 else 814 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 815 816 if (cpumask_test_cpu(cpu, dest) && enable_one) { 817 enable_one = 0; 818 __set_bit(cd->bit, pen); 819 } else { 820 __clear_bit(cd->bit, pen); 821 } 822 /* 823 * Must be visible to octeon_irq_ip{2,3}_ciu() before 824 * enabling the irq. 825 */ 826 wmb(); 827 828 if (cd->line == 0) 829 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 830 else 831 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 832 833 raw_spin_unlock_irqrestore(lock, flags); 834 } 835 return 0; 836 } 837 838 /* 839 * Set affinity for the irq for chips that have the EN*_W1{S,C} 840 * registers. 841 */ 842 static int octeon_irq_ciu_set_affinity_v2(struct irq_data *data, 843 const struct cpumask *dest, 844 bool force) 845 { 846 int cpu; 847 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 848 u64 mask; 849 struct octeon_ciu_chip_data *cd; 850 851 if (!enable_one) 852 return 0; 853 854 cd = irq_data_get_irq_chip_data(data); 855 mask = 1ull << cd->bit; 856 857 if (cd->line == 0) { 858 for_each_online_cpu(cpu) { 859 unsigned long *pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 860 int index = octeon_coreid_for_cpu(cpu) * 2; 861 if (cpumask_test_cpu(cpu, dest) && enable_one) { 862 enable_one = false; 863 set_bit(cd->bit, pen); 864 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 865 } else { 866 clear_bit(cd->bit, pen); 867 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); 868 } 869 } 870 } else { 871 for_each_online_cpu(cpu) { 872 unsigned long *pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 873 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 874 if (cpumask_test_cpu(cpu, dest) && enable_one) { 875 enable_one = false; 876 set_bit(cd->bit, pen); 877 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 878 } else { 879 clear_bit(cd->bit, pen); 880 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask); 881 } 882 } 883 } 884 return 0; 885 } 886 887 static int octeon_irq_ciu_set_affinity_sum2(struct irq_data *data, 888 const struct cpumask *dest, 889 bool force) 890 { 891 int cpu; 892 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 893 u64 mask; 894 struct octeon_ciu_chip_data *cd; 895 896 if (!enable_one) 897 return 0; 898 899 cd = irq_data_get_irq_chip_data(data); 900 mask = 1ull << cd->bit; 901 902 for_each_online_cpu(cpu) { 903 int index = octeon_coreid_for_cpu(cpu); 904 905 if (cpumask_test_cpu(cpu, dest) && enable_one) { 906 enable_one = false; 907 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1S(index), mask); 908 } else { 909 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(index), mask); 910 } 911 } 912 return 0; 913 } 914 #endif 915 916 static unsigned int edge_startup(struct irq_data *data) 917 { 918 /* ack any pending edge-irq at startup, so there is 919 * an _edge_ to fire on when the event reappears. 920 */ 921 data->chip->irq_ack(data); 922 data->chip->irq_enable(data); 923 return 0; 924 } 925 926 /* 927 * Newer octeon chips have support for lockless CIU operation. 928 */ 929 static struct irq_chip octeon_irq_chip_ciu_v2 = { 930 .name = "CIU", 931 .irq_enable = octeon_irq_ciu_enable_v2, 932 .irq_disable = octeon_irq_ciu_disable_all_v2, 933 .irq_mask = octeon_irq_ciu_disable_local_v2, 934 .irq_unmask = octeon_irq_ciu_enable_v2, 935 #ifdef CONFIG_SMP 936 .irq_set_affinity = octeon_irq_ciu_set_affinity_v2, 937 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 938 #endif 939 }; 940 941 static struct irq_chip octeon_irq_chip_ciu_v2_edge = { 942 .name = "CIU", 943 .irq_enable = octeon_irq_ciu_enable_v2, 944 .irq_disable = octeon_irq_ciu_disable_all_v2, 945 .irq_ack = octeon_irq_ciu_ack, 946 .irq_mask = octeon_irq_ciu_disable_local_v2, 947 .irq_unmask = octeon_irq_ciu_enable_v2, 948 #ifdef CONFIG_SMP 949 .irq_set_affinity = octeon_irq_ciu_set_affinity_v2, 950 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 951 #endif 952 }; 953 954 /* 955 * Newer octeon chips have support for lockless CIU operation. 956 */ 957 static struct irq_chip octeon_irq_chip_ciu_sum2 = { 958 .name = "CIU", 959 .irq_enable = octeon_irq_ciu_enable_sum2, 960 .irq_disable = octeon_irq_ciu_disable_all_sum2, 961 .irq_mask = octeon_irq_ciu_disable_local_sum2, 962 .irq_unmask = octeon_irq_ciu_enable_sum2, 963 #ifdef CONFIG_SMP 964 .irq_set_affinity = octeon_irq_ciu_set_affinity_sum2, 965 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 966 #endif 967 }; 968 969 static struct irq_chip octeon_irq_chip_ciu_sum2_edge = { 970 .name = "CIU", 971 .irq_enable = octeon_irq_ciu_enable_sum2, 972 .irq_disable = octeon_irq_ciu_disable_all_sum2, 973 .irq_ack = octeon_irq_ciu_ack_sum2, 974 .irq_mask = octeon_irq_ciu_disable_local_sum2, 975 .irq_unmask = octeon_irq_ciu_enable_sum2, 976 #ifdef CONFIG_SMP 977 .irq_set_affinity = octeon_irq_ciu_set_affinity_sum2, 978 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 979 #endif 980 }; 981 982 static struct irq_chip octeon_irq_chip_ciu = { 983 .name = "CIU", 984 .irq_enable = octeon_irq_ciu_enable, 985 .irq_disable = octeon_irq_ciu_disable_all, 986 .irq_mask = octeon_irq_ciu_disable_local, 987 .irq_unmask = octeon_irq_ciu_enable, 988 #ifdef CONFIG_SMP 989 .irq_set_affinity = octeon_irq_ciu_set_affinity, 990 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 991 #endif 992 }; 993 994 static struct irq_chip octeon_irq_chip_ciu_edge = { 995 .name = "CIU", 996 .irq_enable = octeon_irq_ciu_enable, 997 .irq_disable = octeon_irq_ciu_disable_all, 998 .irq_ack = octeon_irq_ciu_ack, 999 .irq_mask = octeon_irq_ciu_disable_local, 1000 .irq_unmask = octeon_irq_ciu_enable, 1001 #ifdef CONFIG_SMP 1002 .irq_set_affinity = octeon_irq_ciu_set_affinity, 1003 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1004 #endif 1005 }; 1006 1007 /* The mbox versions don't do any affinity or round-robin. */ 1008 static struct irq_chip octeon_irq_chip_ciu_mbox_v2 = { 1009 .name = "CIU-M", 1010 .irq_enable = octeon_irq_ciu_enable_all_v2, 1011 .irq_disable = octeon_irq_ciu_disable_all_v2, 1012 .irq_ack = octeon_irq_ciu_disable_local_v2, 1013 .irq_eoi = octeon_irq_ciu_enable_local_v2, 1014 1015 .irq_cpu_online = octeon_irq_ciu_enable_local_v2, 1016 .irq_cpu_offline = octeon_irq_ciu_disable_local_v2, 1017 .flags = IRQCHIP_ONOFFLINE_ENABLED, 1018 }; 1019 1020 static struct irq_chip octeon_irq_chip_ciu_mbox = { 1021 .name = "CIU-M", 1022 .irq_enable = octeon_irq_ciu_enable_all, 1023 .irq_disable = octeon_irq_ciu_disable_all, 1024 .irq_ack = octeon_irq_ciu_disable_local, 1025 .irq_eoi = octeon_irq_ciu_enable_local, 1026 1027 .irq_cpu_online = octeon_irq_ciu_enable_local, 1028 .irq_cpu_offline = octeon_irq_ciu_disable_local, 1029 .flags = IRQCHIP_ONOFFLINE_ENABLED, 1030 }; 1031 1032 static struct irq_chip octeon_irq_chip_ciu_gpio_v2 = { 1033 .name = "CIU-GPIO", 1034 .irq_enable = octeon_irq_ciu_enable_gpio_v2, 1035 .irq_disable = octeon_irq_ciu_disable_gpio_v2, 1036 .irq_ack = octeon_irq_ciu_gpio_ack, 1037 .irq_mask = octeon_irq_ciu_disable_local_v2, 1038 .irq_unmask = octeon_irq_ciu_enable_v2, 1039 .irq_set_type = octeon_irq_ciu_gpio_set_type, 1040 #ifdef CONFIG_SMP 1041 .irq_set_affinity = octeon_irq_ciu_set_affinity_v2, 1042 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1043 #endif 1044 .flags = IRQCHIP_SET_TYPE_MASKED, 1045 }; 1046 1047 static struct irq_chip octeon_irq_chip_ciu_gpio = { 1048 .name = "CIU-GPIO", 1049 .irq_enable = octeon_irq_ciu_enable_gpio, 1050 .irq_disable = octeon_irq_ciu_disable_gpio, 1051 .irq_mask = octeon_irq_ciu_disable_local, 1052 .irq_unmask = octeon_irq_ciu_enable, 1053 .irq_ack = octeon_irq_ciu_gpio_ack, 1054 .irq_set_type = octeon_irq_ciu_gpio_set_type, 1055 #ifdef CONFIG_SMP 1056 .irq_set_affinity = octeon_irq_ciu_set_affinity, 1057 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1058 #endif 1059 .flags = IRQCHIP_SET_TYPE_MASKED, 1060 }; 1061 1062 /* 1063 * Watchdog interrupts are special. They are associated with a single 1064 * core, so we hardwire the affinity to that core. 1065 */ 1066 static void octeon_irq_ciu_wd_enable(struct irq_data *data) 1067 { 1068 unsigned long flags; 1069 unsigned long *pen; 1070 int coreid = data->irq - OCTEON_IRQ_WDOG0; /* Bit 0-63 of EN1 */ 1071 int cpu = octeon_cpu_for_coreid(coreid); 1072 raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu); 1073 1074 raw_spin_lock_irqsave(lock, flags); 1075 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 1076 __set_bit(coreid, pen); 1077 /* 1078 * Must be visible to octeon_irq_ip{2,3}_ciu() before enabling 1079 * the irq. 1080 */ 1081 wmb(); 1082 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 1083 raw_spin_unlock_irqrestore(lock, flags); 1084 } 1085 1086 /* 1087 * Watchdog interrupts are special. They are associated with a single 1088 * core, so we hardwire the affinity to that core. 1089 */ 1090 static void octeon_irq_ciu1_wd_enable_v2(struct irq_data *data) 1091 { 1092 int coreid = data->irq - OCTEON_IRQ_WDOG0; 1093 int cpu = octeon_cpu_for_coreid(coreid); 1094 1095 set_bit(coreid, &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 1096 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(coreid * 2 + 1), 1ull << coreid); 1097 } 1098 1099 1100 static struct irq_chip octeon_irq_chip_ciu_wd_v2 = { 1101 .name = "CIU-W", 1102 .irq_enable = octeon_irq_ciu1_wd_enable_v2, 1103 .irq_disable = octeon_irq_ciu_disable_all_v2, 1104 .irq_mask = octeon_irq_ciu_disable_local_v2, 1105 .irq_unmask = octeon_irq_ciu_enable_local_v2, 1106 }; 1107 1108 static struct irq_chip octeon_irq_chip_ciu_wd = { 1109 .name = "CIU-W", 1110 .irq_enable = octeon_irq_ciu_wd_enable, 1111 .irq_disable = octeon_irq_ciu_disable_all, 1112 .irq_mask = octeon_irq_ciu_disable_local, 1113 .irq_unmask = octeon_irq_ciu_enable_local, 1114 }; 1115 1116 static bool octeon_irq_ciu_is_edge(unsigned int line, unsigned int bit) 1117 { 1118 bool edge = false; 1119 1120 if (line == 0) 1121 switch (bit) { 1122 case 48 ... 49: /* GMX DRP */ 1123 case 50: /* IPD_DRP */ 1124 case 52 ... 55: /* Timers */ 1125 case 58: /* MPI */ 1126 edge = true; 1127 break; 1128 default: 1129 break; 1130 } 1131 else /* line == 1 */ 1132 switch (bit) { 1133 case 47: /* PTP */ 1134 edge = true; 1135 break; 1136 default: 1137 break; 1138 } 1139 return edge; 1140 } 1141 1142 struct octeon_irq_gpio_domain_data { 1143 unsigned int base_hwirq; 1144 }; 1145 1146 static int octeon_irq_gpio_xlat(struct irq_domain *d, 1147 struct device_node *node, 1148 const u32 *intspec, 1149 unsigned int intsize, 1150 unsigned long *out_hwirq, 1151 unsigned int *out_type) 1152 { 1153 unsigned int type; 1154 unsigned int pin; 1155 unsigned int trigger; 1156 1157 if (irq_domain_get_of_node(d) != node) 1158 return -EINVAL; 1159 1160 if (intsize < 2) 1161 return -EINVAL; 1162 1163 pin = intspec[0]; 1164 if (pin >= 16) 1165 return -EINVAL; 1166 1167 trigger = intspec[1]; 1168 1169 switch (trigger) { 1170 case 1: 1171 type = IRQ_TYPE_EDGE_RISING; 1172 break; 1173 case 2: 1174 type = IRQ_TYPE_EDGE_FALLING; 1175 break; 1176 case 4: 1177 type = IRQ_TYPE_LEVEL_HIGH; 1178 break; 1179 case 8: 1180 type = IRQ_TYPE_LEVEL_LOW; 1181 break; 1182 default: 1183 pr_err("Error: (%s) Invalid irq trigger specification: %x\n", 1184 node->name, 1185 trigger); 1186 type = IRQ_TYPE_LEVEL_LOW; 1187 break; 1188 } 1189 *out_type = type; 1190 *out_hwirq = pin; 1191 1192 return 0; 1193 } 1194 1195 static int octeon_irq_ciu_xlat(struct irq_domain *d, 1196 struct device_node *node, 1197 const u32 *intspec, 1198 unsigned int intsize, 1199 unsigned long *out_hwirq, 1200 unsigned int *out_type) 1201 { 1202 unsigned int ciu, bit; 1203 struct octeon_irq_ciu_domain_data *dd = d->host_data; 1204 1205 ciu = intspec[0]; 1206 bit = intspec[1]; 1207 1208 if (ciu >= dd->num_sum || bit > 63) 1209 return -EINVAL; 1210 1211 *out_hwirq = (ciu << 6) | bit; 1212 *out_type = 0; 1213 1214 return 0; 1215 } 1216 1217 static struct irq_chip *octeon_irq_ciu_chip; 1218 static struct irq_chip *octeon_irq_ciu_chip_edge; 1219 static struct irq_chip *octeon_irq_gpio_chip; 1220 1221 static int octeon_irq_ciu_map(struct irq_domain *d, 1222 unsigned int virq, irq_hw_number_t hw) 1223 { 1224 int rv; 1225 unsigned int line = hw >> 6; 1226 unsigned int bit = hw & 63; 1227 struct octeon_irq_ciu_domain_data *dd = d->host_data; 1228 1229 if (line >= dd->num_sum || octeon_irq_ciu_to_irq[line][bit] != 0) 1230 return -EINVAL; 1231 1232 if (line == 2) { 1233 if (octeon_irq_ciu_is_edge(line, bit)) 1234 rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1235 &octeon_irq_chip_ciu_sum2_edge, 1236 handle_edge_irq); 1237 else 1238 rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1239 &octeon_irq_chip_ciu_sum2, 1240 handle_level_irq); 1241 } else { 1242 if (octeon_irq_ciu_is_edge(line, bit)) 1243 rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1244 octeon_irq_ciu_chip_edge, 1245 handle_edge_irq); 1246 else 1247 rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1248 octeon_irq_ciu_chip, 1249 handle_level_irq); 1250 } 1251 return rv; 1252 } 1253 1254 static int octeon_irq_gpio_map(struct irq_domain *d, 1255 unsigned int virq, irq_hw_number_t hw) 1256 { 1257 struct octeon_irq_gpio_domain_data *gpiod = d->host_data; 1258 unsigned int line, bit; 1259 int r; 1260 1261 line = (hw + gpiod->base_hwirq) >> 6; 1262 bit = (hw + gpiod->base_hwirq) & 63; 1263 if (line > ARRAY_SIZE(octeon_irq_ciu_to_irq) || 1264 octeon_irq_ciu_to_irq[line][bit] != 0) 1265 return -EINVAL; 1266 1267 /* 1268 * Default to handle_level_irq. If the DT contains a different 1269 * trigger type, it will call the irq_set_type callback and 1270 * the handler gets updated. 1271 */ 1272 r = octeon_irq_set_ciu_mapping(virq, line, bit, hw, 1273 octeon_irq_gpio_chip, handle_level_irq); 1274 return r; 1275 } 1276 1277 static struct irq_domain_ops octeon_irq_domain_ciu_ops = { 1278 .map = octeon_irq_ciu_map, 1279 .unmap = octeon_irq_free_cd, 1280 .xlate = octeon_irq_ciu_xlat, 1281 }; 1282 1283 static struct irq_domain_ops octeon_irq_domain_gpio_ops = { 1284 .map = octeon_irq_gpio_map, 1285 .unmap = octeon_irq_free_cd, 1286 .xlate = octeon_irq_gpio_xlat, 1287 }; 1288 1289 static void octeon_irq_ip2_ciu(void) 1290 { 1291 const unsigned long core_id = cvmx_get_core_num(); 1292 u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INTX_SUM0(core_id * 2)); 1293 1294 ciu_sum &= __this_cpu_read(octeon_irq_ciu0_en_mirror); 1295 if (likely(ciu_sum)) { 1296 int bit = fls64(ciu_sum) - 1; 1297 int irq = octeon_irq_ciu_to_irq[0][bit]; 1298 if (likely(irq)) 1299 do_IRQ(irq); 1300 else 1301 spurious_interrupt(); 1302 } else { 1303 spurious_interrupt(); 1304 } 1305 } 1306 1307 static void octeon_irq_ip3_ciu(void) 1308 { 1309 u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INT_SUM1); 1310 1311 ciu_sum &= __this_cpu_read(octeon_irq_ciu1_en_mirror); 1312 if (likely(ciu_sum)) { 1313 int bit = fls64(ciu_sum) - 1; 1314 int irq = octeon_irq_ciu_to_irq[1][bit]; 1315 if (likely(irq)) 1316 do_IRQ(irq); 1317 else 1318 spurious_interrupt(); 1319 } else { 1320 spurious_interrupt(); 1321 } 1322 } 1323 1324 static void octeon_irq_ip4_ciu(void) 1325 { 1326 int coreid = cvmx_get_core_num(); 1327 u64 ciu_sum = cvmx_read_csr(CVMX_CIU_SUM2_PPX_IP4(coreid)); 1328 u64 ciu_en = cvmx_read_csr(CVMX_CIU_EN2_PPX_IP4(coreid)); 1329 1330 ciu_sum &= ciu_en; 1331 if (likely(ciu_sum)) { 1332 int bit = fls64(ciu_sum) - 1; 1333 int irq = octeon_irq_ciu_to_irq[2][bit]; 1334 1335 if (likely(irq)) 1336 do_IRQ(irq); 1337 else 1338 spurious_interrupt(); 1339 } else { 1340 spurious_interrupt(); 1341 } 1342 } 1343 1344 static bool octeon_irq_use_ip4; 1345 1346 static void octeon_irq_local_enable_ip4(void *arg) 1347 { 1348 set_c0_status(STATUSF_IP4); 1349 } 1350 1351 static void octeon_irq_ip4_mask(void) 1352 { 1353 clear_c0_status(STATUSF_IP4); 1354 spurious_interrupt(); 1355 } 1356 1357 static void (*octeon_irq_ip2)(void); 1358 static void (*octeon_irq_ip3)(void); 1359 static void (*octeon_irq_ip4)(void); 1360 1361 void (*octeon_irq_setup_secondary)(void); 1362 1363 void octeon_irq_set_ip4_handler(octeon_irq_ip4_handler_t h) 1364 { 1365 octeon_irq_ip4 = h; 1366 octeon_irq_use_ip4 = true; 1367 on_each_cpu(octeon_irq_local_enable_ip4, NULL, 1); 1368 } 1369 1370 static void octeon_irq_percpu_enable(void) 1371 { 1372 irq_cpu_online(); 1373 } 1374 1375 static void octeon_irq_init_ciu_percpu(void) 1376 { 1377 int coreid = cvmx_get_core_num(); 1378 1379 1380 __this_cpu_write(octeon_irq_ciu0_en_mirror, 0); 1381 __this_cpu_write(octeon_irq_ciu1_en_mirror, 0); 1382 wmb(); 1383 raw_spin_lock_init(this_cpu_ptr(&octeon_irq_ciu_spinlock)); 1384 /* 1385 * Disable All CIU Interrupts. The ones we need will be 1386 * enabled later. Read the SUM register so we know the write 1387 * completed. 1388 */ 1389 cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2)), 0); 1390 cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2 + 1)), 0); 1391 cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2)), 0); 1392 cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2 + 1)), 0); 1393 cvmx_read_csr(CVMX_CIU_INTX_SUM0((coreid * 2))); 1394 } 1395 1396 static void octeon_irq_init_ciu2_percpu(void) 1397 { 1398 u64 regx, ipx; 1399 int coreid = cvmx_get_core_num(); 1400 u64 base = CVMX_CIU2_EN_PPX_IP2_WRKQ(coreid); 1401 1402 /* 1403 * Disable All CIU2 Interrupts. The ones we need will be 1404 * enabled later. Read the SUM register so we know the write 1405 * completed. 1406 * 1407 * There are 9 registers and 3 IPX levels with strides 0x1000 1408 * and 0x200 respectivly. Use loops to clear them. 1409 */ 1410 for (regx = 0; regx <= 0x8000; regx += 0x1000) { 1411 for (ipx = 0; ipx <= 0x400; ipx += 0x200) 1412 cvmx_write_csr(base + regx + ipx, 0); 1413 } 1414 1415 cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(coreid)); 1416 } 1417 1418 static void octeon_irq_setup_secondary_ciu(void) 1419 { 1420 octeon_irq_init_ciu_percpu(); 1421 octeon_irq_percpu_enable(); 1422 1423 /* Enable the CIU lines */ 1424 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 1425 if (octeon_irq_use_ip4) 1426 set_c0_status(STATUSF_IP4); 1427 else 1428 clear_c0_status(STATUSF_IP4); 1429 } 1430 1431 static void octeon_irq_setup_secondary_ciu2(void) 1432 { 1433 octeon_irq_init_ciu2_percpu(); 1434 octeon_irq_percpu_enable(); 1435 1436 /* Enable the CIU lines */ 1437 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 1438 if (octeon_irq_use_ip4) 1439 set_c0_status(STATUSF_IP4); 1440 else 1441 clear_c0_status(STATUSF_IP4); 1442 } 1443 1444 static int __init octeon_irq_init_ciu( 1445 struct device_node *ciu_node, struct device_node *parent) 1446 { 1447 unsigned int i, r; 1448 struct irq_chip *chip; 1449 struct irq_chip *chip_edge; 1450 struct irq_chip *chip_mbox; 1451 struct irq_chip *chip_wd; 1452 struct irq_domain *ciu_domain = NULL; 1453 struct octeon_irq_ciu_domain_data *dd; 1454 1455 dd = kzalloc(sizeof(*dd), GFP_KERNEL); 1456 if (!dd) 1457 return -ENOMEM; 1458 1459 octeon_irq_init_ciu_percpu(); 1460 octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu; 1461 1462 octeon_irq_ip2 = octeon_irq_ip2_ciu; 1463 octeon_irq_ip3 = octeon_irq_ip3_ciu; 1464 if ((OCTEON_IS_OCTEON2() || OCTEON_IS_OCTEON3()) 1465 && !OCTEON_IS_MODEL(OCTEON_CN63XX)) { 1466 octeon_irq_ip4 = octeon_irq_ip4_ciu; 1467 dd->num_sum = 3; 1468 octeon_irq_use_ip4 = true; 1469 } else { 1470 octeon_irq_ip4 = octeon_irq_ip4_mask; 1471 dd->num_sum = 2; 1472 octeon_irq_use_ip4 = false; 1473 } 1474 if (OCTEON_IS_MODEL(OCTEON_CN58XX_PASS2_X) || 1475 OCTEON_IS_MODEL(OCTEON_CN56XX_PASS2_X) || 1476 OCTEON_IS_MODEL(OCTEON_CN52XX_PASS2_X) || 1477 OCTEON_IS_OCTEON2() || OCTEON_IS_OCTEON3()) { 1478 chip = &octeon_irq_chip_ciu_v2; 1479 chip_edge = &octeon_irq_chip_ciu_v2_edge; 1480 chip_mbox = &octeon_irq_chip_ciu_mbox_v2; 1481 chip_wd = &octeon_irq_chip_ciu_wd_v2; 1482 octeon_irq_gpio_chip = &octeon_irq_chip_ciu_gpio_v2; 1483 } else { 1484 chip = &octeon_irq_chip_ciu; 1485 chip_edge = &octeon_irq_chip_ciu_edge; 1486 chip_mbox = &octeon_irq_chip_ciu_mbox; 1487 chip_wd = &octeon_irq_chip_ciu_wd; 1488 octeon_irq_gpio_chip = &octeon_irq_chip_ciu_gpio; 1489 } 1490 octeon_irq_ciu_chip = chip; 1491 octeon_irq_ciu_chip_edge = chip_edge; 1492 1493 /* Mips internal */ 1494 octeon_irq_init_core(); 1495 1496 ciu_domain = irq_domain_add_tree( 1497 ciu_node, &octeon_irq_domain_ciu_ops, dd); 1498 irq_set_default_host(ciu_domain); 1499 1500 /* CIU_0 */ 1501 for (i = 0; i < 16; i++) { 1502 r = octeon_irq_force_ciu_mapping( 1503 ciu_domain, i + OCTEON_IRQ_WORKQ0, 0, i + 0); 1504 if (r) 1505 goto err; 1506 } 1507 1508 r = octeon_irq_set_ciu_mapping( 1509 OCTEON_IRQ_MBOX0, 0, 32, 0, chip_mbox, handle_percpu_irq); 1510 if (r) 1511 goto err; 1512 r = octeon_irq_set_ciu_mapping( 1513 OCTEON_IRQ_MBOX1, 0, 33, 0, chip_mbox, handle_percpu_irq); 1514 if (r) 1515 goto err; 1516 1517 for (i = 0; i < 4; i++) { 1518 r = octeon_irq_force_ciu_mapping( 1519 ciu_domain, i + OCTEON_IRQ_PCI_INT0, 0, i + 36); 1520 if (r) 1521 goto err; 1522 } 1523 for (i = 0; i < 4; i++) { 1524 r = octeon_irq_force_ciu_mapping( 1525 ciu_domain, i + OCTEON_IRQ_PCI_MSI0, 0, i + 40); 1526 if (r) 1527 goto err; 1528 } 1529 1530 r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_TWSI, 0, 45); 1531 if (r) 1532 goto err; 1533 1534 r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_RML, 0, 46); 1535 if (r) 1536 goto err; 1537 1538 for (i = 0; i < 4; i++) { 1539 r = octeon_irq_force_ciu_mapping( 1540 ciu_domain, i + OCTEON_IRQ_TIMER0, 0, i + 52); 1541 if (r) 1542 goto err; 1543 } 1544 1545 r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_USB0, 0, 56); 1546 if (r) 1547 goto err; 1548 1549 r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_TWSI2, 0, 59); 1550 if (r) 1551 goto err; 1552 1553 /* CIU_1 */ 1554 for (i = 0; i < 16; i++) { 1555 r = octeon_irq_set_ciu_mapping( 1556 i + OCTEON_IRQ_WDOG0, 1, i + 0, 0, chip_wd, 1557 handle_level_irq); 1558 if (r) 1559 goto err; 1560 } 1561 1562 r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_USB1, 1, 17); 1563 if (r) 1564 goto err; 1565 1566 /* Enable the CIU lines */ 1567 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 1568 if (octeon_irq_use_ip4) 1569 set_c0_status(STATUSF_IP4); 1570 else 1571 clear_c0_status(STATUSF_IP4); 1572 1573 return 0; 1574 err: 1575 return r; 1576 } 1577 1578 static int __init octeon_irq_init_gpio( 1579 struct device_node *gpio_node, struct device_node *parent) 1580 { 1581 struct octeon_irq_gpio_domain_data *gpiod; 1582 u32 interrupt_cells; 1583 unsigned int base_hwirq; 1584 int r; 1585 1586 r = of_property_read_u32(parent, "#interrupt-cells", &interrupt_cells); 1587 if (r) 1588 return r; 1589 1590 if (interrupt_cells == 1) { 1591 u32 v; 1592 1593 r = of_property_read_u32_index(gpio_node, "interrupts", 0, &v); 1594 if (r) { 1595 pr_warn("No \"interrupts\" property.\n"); 1596 return r; 1597 } 1598 base_hwirq = v; 1599 } else if (interrupt_cells == 2) { 1600 u32 v0, v1; 1601 1602 r = of_property_read_u32_index(gpio_node, "interrupts", 0, &v0); 1603 if (r) { 1604 pr_warn("No \"interrupts\" property.\n"); 1605 return r; 1606 } 1607 r = of_property_read_u32_index(gpio_node, "interrupts", 1, &v1); 1608 if (r) { 1609 pr_warn("No \"interrupts\" property.\n"); 1610 return r; 1611 } 1612 base_hwirq = (v0 << 6) | v1; 1613 } else { 1614 pr_warn("Bad \"#interrupt-cells\" property: %u\n", 1615 interrupt_cells); 1616 return -EINVAL; 1617 } 1618 1619 gpiod = kzalloc(sizeof(*gpiod), GFP_KERNEL); 1620 if (gpiod) { 1621 /* gpio domain host_data is the base hwirq number. */ 1622 gpiod->base_hwirq = base_hwirq; 1623 irq_domain_add_linear( 1624 gpio_node, 16, &octeon_irq_domain_gpio_ops, gpiod); 1625 } else { 1626 pr_warn("Cannot allocate memory for GPIO irq_domain.\n"); 1627 return -ENOMEM; 1628 } 1629 1630 return 0; 1631 } 1632 /* 1633 * Watchdog interrupts are special. They are associated with a single 1634 * core, so we hardwire the affinity to that core. 1635 */ 1636 static void octeon_irq_ciu2_wd_enable(struct irq_data *data) 1637 { 1638 u64 mask; 1639 u64 en_addr; 1640 int coreid = data->irq - OCTEON_IRQ_WDOG0; 1641 struct octeon_ciu_chip_data *cd; 1642 1643 cd = irq_data_get_irq_chip_data(data); 1644 mask = 1ull << (cd->bit); 1645 1646 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) + 1647 (0x1000ull * cd->line); 1648 cvmx_write_csr(en_addr, mask); 1649 1650 } 1651 1652 static void octeon_irq_ciu2_enable(struct irq_data *data) 1653 { 1654 u64 mask; 1655 u64 en_addr; 1656 int cpu = next_cpu_for_irq(data); 1657 int coreid = octeon_coreid_for_cpu(cpu); 1658 struct octeon_ciu_chip_data *cd; 1659 1660 cd = irq_data_get_irq_chip_data(data); 1661 mask = 1ull << (cd->bit); 1662 1663 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) + 1664 (0x1000ull * cd->line); 1665 cvmx_write_csr(en_addr, mask); 1666 } 1667 1668 static void octeon_irq_ciu2_enable_local(struct irq_data *data) 1669 { 1670 u64 mask; 1671 u64 en_addr; 1672 int coreid = cvmx_get_core_num(); 1673 struct octeon_ciu_chip_data *cd; 1674 1675 cd = irq_data_get_irq_chip_data(data); 1676 mask = 1ull << (cd->bit); 1677 1678 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) + 1679 (0x1000ull * cd->line); 1680 cvmx_write_csr(en_addr, mask); 1681 1682 } 1683 1684 static void octeon_irq_ciu2_disable_local(struct irq_data *data) 1685 { 1686 u64 mask; 1687 u64 en_addr; 1688 int coreid = cvmx_get_core_num(); 1689 struct octeon_ciu_chip_data *cd; 1690 1691 cd = irq_data_get_irq_chip_data(data); 1692 mask = 1ull << (cd->bit); 1693 1694 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C(coreid) + 1695 (0x1000ull * cd->line); 1696 cvmx_write_csr(en_addr, mask); 1697 1698 } 1699 1700 static void octeon_irq_ciu2_ack(struct irq_data *data) 1701 { 1702 u64 mask; 1703 u64 en_addr; 1704 int coreid = cvmx_get_core_num(); 1705 struct octeon_ciu_chip_data *cd; 1706 1707 cd = irq_data_get_irq_chip_data(data); 1708 mask = 1ull << (cd->bit); 1709 1710 en_addr = CVMX_CIU2_RAW_PPX_IP2_WRKQ(coreid) + (0x1000ull * cd->line); 1711 cvmx_write_csr(en_addr, mask); 1712 1713 } 1714 1715 static void octeon_irq_ciu2_disable_all(struct irq_data *data) 1716 { 1717 int cpu; 1718 u64 mask; 1719 struct octeon_ciu_chip_data *cd; 1720 1721 cd = irq_data_get_irq_chip_data(data); 1722 mask = 1ull << (cd->bit); 1723 1724 for_each_online_cpu(cpu) { 1725 u64 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C( 1726 octeon_coreid_for_cpu(cpu)) + (0x1000ull * cd->line); 1727 cvmx_write_csr(en_addr, mask); 1728 } 1729 } 1730 1731 static void octeon_irq_ciu2_mbox_enable_all(struct irq_data *data) 1732 { 1733 int cpu; 1734 u64 mask; 1735 1736 mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0); 1737 1738 for_each_online_cpu(cpu) { 1739 u64 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1S( 1740 octeon_coreid_for_cpu(cpu)); 1741 cvmx_write_csr(en_addr, mask); 1742 } 1743 } 1744 1745 static void octeon_irq_ciu2_mbox_disable_all(struct irq_data *data) 1746 { 1747 int cpu; 1748 u64 mask; 1749 1750 mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0); 1751 1752 for_each_online_cpu(cpu) { 1753 u64 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1C( 1754 octeon_coreid_for_cpu(cpu)); 1755 cvmx_write_csr(en_addr, mask); 1756 } 1757 } 1758 1759 static void octeon_irq_ciu2_mbox_enable_local(struct irq_data *data) 1760 { 1761 u64 mask; 1762 u64 en_addr; 1763 int coreid = cvmx_get_core_num(); 1764 1765 mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0); 1766 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1S(coreid); 1767 cvmx_write_csr(en_addr, mask); 1768 } 1769 1770 static void octeon_irq_ciu2_mbox_disable_local(struct irq_data *data) 1771 { 1772 u64 mask; 1773 u64 en_addr; 1774 int coreid = cvmx_get_core_num(); 1775 1776 mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0); 1777 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1C(coreid); 1778 cvmx_write_csr(en_addr, mask); 1779 } 1780 1781 #ifdef CONFIG_SMP 1782 static int octeon_irq_ciu2_set_affinity(struct irq_data *data, 1783 const struct cpumask *dest, bool force) 1784 { 1785 int cpu; 1786 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 1787 u64 mask; 1788 struct octeon_ciu_chip_data *cd; 1789 1790 if (!enable_one) 1791 return 0; 1792 1793 cd = irq_data_get_irq_chip_data(data); 1794 mask = 1ull << cd->bit; 1795 1796 for_each_online_cpu(cpu) { 1797 u64 en_addr; 1798 if (cpumask_test_cpu(cpu, dest) && enable_one) { 1799 enable_one = false; 1800 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S( 1801 octeon_coreid_for_cpu(cpu)) + 1802 (0x1000ull * cd->line); 1803 } else { 1804 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C( 1805 octeon_coreid_for_cpu(cpu)) + 1806 (0x1000ull * cd->line); 1807 } 1808 cvmx_write_csr(en_addr, mask); 1809 } 1810 1811 return 0; 1812 } 1813 #endif 1814 1815 static void octeon_irq_ciu2_enable_gpio(struct irq_data *data) 1816 { 1817 octeon_irq_gpio_setup(data); 1818 octeon_irq_ciu2_enable(data); 1819 } 1820 1821 static void octeon_irq_ciu2_disable_gpio(struct irq_data *data) 1822 { 1823 struct octeon_ciu_chip_data *cd; 1824 1825 cd = irq_data_get_irq_chip_data(data); 1826 1827 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0); 1828 1829 octeon_irq_ciu2_disable_all(data); 1830 } 1831 1832 static struct irq_chip octeon_irq_chip_ciu2 = { 1833 .name = "CIU2-E", 1834 .irq_enable = octeon_irq_ciu2_enable, 1835 .irq_disable = octeon_irq_ciu2_disable_all, 1836 .irq_mask = octeon_irq_ciu2_disable_local, 1837 .irq_unmask = octeon_irq_ciu2_enable, 1838 #ifdef CONFIG_SMP 1839 .irq_set_affinity = octeon_irq_ciu2_set_affinity, 1840 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1841 #endif 1842 }; 1843 1844 static struct irq_chip octeon_irq_chip_ciu2_edge = { 1845 .name = "CIU2-E", 1846 .irq_enable = octeon_irq_ciu2_enable, 1847 .irq_disable = octeon_irq_ciu2_disable_all, 1848 .irq_ack = octeon_irq_ciu2_ack, 1849 .irq_mask = octeon_irq_ciu2_disable_local, 1850 .irq_unmask = octeon_irq_ciu2_enable, 1851 #ifdef CONFIG_SMP 1852 .irq_set_affinity = octeon_irq_ciu2_set_affinity, 1853 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1854 #endif 1855 }; 1856 1857 static struct irq_chip octeon_irq_chip_ciu2_mbox = { 1858 .name = "CIU2-M", 1859 .irq_enable = octeon_irq_ciu2_mbox_enable_all, 1860 .irq_disable = octeon_irq_ciu2_mbox_disable_all, 1861 .irq_ack = octeon_irq_ciu2_mbox_disable_local, 1862 .irq_eoi = octeon_irq_ciu2_mbox_enable_local, 1863 1864 .irq_cpu_online = octeon_irq_ciu2_mbox_enable_local, 1865 .irq_cpu_offline = octeon_irq_ciu2_mbox_disable_local, 1866 .flags = IRQCHIP_ONOFFLINE_ENABLED, 1867 }; 1868 1869 static struct irq_chip octeon_irq_chip_ciu2_wd = { 1870 .name = "CIU2-W", 1871 .irq_enable = octeon_irq_ciu2_wd_enable, 1872 .irq_disable = octeon_irq_ciu2_disable_all, 1873 .irq_mask = octeon_irq_ciu2_disable_local, 1874 .irq_unmask = octeon_irq_ciu2_enable_local, 1875 }; 1876 1877 static struct irq_chip octeon_irq_chip_ciu2_gpio = { 1878 .name = "CIU-GPIO", 1879 .irq_enable = octeon_irq_ciu2_enable_gpio, 1880 .irq_disable = octeon_irq_ciu2_disable_gpio, 1881 .irq_ack = octeon_irq_ciu_gpio_ack, 1882 .irq_mask = octeon_irq_ciu2_disable_local, 1883 .irq_unmask = octeon_irq_ciu2_enable, 1884 .irq_set_type = octeon_irq_ciu_gpio_set_type, 1885 #ifdef CONFIG_SMP 1886 .irq_set_affinity = octeon_irq_ciu2_set_affinity, 1887 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 1888 #endif 1889 .flags = IRQCHIP_SET_TYPE_MASKED, 1890 }; 1891 1892 static int octeon_irq_ciu2_xlat(struct irq_domain *d, 1893 struct device_node *node, 1894 const u32 *intspec, 1895 unsigned int intsize, 1896 unsigned long *out_hwirq, 1897 unsigned int *out_type) 1898 { 1899 unsigned int ciu, bit; 1900 1901 ciu = intspec[0]; 1902 bit = intspec[1]; 1903 1904 *out_hwirq = (ciu << 6) | bit; 1905 *out_type = 0; 1906 1907 return 0; 1908 } 1909 1910 static bool octeon_irq_ciu2_is_edge(unsigned int line, unsigned int bit) 1911 { 1912 bool edge = false; 1913 1914 if (line == 3) /* MIO */ 1915 switch (bit) { 1916 case 2: /* IPD_DRP */ 1917 case 8 ... 11: /* Timers */ 1918 case 48: /* PTP */ 1919 edge = true; 1920 break; 1921 default: 1922 break; 1923 } 1924 else if (line == 6) /* PKT */ 1925 switch (bit) { 1926 case 52 ... 53: /* ILK_DRP */ 1927 case 8 ... 12: /* GMX_DRP */ 1928 edge = true; 1929 break; 1930 default: 1931 break; 1932 } 1933 return edge; 1934 } 1935 1936 static int octeon_irq_ciu2_map(struct irq_domain *d, 1937 unsigned int virq, irq_hw_number_t hw) 1938 { 1939 unsigned int line = hw >> 6; 1940 unsigned int bit = hw & 63; 1941 1942 /* 1943 * Don't map irq if it is reserved for GPIO. 1944 * (Line 7 are the GPIO lines.) 1945 */ 1946 if (line == 7) 1947 return 0; 1948 1949 if (line > 7 || octeon_irq_ciu_to_irq[line][bit] != 0) 1950 return -EINVAL; 1951 1952 if (octeon_irq_ciu2_is_edge(line, bit)) 1953 octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1954 &octeon_irq_chip_ciu2_edge, 1955 handle_edge_irq); 1956 else 1957 octeon_irq_set_ciu_mapping(virq, line, bit, 0, 1958 &octeon_irq_chip_ciu2, 1959 handle_level_irq); 1960 1961 return 0; 1962 } 1963 1964 static struct irq_domain_ops octeon_irq_domain_ciu2_ops = { 1965 .map = octeon_irq_ciu2_map, 1966 .unmap = octeon_irq_free_cd, 1967 .xlate = octeon_irq_ciu2_xlat, 1968 }; 1969 1970 static void octeon_irq_ciu2(void) 1971 { 1972 int line; 1973 int bit; 1974 int irq; 1975 u64 src_reg, src, sum; 1976 const unsigned long core_id = cvmx_get_core_num(); 1977 1978 sum = cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(core_id)) & 0xfful; 1979 1980 if (unlikely(!sum)) 1981 goto spurious; 1982 1983 line = fls64(sum) - 1; 1984 src_reg = CVMX_CIU2_SRC_PPX_IP2_WRKQ(core_id) + (0x1000 * line); 1985 src = cvmx_read_csr(src_reg); 1986 1987 if (unlikely(!src)) 1988 goto spurious; 1989 1990 bit = fls64(src) - 1; 1991 irq = octeon_irq_ciu_to_irq[line][bit]; 1992 if (unlikely(!irq)) 1993 goto spurious; 1994 1995 do_IRQ(irq); 1996 goto out; 1997 1998 spurious: 1999 spurious_interrupt(); 2000 out: 2001 /* CN68XX pass 1.x has an errata that accessing the ACK registers 2002 can stop interrupts from propagating */ 2003 if (OCTEON_IS_MODEL(OCTEON_CN68XX)) 2004 cvmx_read_csr(CVMX_CIU2_INTR_CIU_READY); 2005 else 2006 cvmx_read_csr(CVMX_CIU2_ACK_PPX_IP2(core_id)); 2007 return; 2008 } 2009 2010 static void octeon_irq_ciu2_mbox(void) 2011 { 2012 int line; 2013 2014 const unsigned long core_id = cvmx_get_core_num(); 2015 u64 sum = cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP3(core_id)) >> 60; 2016 2017 if (unlikely(!sum)) 2018 goto spurious; 2019 2020 line = fls64(sum) - 1; 2021 2022 do_IRQ(OCTEON_IRQ_MBOX0 + line); 2023 goto out; 2024 2025 spurious: 2026 spurious_interrupt(); 2027 out: 2028 /* CN68XX pass 1.x has an errata that accessing the ACK registers 2029 can stop interrupts from propagating */ 2030 if (OCTEON_IS_MODEL(OCTEON_CN68XX)) 2031 cvmx_read_csr(CVMX_CIU2_INTR_CIU_READY); 2032 else 2033 cvmx_read_csr(CVMX_CIU2_ACK_PPX_IP3(core_id)); 2034 return; 2035 } 2036 2037 static int __init octeon_irq_init_ciu2( 2038 struct device_node *ciu_node, struct device_node *parent) 2039 { 2040 unsigned int i, r; 2041 struct irq_domain *ciu_domain = NULL; 2042 2043 octeon_irq_init_ciu2_percpu(); 2044 octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu2; 2045 2046 octeon_irq_gpio_chip = &octeon_irq_chip_ciu2_gpio; 2047 octeon_irq_ip2 = octeon_irq_ciu2; 2048 octeon_irq_ip3 = octeon_irq_ciu2_mbox; 2049 octeon_irq_ip4 = octeon_irq_ip4_mask; 2050 2051 /* Mips internal */ 2052 octeon_irq_init_core(); 2053 2054 ciu_domain = irq_domain_add_tree( 2055 ciu_node, &octeon_irq_domain_ciu2_ops, NULL); 2056 irq_set_default_host(ciu_domain); 2057 2058 /* CUI2 */ 2059 for (i = 0; i < 64; i++) { 2060 r = octeon_irq_force_ciu_mapping( 2061 ciu_domain, i + OCTEON_IRQ_WORKQ0, 0, i); 2062 if (r) 2063 goto err; 2064 } 2065 2066 for (i = 0; i < 32; i++) { 2067 r = octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_WDOG0, 1, i, 0, 2068 &octeon_irq_chip_ciu2_wd, handle_level_irq); 2069 if (r) 2070 goto err; 2071 } 2072 2073 for (i = 0; i < 4; i++) { 2074 r = octeon_irq_force_ciu_mapping( 2075 ciu_domain, i + OCTEON_IRQ_TIMER0, 3, i + 8); 2076 if (r) 2077 goto err; 2078 } 2079 2080 r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_USB0, 3, 44); 2081 if (r) 2082 goto err; 2083 2084 for (i = 0; i < 4; i++) { 2085 r = octeon_irq_force_ciu_mapping( 2086 ciu_domain, i + OCTEON_IRQ_PCI_INT0, 4, i); 2087 if (r) 2088 goto err; 2089 } 2090 2091 for (i = 0; i < 4; i++) { 2092 r = octeon_irq_force_ciu_mapping( 2093 ciu_domain, i + OCTEON_IRQ_PCI_MSI0, 4, i + 8); 2094 if (r) 2095 goto err; 2096 } 2097 2098 irq_set_chip_and_handler(OCTEON_IRQ_MBOX0, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq); 2099 irq_set_chip_and_handler(OCTEON_IRQ_MBOX1, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq); 2100 irq_set_chip_and_handler(OCTEON_IRQ_MBOX2, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq); 2101 irq_set_chip_and_handler(OCTEON_IRQ_MBOX3, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq); 2102 2103 /* Enable the CIU lines */ 2104 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 2105 clear_c0_status(STATUSF_IP4); 2106 return 0; 2107 err: 2108 return r; 2109 } 2110 2111 struct octeon_irq_cib_host_data { 2112 raw_spinlock_t lock; 2113 u64 raw_reg; 2114 u64 en_reg; 2115 int max_bits; 2116 }; 2117 2118 struct octeon_irq_cib_chip_data { 2119 struct octeon_irq_cib_host_data *host_data; 2120 int bit; 2121 }; 2122 2123 static void octeon_irq_cib_enable(struct irq_data *data) 2124 { 2125 unsigned long flags; 2126 u64 en; 2127 struct octeon_irq_cib_chip_data *cd = irq_data_get_irq_chip_data(data); 2128 struct octeon_irq_cib_host_data *host_data = cd->host_data; 2129 2130 raw_spin_lock_irqsave(&host_data->lock, flags); 2131 en = cvmx_read_csr(host_data->en_reg); 2132 en |= 1ull << cd->bit; 2133 cvmx_write_csr(host_data->en_reg, en); 2134 raw_spin_unlock_irqrestore(&host_data->lock, flags); 2135 } 2136 2137 static void octeon_irq_cib_disable(struct irq_data *data) 2138 { 2139 unsigned long flags; 2140 u64 en; 2141 struct octeon_irq_cib_chip_data *cd = irq_data_get_irq_chip_data(data); 2142 struct octeon_irq_cib_host_data *host_data = cd->host_data; 2143 2144 raw_spin_lock_irqsave(&host_data->lock, flags); 2145 en = cvmx_read_csr(host_data->en_reg); 2146 en &= ~(1ull << cd->bit); 2147 cvmx_write_csr(host_data->en_reg, en); 2148 raw_spin_unlock_irqrestore(&host_data->lock, flags); 2149 } 2150 2151 static int octeon_irq_cib_set_type(struct irq_data *data, unsigned int t) 2152 { 2153 irqd_set_trigger_type(data, t); 2154 return IRQ_SET_MASK_OK; 2155 } 2156 2157 static struct irq_chip octeon_irq_chip_cib = { 2158 .name = "CIB", 2159 .irq_enable = octeon_irq_cib_enable, 2160 .irq_disable = octeon_irq_cib_disable, 2161 .irq_mask = octeon_irq_cib_disable, 2162 .irq_unmask = octeon_irq_cib_enable, 2163 .irq_set_type = octeon_irq_cib_set_type, 2164 }; 2165 2166 static int octeon_irq_cib_xlat(struct irq_domain *d, 2167 struct device_node *node, 2168 const u32 *intspec, 2169 unsigned int intsize, 2170 unsigned long *out_hwirq, 2171 unsigned int *out_type) 2172 { 2173 unsigned int type = 0; 2174 2175 if (intsize == 2) 2176 type = intspec[1]; 2177 2178 switch (type) { 2179 case 0: /* unofficial value, but we might as well let it work. */ 2180 case 4: /* official value for level triggering. */ 2181 *out_type = IRQ_TYPE_LEVEL_HIGH; 2182 break; 2183 case 1: /* official value for edge triggering. */ 2184 *out_type = IRQ_TYPE_EDGE_RISING; 2185 break; 2186 default: /* Nothing else is acceptable. */ 2187 return -EINVAL; 2188 } 2189 2190 *out_hwirq = intspec[0]; 2191 2192 return 0; 2193 } 2194 2195 static int octeon_irq_cib_map(struct irq_domain *d, 2196 unsigned int virq, irq_hw_number_t hw) 2197 { 2198 struct octeon_irq_cib_host_data *host_data = d->host_data; 2199 struct octeon_irq_cib_chip_data *cd; 2200 2201 if (hw >= host_data->max_bits) { 2202 pr_err("ERROR: %s mapping %u is to big!\n", 2203 irq_domain_get_of_node(d)->name, (unsigned)hw); 2204 return -EINVAL; 2205 } 2206 2207 cd = kzalloc(sizeof(*cd), GFP_KERNEL); 2208 cd->host_data = host_data; 2209 cd->bit = hw; 2210 2211 irq_set_chip_and_handler(virq, &octeon_irq_chip_cib, 2212 handle_simple_irq); 2213 irq_set_chip_data(virq, cd); 2214 return 0; 2215 } 2216 2217 static struct irq_domain_ops octeon_irq_domain_cib_ops = { 2218 .map = octeon_irq_cib_map, 2219 .unmap = octeon_irq_free_cd, 2220 .xlate = octeon_irq_cib_xlat, 2221 }; 2222 2223 /* Chain to real handler. */ 2224 static irqreturn_t octeon_irq_cib_handler(int my_irq, void *data) 2225 { 2226 u64 en; 2227 u64 raw; 2228 u64 bits; 2229 int i; 2230 int irq; 2231 struct irq_domain *cib_domain = data; 2232 struct octeon_irq_cib_host_data *host_data = cib_domain->host_data; 2233 2234 en = cvmx_read_csr(host_data->en_reg); 2235 raw = cvmx_read_csr(host_data->raw_reg); 2236 2237 bits = en & raw; 2238 2239 for (i = 0; i < host_data->max_bits; i++) { 2240 if ((bits & 1ull << i) == 0) 2241 continue; 2242 irq = irq_find_mapping(cib_domain, i); 2243 if (!irq) { 2244 unsigned long flags; 2245 2246 pr_err("ERROR: CIB bit %d@%llx IRQ unhandled, disabling\n", 2247 i, host_data->raw_reg); 2248 raw_spin_lock_irqsave(&host_data->lock, flags); 2249 en = cvmx_read_csr(host_data->en_reg); 2250 en &= ~(1ull << i); 2251 cvmx_write_csr(host_data->en_reg, en); 2252 cvmx_write_csr(host_data->raw_reg, 1ull << i); 2253 raw_spin_unlock_irqrestore(&host_data->lock, flags); 2254 } else { 2255 struct irq_desc *desc = irq_to_desc(irq); 2256 struct irq_data *irq_data = irq_desc_get_irq_data(desc); 2257 /* If edge, acknowledge the bit we will be sending. */ 2258 if (irqd_get_trigger_type(irq_data) & 2259 IRQ_TYPE_EDGE_BOTH) 2260 cvmx_write_csr(host_data->raw_reg, 1ull << i); 2261 generic_handle_irq_desc(desc); 2262 } 2263 } 2264 2265 return IRQ_HANDLED; 2266 } 2267 2268 static int __init octeon_irq_init_cib(struct device_node *ciu_node, 2269 struct device_node *parent) 2270 { 2271 const __be32 *addr; 2272 u32 val; 2273 struct octeon_irq_cib_host_data *host_data; 2274 int parent_irq; 2275 int r; 2276 struct irq_domain *cib_domain; 2277 2278 parent_irq = irq_of_parse_and_map(ciu_node, 0); 2279 if (!parent_irq) { 2280 pr_err("ERROR: Couldn't acquire parent_irq for %s\n.", 2281 ciu_node->name); 2282 return -EINVAL; 2283 } 2284 2285 host_data = kzalloc(sizeof(*host_data), GFP_KERNEL); 2286 raw_spin_lock_init(&host_data->lock); 2287 2288 addr = of_get_address(ciu_node, 0, NULL, NULL); 2289 if (!addr) { 2290 pr_err("ERROR: Couldn't acquire reg(0) %s\n.", ciu_node->name); 2291 return -EINVAL; 2292 } 2293 host_data->raw_reg = (u64)phys_to_virt( 2294 of_translate_address(ciu_node, addr)); 2295 2296 addr = of_get_address(ciu_node, 1, NULL, NULL); 2297 if (!addr) { 2298 pr_err("ERROR: Couldn't acquire reg(1) %s\n.", ciu_node->name); 2299 return -EINVAL; 2300 } 2301 host_data->en_reg = (u64)phys_to_virt( 2302 of_translate_address(ciu_node, addr)); 2303 2304 r = of_property_read_u32(ciu_node, "cavium,max-bits", &val); 2305 if (r) { 2306 pr_err("ERROR: Couldn't read cavium,max-bits from %s\n.", 2307 ciu_node->name); 2308 return r; 2309 } 2310 host_data->max_bits = val; 2311 2312 cib_domain = irq_domain_add_linear(ciu_node, host_data->max_bits, 2313 &octeon_irq_domain_cib_ops, 2314 host_data); 2315 if (!cib_domain) { 2316 pr_err("ERROR: Couldn't irq_domain_add_linear()\n."); 2317 return -ENOMEM; 2318 } 2319 2320 cvmx_write_csr(host_data->en_reg, 0); /* disable all IRQs */ 2321 cvmx_write_csr(host_data->raw_reg, ~0); /* ack any outstanding */ 2322 2323 r = request_irq(parent_irq, octeon_irq_cib_handler, 2324 IRQF_NO_THREAD, "cib", cib_domain); 2325 if (r) { 2326 pr_err("request_irq cib failed %d\n", r); 2327 return r; 2328 } 2329 pr_info("CIB interrupt controller probed: %llx %d\n", 2330 host_data->raw_reg, host_data->max_bits); 2331 return 0; 2332 } 2333 2334 int octeon_irq_ciu3_xlat(struct irq_domain *d, 2335 struct device_node *node, 2336 const u32 *intspec, 2337 unsigned int intsize, 2338 unsigned long *out_hwirq, 2339 unsigned int *out_type) 2340 { 2341 struct octeon_ciu3_info *ciu3_info = d->host_data; 2342 unsigned int hwirq, type, intsn_major; 2343 union cvmx_ciu3_iscx_ctl isc; 2344 2345 if (intsize < 2) 2346 return -EINVAL; 2347 hwirq = intspec[0]; 2348 type = intspec[1]; 2349 2350 if (hwirq >= (1 << 20)) 2351 return -EINVAL; 2352 2353 intsn_major = hwirq >> 12; 2354 switch (intsn_major) { 2355 case 0x04: /* Software handled separately. */ 2356 return -EINVAL; 2357 default: 2358 break; 2359 } 2360 2361 isc.u64 = cvmx_read_csr(ciu3_info->ciu3_addr + CIU3_ISC_CTL(hwirq)); 2362 if (!isc.s.imp) 2363 return -EINVAL; 2364 2365 switch (type) { 2366 case 4: /* official value for level triggering. */ 2367 *out_type = IRQ_TYPE_LEVEL_HIGH; 2368 break; 2369 case 0: /* unofficial value, but we might as well let it work. */ 2370 case 1: /* official value for edge triggering. */ 2371 *out_type = IRQ_TYPE_EDGE_RISING; 2372 break; 2373 default: /* Nothing else is acceptable. */ 2374 return -EINVAL; 2375 } 2376 2377 *out_hwirq = hwirq; 2378 2379 return 0; 2380 } 2381 2382 void octeon_irq_ciu3_enable(struct irq_data *data) 2383 { 2384 int cpu; 2385 union cvmx_ciu3_iscx_ctl isc_ctl; 2386 union cvmx_ciu3_iscx_w1c isc_w1c; 2387 u64 isc_ctl_addr; 2388 2389 struct octeon_ciu_chip_data *cd; 2390 2391 cpu = next_cpu_for_irq(data); 2392 2393 cd = irq_data_get_irq_chip_data(data); 2394 2395 isc_w1c.u64 = 0; 2396 isc_w1c.s.en = 1; 2397 cvmx_write_csr(cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn), isc_w1c.u64); 2398 2399 isc_ctl_addr = cd->ciu3_addr + CIU3_ISC_CTL(cd->intsn); 2400 isc_ctl.u64 = 0; 2401 isc_ctl.s.en = 1; 2402 isc_ctl.s.idt = per_cpu(octeon_irq_ciu3_idt_ip2, cpu); 2403 cvmx_write_csr(isc_ctl_addr, isc_ctl.u64); 2404 cvmx_read_csr(isc_ctl_addr); 2405 } 2406 2407 void octeon_irq_ciu3_disable(struct irq_data *data) 2408 { 2409 u64 isc_ctl_addr; 2410 union cvmx_ciu3_iscx_w1c isc_w1c; 2411 2412 struct octeon_ciu_chip_data *cd; 2413 2414 cd = irq_data_get_irq_chip_data(data); 2415 2416 isc_w1c.u64 = 0; 2417 isc_w1c.s.en = 1; 2418 2419 isc_ctl_addr = cd->ciu3_addr + CIU3_ISC_CTL(cd->intsn); 2420 cvmx_write_csr(cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn), isc_w1c.u64); 2421 cvmx_write_csr(isc_ctl_addr, 0); 2422 cvmx_read_csr(isc_ctl_addr); 2423 } 2424 2425 void octeon_irq_ciu3_ack(struct irq_data *data) 2426 { 2427 u64 isc_w1c_addr; 2428 union cvmx_ciu3_iscx_w1c isc_w1c; 2429 struct octeon_ciu_chip_data *cd; 2430 u32 trigger_type = irqd_get_trigger_type(data); 2431 2432 /* 2433 * We use a single irq_chip, so we have to do nothing to ack a 2434 * level interrupt. 2435 */ 2436 if (!(trigger_type & IRQ_TYPE_EDGE_BOTH)) 2437 return; 2438 2439 cd = irq_data_get_irq_chip_data(data); 2440 2441 isc_w1c.u64 = 0; 2442 isc_w1c.s.raw = 1; 2443 2444 isc_w1c_addr = cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn); 2445 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2446 cvmx_read_csr(isc_w1c_addr); 2447 } 2448 2449 void octeon_irq_ciu3_mask(struct irq_data *data) 2450 { 2451 union cvmx_ciu3_iscx_w1c isc_w1c; 2452 u64 isc_w1c_addr; 2453 struct octeon_ciu_chip_data *cd; 2454 2455 cd = irq_data_get_irq_chip_data(data); 2456 2457 isc_w1c.u64 = 0; 2458 isc_w1c.s.en = 1; 2459 2460 isc_w1c_addr = cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn); 2461 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2462 cvmx_read_csr(isc_w1c_addr); 2463 } 2464 2465 void octeon_irq_ciu3_mask_ack(struct irq_data *data) 2466 { 2467 union cvmx_ciu3_iscx_w1c isc_w1c; 2468 u64 isc_w1c_addr; 2469 struct octeon_ciu_chip_data *cd; 2470 u32 trigger_type = irqd_get_trigger_type(data); 2471 2472 cd = irq_data_get_irq_chip_data(data); 2473 2474 isc_w1c.u64 = 0; 2475 isc_w1c.s.en = 1; 2476 2477 /* 2478 * We use a single irq_chip, so only ack an edge (!level) 2479 * interrupt. 2480 */ 2481 if (trigger_type & IRQ_TYPE_EDGE_BOTH) 2482 isc_w1c.s.raw = 1; 2483 2484 isc_w1c_addr = cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn); 2485 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2486 cvmx_read_csr(isc_w1c_addr); 2487 } 2488 2489 #ifdef CONFIG_SMP 2490 int octeon_irq_ciu3_set_affinity(struct irq_data *data, 2491 const struct cpumask *dest, bool force) 2492 { 2493 union cvmx_ciu3_iscx_ctl isc_ctl; 2494 union cvmx_ciu3_iscx_w1c isc_w1c; 2495 u64 isc_ctl_addr; 2496 int cpu; 2497 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 2498 struct octeon_ciu_chip_data *cd = irq_data_get_irq_chip_data(data); 2499 2500 if (!cpumask_subset(dest, cpumask_of_node(cd->ciu_node))) 2501 return -EINVAL; 2502 2503 if (!enable_one) 2504 return IRQ_SET_MASK_OK; 2505 2506 cd = irq_data_get_irq_chip_data(data); 2507 cpu = cpumask_first(dest); 2508 if (cpu >= nr_cpu_ids) 2509 cpu = smp_processor_id(); 2510 cd->current_cpu = cpu; 2511 2512 isc_w1c.u64 = 0; 2513 isc_w1c.s.en = 1; 2514 cvmx_write_csr(cd->ciu3_addr + CIU3_ISC_W1C(cd->intsn), isc_w1c.u64); 2515 2516 isc_ctl_addr = cd->ciu3_addr + CIU3_ISC_CTL(cd->intsn); 2517 isc_ctl.u64 = 0; 2518 isc_ctl.s.en = 1; 2519 isc_ctl.s.idt = per_cpu(octeon_irq_ciu3_idt_ip2, cpu); 2520 cvmx_write_csr(isc_ctl_addr, isc_ctl.u64); 2521 cvmx_read_csr(isc_ctl_addr); 2522 2523 return IRQ_SET_MASK_OK; 2524 } 2525 #endif 2526 2527 static struct irq_chip octeon_irq_chip_ciu3 = { 2528 .name = "CIU3", 2529 .irq_startup = edge_startup, 2530 .irq_enable = octeon_irq_ciu3_enable, 2531 .irq_disable = octeon_irq_ciu3_disable, 2532 .irq_ack = octeon_irq_ciu3_ack, 2533 .irq_mask = octeon_irq_ciu3_mask, 2534 .irq_mask_ack = octeon_irq_ciu3_mask_ack, 2535 .irq_unmask = octeon_irq_ciu3_enable, 2536 .irq_set_type = octeon_irq_ciu_set_type, 2537 #ifdef CONFIG_SMP 2538 .irq_set_affinity = octeon_irq_ciu3_set_affinity, 2539 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 2540 #endif 2541 }; 2542 2543 int octeon_irq_ciu3_mapx(struct irq_domain *d, unsigned int virq, 2544 irq_hw_number_t hw, struct irq_chip *chip) 2545 { 2546 struct octeon_ciu3_info *ciu3_info = d->host_data; 2547 struct octeon_ciu_chip_data *cd = kzalloc_node(sizeof(*cd), GFP_KERNEL, 2548 ciu3_info->node); 2549 if (!cd) 2550 return -ENOMEM; 2551 cd->intsn = hw; 2552 cd->current_cpu = -1; 2553 cd->ciu3_addr = ciu3_info->ciu3_addr; 2554 cd->ciu_node = ciu3_info->node; 2555 irq_set_chip_and_handler(virq, chip, handle_edge_irq); 2556 irq_set_chip_data(virq, cd); 2557 2558 return 0; 2559 } 2560 2561 static int octeon_irq_ciu3_map(struct irq_domain *d, 2562 unsigned int virq, irq_hw_number_t hw) 2563 { 2564 return octeon_irq_ciu3_mapx(d, virq, hw, &octeon_irq_chip_ciu3); 2565 } 2566 2567 static struct irq_domain_ops octeon_dflt_domain_ciu3_ops = { 2568 .map = octeon_irq_ciu3_map, 2569 .unmap = octeon_irq_free_cd, 2570 .xlate = octeon_irq_ciu3_xlat, 2571 }; 2572 2573 static void octeon_irq_ciu3_ip2(void) 2574 { 2575 union cvmx_ciu3_destx_pp_int dest_pp_int; 2576 struct octeon_ciu3_info *ciu3_info; 2577 u64 ciu3_addr; 2578 2579 ciu3_info = __this_cpu_read(octeon_ciu3_info); 2580 ciu3_addr = ciu3_info->ciu3_addr; 2581 2582 dest_pp_int.u64 = cvmx_read_csr(ciu3_addr + CIU3_DEST_PP_INT(3 * cvmx_get_local_core_num())); 2583 2584 if (likely(dest_pp_int.s.intr)) { 2585 irq_hw_number_t intsn = dest_pp_int.s.intsn; 2586 irq_hw_number_t hw; 2587 struct irq_domain *domain; 2588 /* Get the domain to use from the major block */ 2589 int block = intsn >> 12; 2590 int ret; 2591 2592 domain = ciu3_info->domain[block]; 2593 if (ciu3_info->intsn2hw[block]) 2594 hw = ciu3_info->intsn2hw[block](domain, intsn); 2595 else 2596 hw = intsn; 2597 2598 ret = handle_domain_irq(domain, hw, NULL); 2599 if (ret < 0) { 2600 union cvmx_ciu3_iscx_w1c isc_w1c; 2601 u64 isc_w1c_addr = ciu3_addr + CIU3_ISC_W1C(intsn); 2602 2603 isc_w1c.u64 = 0; 2604 isc_w1c.s.en = 1; 2605 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2606 cvmx_read_csr(isc_w1c_addr); 2607 spurious_interrupt(); 2608 } 2609 } else { 2610 spurious_interrupt(); 2611 } 2612 } 2613 2614 /* 2615 * 10 mbox per core starting from zero. 2616 * Base mbox is core * 10 2617 */ 2618 static unsigned int octeon_irq_ciu3_base_mbox_intsn(int core) 2619 { 2620 /* SW (mbox) are 0x04 in bits 12..19 */ 2621 return 0x04000 + CIU3_MBOX_PER_CORE * core; 2622 } 2623 2624 static unsigned int octeon_irq_ciu3_mbox_intsn_for_core(int core, unsigned int mbox) 2625 { 2626 return octeon_irq_ciu3_base_mbox_intsn(core) + mbox; 2627 } 2628 2629 static unsigned int octeon_irq_ciu3_mbox_intsn_for_cpu(int cpu, unsigned int mbox) 2630 { 2631 int local_core = octeon_coreid_for_cpu(cpu) & 0x3f; 2632 2633 return octeon_irq_ciu3_mbox_intsn_for_core(local_core, mbox); 2634 } 2635 2636 static void octeon_irq_ciu3_mbox(void) 2637 { 2638 union cvmx_ciu3_destx_pp_int dest_pp_int; 2639 struct octeon_ciu3_info *ciu3_info; 2640 u64 ciu3_addr; 2641 int core = cvmx_get_local_core_num(); 2642 2643 ciu3_info = __this_cpu_read(octeon_ciu3_info); 2644 ciu3_addr = ciu3_info->ciu3_addr; 2645 2646 dest_pp_int.u64 = cvmx_read_csr(ciu3_addr + CIU3_DEST_PP_INT(1 + 3 * core)); 2647 2648 if (likely(dest_pp_int.s.intr)) { 2649 irq_hw_number_t intsn = dest_pp_int.s.intsn; 2650 int mbox = intsn - octeon_irq_ciu3_base_mbox_intsn(core); 2651 2652 if (likely(mbox >= 0 && mbox < CIU3_MBOX_PER_CORE)) { 2653 do_IRQ(mbox + OCTEON_IRQ_MBOX0); 2654 } else { 2655 union cvmx_ciu3_iscx_w1c isc_w1c; 2656 u64 isc_w1c_addr = ciu3_addr + CIU3_ISC_W1C(intsn); 2657 2658 isc_w1c.u64 = 0; 2659 isc_w1c.s.en = 1; 2660 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2661 cvmx_read_csr(isc_w1c_addr); 2662 spurious_interrupt(); 2663 } 2664 } else { 2665 spurious_interrupt(); 2666 } 2667 } 2668 2669 void octeon_ciu3_mbox_send(int cpu, unsigned int mbox) 2670 { 2671 struct octeon_ciu3_info *ciu3_info; 2672 unsigned int intsn; 2673 union cvmx_ciu3_iscx_w1s isc_w1s; 2674 u64 isc_w1s_addr; 2675 2676 if (WARN_ON_ONCE(mbox >= CIU3_MBOX_PER_CORE)) 2677 return; 2678 2679 intsn = octeon_irq_ciu3_mbox_intsn_for_cpu(cpu, mbox); 2680 ciu3_info = per_cpu(octeon_ciu3_info, cpu); 2681 isc_w1s_addr = ciu3_info->ciu3_addr + CIU3_ISC_W1S(intsn); 2682 2683 isc_w1s.u64 = 0; 2684 isc_w1s.s.raw = 1; 2685 2686 cvmx_write_csr(isc_w1s_addr, isc_w1s.u64); 2687 cvmx_read_csr(isc_w1s_addr); 2688 } 2689 2690 static void octeon_irq_ciu3_mbox_set_enable(struct irq_data *data, int cpu, bool en) 2691 { 2692 struct octeon_ciu3_info *ciu3_info; 2693 unsigned int intsn; 2694 u64 isc_ctl_addr, isc_w1c_addr; 2695 union cvmx_ciu3_iscx_ctl isc_ctl; 2696 unsigned int mbox = data->irq - OCTEON_IRQ_MBOX0; 2697 2698 intsn = octeon_irq_ciu3_mbox_intsn_for_cpu(cpu, mbox); 2699 ciu3_info = per_cpu(octeon_ciu3_info, cpu); 2700 isc_w1c_addr = ciu3_info->ciu3_addr + CIU3_ISC_W1C(intsn); 2701 isc_ctl_addr = ciu3_info->ciu3_addr + CIU3_ISC_CTL(intsn); 2702 2703 isc_ctl.u64 = 0; 2704 isc_ctl.s.en = 1; 2705 2706 cvmx_write_csr(isc_w1c_addr, isc_ctl.u64); 2707 cvmx_write_csr(isc_ctl_addr, 0); 2708 if (en) { 2709 unsigned int idt = per_cpu(octeon_irq_ciu3_idt_ip3, cpu); 2710 2711 isc_ctl.u64 = 0; 2712 isc_ctl.s.en = 1; 2713 isc_ctl.s.idt = idt; 2714 cvmx_write_csr(isc_ctl_addr, isc_ctl.u64); 2715 } 2716 cvmx_read_csr(isc_ctl_addr); 2717 } 2718 2719 static void octeon_irq_ciu3_mbox_enable(struct irq_data *data) 2720 { 2721 int cpu; 2722 unsigned int mbox = data->irq - OCTEON_IRQ_MBOX0; 2723 2724 WARN_ON(mbox >= CIU3_MBOX_PER_CORE); 2725 2726 for_each_online_cpu(cpu) 2727 octeon_irq_ciu3_mbox_set_enable(data, cpu, true); 2728 } 2729 2730 static void octeon_irq_ciu3_mbox_disable(struct irq_data *data) 2731 { 2732 int cpu; 2733 unsigned int mbox = data->irq - OCTEON_IRQ_MBOX0; 2734 2735 WARN_ON(mbox >= CIU3_MBOX_PER_CORE); 2736 2737 for_each_online_cpu(cpu) 2738 octeon_irq_ciu3_mbox_set_enable(data, cpu, false); 2739 } 2740 2741 static void octeon_irq_ciu3_mbox_ack(struct irq_data *data) 2742 { 2743 struct octeon_ciu3_info *ciu3_info; 2744 unsigned int intsn; 2745 u64 isc_w1c_addr; 2746 union cvmx_ciu3_iscx_w1c isc_w1c; 2747 unsigned int mbox = data->irq - OCTEON_IRQ_MBOX0; 2748 2749 intsn = octeon_irq_ciu3_mbox_intsn_for_core(cvmx_get_local_core_num(), mbox); 2750 2751 isc_w1c.u64 = 0; 2752 isc_w1c.s.raw = 1; 2753 2754 ciu3_info = __this_cpu_read(octeon_ciu3_info); 2755 isc_w1c_addr = ciu3_info->ciu3_addr + CIU3_ISC_W1C(intsn); 2756 cvmx_write_csr(isc_w1c_addr, isc_w1c.u64); 2757 cvmx_read_csr(isc_w1c_addr); 2758 } 2759 2760 static void octeon_irq_ciu3_mbox_cpu_online(struct irq_data *data) 2761 { 2762 octeon_irq_ciu3_mbox_set_enable(data, smp_processor_id(), true); 2763 } 2764 2765 static void octeon_irq_ciu3_mbox_cpu_offline(struct irq_data *data) 2766 { 2767 octeon_irq_ciu3_mbox_set_enable(data, smp_processor_id(), false); 2768 } 2769 2770 static int octeon_irq_ciu3_alloc_resources(struct octeon_ciu3_info *ciu3_info) 2771 { 2772 u64 b = ciu3_info->ciu3_addr; 2773 int idt_ip2, idt_ip3, idt_ip4; 2774 int unused_idt2; 2775 int core = cvmx_get_local_core_num(); 2776 int i; 2777 2778 __this_cpu_write(octeon_ciu3_info, ciu3_info); 2779 2780 /* 2781 * 4 idt per core starting from 1 because zero is reserved. 2782 * Base idt per core is 4 * core + 1 2783 */ 2784 idt_ip2 = core * 4 + 1; 2785 idt_ip3 = core * 4 + 2; 2786 idt_ip4 = core * 4 + 3; 2787 unused_idt2 = core * 4 + 4; 2788 __this_cpu_write(octeon_irq_ciu3_idt_ip2, idt_ip2); 2789 __this_cpu_write(octeon_irq_ciu3_idt_ip3, idt_ip3); 2790 2791 /* ip2 interrupts for this CPU */ 2792 cvmx_write_csr(b + CIU3_IDT_CTL(idt_ip2), 0); 2793 cvmx_write_csr(b + CIU3_IDT_PP(idt_ip2, 0), 1ull << core); 2794 cvmx_write_csr(b + CIU3_IDT_IO(idt_ip2), 0); 2795 2796 /* ip3 interrupts for this CPU */ 2797 cvmx_write_csr(b + CIU3_IDT_CTL(idt_ip3), 1); 2798 cvmx_write_csr(b + CIU3_IDT_PP(idt_ip3, 0), 1ull << core); 2799 cvmx_write_csr(b + CIU3_IDT_IO(idt_ip3), 0); 2800 2801 /* ip4 interrupts for this CPU */ 2802 cvmx_write_csr(b + CIU3_IDT_CTL(idt_ip4), 2); 2803 cvmx_write_csr(b + CIU3_IDT_PP(idt_ip4, 0), 0); 2804 cvmx_write_csr(b + CIU3_IDT_IO(idt_ip4), 0); 2805 2806 cvmx_write_csr(b + CIU3_IDT_CTL(unused_idt2), 0); 2807 cvmx_write_csr(b + CIU3_IDT_PP(unused_idt2, 0), 0); 2808 cvmx_write_csr(b + CIU3_IDT_IO(unused_idt2), 0); 2809 2810 for (i = 0; i < CIU3_MBOX_PER_CORE; i++) { 2811 unsigned int intsn = octeon_irq_ciu3_mbox_intsn_for_core(core, i); 2812 2813 cvmx_write_csr(b + CIU3_ISC_W1C(intsn), 2); 2814 cvmx_write_csr(b + CIU3_ISC_CTL(intsn), 0); 2815 } 2816 2817 return 0; 2818 } 2819 2820 static void octeon_irq_setup_secondary_ciu3(void) 2821 { 2822 struct octeon_ciu3_info *ciu3_info; 2823 2824 ciu3_info = octeon_ciu3_info_per_node[cvmx_get_node_num()]; 2825 octeon_irq_ciu3_alloc_resources(ciu3_info); 2826 irq_cpu_online(); 2827 2828 /* Enable the CIU lines */ 2829 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 2830 if (octeon_irq_use_ip4) 2831 set_c0_status(STATUSF_IP4); 2832 else 2833 clear_c0_status(STATUSF_IP4); 2834 } 2835 2836 static struct irq_chip octeon_irq_chip_ciu3_mbox = { 2837 .name = "CIU3-M", 2838 .irq_enable = octeon_irq_ciu3_mbox_enable, 2839 .irq_disable = octeon_irq_ciu3_mbox_disable, 2840 .irq_ack = octeon_irq_ciu3_mbox_ack, 2841 2842 .irq_cpu_online = octeon_irq_ciu3_mbox_cpu_online, 2843 .irq_cpu_offline = octeon_irq_ciu3_mbox_cpu_offline, 2844 .flags = IRQCHIP_ONOFFLINE_ENABLED, 2845 }; 2846 2847 static int __init octeon_irq_init_ciu3(struct device_node *ciu_node, 2848 struct device_node *parent) 2849 { 2850 int i; 2851 int node; 2852 struct irq_domain *domain; 2853 struct octeon_ciu3_info *ciu3_info; 2854 const __be32 *zero_addr; 2855 u64 base_addr; 2856 union cvmx_ciu3_const consts; 2857 2858 node = 0; /* of_node_to_nid(ciu_node); */ 2859 ciu3_info = kzalloc_node(sizeof(*ciu3_info), GFP_KERNEL, node); 2860 2861 if (!ciu3_info) 2862 return -ENOMEM; 2863 2864 zero_addr = of_get_address(ciu_node, 0, NULL, NULL); 2865 if (WARN_ON(!zero_addr)) 2866 return -EINVAL; 2867 2868 base_addr = of_translate_address(ciu_node, zero_addr); 2869 base_addr = (u64)phys_to_virt(base_addr); 2870 2871 ciu3_info->ciu3_addr = base_addr; 2872 ciu3_info->node = node; 2873 2874 consts.u64 = cvmx_read_csr(base_addr + CIU3_CONST); 2875 2876 octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu3; 2877 2878 octeon_irq_ip2 = octeon_irq_ciu3_ip2; 2879 octeon_irq_ip3 = octeon_irq_ciu3_mbox; 2880 octeon_irq_ip4 = octeon_irq_ip4_mask; 2881 2882 if (node == cvmx_get_node_num()) { 2883 /* Mips internal */ 2884 octeon_irq_init_core(); 2885 2886 /* Only do per CPU things if it is the CIU of the boot node. */ 2887 i = irq_alloc_descs_from(OCTEON_IRQ_MBOX0, 8, node); 2888 WARN_ON(i < 0); 2889 2890 for (i = 0; i < 8; i++) 2891 irq_set_chip_and_handler(i + OCTEON_IRQ_MBOX0, 2892 &octeon_irq_chip_ciu3_mbox, handle_percpu_irq); 2893 } 2894 2895 /* 2896 * Initialize all domains to use the default domain. Specific major 2897 * blocks will overwrite the default domain as needed. 2898 */ 2899 domain = irq_domain_add_tree(ciu_node, &octeon_dflt_domain_ciu3_ops, 2900 ciu3_info); 2901 for (i = 0; i < MAX_CIU3_DOMAINS; i++) 2902 ciu3_info->domain[i] = domain; 2903 2904 octeon_ciu3_info_per_node[node] = ciu3_info; 2905 2906 if (node == cvmx_get_node_num()) { 2907 /* Only do per CPU things if it is the CIU of the boot node. */ 2908 octeon_irq_ciu3_alloc_resources(ciu3_info); 2909 if (node == 0) 2910 irq_set_default_host(domain); 2911 2912 octeon_irq_use_ip4 = false; 2913 /* Enable the CIU lines */ 2914 set_c0_status(STATUSF_IP2 | STATUSF_IP3); 2915 clear_c0_status(STATUSF_IP4); 2916 } 2917 2918 return 0; 2919 } 2920 2921 static struct of_device_id ciu_types[] __initdata = { 2922 {.compatible = "cavium,octeon-3860-ciu", .data = octeon_irq_init_ciu}, 2923 {.compatible = "cavium,octeon-3860-gpio", .data = octeon_irq_init_gpio}, 2924 {.compatible = "cavium,octeon-6880-ciu2", .data = octeon_irq_init_ciu2}, 2925 {.compatible = "cavium,octeon-7890-ciu3", .data = octeon_irq_init_ciu3}, 2926 {.compatible = "cavium,octeon-7130-cib", .data = octeon_irq_init_cib}, 2927 {} 2928 }; 2929 2930 void __init arch_init_irq(void) 2931 { 2932 #ifdef CONFIG_SMP 2933 /* Set the default affinity to the boot cpu. */ 2934 cpumask_clear(irq_default_affinity); 2935 cpumask_set_cpu(smp_processor_id(), irq_default_affinity); 2936 #endif 2937 of_irq_init(ciu_types); 2938 } 2939 2940 asmlinkage void plat_irq_dispatch(void) 2941 { 2942 unsigned long cop0_cause; 2943 unsigned long cop0_status; 2944 2945 while (1) { 2946 cop0_cause = read_c0_cause(); 2947 cop0_status = read_c0_status(); 2948 cop0_cause &= cop0_status; 2949 cop0_cause &= ST0_IM; 2950 2951 if (cop0_cause & STATUSF_IP2) 2952 octeon_irq_ip2(); 2953 else if (cop0_cause & STATUSF_IP3) 2954 octeon_irq_ip3(); 2955 else if (cop0_cause & STATUSF_IP4) 2956 octeon_irq_ip4(); 2957 else if (cop0_cause) 2958 do_IRQ(fls(cop0_cause) - 9 + MIPS_CPU_IRQ_BASE); 2959 else 2960 break; 2961 } 2962 } 2963 2964 #ifdef CONFIG_HOTPLUG_CPU 2965 2966 void octeon_fixup_irqs(void) 2967 { 2968 irq_cpu_offline(); 2969 } 2970 2971 #endif /* CONFIG_HOTPLUG_CPU */ 2972