1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 2004-2008, 2009, 2010, 2011 Cavium Networks 7 */ 8 9 #include <linux/interrupt.h> 10 #include <linux/bitops.h> 11 #include <linux/percpu.h> 12 #include <linux/irq.h> 13 #include <linux/smp.h> 14 15 #include <asm/octeon/octeon.h> 16 17 static DEFINE_RAW_SPINLOCK(octeon_irq_ciu0_lock); 18 static DEFINE_RAW_SPINLOCK(octeon_irq_ciu1_lock); 19 20 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu0_en_mirror); 21 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu1_en_mirror); 22 23 static __read_mostly u8 octeon_irq_ciu_to_irq[8][64]; 24 25 union octeon_ciu_chip_data { 26 void *p; 27 unsigned long l; 28 struct { 29 unsigned int line:6; 30 unsigned int bit:6; 31 } s; 32 }; 33 34 struct octeon_core_chip_data { 35 struct mutex core_irq_mutex; 36 bool current_en; 37 bool desired_en; 38 u8 bit; 39 }; 40 41 #define MIPS_CORE_IRQ_LINES 8 42 43 static struct octeon_core_chip_data octeon_irq_core_chip_data[MIPS_CORE_IRQ_LINES]; 44 45 static void __init octeon_irq_set_ciu_mapping(int irq, int line, int bit, 46 struct irq_chip *chip, 47 irq_flow_handler_t handler) 48 { 49 union octeon_ciu_chip_data cd; 50 51 irq_set_chip_and_handler(irq, chip, handler); 52 53 cd.l = 0; 54 cd.s.line = line; 55 cd.s.bit = bit; 56 57 irq_set_chip_data(irq, cd.p); 58 octeon_irq_ciu_to_irq[line][bit] = irq; 59 } 60 61 static int octeon_coreid_for_cpu(int cpu) 62 { 63 #ifdef CONFIG_SMP 64 return cpu_logical_map(cpu); 65 #else 66 return cvmx_get_core_num(); 67 #endif 68 } 69 70 static int octeon_cpu_for_coreid(int coreid) 71 { 72 #ifdef CONFIG_SMP 73 return cpu_number_map(coreid); 74 #else 75 return smp_processor_id(); 76 #endif 77 } 78 79 static void octeon_irq_core_ack(struct irq_data *data) 80 { 81 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 82 unsigned int bit = cd->bit; 83 84 /* 85 * We don't need to disable IRQs to make these atomic since 86 * they are already disabled earlier in the low level 87 * interrupt code. 88 */ 89 clear_c0_status(0x100 << bit); 90 /* The two user interrupts must be cleared manually. */ 91 if (bit < 2) 92 clear_c0_cause(0x100 << bit); 93 } 94 95 static void octeon_irq_core_eoi(struct irq_data *data) 96 { 97 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 98 99 /* 100 * We don't need to disable IRQs to make these atomic since 101 * they are already disabled earlier in the low level 102 * interrupt code. 103 */ 104 set_c0_status(0x100 << cd->bit); 105 } 106 107 static void octeon_irq_core_set_enable_local(void *arg) 108 { 109 struct irq_data *data = arg; 110 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 111 unsigned int mask = 0x100 << cd->bit; 112 113 /* 114 * Interrupts are already disabled, so these are atomic. 115 */ 116 if (cd->desired_en) 117 set_c0_status(mask); 118 else 119 clear_c0_status(mask); 120 121 } 122 123 static void octeon_irq_core_disable(struct irq_data *data) 124 { 125 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 126 cd->desired_en = false; 127 } 128 129 static void octeon_irq_core_enable(struct irq_data *data) 130 { 131 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 132 cd->desired_en = true; 133 } 134 135 static void octeon_irq_core_bus_lock(struct irq_data *data) 136 { 137 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 138 139 mutex_lock(&cd->core_irq_mutex); 140 } 141 142 static void octeon_irq_core_bus_sync_unlock(struct irq_data *data) 143 { 144 struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); 145 146 if (cd->desired_en != cd->current_en) { 147 on_each_cpu(octeon_irq_core_set_enable_local, data, 1); 148 149 cd->current_en = cd->desired_en; 150 } 151 152 mutex_unlock(&cd->core_irq_mutex); 153 } 154 155 static struct irq_chip octeon_irq_chip_core = { 156 .name = "Core", 157 .irq_enable = octeon_irq_core_enable, 158 .irq_disable = octeon_irq_core_disable, 159 .irq_ack = octeon_irq_core_ack, 160 .irq_eoi = octeon_irq_core_eoi, 161 .irq_bus_lock = octeon_irq_core_bus_lock, 162 .irq_bus_sync_unlock = octeon_irq_core_bus_sync_unlock, 163 164 .irq_cpu_online = octeon_irq_core_eoi, 165 .irq_cpu_offline = octeon_irq_core_ack, 166 .flags = IRQCHIP_ONOFFLINE_ENABLED, 167 }; 168 169 static void __init octeon_irq_init_core(void) 170 { 171 int i; 172 int irq; 173 struct octeon_core_chip_data *cd; 174 175 for (i = 0; i < MIPS_CORE_IRQ_LINES; i++) { 176 cd = &octeon_irq_core_chip_data[i]; 177 cd->current_en = false; 178 cd->desired_en = false; 179 cd->bit = i; 180 mutex_init(&cd->core_irq_mutex); 181 182 irq = OCTEON_IRQ_SW0 + i; 183 switch (irq) { 184 case OCTEON_IRQ_TIMER: 185 case OCTEON_IRQ_SW0: 186 case OCTEON_IRQ_SW1: 187 case OCTEON_IRQ_5: 188 case OCTEON_IRQ_PERF: 189 irq_set_chip_data(irq, cd); 190 irq_set_chip_and_handler(irq, &octeon_irq_chip_core, 191 handle_percpu_irq); 192 break; 193 default: 194 break; 195 } 196 } 197 } 198 199 static int next_cpu_for_irq(struct irq_data *data) 200 { 201 202 #ifdef CONFIG_SMP 203 int cpu; 204 int weight = cpumask_weight(data->affinity); 205 206 if (weight > 1) { 207 cpu = smp_processor_id(); 208 for (;;) { 209 cpu = cpumask_next(cpu, data->affinity); 210 if (cpu >= nr_cpu_ids) { 211 cpu = -1; 212 continue; 213 } else if (cpumask_test_cpu(cpu, cpu_online_mask)) { 214 break; 215 } 216 } 217 } else if (weight == 1) { 218 cpu = cpumask_first(data->affinity); 219 } else { 220 cpu = smp_processor_id(); 221 } 222 return cpu; 223 #else 224 return smp_processor_id(); 225 #endif 226 } 227 228 static void octeon_irq_ciu_enable(struct irq_data *data) 229 { 230 int cpu = next_cpu_for_irq(data); 231 int coreid = octeon_coreid_for_cpu(cpu); 232 unsigned long *pen; 233 unsigned long flags; 234 union octeon_ciu_chip_data cd; 235 236 cd.p = irq_data_get_irq_chip_data(data); 237 238 if (cd.s.line == 0) { 239 raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags); 240 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 241 set_bit(cd.s.bit, pen); 242 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 243 raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags); 244 } else { 245 raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags); 246 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 247 set_bit(cd.s.bit, pen); 248 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 249 raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags); 250 } 251 } 252 253 static void octeon_irq_ciu_enable_local(struct irq_data *data) 254 { 255 unsigned long *pen; 256 unsigned long flags; 257 union octeon_ciu_chip_data cd; 258 259 cd.p = irq_data_get_irq_chip_data(data); 260 261 if (cd.s.line == 0) { 262 raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags); 263 pen = &__get_cpu_var(octeon_irq_ciu0_en_mirror); 264 set_bit(cd.s.bit, pen); 265 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen); 266 raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags); 267 } else { 268 raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags); 269 pen = &__get_cpu_var(octeon_irq_ciu1_en_mirror); 270 set_bit(cd.s.bit, pen); 271 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen); 272 raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags); 273 } 274 } 275 276 static void octeon_irq_ciu_disable_local(struct irq_data *data) 277 { 278 unsigned long *pen; 279 unsigned long flags; 280 union octeon_ciu_chip_data cd; 281 282 cd.p = irq_data_get_irq_chip_data(data); 283 284 if (cd.s.line == 0) { 285 raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags); 286 pen = &__get_cpu_var(octeon_irq_ciu0_en_mirror); 287 clear_bit(cd.s.bit, pen); 288 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen); 289 raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags); 290 } else { 291 raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags); 292 pen = &__get_cpu_var(octeon_irq_ciu1_en_mirror); 293 clear_bit(cd.s.bit, pen); 294 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen); 295 raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags); 296 } 297 } 298 299 static void octeon_irq_ciu_disable_all(struct irq_data *data) 300 { 301 unsigned long flags; 302 unsigned long *pen; 303 int cpu; 304 union octeon_ciu_chip_data cd; 305 306 wmb(); /* Make sure flag changes arrive before register updates. */ 307 308 cd.p = irq_data_get_irq_chip_data(data); 309 310 if (cd.s.line == 0) { 311 raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags); 312 for_each_online_cpu(cpu) { 313 int coreid = octeon_coreid_for_cpu(cpu); 314 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 315 clear_bit(cd.s.bit, pen); 316 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 317 } 318 raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags); 319 } else { 320 raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags); 321 for_each_online_cpu(cpu) { 322 int coreid = octeon_coreid_for_cpu(cpu); 323 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 324 clear_bit(cd.s.bit, pen); 325 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 326 } 327 raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags); 328 } 329 } 330 331 static void octeon_irq_ciu_enable_all(struct irq_data *data) 332 { 333 unsigned long flags; 334 unsigned long *pen; 335 int cpu; 336 union octeon_ciu_chip_data cd; 337 338 cd.p = irq_data_get_irq_chip_data(data); 339 340 if (cd.s.line == 0) { 341 raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags); 342 for_each_online_cpu(cpu) { 343 int coreid = octeon_coreid_for_cpu(cpu); 344 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 345 set_bit(cd.s.bit, pen); 346 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 347 } 348 raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags); 349 } else { 350 raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags); 351 for_each_online_cpu(cpu) { 352 int coreid = octeon_coreid_for_cpu(cpu); 353 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 354 set_bit(cd.s.bit, pen); 355 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 356 } 357 raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags); 358 } 359 } 360 361 /* 362 * Enable the irq on the next core in the affinity set for chips that 363 * have the EN*_W1{S,C} registers. 364 */ 365 static void octeon_irq_ciu_enable_v2(struct irq_data *data) 366 { 367 u64 mask; 368 int cpu = next_cpu_for_irq(data); 369 union octeon_ciu_chip_data cd; 370 371 cd.p = irq_data_get_irq_chip_data(data); 372 mask = 1ull << (cd.s.bit); 373 374 /* 375 * Called under the desc lock, so these should never get out 376 * of sync. 377 */ 378 if (cd.s.line == 0) { 379 int index = octeon_coreid_for_cpu(cpu) * 2; 380 set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu)); 381 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 382 } else { 383 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 384 set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 385 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 386 } 387 } 388 389 /* 390 * Enable the irq on the current CPU for chips that 391 * have the EN*_W1{S,C} registers. 392 */ 393 static void octeon_irq_ciu_enable_local_v2(struct irq_data *data) 394 { 395 u64 mask; 396 union octeon_ciu_chip_data cd; 397 398 cd.p = irq_data_get_irq_chip_data(data); 399 mask = 1ull << (cd.s.bit); 400 401 if (cd.s.line == 0) { 402 int index = cvmx_get_core_num() * 2; 403 set_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu0_en_mirror)); 404 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 405 } else { 406 int index = cvmx_get_core_num() * 2 + 1; 407 set_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu1_en_mirror)); 408 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 409 } 410 } 411 412 static void octeon_irq_ciu_disable_local_v2(struct irq_data *data) 413 { 414 u64 mask; 415 union octeon_ciu_chip_data cd; 416 417 cd.p = irq_data_get_irq_chip_data(data); 418 mask = 1ull << (cd.s.bit); 419 420 if (cd.s.line == 0) { 421 int index = cvmx_get_core_num() * 2; 422 clear_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu0_en_mirror)); 423 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); 424 } else { 425 int index = cvmx_get_core_num() * 2 + 1; 426 clear_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu1_en_mirror)); 427 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask); 428 } 429 } 430 431 /* 432 * Write to the W1C bit in CVMX_CIU_INTX_SUM0 to clear the irq. 433 */ 434 static void octeon_irq_ciu_ack(struct irq_data *data) 435 { 436 u64 mask; 437 union octeon_ciu_chip_data cd; 438 439 cd.p = data->chip_data; 440 mask = 1ull << (cd.s.bit); 441 442 if (cd.s.line == 0) { 443 int index = cvmx_get_core_num() * 2; 444 cvmx_write_csr(CVMX_CIU_INTX_SUM0(index), mask); 445 } else { 446 cvmx_write_csr(CVMX_CIU_INT_SUM1, mask); 447 } 448 } 449 450 /* 451 * Disable the irq on the all cores for chips that have the EN*_W1{S,C} 452 * registers. 453 */ 454 static void octeon_irq_ciu_disable_all_v2(struct irq_data *data) 455 { 456 int cpu; 457 u64 mask; 458 union octeon_ciu_chip_data cd; 459 460 wmb(); /* Make sure flag changes arrive before register updates. */ 461 462 cd.p = data->chip_data; 463 mask = 1ull << (cd.s.bit); 464 465 if (cd.s.line == 0) { 466 for_each_online_cpu(cpu) { 467 int index = octeon_coreid_for_cpu(cpu) * 2; 468 clear_bit(cd.s.bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu)); 469 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); 470 } 471 } else { 472 for_each_online_cpu(cpu) { 473 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 474 clear_bit(cd.s.bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 475 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask); 476 } 477 } 478 } 479 480 /* 481 * Enable the irq on the all cores for chips that have the EN*_W1{S,C} 482 * registers. 483 */ 484 static void octeon_irq_ciu_enable_all_v2(struct irq_data *data) 485 { 486 int cpu; 487 u64 mask; 488 union octeon_ciu_chip_data cd; 489 490 cd.p = data->chip_data; 491 mask = 1ull << (cd.s.bit); 492 493 if (cd.s.line == 0) { 494 for_each_online_cpu(cpu) { 495 int index = octeon_coreid_for_cpu(cpu) * 2; 496 set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu)); 497 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 498 } 499 } else { 500 for_each_online_cpu(cpu) { 501 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 502 set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 503 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 504 } 505 } 506 } 507 508 #ifdef CONFIG_SMP 509 510 static void octeon_irq_cpu_offline_ciu(struct irq_data *data) 511 { 512 int cpu = smp_processor_id(); 513 cpumask_t new_affinity; 514 515 if (!cpumask_test_cpu(cpu, data->affinity)) 516 return; 517 518 if (cpumask_weight(data->affinity) > 1) { 519 /* 520 * It has multi CPU affinity, just remove this CPU 521 * from the affinity set. 522 */ 523 cpumask_copy(&new_affinity, data->affinity); 524 cpumask_clear_cpu(cpu, &new_affinity); 525 } else { 526 /* Otherwise, put it on lowest numbered online CPU. */ 527 cpumask_clear(&new_affinity); 528 cpumask_set_cpu(cpumask_first(cpu_online_mask), &new_affinity); 529 } 530 __irq_set_affinity_locked(data, &new_affinity); 531 } 532 533 static int octeon_irq_ciu_set_affinity(struct irq_data *data, 534 const struct cpumask *dest, bool force) 535 { 536 int cpu; 537 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 538 unsigned long flags; 539 union octeon_ciu_chip_data cd; 540 541 cd.p = data->chip_data; 542 543 /* 544 * For non-v2 CIU, we will allow only single CPU affinity. 545 * This removes the need to do locking in the .ack/.eoi 546 * functions. 547 */ 548 if (cpumask_weight(dest) != 1) 549 return -EINVAL; 550 551 if (!enable_one) 552 return 0; 553 554 if (cd.s.line == 0) { 555 raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags); 556 for_each_online_cpu(cpu) { 557 int coreid = octeon_coreid_for_cpu(cpu); 558 unsigned long *pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 559 560 if (cpumask_test_cpu(cpu, dest) && enable_one) { 561 enable_one = false; 562 set_bit(cd.s.bit, pen); 563 } else { 564 clear_bit(cd.s.bit, pen); 565 } 566 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); 567 } 568 raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags); 569 } else { 570 raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags); 571 for_each_online_cpu(cpu) { 572 int coreid = octeon_coreid_for_cpu(cpu); 573 unsigned long *pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 574 575 if (cpumask_test_cpu(cpu, dest) && enable_one) { 576 enable_one = false; 577 set_bit(cd.s.bit, pen); 578 } else { 579 clear_bit(cd.s.bit, pen); 580 } 581 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 582 } 583 raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags); 584 } 585 return 0; 586 } 587 588 /* 589 * Set affinity for the irq for chips that have the EN*_W1{S,C} 590 * registers. 591 */ 592 static int octeon_irq_ciu_set_affinity_v2(struct irq_data *data, 593 const struct cpumask *dest, 594 bool force) 595 { 596 int cpu; 597 bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data); 598 u64 mask; 599 union octeon_ciu_chip_data cd; 600 601 if (!enable_one) 602 return 0; 603 604 cd.p = data->chip_data; 605 mask = 1ull << cd.s.bit; 606 607 if (cd.s.line == 0) { 608 for_each_online_cpu(cpu) { 609 unsigned long *pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); 610 int index = octeon_coreid_for_cpu(cpu) * 2; 611 if (cpumask_test_cpu(cpu, dest) && enable_one) { 612 enable_one = false; 613 set_bit(cd.s.bit, pen); 614 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); 615 } else { 616 clear_bit(cd.s.bit, pen); 617 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); 618 } 619 } 620 } else { 621 for_each_online_cpu(cpu) { 622 unsigned long *pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 623 int index = octeon_coreid_for_cpu(cpu) * 2 + 1; 624 if (cpumask_test_cpu(cpu, dest) && enable_one) { 625 enable_one = false; 626 set_bit(cd.s.bit, pen); 627 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); 628 } else { 629 clear_bit(cd.s.bit, pen); 630 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask); 631 } 632 } 633 } 634 return 0; 635 } 636 #endif 637 638 /* 639 * The v1 CIU code already masks things, so supply a dummy version to 640 * the core chip code. 641 */ 642 static void octeon_irq_dummy_mask(struct irq_data *data) 643 { 644 } 645 646 /* 647 * Newer octeon chips have support for lockless CIU operation. 648 */ 649 static struct irq_chip octeon_irq_chip_ciu_v2 = { 650 .name = "CIU", 651 .irq_enable = octeon_irq_ciu_enable_v2, 652 .irq_disable = octeon_irq_ciu_disable_all_v2, 653 .irq_mask = octeon_irq_ciu_disable_local_v2, 654 .irq_unmask = octeon_irq_ciu_enable_v2, 655 #ifdef CONFIG_SMP 656 .irq_set_affinity = octeon_irq_ciu_set_affinity_v2, 657 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 658 #endif 659 }; 660 661 static struct irq_chip octeon_irq_chip_ciu_edge_v2 = { 662 .name = "CIU-E", 663 .irq_enable = octeon_irq_ciu_enable_v2, 664 .irq_disable = octeon_irq_ciu_disable_all_v2, 665 .irq_ack = octeon_irq_ciu_ack, 666 .irq_mask = octeon_irq_ciu_disable_local_v2, 667 .irq_unmask = octeon_irq_ciu_enable_v2, 668 #ifdef CONFIG_SMP 669 .irq_set_affinity = octeon_irq_ciu_set_affinity_v2, 670 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 671 #endif 672 }; 673 674 static struct irq_chip octeon_irq_chip_ciu = { 675 .name = "CIU", 676 .irq_enable = octeon_irq_ciu_enable, 677 .irq_disable = octeon_irq_ciu_disable_all, 678 .irq_mask = octeon_irq_dummy_mask, 679 #ifdef CONFIG_SMP 680 .irq_set_affinity = octeon_irq_ciu_set_affinity, 681 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 682 #endif 683 }; 684 685 static struct irq_chip octeon_irq_chip_ciu_edge = { 686 .name = "CIU-E", 687 .irq_enable = octeon_irq_ciu_enable, 688 .irq_disable = octeon_irq_ciu_disable_all, 689 .irq_mask = octeon_irq_dummy_mask, 690 .irq_ack = octeon_irq_ciu_ack, 691 #ifdef CONFIG_SMP 692 .irq_set_affinity = octeon_irq_ciu_set_affinity, 693 .irq_cpu_offline = octeon_irq_cpu_offline_ciu, 694 #endif 695 }; 696 697 /* The mbox versions don't do any affinity or round-robin. */ 698 static struct irq_chip octeon_irq_chip_ciu_mbox_v2 = { 699 .name = "CIU-M", 700 .irq_enable = octeon_irq_ciu_enable_all_v2, 701 .irq_disable = octeon_irq_ciu_disable_all_v2, 702 .irq_ack = octeon_irq_ciu_disable_local_v2, 703 .irq_eoi = octeon_irq_ciu_enable_local_v2, 704 705 .irq_cpu_online = octeon_irq_ciu_enable_local_v2, 706 .irq_cpu_offline = octeon_irq_ciu_disable_local_v2, 707 .flags = IRQCHIP_ONOFFLINE_ENABLED, 708 }; 709 710 static struct irq_chip octeon_irq_chip_ciu_mbox = { 711 .name = "CIU-M", 712 .irq_enable = octeon_irq_ciu_enable_all, 713 .irq_disable = octeon_irq_ciu_disable_all, 714 715 .irq_cpu_online = octeon_irq_ciu_enable_local, 716 .irq_cpu_offline = octeon_irq_ciu_disable_local, 717 .flags = IRQCHIP_ONOFFLINE_ENABLED, 718 }; 719 720 /* 721 * Watchdog interrupts are special. They are associated with a single 722 * core, so we hardwire the affinity to that core. 723 */ 724 static void octeon_irq_ciu_wd_enable(struct irq_data *data) 725 { 726 unsigned long flags; 727 unsigned long *pen; 728 int coreid = data->irq - OCTEON_IRQ_WDOG0; /* Bit 0-63 of EN1 */ 729 int cpu = octeon_cpu_for_coreid(coreid); 730 731 raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags); 732 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); 733 set_bit(coreid, pen); 734 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); 735 raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags); 736 } 737 738 /* 739 * Watchdog interrupts are special. They are associated with a single 740 * core, so we hardwire the affinity to that core. 741 */ 742 static void octeon_irq_ciu1_wd_enable_v2(struct irq_data *data) 743 { 744 int coreid = data->irq - OCTEON_IRQ_WDOG0; 745 int cpu = octeon_cpu_for_coreid(coreid); 746 747 set_bit(coreid, &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); 748 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(coreid * 2 + 1), 1ull << coreid); 749 } 750 751 752 static struct irq_chip octeon_irq_chip_ciu_wd_v2 = { 753 .name = "CIU-W", 754 .irq_enable = octeon_irq_ciu1_wd_enable_v2, 755 .irq_disable = octeon_irq_ciu_disable_all_v2, 756 .irq_mask = octeon_irq_ciu_disable_local_v2, 757 .irq_unmask = octeon_irq_ciu_enable_local_v2, 758 }; 759 760 static struct irq_chip octeon_irq_chip_ciu_wd = { 761 .name = "CIU-W", 762 .irq_enable = octeon_irq_ciu_wd_enable, 763 .irq_disable = octeon_irq_ciu_disable_all, 764 .irq_mask = octeon_irq_dummy_mask, 765 }; 766 767 static void octeon_irq_ip2_v1(void) 768 { 769 const unsigned long core_id = cvmx_get_core_num(); 770 u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INTX_SUM0(core_id * 2)); 771 772 ciu_sum &= __get_cpu_var(octeon_irq_ciu0_en_mirror); 773 clear_c0_status(STATUSF_IP2); 774 if (likely(ciu_sum)) { 775 int bit = fls64(ciu_sum) - 1; 776 int irq = octeon_irq_ciu_to_irq[0][bit]; 777 if (likely(irq)) 778 do_IRQ(irq); 779 else 780 spurious_interrupt(); 781 } else { 782 spurious_interrupt(); 783 } 784 set_c0_status(STATUSF_IP2); 785 } 786 787 static void octeon_irq_ip2_v2(void) 788 { 789 const unsigned long core_id = cvmx_get_core_num(); 790 u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INTX_SUM0(core_id * 2)); 791 792 ciu_sum &= __get_cpu_var(octeon_irq_ciu0_en_mirror); 793 if (likely(ciu_sum)) { 794 int bit = fls64(ciu_sum) - 1; 795 int irq = octeon_irq_ciu_to_irq[0][bit]; 796 if (likely(irq)) 797 do_IRQ(irq); 798 else 799 spurious_interrupt(); 800 } else { 801 spurious_interrupt(); 802 } 803 } 804 static void octeon_irq_ip3_v1(void) 805 { 806 u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INT_SUM1); 807 808 ciu_sum &= __get_cpu_var(octeon_irq_ciu1_en_mirror); 809 clear_c0_status(STATUSF_IP3); 810 if (likely(ciu_sum)) { 811 int bit = fls64(ciu_sum) - 1; 812 int irq = octeon_irq_ciu_to_irq[1][bit]; 813 if (likely(irq)) 814 do_IRQ(irq); 815 else 816 spurious_interrupt(); 817 } else { 818 spurious_interrupt(); 819 } 820 set_c0_status(STATUSF_IP3); 821 } 822 823 static void octeon_irq_ip3_v2(void) 824 { 825 u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INT_SUM1); 826 827 ciu_sum &= __get_cpu_var(octeon_irq_ciu1_en_mirror); 828 if (likely(ciu_sum)) { 829 int bit = fls64(ciu_sum) - 1; 830 int irq = octeon_irq_ciu_to_irq[1][bit]; 831 if (likely(irq)) 832 do_IRQ(irq); 833 else 834 spurious_interrupt(); 835 } else { 836 spurious_interrupt(); 837 } 838 } 839 840 static void octeon_irq_ip4_mask(void) 841 { 842 clear_c0_status(STATUSF_IP4); 843 spurious_interrupt(); 844 } 845 846 static void (*octeon_irq_ip2)(void); 847 static void (*octeon_irq_ip3)(void); 848 static void (*octeon_irq_ip4)(void); 849 850 void __cpuinitdata (*octeon_irq_setup_secondary)(void); 851 852 static void __cpuinit octeon_irq_percpu_enable(void) 853 { 854 irq_cpu_online(); 855 } 856 857 static void __cpuinit octeon_irq_init_ciu_percpu(void) 858 { 859 int coreid = cvmx_get_core_num(); 860 /* 861 * Disable All CIU Interrupts. The ones we need will be 862 * enabled later. Read the SUM register so we know the write 863 * completed. 864 */ 865 cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2)), 0); 866 cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2 + 1)), 0); 867 cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2)), 0); 868 cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2 + 1)), 0); 869 cvmx_read_csr(CVMX_CIU_INTX_SUM0((coreid * 2))); 870 } 871 872 static void __cpuinit octeon_irq_setup_secondary_ciu(void) 873 { 874 875 __get_cpu_var(octeon_irq_ciu0_en_mirror) = 0; 876 __get_cpu_var(octeon_irq_ciu1_en_mirror) = 0; 877 878 octeon_irq_init_ciu_percpu(); 879 octeon_irq_percpu_enable(); 880 881 /* Enable the CIU lines */ 882 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 883 clear_c0_status(STATUSF_IP4); 884 } 885 886 static void __init octeon_irq_init_ciu(void) 887 { 888 unsigned int i; 889 struct irq_chip *chip; 890 struct irq_chip *chip_edge; 891 struct irq_chip *chip_mbox; 892 struct irq_chip *chip_wd; 893 894 octeon_irq_init_ciu_percpu(); 895 octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu; 896 897 if (OCTEON_IS_MODEL(OCTEON_CN58XX_PASS2_X) || 898 OCTEON_IS_MODEL(OCTEON_CN56XX_PASS2_X) || 899 OCTEON_IS_MODEL(OCTEON_CN52XX_PASS2_X) || 900 OCTEON_IS_MODEL(OCTEON_CN6XXX)) { 901 octeon_irq_ip2 = octeon_irq_ip2_v2; 902 octeon_irq_ip3 = octeon_irq_ip3_v2; 903 chip = &octeon_irq_chip_ciu_v2; 904 chip_edge = &octeon_irq_chip_ciu_edge_v2; 905 chip_mbox = &octeon_irq_chip_ciu_mbox_v2; 906 chip_wd = &octeon_irq_chip_ciu_wd_v2; 907 } else { 908 octeon_irq_ip2 = octeon_irq_ip2_v1; 909 octeon_irq_ip3 = octeon_irq_ip3_v1; 910 chip = &octeon_irq_chip_ciu; 911 chip_edge = &octeon_irq_chip_ciu_edge; 912 chip_mbox = &octeon_irq_chip_ciu_mbox; 913 chip_wd = &octeon_irq_chip_ciu_wd; 914 } 915 octeon_irq_ip4 = octeon_irq_ip4_mask; 916 917 /* Mips internal */ 918 octeon_irq_init_core(); 919 920 /* CIU_0 */ 921 for (i = 0; i < 16; i++) 922 octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_WORKQ0, 0, i + 0, chip, handle_level_irq); 923 for (i = 0; i < 16; i++) 924 octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_GPIO0, 0, i + 16, chip, handle_level_irq); 925 926 octeon_irq_set_ciu_mapping(OCTEON_IRQ_MBOX0, 0, 32, chip_mbox, handle_percpu_irq); 927 octeon_irq_set_ciu_mapping(OCTEON_IRQ_MBOX1, 0, 33, chip_mbox, handle_percpu_irq); 928 929 octeon_irq_set_ciu_mapping(OCTEON_IRQ_UART0, 0, 34, chip, handle_level_irq); 930 octeon_irq_set_ciu_mapping(OCTEON_IRQ_UART1, 0, 35, chip, handle_level_irq); 931 932 for (i = 0; i < 4; i++) 933 octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_PCI_INT0, 0, i + 36, chip, handle_level_irq); 934 for (i = 0; i < 4; i++) 935 octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_PCI_MSI0, 0, i + 40, chip, handle_level_irq); 936 937 octeon_irq_set_ciu_mapping(OCTEON_IRQ_TWSI, 0, 45, chip, handle_level_irq); 938 octeon_irq_set_ciu_mapping(OCTEON_IRQ_RML, 0, 46, chip, handle_level_irq); 939 octeon_irq_set_ciu_mapping(OCTEON_IRQ_TRACE0, 0, 47, chip, handle_level_irq); 940 941 for (i = 0; i < 2; i++) 942 octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_GMX_DRP0, 0, i + 48, chip_edge, handle_edge_irq); 943 944 octeon_irq_set_ciu_mapping(OCTEON_IRQ_IPD_DRP, 0, 50, chip_edge, handle_edge_irq); 945 octeon_irq_set_ciu_mapping(OCTEON_IRQ_KEY_ZERO, 0, 51, chip_edge, handle_edge_irq); 946 947 for (i = 0; i < 4; i++) 948 octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_TIMER0, 0, i + 52, chip_edge, handle_edge_irq); 949 950 octeon_irq_set_ciu_mapping(OCTEON_IRQ_USB0, 0, 56, chip, handle_level_irq); 951 octeon_irq_set_ciu_mapping(OCTEON_IRQ_PCM, 0, 57, chip, handle_level_irq); 952 octeon_irq_set_ciu_mapping(OCTEON_IRQ_MPI, 0, 58, chip, handle_level_irq); 953 octeon_irq_set_ciu_mapping(OCTEON_IRQ_TWSI2, 0, 59, chip, handle_level_irq); 954 octeon_irq_set_ciu_mapping(OCTEON_IRQ_POWIQ, 0, 60, chip, handle_level_irq); 955 octeon_irq_set_ciu_mapping(OCTEON_IRQ_IPDPPTHR, 0, 61, chip, handle_level_irq); 956 octeon_irq_set_ciu_mapping(OCTEON_IRQ_MII0, 0, 62, chip, handle_level_irq); 957 octeon_irq_set_ciu_mapping(OCTEON_IRQ_BOOTDMA, 0, 63, chip, handle_level_irq); 958 959 /* CIU_1 */ 960 for (i = 0; i < 16; i++) 961 octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_WDOG0, 1, i + 0, chip_wd, handle_level_irq); 962 963 octeon_irq_set_ciu_mapping(OCTEON_IRQ_UART2, 1, 16, chip, handle_level_irq); 964 octeon_irq_set_ciu_mapping(OCTEON_IRQ_USB1, 1, 17, chip, handle_level_irq); 965 octeon_irq_set_ciu_mapping(OCTEON_IRQ_MII1, 1, 18, chip, handle_level_irq); 966 octeon_irq_set_ciu_mapping(OCTEON_IRQ_NAND, 1, 19, chip, handle_level_irq); 967 octeon_irq_set_ciu_mapping(OCTEON_IRQ_MIO, 1, 20, chip, handle_level_irq); 968 octeon_irq_set_ciu_mapping(OCTEON_IRQ_IOB, 1, 21, chip, handle_level_irq); 969 octeon_irq_set_ciu_mapping(OCTEON_IRQ_FPA, 1, 22, chip, handle_level_irq); 970 octeon_irq_set_ciu_mapping(OCTEON_IRQ_POW, 1, 23, chip, handle_level_irq); 971 octeon_irq_set_ciu_mapping(OCTEON_IRQ_L2C, 1, 24, chip, handle_level_irq); 972 octeon_irq_set_ciu_mapping(OCTEON_IRQ_IPD, 1, 25, chip, handle_level_irq); 973 octeon_irq_set_ciu_mapping(OCTEON_IRQ_PIP, 1, 26, chip, handle_level_irq); 974 octeon_irq_set_ciu_mapping(OCTEON_IRQ_PKO, 1, 27, chip, handle_level_irq); 975 octeon_irq_set_ciu_mapping(OCTEON_IRQ_ZIP, 1, 28, chip, handle_level_irq); 976 octeon_irq_set_ciu_mapping(OCTEON_IRQ_TIM, 1, 29, chip, handle_level_irq); 977 octeon_irq_set_ciu_mapping(OCTEON_IRQ_RAD, 1, 30, chip, handle_level_irq); 978 octeon_irq_set_ciu_mapping(OCTEON_IRQ_KEY, 1, 31, chip, handle_level_irq); 979 octeon_irq_set_ciu_mapping(OCTEON_IRQ_DFA, 1, 32, chip, handle_level_irq); 980 octeon_irq_set_ciu_mapping(OCTEON_IRQ_USBCTL, 1, 33, chip, handle_level_irq); 981 octeon_irq_set_ciu_mapping(OCTEON_IRQ_SLI, 1, 34, chip, handle_level_irq); 982 octeon_irq_set_ciu_mapping(OCTEON_IRQ_DPI, 1, 35, chip, handle_level_irq); 983 984 octeon_irq_set_ciu_mapping(OCTEON_IRQ_AGX0, 1, 36, chip, handle_level_irq); 985 986 octeon_irq_set_ciu_mapping(OCTEON_IRQ_AGL, 1, 46, chip, handle_level_irq); 987 988 octeon_irq_set_ciu_mapping(OCTEON_IRQ_PTP, 1, 47, chip_edge, handle_edge_irq); 989 990 octeon_irq_set_ciu_mapping(OCTEON_IRQ_PEM0, 1, 48, chip, handle_level_irq); 991 octeon_irq_set_ciu_mapping(OCTEON_IRQ_PEM1, 1, 49, chip, handle_level_irq); 992 octeon_irq_set_ciu_mapping(OCTEON_IRQ_SRIO0, 1, 50, chip, handle_level_irq); 993 octeon_irq_set_ciu_mapping(OCTEON_IRQ_SRIO1, 1, 51, chip, handle_level_irq); 994 octeon_irq_set_ciu_mapping(OCTEON_IRQ_LMC0, 1, 52, chip, handle_level_irq); 995 octeon_irq_set_ciu_mapping(OCTEON_IRQ_DFM, 1, 56, chip, handle_level_irq); 996 octeon_irq_set_ciu_mapping(OCTEON_IRQ_RST, 1, 63, chip, handle_level_irq); 997 998 /* Enable the CIU lines */ 999 set_c0_status(STATUSF_IP3 | STATUSF_IP2); 1000 clear_c0_status(STATUSF_IP4); 1001 } 1002 1003 void __init arch_init_irq(void) 1004 { 1005 #ifdef CONFIG_SMP 1006 /* Set the default affinity to the boot cpu. */ 1007 cpumask_clear(irq_default_affinity); 1008 cpumask_set_cpu(smp_processor_id(), irq_default_affinity); 1009 #endif 1010 octeon_irq_init_ciu(); 1011 } 1012 1013 asmlinkage void plat_irq_dispatch(void) 1014 { 1015 unsigned long cop0_cause; 1016 unsigned long cop0_status; 1017 1018 while (1) { 1019 cop0_cause = read_c0_cause(); 1020 cop0_status = read_c0_status(); 1021 cop0_cause &= cop0_status; 1022 cop0_cause &= ST0_IM; 1023 1024 if (unlikely(cop0_cause & STATUSF_IP2)) 1025 octeon_irq_ip2(); 1026 else if (unlikely(cop0_cause & STATUSF_IP3)) 1027 octeon_irq_ip3(); 1028 else if (unlikely(cop0_cause & STATUSF_IP4)) 1029 octeon_irq_ip4(); 1030 else if (likely(cop0_cause)) 1031 do_IRQ(fls(cop0_cause) - 9 + MIPS_CPU_IRQ_BASE); 1032 else 1033 break; 1034 } 1035 } 1036 1037 #ifdef CONFIG_HOTPLUG_CPU 1038 1039 void fixup_irqs(void) 1040 { 1041 irq_cpu_offline(); 1042 } 1043 1044 #endif /* CONFIG_HOTPLUG_CPU */ 1045