1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * linux/arch/ia64/kernel/irq_ia64.c 4 * 5 * Copyright (C) 1998-2001 Hewlett-Packard Co 6 * Stephane Eranian <eranian@hpl.hp.com> 7 * David Mosberger-Tang <davidm@hpl.hp.com> 8 * 9 * 6/10/99: Updated to bring in sync with x86 version to facilitate 10 * support for SMP and different interrupt controllers. 11 * 12 * 09/15/00 Goutham Rao <goutham.rao@intel.com> Implemented pci_irq_to_vector 13 * PCI to vector allocation routine. 14 * 04/14/2004 Ashok Raj <ashok.raj@intel.com> 15 * Added CPU Hotplug handling for IPF. 16 */ 17 18 #include <linux/module.h> 19 20 #include <linux/jiffies.h> 21 #include <linux/errno.h> 22 #include <linux/init.h> 23 #include <linux/interrupt.h> 24 #include <linux/ioport.h> 25 #include <linux/kernel_stat.h> 26 #include <linux/ptrace.h> 27 #include <linux/signal.h> 28 #include <linux/smp.h> 29 #include <linux/threads.h> 30 #include <linux/bitops.h> 31 #include <linux/irq.h> 32 #include <linux/ratelimit.h> 33 #include <linux/acpi.h> 34 #include <linux/sched.h> 35 36 #include <asm/delay.h> 37 #include <asm/intrinsics.h> 38 #include <asm/io.h> 39 #include <asm/hw_irq.h> 40 #include <asm/pgtable.h> 41 #include <asm/tlbflush.h> 42 43 #ifdef CONFIG_PERFMON 44 # include <asm/perfmon.h> 45 #endif 46 47 #define IRQ_DEBUG 0 48 49 #define IRQ_VECTOR_UNASSIGNED (0) 50 51 #define IRQ_UNUSED (0) 52 #define IRQ_USED (1) 53 #define IRQ_RSVD (2) 54 55 int ia64_first_device_vector = IA64_DEF_FIRST_DEVICE_VECTOR; 56 int ia64_last_device_vector = IA64_DEF_LAST_DEVICE_VECTOR; 57 58 /* default base addr of IPI table */ 59 void __iomem *ipi_base_addr = ((void __iomem *) 60 (__IA64_UNCACHED_OFFSET | IA64_IPI_DEFAULT_BASE_ADDR)); 61 62 static cpumask_t vector_allocation_domain(int cpu); 63 64 /* 65 * Legacy IRQ to IA-64 vector translation table. 66 */ 67 __u8 isa_irq_to_vector_map[16] = { 68 /* 8259 IRQ translation, first 16 entries */ 69 0x2f, 0x20, 0x2e, 0x2d, 0x2c, 0x2b, 0x2a, 0x29, 70 0x28, 0x27, 0x26, 0x25, 0x24, 0x23, 0x22, 0x21 71 }; 72 EXPORT_SYMBOL(isa_irq_to_vector_map); 73 74 DEFINE_SPINLOCK(vector_lock); 75 76 struct irq_cfg irq_cfg[NR_IRQS] __read_mostly = { 77 [0 ... NR_IRQS - 1] = { 78 .vector = IRQ_VECTOR_UNASSIGNED, 79 .domain = CPU_MASK_NONE 80 } 81 }; 82 83 DEFINE_PER_CPU(int[IA64_NUM_VECTORS], vector_irq) = { 84 [0 ... IA64_NUM_VECTORS - 1] = -1 85 }; 86 87 static cpumask_t vector_table[IA64_NUM_VECTORS] = { 88 [0 ... IA64_NUM_VECTORS - 1] = CPU_MASK_NONE 89 }; 90 91 static int irq_status[NR_IRQS] = { 92 [0 ... NR_IRQS -1] = IRQ_UNUSED 93 }; 94 95 static inline int find_unassigned_irq(void) 96 { 97 int irq; 98 99 for (irq = IA64_FIRST_DEVICE_VECTOR; irq < NR_IRQS; irq++) 100 if (irq_status[irq] == IRQ_UNUSED) 101 return irq; 102 return -ENOSPC; 103 } 104 105 static inline int find_unassigned_vector(cpumask_t domain) 106 { 107 cpumask_t mask; 108 int pos, vector; 109 110 cpumask_and(&mask, &domain, cpu_online_mask); 111 if (cpumask_empty(&mask)) 112 return -EINVAL; 113 114 for (pos = 0; pos < IA64_NUM_DEVICE_VECTORS; pos++) { 115 vector = IA64_FIRST_DEVICE_VECTOR + pos; 116 cpumask_and(&mask, &domain, &vector_table[vector]); 117 if (!cpumask_empty(&mask)) 118 continue; 119 return vector; 120 } 121 return -ENOSPC; 122 } 123 124 static int __bind_irq_vector(int irq, int vector, cpumask_t domain) 125 { 126 cpumask_t mask; 127 int cpu; 128 struct irq_cfg *cfg = &irq_cfg[irq]; 129 130 BUG_ON((unsigned)irq >= NR_IRQS); 131 BUG_ON((unsigned)vector >= IA64_NUM_VECTORS); 132 133 cpumask_and(&mask, &domain, cpu_online_mask); 134 if (cpumask_empty(&mask)) 135 return -EINVAL; 136 if ((cfg->vector == vector) && cpumask_equal(&cfg->domain, &domain)) 137 return 0; 138 if (cfg->vector != IRQ_VECTOR_UNASSIGNED) 139 return -EBUSY; 140 for_each_cpu(cpu, &mask) 141 per_cpu(vector_irq, cpu)[vector] = irq; 142 cfg->vector = vector; 143 cfg->domain = domain; 144 irq_status[irq] = IRQ_USED; 145 cpumask_or(&vector_table[vector], &vector_table[vector], &domain); 146 return 0; 147 } 148 149 int bind_irq_vector(int irq, int vector, cpumask_t domain) 150 { 151 unsigned long flags; 152 int ret; 153 154 spin_lock_irqsave(&vector_lock, flags); 155 ret = __bind_irq_vector(irq, vector, domain); 156 spin_unlock_irqrestore(&vector_lock, flags); 157 return ret; 158 } 159 160 static void __clear_irq_vector(int irq) 161 { 162 int vector, cpu; 163 cpumask_t domain; 164 struct irq_cfg *cfg = &irq_cfg[irq]; 165 166 BUG_ON((unsigned)irq >= NR_IRQS); 167 BUG_ON(cfg->vector == IRQ_VECTOR_UNASSIGNED); 168 vector = cfg->vector; 169 domain = cfg->domain; 170 for_each_cpu_and(cpu, &cfg->domain, cpu_online_mask) 171 per_cpu(vector_irq, cpu)[vector] = -1; 172 cfg->vector = IRQ_VECTOR_UNASSIGNED; 173 cfg->domain = CPU_MASK_NONE; 174 irq_status[irq] = IRQ_UNUSED; 175 cpumask_andnot(&vector_table[vector], &vector_table[vector], &domain); 176 } 177 178 static void clear_irq_vector(int irq) 179 { 180 unsigned long flags; 181 182 spin_lock_irqsave(&vector_lock, flags); 183 __clear_irq_vector(irq); 184 spin_unlock_irqrestore(&vector_lock, flags); 185 } 186 187 int 188 ia64_native_assign_irq_vector (int irq) 189 { 190 unsigned long flags; 191 int vector, cpu; 192 cpumask_t domain = CPU_MASK_NONE; 193 194 vector = -ENOSPC; 195 196 spin_lock_irqsave(&vector_lock, flags); 197 for_each_online_cpu(cpu) { 198 domain = vector_allocation_domain(cpu); 199 vector = find_unassigned_vector(domain); 200 if (vector >= 0) 201 break; 202 } 203 if (vector < 0) 204 goto out; 205 if (irq == AUTO_ASSIGN) 206 irq = vector; 207 BUG_ON(__bind_irq_vector(irq, vector, domain)); 208 out: 209 spin_unlock_irqrestore(&vector_lock, flags); 210 return vector; 211 } 212 213 void 214 ia64_native_free_irq_vector (int vector) 215 { 216 if (vector < IA64_FIRST_DEVICE_VECTOR || 217 vector > IA64_LAST_DEVICE_VECTOR) 218 return; 219 clear_irq_vector(vector); 220 } 221 222 int 223 reserve_irq_vector (int vector) 224 { 225 if (vector < IA64_FIRST_DEVICE_VECTOR || 226 vector > IA64_LAST_DEVICE_VECTOR) 227 return -EINVAL; 228 return !!bind_irq_vector(vector, vector, CPU_MASK_ALL); 229 } 230 231 /* 232 * Initialize vector_irq on a new cpu. This function must be called 233 * with vector_lock held. 234 */ 235 void __setup_vector_irq(int cpu) 236 { 237 int irq, vector; 238 239 /* Clear vector_irq */ 240 for (vector = 0; vector < IA64_NUM_VECTORS; ++vector) 241 per_cpu(vector_irq, cpu)[vector] = -1; 242 /* Mark the inuse vectors */ 243 for (irq = 0; irq < NR_IRQS; ++irq) { 244 if (!cpumask_test_cpu(cpu, &irq_cfg[irq].domain)) 245 continue; 246 vector = irq_to_vector(irq); 247 per_cpu(vector_irq, cpu)[vector] = irq; 248 } 249 } 250 251 #ifdef CONFIG_SMP 252 253 static enum vector_domain_type { 254 VECTOR_DOMAIN_NONE, 255 VECTOR_DOMAIN_PERCPU 256 } vector_domain_type = VECTOR_DOMAIN_NONE; 257 258 static cpumask_t vector_allocation_domain(int cpu) 259 { 260 if (vector_domain_type == VECTOR_DOMAIN_PERCPU) 261 return *cpumask_of(cpu); 262 return CPU_MASK_ALL; 263 } 264 265 static int __irq_prepare_move(int irq, int cpu) 266 { 267 struct irq_cfg *cfg = &irq_cfg[irq]; 268 int vector; 269 cpumask_t domain; 270 271 if (cfg->move_in_progress || cfg->move_cleanup_count) 272 return -EBUSY; 273 if (cfg->vector == IRQ_VECTOR_UNASSIGNED || !cpu_online(cpu)) 274 return -EINVAL; 275 if (cpumask_test_cpu(cpu, &cfg->domain)) 276 return 0; 277 domain = vector_allocation_domain(cpu); 278 vector = find_unassigned_vector(domain); 279 if (vector < 0) 280 return -ENOSPC; 281 cfg->move_in_progress = 1; 282 cfg->old_domain = cfg->domain; 283 cfg->vector = IRQ_VECTOR_UNASSIGNED; 284 cfg->domain = CPU_MASK_NONE; 285 BUG_ON(__bind_irq_vector(irq, vector, domain)); 286 return 0; 287 } 288 289 int irq_prepare_move(int irq, int cpu) 290 { 291 unsigned long flags; 292 int ret; 293 294 spin_lock_irqsave(&vector_lock, flags); 295 ret = __irq_prepare_move(irq, cpu); 296 spin_unlock_irqrestore(&vector_lock, flags); 297 return ret; 298 } 299 300 void irq_complete_move(unsigned irq) 301 { 302 struct irq_cfg *cfg = &irq_cfg[irq]; 303 cpumask_t cleanup_mask; 304 int i; 305 306 if (likely(!cfg->move_in_progress)) 307 return; 308 309 if (unlikely(cpumask_test_cpu(smp_processor_id(), &cfg->old_domain))) 310 return; 311 312 cpumask_and(&cleanup_mask, &cfg->old_domain, cpu_online_mask); 313 cfg->move_cleanup_count = cpumask_weight(&cleanup_mask); 314 for_each_cpu(i, &cleanup_mask) 315 ia64_send_ipi(i, IA64_IRQ_MOVE_VECTOR, IA64_IPI_DM_INT, 0); 316 cfg->move_in_progress = 0; 317 } 318 319 static irqreturn_t smp_irq_move_cleanup_interrupt(int irq, void *dev_id) 320 { 321 int me = smp_processor_id(); 322 ia64_vector vector; 323 unsigned long flags; 324 325 for (vector = IA64_FIRST_DEVICE_VECTOR; 326 vector < IA64_LAST_DEVICE_VECTOR; vector++) { 327 int irq; 328 struct irq_desc *desc; 329 struct irq_cfg *cfg; 330 irq = __this_cpu_read(vector_irq[vector]); 331 if (irq < 0) 332 continue; 333 334 desc = irq_to_desc(irq); 335 cfg = irq_cfg + irq; 336 raw_spin_lock(&desc->lock); 337 if (!cfg->move_cleanup_count) 338 goto unlock; 339 340 if (!cpumask_test_cpu(me, &cfg->old_domain)) 341 goto unlock; 342 343 spin_lock_irqsave(&vector_lock, flags); 344 __this_cpu_write(vector_irq[vector], -1); 345 cpumask_clear_cpu(me, &vector_table[vector]); 346 spin_unlock_irqrestore(&vector_lock, flags); 347 cfg->move_cleanup_count--; 348 unlock: 349 raw_spin_unlock(&desc->lock); 350 } 351 return IRQ_HANDLED; 352 } 353 354 static struct irqaction irq_move_irqaction = { 355 .handler = smp_irq_move_cleanup_interrupt, 356 .name = "irq_move" 357 }; 358 359 static int __init parse_vector_domain(char *arg) 360 { 361 if (!arg) 362 return -EINVAL; 363 if (!strcmp(arg, "percpu")) { 364 vector_domain_type = VECTOR_DOMAIN_PERCPU; 365 no_int_routing = 1; 366 } 367 return 0; 368 } 369 early_param("vector", parse_vector_domain); 370 #else 371 static cpumask_t vector_allocation_domain(int cpu) 372 { 373 return CPU_MASK_ALL; 374 } 375 #endif 376 377 378 void destroy_and_reserve_irq(unsigned int irq) 379 { 380 unsigned long flags; 381 382 irq_init_desc(irq); 383 spin_lock_irqsave(&vector_lock, flags); 384 __clear_irq_vector(irq); 385 irq_status[irq] = IRQ_RSVD; 386 spin_unlock_irqrestore(&vector_lock, flags); 387 } 388 389 /* 390 * Dynamic irq allocate and deallocation for MSI 391 */ 392 int create_irq(void) 393 { 394 unsigned long flags; 395 int irq, vector, cpu; 396 cpumask_t domain = CPU_MASK_NONE; 397 398 irq = vector = -ENOSPC; 399 spin_lock_irqsave(&vector_lock, flags); 400 for_each_online_cpu(cpu) { 401 domain = vector_allocation_domain(cpu); 402 vector = find_unassigned_vector(domain); 403 if (vector >= 0) 404 break; 405 } 406 if (vector < 0) 407 goto out; 408 irq = find_unassigned_irq(); 409 if (irq < 0) 410 goto out; 411 BUG_ON(__bind_irq_vector(irq, vector, domain)); 412 out: 413 spin_unlock_irqrestore(&vector_lock, flags); 414 if (irq >= 0) 415 irq_init_desc(irq); 416 return irq; 417 } 418 419 void destroy_irq(unsigned int irq) 420 { 421 irq_init_desc(irq); 422 clear_irq_vector(irq); 423 } 424 425 #ifdef CONFIG_SMP 426 # define IS_RESCHEDULE(vec) (vec == IA64_IPI_RESCHEDULE) 427 # define IS_LOCAL_TLB_FLUSH(vec) (vec == IA64_IPI_LOCAL_TLB_FLUSH) 428 #else 429 # define IS_RESCHEDULE(vec) (0) 430 # define IS_LOCAL_TLB_FLUSH(vec) (0) 431 #endif 432 /* 433 * That's where the IVT branches when we get an external 434 * interrupt. This branches to the correct hardware IRQ handler via 435 * function ptr. 436 */ 437 void 438 ia64_handle_irq (ia64_vector vector, struct pt_regs *regs) 439 { 440 struct pt_regs *old_regs = set_irq_regs(regs); 441 unsigned long saved_tpr; 442 443 #if IRQ_DEBUG 444 { 445 unsigned long bsp, sp; 446 447 /* 448 * Note: if the interrupt happened while executing in 449 * the context switch routine (ia64_switch_to), we may 450 * get a spurious stack overflow here. This is 451 * because the register and the memory stack are not 452 * switched atomically. 453 */ 454 bsp = ia64_getreg(_IA64_REG_AR_BSP); 455 sp = ia64_getreg(_IA64_REG_SP); 456 457 if ((sp - bsp) < 1024) { 458 static DEFINE_RATELIMIT_STATE(ratelimit, 5 * HZ, 5); 459 460 if (__ratelimit(&ratelimit)) { 461 printk("ia64_handle_irq: DANGER: less than " 462 "1KB of free stack space!!\n" 463 "(bsp=0x%lx, sp=%lx)\n", bsp, sp); 464 } 465 } 466 } 467 #endif /* IRQ_DEBUG */ 468 469 /* 470 * Always set TPR to limit maximum interrupt nesting depth to 471 * 16 (without this, it would be ~240, which could easily lead 472 * to kernel stack overflows). 473 */ 474 irq_enter(); 475 saved_tpr = ia64_getreg(_IA64_REG_CR_TPR); 476 ia64_srlz_d(); 477 while (vector != IA64_SPURIOUS_INT_VECTOR) { 478 int irq = local_vector_to_irq(vector); 479 480 if (unlikely(IS_LOCAL_TLB_FLUSH(vector))) { 481 smp_local_flush_tlb(); 482 kstat_incr_irq_this_cpu(irq); 483 } else if (unlikely(IS_RESCHEDULE(vector))) { 484 scheduler_ipi(); 485 kstat_incr_irq_this_cpu(irq); 486 } else { 487 ia64_setreg(_IA64_REG_CR_TPR, vector); 488 ia64_srlz_d(); 489 490 if (unlikely(irq < 0)) { 491 printk(KERN_ERR "%s: Unexpected interrupt " 492 "vector %d on CPU %d is not mapped " 493 "to any IRQ!\n", __func__, vector, 494 smp_processor_id()); 495 } else 496 generic_handle_irq(irq); 497 498 /* 499 * Disable interrupts and send EOI: 500 */ 501 local_irq_disable(); 502 ia64_setreg(_IA64_REG_CR_TPR, saved_tpr); 503 } 504 ia64_eoi(); 505 vector = ia64_get_ivr(); 506 } 507 /* 508 * This must be done *after* the ia64_eoi(). For example, the keyboard softirq 509 * handler needs to be able to wait for further keyboard interrupts, which can't 510 * come through until ia64_eoi() has been done. 511 */ 512 irq_exit(); 513 set_irq_regs(old_regs); 514 } 515 516 #ifdef CONFIG_HOTPLUG_CPU 517 /* 518 * This function emulates a interrupt processing when a cpu is about to be 519 * brought down. 520 */ 521 void ia64_process_pending_intr(void) 522 { 523 ia64_vector vector; 524 unsigned long saved_tpr; 525 extern unsigned int vectors_in_migration[NR_IRQS]; 526 527 vector = ia64_get_ivr(); 528 529 irq_enter(); 530 saved_tpr = ia64_getreg(_IA64_REG_CR_TPR); 531 ia64_srlz_d(); 532 533 /* 534 * Perform normal interrupt style processing 535 */ 536 while (vector != IA64_SPURIOUS_INT_VECTOR) { 537 int irq = local_vector_to_irq(vector); 538 539 if (unlikely(IS_LOCAL_TLB_FLUSH(vector))) { 540 smp_local_flush_tlb(); 541 kstat_incr_irq_this_cpu(irq); 542 } else if (unlikely(IS_RESCHEDULE(vector))) { 543 kstat_incr_irq_this_cpu(irq); 544 } else { 545 struct pt_regs *old_regs = set_irq_regs(NULL); 546 547 ia64_setreg(_IA64_REG_CR_TPR, vector); 548 ia64_srlz_d(); 549 550 /* 551 * Now try calling normal ia64_handle_irq as it would have got called 552 * from a real intr handler. Try passing null for pt_regs, hopefully 553 * it will work. I hope it works!. 554 * Probably could shared code. 555 */ 556 if (unlikely(irq < 0)) { 557 printk(KERN_ERR "%s: Unexpected interrupt " 558 "vector %d on CPU %d not being mapped " 559 "to any IRQ!!\n", __func__, vector, 560 smp_processor_id()); 561 } else { 562 vectors_in_migration[irq]=0; 563 generic_handle_irq(irq); 564 } 565 set_irq_regs(old_regs); 566 567 /* 568 * Disable interrupts and send EOI 569 */ 570 local_irq_disable(); 571 ia64_setreg(_IA64_REG_CR_TPR, saved_tpr); 572 } 573 ia64_eoi(); 574 vector = ia64_get_ivr(); 575 } 576 irq_exit(); 577 } 578 #endif 579 580 581 #ifdef CONFIG_SMP 582 583 static irqreturn_t dummy_handler (int irq, void *dev_id) 584 { 585 BUG(); 586 } 587 588 static struct irqaction ipi_irqaction = { 589 .handler = handle_IPI, 590 .name = "IPI" 591 }; 592 593 /* 594 * KVM uses this interrupt to force a cpu out of guest mode 595 */ 596 static struct irqaction resched_irqaction = { 597 .handler = dummy_handler, 598 .name = "resched" 599 }; 600 601 static struct irqaction tlb_irqaction = { 602 .handler = dummy_handler, 603 .name = "tlb_flush" 604 }; 605 606 #endif 607 608 void 609 ia64_native_register_percpu_irq (ia64_vector vec, struct irqaction *action) 610 { 611 unsigned int irq; 612 613 irq = vec; 614 BUG_ON(bind_irq_vector(irq, vec, CPU_MASK_ALL)); 615 irq_set_status_flags(irq, IRQ_PER_CPU); 616 irq_set_chip(irq, &irq_type_ia64_lsapic); 617 if (action) 618 setup_irq(irq, action); 619 irq_set_handler(irq, handle_percpu_irq); 620 } 621 622 void __init 623 ia64_native_register_ipi(void) 624 { 625 #ifdef CONFIG_SMP 626 register_percpu_irq(IA64_IPI_VECTOR, &ipi_irqaction); 627 register_percpu_irq(IA64_IPI_RESCHEDULE, &resched_irqaction); 628 register_percpu_irq(IA64_IPI_LOCAL_TLB_FLUSH, &tlb_irqaction); 629 #endif 630 } 631 632 void __init 633 init_IRQ (void) 634 { 635 acpi_boot_init(); 636 ia64_register_ipi(); 637 register_percpu_irq(IA64_SPURIOUS_INT_VECTOR, NULL); 638 #ifdef CONFIG_SMP 639 if (vector_domain_type != VECTOR_DOMAIN_NONE) 640 register_percpu_irq(IA64_IRQ_MOVE_VECTOR, &irq_move_irqaction); 641 #endif 642 #ifdef CONFIG_PERFMON 643 pfm_init_percpu(); 644 #endif 645 } 646 647 void 648 ia64_send_ipi (int cpu, int vector, int delivery_mode, int redirect) 649 { 650 void __iomem *ipi_addr; 651 unsigned long ipi_data; 652 unsigned long phys_cpu_id; 653 654 phys_cpu_id = cpu_physical_id(cpu); 655 656 /* 657 * cpu number is in 8bit ID and 8bit EID 658 */ 659 660 ipi_data = (delivery_mode << 8) | (vector & 0xff); 661 ipi_addr = ipi_base_addr + ((phys_cpu_id << 4) | ((redirect & 1) << 3)); 662 663 writeq(ipi_data, ipi_addr); 664 } 665