1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * linux/arch/ia64/kernel/irq_ia64.c 4 * 5 * Copyright (C) 1998-2001 Hewlett-Packard Co 6 * Stephane Eranian <eranian@hpl.hp.com> 7 * David Mosberger-Tang <davidm@hpl.hp.com> 8 * 9 * 6/10/99: Updated to bring in sync with x86 version to facilitate 10 * support for SMP and different interrupt controllers. 11 * 12 * 09/15/00 Goutham Rao <goutham.rao@intel.com> Implemented pci_irq_to_vector 13 * PCI to vector allocation routine. 14 * 04/14/2004 Ashok Raj <ashok.raj@intel.com> 15 * Added CPU Hotplug handling for IPF. 16 */ 17 18 #include <linux/module.h> 19 20 #include <linux/jiffies.h> 21 #include <linux/errno.h> 22 #include <linux/init.h> 23 #include <linux/interrupt.h> 24 #include <linux/ioport.h> 25 #include <linux/kernel_stat.h> 26 #include <linux/ptrace.h> 27 #include <linux/signal.h> 28 #include <linux/smp.h> 29 #include <linux/threads.h> 30 #include <linux/bitops.h> 31 #include <linux/irq.h> 32 #include <linux/ratelimit.h> 33 #include <linux/acpi.h> 34 #include <linux/sched.h> 35 36 #include <asm/delay.h> 37 #include <asm/intrinsics.h> 38 #include <asm/io.h> 39 #include <asm/hw_irq.h> 40 #include <asm/machvec.h> 41 #include <asm/pgtable.h> 42 #include <asm/tlbflush.h> 43 44 #ifdef CONFIG_PERFMON 45 # include <asm/perfmon.h> 46 #endif 47 48 #define IRQ_DEBUG 0 49 50 #define IRQ_VECTOR_UNASSIGNED (0) 51 52 #define IRQ_UNUSED (0) 53 #define IRQ_USED (1) 54 #define IRQ_RSVD (2) 55 56 /* These can be overridden in platform_irq_init */ 57 int ia64_first_device_vector = IA64_DEF_FIRST_DEVICE_VECTOR; 58 int ia64_last_device_vector = IA64_DEF_LAST_DEVICE_VECTOR; 59 60 /* default base addr of IPI table */ 61 void __iomem *ipi_base_addr = ((void __iomem *) 62 (__IA64_UNCACHED_OFFSET | IA64_IPI_DEFAULT_BASE_ADDR)); 63 64 static cpumask_t vector_allocation_domain(int cpu); 65 66 /* 67 * Legacy IRQ to IA-64 vector translation table. 68 */ 69 __u8 isa_irq_to_vector_map[16] = { 70 /* 8259 IRQ translation, first 16 entries */ 71 0x2f, 0x20, 0x2e, 0x2d, 0x2c, 0x2b, 0x2a, 0x29, 72 0x28, 0x27, 0x26, 0x25, 0x24, 0x23, 0x22, 0x21 73 }; 74 EXPORT_SYMBOL(isa_irq_to_vector_map); 75 76 DEFINE_SPINLOCK(vector_lock); 77 78 struct irq_cfg irq_cfg[NR_IRQS] __read_mostly = { 79 [0 ... NR_IRQS - 1] = { 80 .vector = IRQ_VECTOR_UNASSIGNED, 81 .domain = CPU_MASK_NONE 82 } 83 }; 84 85 DEFINE_PER_CPU(int[IA64_NUM_VECTORS], vector_irq) = { 86 [0 ... IA64_NUM_VECTORS - 1] = -1 87 }; 88 89 static cpumask_t vector_table[IA64_NUM_VECTORS] = { 90 [0 ... IA64_NUM_VECTORS - 1] = CPU_MASK_NONE 91 }; 92 93 static int irq_status[NR_IRQS] = { 94 [0 ... NR_IRQS -1] = IRQ_UNUSED 95 }; 96 97 static inline int find_unassigned_irq(void) 98 { 99 int irq; 100 101 for (irq = IA64_FIRST_DEVICE_VECTOR; irq < NR_IRQS; irq++) 102 if (irq_status[irq] == IRQ_UNUSED) 103 return irq; 104 return -ENOSPC; 105 } 106 107 static inline int find_unassigned_vector(cpumask_t domain) 108 { 109 cpumask_t mask; 110 int pos, vector; 111 112 cpumask_and(&mask, &domain, cpu_online_mask); 113 if (cpumask_empty(&mask)) 114 return -EINVAL; 115 116 for (pos = 0; pos < IA64_NUM_DEVICE_VECTORS; pos++) { 117 vector = IA64_FIRST_DEVICE_VECTOR + pos; 118 cpumask_and(&mask, &domain, &vector_table[vector]); 119 if (!cpumask_empty(&mask)) 120 continue; 121 return vector; 122 } 123 return -ENOSPC; 124 } 125 126 static int __bind_irq_vector(int irq, int vector, cpumask_t domain) 127 { 128 cpumask_t mask; 129 int cpu; 130 struct irq_cfg *cfg = &irq_cfg[irq]; 131 132 BUG_ON((unsigned)irq >= NR_IRQS); 133 BUG_ON((unsigned)vector >= IA64_NUM_VECTORS); 134 135 cpumask_and(&mask, &domain, cpu_online_mask); 136 if (cpumask_empty(&mask)) 137 return -EINVAL; 138 if ((cfg->vector == vector) && cpumask_equal(&cfg->domain, &domain)) 139 return 0; 140 if (cfg->vector != IRQ_VECTOR_UNASSIGNED) 141 return -EBUSY; 142 for_each_cpu(cpu, &mask) 143 per_cpu(vector_irq, cpu)[vector] = irq; 144 cfg->vector = vector; 145 cfg->domain = domain; 146 irq_status[irq] = IRQ_USED; 147 cpumask_or(&vector_table[vector], &vector_table[vector], &domain); 148 return 0; 149 } 150 151 int bind_irq_vector(int irq, int vector, cpumask_t domain) 152 { 153 unsigned long flags; 154 int ret; 155 156 spin_lock_irqsave(&vector_lock, flags); 157 ret = __bind_irq_vector(irq, vector, domain); 158 spin_unlock_irqrestore(&vector_lock, flags); 159 return ret; 160 } 161 162 static void __clear_irq_vector(int irq) 163 { 164 int vector, cpu; 165 cpumask_t domain; 166 struct irq_cfg *cfg = &irq_cfg[irq]; 167 168 BUG_ON((unsigned)irq >= NR_IRQS); 169 BUG_ON(cfg->vector == IRQ_VECTOR_UNASSIGNED); 170 vector = cfg->vector; 171 domain = cfg->domain; 172 for_each_cpu_and(cpu, &cfg->domain, cpu_online_mask) 173 per_cpu(vector_irq, cpu)[vector] = -1; 174 cfg->vector = IRQ_VECTOR_UNASSIGNED; 175 cfg->domain = CPU_MASK_NONE; 176 irq_status[irq] = IRQ_UNUSED; 177 cpumask_andnot(&vector_table[vector], &vector_table[vector], &domain); 178 } 179 180 static void clear_irq_vector(int irq) 181 { 182 unsigned long flags; 183 184 spin_lock_irqsave(&vector_lock, flags); 185 __clear_irq_vector(irq); 186 spin_unlock_irqrestore(&vector_lock, flags); 187 } 188 189 int 190 ia64_native_assign_irq_vector (int irq) 191 { 192 unsigned long flags; 193 int vector, cpu; 194 cpumask_t domain = CPU_MASK_NONE; 195 196 vector = -ENOSPC; 197 198 spin_lock_irqsave(&vector_lock, flags); 199 for_each_online_cpu(cpu) { 200 domain = vector_allocation_domain(cpu); 201 vector = find_unassigned_vector(domain); 202 if (vector >= 0) 203 break; 204 } 205 if (vector < 0) 206 goto out; 207 if (irq == AUTO_ASSIGN) 208 irq = vector; 209 BUG_ON(__bind_irq_vector(irq, vector, domain)); 210 out: 211 spin_unlock_irqrestore(&vector_lock, flags); 212 return vector; 213 } 214 215 void 216 ia64_native_free_irq_vector (int vector) 217 { 218 if (vector < IA64_FIRST_DEVICE_VECTOR || 219 vector > IA64_LAST_DEVICE_VECTOR) 220 return; 221 clear_irq_vector(vector); 222 } 223 224 int 225 reserve_irq_vector (int vector) 226 { 227 if (vector < IA64_FIRST_DEVICE_VECTOR || 228 vector > IA64_LAST_DEVICE_VECTOR) 229 return -EINVAL; 230 return !!bind_irq_vector(vector, vector, CPU_MASK_ALL); 231 } 232 233 /* 234 * Initialize vector_irq on a new cpu. This function must be called 235 * with vector_lock held. 236 */ 237 void __setup_vector_irq(int cpu) 238 { 239 int irq, vector; 240 241 /* Clear vector_irq */ 242 for (vector = 0; vector < IA64_NUM_VECTORS; ++vector) 243 per_cpu(vector_irq, cpu)[vector] = -1; 244 /* Mark the inuse vectors */ 245 for (irq = 0; irq < NR_IRQS; ++irq) { 246 if (!cpumask_test_cpu(cpu, &irq_cfg[irq].domain)) 247 continue; 248 vector = irq_to_vector(irq); 249 per_cpu(vector_irq, cpu)[vector] = irq; 250 } 251 } 252 253 #if defined(CONFIG_SMP) && (defined(CONFIG_IA64_GENERIC) || defined(CONFIG_IA64_DIG)) 254 255 static enum vector_domain_type { 256 VECTOR_DOMAIN_NONE, 257 VECTOR_DOMAIN_PERCPU 258 } vector_domain_type = VECTOR_DOMAIN_NONE; 259 260 static cpumask_t vector_allocation_domain(int cpu) 261 { 262 if (vector_domain_type == VECTOR_DOMAIN_PERCPU) 263 return *cpumask_of(cpu); 264 return CPU_MASK_ALL; 265 } 266 267 static int __irq_prepare_move(int irq, int cpu) 268 { 269 struct irq_cfg *cfg = &irq_cfg[irq]; 270 int vector; 271 cpumask_t domain; 272 273 if (cfg->move_in_progress || cfg->move_cleanup_count) 274 return -EBUSY; 275 if (cfg->vector == IRQ_VECTOR_UNASSIGNED || !cpu_online(cpu)) 276 return -EINVAL; 277 if (cpumask_test_cpu(cpu, &cfg->domain)) 278 return 0; 279 domain = vector_allocation_domain(cpu); 280 vector = find_unassigned_vector(domain); 281 if (vector < 0) 282 return -ENOSPC; 283 cfg->move_in_progress = 1; 284 cfg->old_domain = cfg->domain; 285 cfg->vector = IRQ_VECTOR_UNASSIGNED; 286 cfg->domain = CPU_MASK_NONE; 287 BUG_ON(__bind_irq_vector(irq, vector, domain)); 288 return 0; 289 } 290 291 int irq_prepare_move(int irq, int cpu) 292 { 293 unsigned long flags; 294 int ret; 295 296 spin_lock_irqsave(&vector_lock, flags); 297 ret = __irq_prepare_move(irq, cpu); 298 spin_unlock_irqrestore(&vector_lock, flags); 299 return ret; 300 } 301 302 void irq_complete_move(unsigned irq) 303 { 304 struct irq_cfg *cfg = &irq_cfg[irq]; 305 cpumask_t cleanup_mask; 306 int i; 307 308 if (likely(!cfg->move_in_progress)) 309 return; 310 311 if (unlikely(cpumask_test_cpu(smp_processor_id(), &cfg->old_domain))) 312 return; 313 314 cpumask_and(&cleanup_mask, &cfg->old_domain, cpu_online_mask); 315 cfg->move_cleanup_count = cpumask_weight(&cleanup_mask); 316 for_each_cpu(i, &cleanup_mask) 317 platform_send_ipi(i, IA64_IRQ_MOVE_VECTOR, IA64_IPI_DM_INT, 0); 318 cfg->move_in_progress = 0; 319 } 320 321 static irqreturn_t smp_irq_move_cleanup_interrupt(int irq, void *dev_id) 322 { 323 int me = smp_processor_id(); 324 ia64_vector vector; 325 unsigned long flags; 326 327 for (vector = IA64_FIRST_DEVICE_VECTOR; 328 vector < IA64_LAST_DEVICE_VECTOR; vector++) { 329 int irq; 330 struct irq_desc *desc; 331 struct irq_cfg *cfg; 332 irq = __this_cpu_read(vector_irq[vector]); 333 if (irq < 0) 334 continue; 335 336 desc = irq_to_desc(irq); 337 cfg = irq_cfg + irq; 338 raw_spin_lock(&desc->lock); 339 if (!cfg->move_cleanup_count) 340 goto unlock; 341 342 if (!cpumask_test_cpu(me, &cfg->old_domain)) 343 goto unlock; 344 345 spin_lock_irqsave(&vector_lock, flags); 346 __this_cpu_write(vector_irq[vector], -1); 347 cpumask_clear_cpu(me, &vector_table[vector]); 348 spin_unlock_irqrestore(&vector_lock, flags); 349 cfg->move_cleanup_count--; 350 unlock: 351 raw_spin_unlock(&desc->lock); 352 } 353 return IRQ_HANDLED; 354 } 355 356 static struct irqaction irq_move_irqaction = { 357 .handler = smp_irq_move_cleanup_interrupt, 358 .name = "irq_move" 359 }; 360 361 static int __init parse_vector_domain(char *arg) 362 { 363 if (!arg) 364 return -EINVAL; 365 if (!strcmp(arg, "percpu")) { 366 vector_domain_type = VECTOR_DOMAIN_PERCPU; 367 no_int_routing = 1; 368 } 369 return 0; 370 } 371 early_param("vector", parse_vector_domain); 372 #else 373 static cpumask_t vector_allocation_domain(int cpu) 374 { 375 return CPU_MASK_ALL; 376 } 377 #endif 378 379 380 void destroy_and_reserve_irq(unsigned int irq) 381 { 382 unsigned long flags; 383 384 irq_init_desc(irq); 385 spin_lock_irqsave(&vector_lock, flags); 386 __clear_irq_vector(irq); 387 irq_status[irq] = IRQ_RSVD; 388 spin_unlock_irqrestore(&vector_lock, flags); 389 } 390 391 /* 392 * Dynamic irq allocate and deallocation for MSI 393 */ 394 int create_irq(void) 395 { 396 unsigned long flags; 397 int irq, vector, cpu; 398 cpumask_t domain = CPU_MASK_NONE; 399 400 irq = vector = -ENOSPC; 401 spin_lock_irqsave(&vector_lock, flags); 402 for_each_online_cpu(cpu) { 403 domain = vector_allocation_domain(cpu); 404 vector = find_unassigned_vector(domain); 405 if (vector >= 0) 406 break; 407 } 408 if (vector < 0) 409 goto out; 410 irq = find_unassigned_irq(); 411 if (irq < 0) 412 goto out; 413 BUG_ON(__bind_irq_vector(irq, vector, domain)); 414 out: 415 spin_unlock_irqrestore(&vector_lock, flags); 416 if (irq >= 0) 417 irq_init_desc(irq); 418 return irq; 419 } 420 421 void destroy_irq(unsigned int irq) 422 { 423 irq_init_desc(irq); 424 clear_irq_vector(irq); 425 } 426 427 #ifdef CONFIG_SMP 428 # define IS_RESCHEDULE(vec) (vec == IA64_IPI_RESCHEDULE) 429 # define IS_LOCAL_TLB_FLUSH(vec) (vec == IA64_IPI_LOCAL_TLB_FLUSH) 430 #else 431 # define IS_RESCHEDULE(vec) (0) 432 # define IS_LOCAL_TLB_FLUSH(vec) (0) 433 #endif 434 /* 435 * That's where the IVT branches when we get an external 436 * interrupt. This branches to the correct hardware IRQ handler via 437 * function ptr. 438 */ 439 void 440 ia64_handle_irq (ia64_vector vector, struct pt_regs *regs) 441 { 442 struct pt_regs *old_regs = set_irq_regs(regs); 443 unsigned long saved_tpr; 444 445 #if IRQ_DEBUG 446 { 447 unsigned long bsp, sp; 448 449 /* 450 * Note: if the interrupt happened while executing in 451 * the context switch routine (ia64_switch_to), we may 452 * get a spurious stack overflow here. This is 453 * because the register and the memory stack are not 454 * switched atomically. 455 */ 456 bsp = ia64_getreg(_IA64_REG_AR_BSP); 457 sp = ia64_getreg(_IA64_REG_SP); 458 459 if ((sp - bsp) < 1024) { 460 static DEFINE_RATELIMIT_STATE(ratelimit, 5 * HZ, 5); 461 462 if (__ratelimit(&ratelimit)) { 463 printk("ia64_handle_irq: DANGER: less than " 464 "1KB of free stack space!!\n" 465 "(bsp=0x%lx, sp=%lx)\n", bsp, sp); 466 } 467 } 468 } 469 #endif /* IRQ_DEBUG */ 470 471 /* 472 * Always set TPR to limit maximum interrupt nesting depth to 473 * 16 (without this, it would be ~240, which could easily lead 474 * to kernel stack overflows). 475 */ 476 irq_enter(); 477 saved_tpr = ia64_getreg(_IA64_REG_CR_TPR); 478 ia64_srlz_d(); 479 while (vector != IA64_SPURIOUS_INT_VECTOR) { 480 int irq = local_vector_to_irq(vector); 481 482 if (unlikely(IS_LOCAL_TLB_FLUSH(vector))) { 483 smp_local_flush_tlb(); 484 kstat_incr_irq_this_cpu(irq); 485 } else if (unlikely(IS_RESCHEDULE(vector))) { 486 scheduler_ipi(); 487 kstat_incr_irq_this_cpu(irq); 488 } else { 489 ia64_setreg(_IA64_REG_CR_TPR, vector); 490 ia64_srlz_d(); 491 492 if (unlikely(irq < 0)) { 493 printk(KERN_ERR "%s: Unexpected interrupt " 494 "vector %d on CPU %d is not mapped " 495 "to any IRQ!\n", __func__, vector, 496 smp_processor_id()); 497 } else 498 generic_handle_irq(irq); 499 500 /* 501 * Disable interrupts and send EOI: 502 */ 503 local_irq_disable(); 504 ia64_setreg(_IA64_REG_CR_TPR, saved_tpr); 505 } 506 ia64_eoi(); 507 vector = ia64_get_ivr(); 508 } 509 /* 510 * This must be done *after* the ia64_eoi(). For example, the keyboard softirq 511 * handler needs to be able to wait for further keyboard interrupts, which can't 512 * come through until ia64_eoi() has been done. 513 */ 514 irq_exit(); 515 set_irq_regs(old_regs); 516 } 517 518 #ifdef CONFIG_HOTPLUG_CPU 519 /* 520 * This function emulates a interrupt processing when a cpu is about to be 521 * brought down. 522 */ 523 void ia64_process_pending_intr(void) 524 { 525 ia64_vector vector; 526 unsigned long saved_tpr; 527 extern unsigned int vectors_in_migration[NR_IRQS]; 528 529 vector = ia64_get_ivr(); 530 531 irq_enter(); 532 saved_tpr = ia64_getreg(_IA64_REG_CR_TPR); 533 ia64_srlz_d(); 534 535 /* 536 * Perform normal interrupt style processing 537 */ 538 while (vector != IA64_SPURIOUS_INT_VECTOR) { 539 int irq = local_vector_to_irq(vector); 540 541 if (unlikely(IS_LOCAL_TLB_FLUSH(vector))) { 542 smp_local_flush_tlb(); 543 kstat_incr_irq_this_cpu(irq); 544 } else if (unlikely(IS_RESCHEDULE(vector))) { 545 kstat_incr_irq_this_cpu(irq); 546 } else { 547 struct pt_regs *old_regs = set_irq_regs(NULL); 548 549 ia64_setreg(_IA64_REG_CR_TPR, vector); 550 ia64_srlz_d(); 551 552 /* 553 * Now try calling normal ia64_handle_irq as it would have got called 554 * from a real intr handler. Try passing null for pt_regs, hopefully 555 * it will work. I hope it works!. 556 * Probably could shared code. 557 */ 558 if (unlikely(irq < 0)) { 559 printk(KERN_ERR "%s: Unexpected interrupt " 560 "vector %d on CPU %d not being mapped " 561 "to any IRQ!!\n", __func__, vector, 562 smp_processor_id()); 563 } else { 564 vectors_in_migration[irq]=0; 565 generic_handle_irq(irq); 566 } 567 set_irq_regs(old_regs); 568 569 /* 570 * Disable interrupts and send EOI 571 */ 572 local_irq_disable(); 573 ia64_setreg(_IA64_REG_CR_TPR, saved_tpr); 574 } 575 ia64_eoi(); 576 vector = ia64_get_ivr(); 577 } 578 irq_exit(); 579 } 580 #endif 581 582 583 #ifdef CONFIG_SMP 584 585 static irqreturn_t dummy_handler (int irq, void *dev_id) 586 { 587 BUG(); 588 } 589 590 static struct irqaction ipi_irqaction = { 591 .handler = handle_IPI, 592 .name = "IPI" 593 }; 594 595 /* 596 * KVM uses this interrupt to force a cpu out of guest mode 597 */ 598 static struct irqaction resched_irqaction = { 599 .handler = dummy_handler, 600 .name = "resched" 601 }; 602 603 static struct irqaction tlb_irqaction = { 604 .handler = dummy_handler, 605 .name = "tlb_flush" 606 }; 607 608 #endif 609 610 void 611 ia64_native_register_percpu_irq (ia64_vector vec, struct irqaction *action) 612 { 613 unsigned int irq; 614 615 irq = vec; 616 BUG_ON(bind_irq_vector(irq, vec, CPU_MASK_ALL)); 617 irq_set_status_flags(irq, IRQ_PER_CPU); 618 irq_set_chip(irq, &irq_type_ia64_lsapic); 619 if (action) 620 setup_irq(irq, action); 621 irq_set_handler(irq, handle_percpu_irq); 622 } 623 624 void __init 625 ia64_native_register_ipi(void) 626 { 627 #ifdef CONFIG_SMP 628 register_percpu_irq(IA64_IPI_VECTOR, &ipi_irqaction); 629 register_percpu_irq(IA64_IPI_RESCHEDULE, &resched_irqaction); 630 register_percpu_irq(IA64_IPI_LOCAL_TLB_FLUSH, &tlb_irqaction); 631 #endif 632 } 633 634 void __init 635 init_IRQ (void) 636 { 637 #ifdef CONFIG_ACPI 638 acpi_boot_init(); 639 #endif 640 ia64_register_ipi(); 641 register_percpu_irq(IA64_SPURIOUS_INT_VECTOR, NULL); 642 #ifdef CONFIG_SMP 643 #if defined(CONFIG_IA64_GENERIC) || defined(CONFIG_IA64_DIG) 644 if (vector_domain_type != VECTOR_DOMAIN_NONE) 645 register_percpu_irq(IA64_IRQ_MOVE_VECTOR, &irq_move_irqaction); 646 #endif 647 #endif 648 #ifdef CONFIG_PERFMON 649 pfm_init_percpu(); 650 #endif 651 platform_irq_init(); 652 } 653 654 void 655 ia64_send_ipi (int cpu, int vector, int delivery_mode, int redirect) 656 { 657 void __iomem *ipi_addr; 658 unsigned long ipi_data; 659 unsigned long phys_cpu_id; 660 661 phys_cpu_id = cpu_physical_id(cpu); 662 663 /* 664 * cpu number is in 8bit ID and 8bit EID 665 */ 666 667 ipi_data = (delivery_mode << 8) | (vector & 0xff); 668 ipi_addr = ipi_base_addr + ((phys_cpu_id << 4) | ((redirect & 1) << 3)); 669 670 writeq(ipi_data, ipi_addr); 671 } 672