1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar 4 * Copyright (C) 2005-2006, Thomas Gleixner, Russell King 5 * 6 * This file contains the interrupt descriptor management code. Detailed 7 * information is available in Documentation/core-api/genericirq.rst 8 * 9 */ 10 #include <linux/irq.h> 11 #include <linux/slab.h> 12 #include <linux/export.h> 13 #include <linux/interrupt.h> 14 #include <linux/kernel_stat.h> 15 #include <linux/radix-tree.h> 16 #include <linux/bitmap.h> 17 #include <linux/irqdomain.h> 18 #include <linux/sysfs.h> 19 20 #include "internals.h" 21 22 /* 23 * lockdep: we want to handle all irq_desc locks as a single lock-class: 24 */ 25 static struct lock_class_key irq_desc_lock_class; 26 27 #if defined(CONFIG_SMP) 28 static int __init irq_affinity_setup(char *str) 29 { 30 alloc_bootmem_cpumask_var(&irq_default_affinity); 31 cpulist_parse(str, irq_default_affinity); 32 /* 33 * Set at least the boot cpu. We don't want to end up with 34 * bugreports caused by random commandline masks 35 */ 36 cpumask_set_cpu(smp_processor_id(), irq_default_affinity); 37 return 1; 38 } 39 __setup("irqaffinity=", irq_affinity_setup); 40 41 static void __init init_irq_default_affinity(void) 42 { 43 if (!cpumask_available(irq_default_affinity)) 44 zalloc_cpumask_var(&irq_default_affinity, GFP_NOWAIT); 45 if (cpumask_empty(irq_default_affinity)) 46 cpumask_setall(irq_default_affinity); 47 } 48 #else 49 static void __init init_irq_default_affinity(void) 50 { 51 } 52 #endif 53 54 #ifdef CONFIG_SMP 55 static int alloc_masks(struct irq_desc *desc, int node) 56 { 57 if (!zalloc_cpumask_var_node(&desc->irq_common_data.affinity, 58 GFP_KERNEL, node)) 59 return -ENOMEM; 60 61 #ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK 62 if (!zalloc_cpumask_var_node(&desc->irq_common_data.effective_affinity, 63 GFP_KERNEL, node)) { 64 free_cpumask_var(desc->irq_common_data.affinity); 65 return -ENOMEM; 66 } 67 #endif 68 69 #ifdef CONFIG_GENERIC_PENDING_IRQ 70 if (!zalloc_cpumask_var_node(&desc->pending_mask, GFP_KERNEL, node)) { 71 #ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK 72 free_cpumask_var(desc->irq_common_data.effective_affinity); 73 #endif 74 free_cpumask_var(desc->irq_common_data.affinity); 75 return -ENOMEM; 76 } 77 #endif 78 return 0; 79 } 80 81 static void desc_smp_init(struct irq_desc *desc, int node, 82 const struct cpumask *affinity) 83 { 84 if (!affinity) 85 affinity = irq_default_affinity; 86 cpumask_copy(desc->irq_common_data.affinity, affinity); 87 88 #ifdef CONFIG_GENERIC_PENDING_IRQ 89 cpumask_clear(desc->pending_mask); 90 #endif 91 #ifdef CONFIG_NUMA 92 desc->irq_common_data.node = node; 93 #endif 94 } 95 96 #else 97 static inline int 98 alloc_masks(struct irq_desc *desc, int node) { return 0; } 99 static inline void 100 desc_smp_init(struct irq_desc *desc, int node, const struct cpumask *affinity) { } 101 #endif 102 103 static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node, 104 const struct cpumask *affinity, struct module *owner) 105 { 106 int cpu; 107 108 desc->irq_common_data.handler_data = NULL; 109 desc->irq_common_data.msi_desc = NULL; 110 111 desc->irq_data.common = &desc->irq_common_data; 112 desc->irq_data.irq = irq; 113 desc->irq_data.chip = &no_irq_chip; 114 desc->irq_data.chip_data = NULL; 115 irq_settings_clr_and_set(desc, ~0, _IRQ_DEFAULT_INIT_FLAGS); 116 irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED); 117 irqd_set(&desc->irq_data, IRQD_IRQ_MASKED); 118 desc->handle_irq = handle_bad_irq; 119 desc->depth = 1; 120 desc->irq_count = 0; 121 desc->irqs_unhandled = 0; 122 desc->tot_count = 0; 123 desc->name = NULL; 124 desc->owner = owner; 125 for_each_possible_cpu(cpu) 126 *per_cpu_ptr(desc->kstat_irqs, cpu) = 0; 127 desc_smp_init(desc, node, affinity); 128 } 129 130 int nr_irqs = NR_IRQS; 131 EXPORT_SYMBOL_GPL(nr_irqs); 132 133 static DEFINE_MUTEX(sparse_irq_lock); 134 static DECLARE_BITMAP(allocated_irqs, IRQ_BITMAP_BITS); 135 136 #ifdef CONFIG_SPARSE_IRQ 137 138 static void irq_kobj_release(struct kobject *kobj); 139 140 #ifdef CONFIG_SYSFS 141 static struct kobject *irq_kobj_base; 142 143 #define IRQ_ATTR_RO(_name) \ 144 static struct kobj_attribute _name##_attr = __ATTR_RO(_name) 145 146 static ssize_t per_cpu_count_show(struct kobject *kobj, 147 struct kobj_attribute *attr, char *buf) 148 { 149 struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); 150 ssize_t ret = 0; 151 char *p = ""; 152 int cpu; 153 154 for_each_possible_cpu(cpu) { 155 unsigned int c = irq_desc_kstat_cpu(desc, cpu); 156 157 ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s%u", p, c); 158 p = ","; 159 } 160 161 ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); 162 return ret; 163 } 164 IRQ_ATTR_RO(per_cpu_count); 165 166 static ssize_t chip_name_show(struct kobject *kobj, 167 struct kobj_attribute *attr, char *buf) 168 { 169 struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); 170 ssize_t ret = 0; 171 172 raw_spin_lock_irq(&desc->lock); 173 if (desc->irq_data.chip && desc->irq_data.chip->name) { 174 ret = scnprintf(buf, PAGE_SIZE, "%s\n", 175 desc->irq_data.chip->name); 176 } 177 raw_spin_unlock_irq(&desc->lock); 178 179 return ret; 180 } 181 IRQ_ATTR_RO(chip_name); 182 183 static ssize_t hwirq_show(struct kobject *kobj, 184 struct kobj_attribute *attr, char *buf) 185 { 186 struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); 187 ssize_t ret = 0; 188 189 raw_spin_lock_irq(&desc->lock); 190 if (desc->irq_data.domain) 191 ret = sprintf(buf, "%d\n", (int)desc->irq_data.hwirq); 192 raw_spin_unlock_irq(&desc->lock); 193 194 return ret; 195 } 196 IRQ_ATTR_RO(hwirq); 197 198 static ssize_t type_show(struct kobject *kobj, 199 struct kobj_attribute *attr, char *buf) 200 { 201 struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); 202 ssize_t ret = 0; 203 204 raw_spin_lock_irq(&desc->lock); 205 ret = sprintf(buf, "%s\n", 206 irqd_is_level_type(&desc->irq_data) ? "level" : "edge"); 207 raw_spin_unlock_irq(&desc->lock); 208 209 return ret; 210 211 } 212 IRQ_ATTR_RO(type); 213 214 static ssize_t wakeup_show(struct kobject *kobj, 215 struct kobj_attribute *attr, char *buf) 216 { 217 struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); 218 ssize_t ret = 0; 219 220 raw_spin_lock_irq(&desc->lock); 221 ret = sprintf(buf, "%s\n", 222 irqd_is_wakeup_set(&desc->irq_data) ? "enabled" : "disabled"); 223 raw_spin_unlock_irq(&desc->lock); 224 225 return ret; 226 227 } 228 IRQ_ATTR_RO(wakeup); 229 230 static ssize_t name_show(struct kobject *kobj, 231 struct kobj_attribute *attr, char *buf) 232 { 233 struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); 234 ssize_t ret = 0; 235 236 raw_spin_lock_irq(&desc->lock); 237 if (desc->name) 238 ret = scnprintf(buf, PAGE_SIZE, "%s\n", desc->name); 239 raw_spin_unlock_irq(&desc->lock); 240 241 return ret; 242 } 243 IRQ_ATTR_RO(name); 244 245 static ssize_t actions_show(struct kobject *kobj, 246 struct kobj_attribute *attr, char *buf) 247 { 248 struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); 249 struct irqaction *action; 250 ssize_t ret = 0; 251 char *p = ""; 252 253 raw_spin_lock_irq(&desc->lock); 254 for (action = desc->action; action != NULL; action = action->next) { 255 ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s%s", 256 p, action->name); 257 p = ","; 258 } 259 raw_spin_unlock_irq(&desc->lock); 260 261 if (ret) 262 ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); 263 264 return ret; 265 } 266 IRQ_ATTR_RO(actions); 267 268 static struct attribute *irq_attrs[] = { 269 &per_cpu_count_attr.attr, 270 &chip_name_attr.attr, 271 &hwirq_attr.attr, 272 &type_attr.attr, 273 &wakeup_attr.attr, 274 &name_attr.attr, 275 &actions_attr.attr, 276 NULL 277 }; 278 ATTRIBUTE_GROUPS(irq); 279 280 static struct kobj_type irq_kobj_type = { 281 .release = irq_kobj_release, 282 .sysfs_ops = &kobj_sysfs_ops, 283 .default_groups = irq_groups, 284 }; 285 286 static void irq_sysfs_add(int irq, struct irq_desc *desc) 287 { 288 if (irq_kobj_base) { 289 /* 290 * Continue even in case of failure as this is nothing 291 * crucial. 292 */ 293 if (kobject_add(&desc->kobj, irq_kobj_base, "%d", irq)) 294 pr_warn("Failed to add kobject for irq %d\n", irq); 295 } 296 } 297 298 static void irq_sysfs_del(struct irq_desc *desc) 299 { 300 /* 301 * If irq_sysfs_init() has not yet been invoked (early boot), then 302 * irq_kobj_base is NULL and the descriptor was never added. 303 * kobject_del() complains about a object with no parent, so make 304 * it conditional. 305 */ 306 if (irq_kobj_base) 307 kobject_del(&desc->kobj); 308 } 309 310 static int __init irq_sysfs_init(void) 311 { 312 struct irq_desc *desc; 313 int irq; 314 315 /* Prevent concurrent irq alloc/free */ 316 irq_lock_sparse(); 317 318 irq_kobj_base = kobject_create_and_add("irq", kernel_kobj); 319 if (!irq_kobj_base) { 320 irq_unlock_sparse(); 321 return -ENOMEM; 322 } 323 324 /* Add the already allocated interrupts */ 325 for_each_irq_desc(irq, desc) 326 irq_sysfs_add(irq, desc); 327 irq_unlock_sparse(); 328 329 return 0; 330 } 331 postcore_initcall(irq_sysfs_init); 332 333 #else /* !CONFIG_SYSFS */ 334 335 static struct kobj_type irq_kobj_type = { 336 .release = irq_kobj_release, 337 }; 338 339 static void irq_sysfs_add(int irq, struct irq_desc *desc) {} 340 static void irq_sysfs_del(struct irq_desc *desc) {} 341 342 #endif /* CONFIG_SYSFS */ 343 344 static RADIX_TREE(irq_desc_tree, GFP_KERNEL); 345 346 static void irq_insert_desc(unsigned int irq, struct irq_desc *desc) 347 { 348 radix_tree_insert(&irq_desc_tree, irq, desc); 349 } 350 351 struct irq_desc *irq_to_desc(unsigned int irq) 352 { 353 return radix_tree_lookup(&irq_desc_tree, irq); 354 } 355 #ifdef CONFIG_KVM_BOOK3S_64_HV_MODULE 356 EXPORT_SYMBOL_GPL(irq_to_desc); 357 #endif 358 359 static void delete_irq_desc(unsigned int irq) 360 { 361 radix_tree_delete(&irq_desc_tree, irq); 362 } 363 364 #ifdef CONFIG_SMP 365 static void free_masks(struct irq_desc *desc) 366 { 367 #ifdef CONFIG_GENERIC_PENDING_IRQ 368 free_cpumask_var(desc->pending_mask); 369 #endif 370 free_cpumask_var(desc->irq_common_data.affinity); 371 #ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK 372 free_cpumask_var(desc->irq_common_data.effective_affinity); 373 #endif 374 } 375 #else 376 static inline void free_masks(struct irq_desc *desc) { } 377 #endif 378 379 void irq_lock_sparse(void) 380 { 381 mutex_lock(&sparse_irq_lock); 382 } 383 384 void irq_unlock_sparse(void) 385 { 386 mutex_unlock(&sparse_irq_lock); 387 } 388 389 static struct irq_desc *alloc_desc(int irq, int node, unsigned int flags, 390 const struct cpumask *affinity, 391 struct module *owner) 392 { 393 struct irq_desc *desc; 394 395 desc = kzalloc_node(sizeof(*desc), GFP_KERNEL, node); 396 if (!desc) 397 return NULL; 398 /* allocate based on nr_cpu_ids */ 399 desc->kstat_irqs = alloc_percpu(unsigned int); 400 if (!desc->kstat_irqs) 401 goto err_desc; 402 403 if (alloc_masks(desc, node)) 404 goto err_kstat; 405 406 raw_spin_lock_init(&desc->lock); 407 lockdep_set_class(&desc->lock, &irq_desc_lock_class); 408 mutex_init(&desc->request_mutex); 409 init_rcu_head(&desc->rcu); 410 411 desc_set_defaults(irq, desc, node, affinity, owner); 412 irqd_set(&desc->irq_data, flags); 413 kobject_init(&desc->kobj, &irq_kobj_type); 414 415 return desc; 416 417 err_kstat: 418 free_percpu(desc->kstat_irqs); 419 err_desc: 420 kfree(desc); 421 return NULL; 422 } 423 424 static void irq_kobj_release(struct kobject *kobj) 425 { 426 struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); 427 428 free_masks(desc); 429 free_percpu(desc->kstat_irqs); 430 kfree(desc); 431 } 432 433 static void delayed_free_desc(struct rcu_head *rhp) 434 { 435 struct irq_desc *desc = container_of(rhp, struct irq_desc, rcu); 436 437 kobject_put(&desc->kobj); 438 } 439 440 static void free_desc(unsigned int irq) 441 { 442 struct irq_desc *desc = irq_to_desc(irq); 443 444 irq_remove_debugfs_entry(desc); 445 unregister_irq_proc(irq, desc); 446 447 /* 448 * sparse_irq_lock protects also show_interrupts() and 449 * kstat_irq_usr(). Once we deleted the descriptor from the 450 * sparse tree we can free it. Access in proc will fail to 451 * lookup the descriptor. 452 * 453 * The sysfs entry must be serialized against a concurrent 454 * irq_sysfs_init() as well. 455 */ 456 irq_sysfs_del(desc); 457 delete_irq_desc(irq); 458 459 /* 460 * We free the descriptor, masks and stat fields via RCU. That 461 * allows demultiplex interrupts to do rcu based management of 462 * the child interrupts. 463 * This also allows us to use rcu in kstat_irqs_usr(). 464 */ 465 call_rcu(&desc->rcu, delayed_free_desc); 466 } 467 468 static int alloc_descs(unsigned int start, unsigned int cnt, int node, 469 const struct irq_affinity_desc *affinity, 470 struct module *owner) 471 { 472 struct irq_desc *desc; 473 int i; 474 475 /* Validate affinity mask(s) */ 476 if (affinity) { 477 for (i = 0; i < cnt; i++) { 478 if (cpumask_empty(&affinity[i].mask)) 479 return -EINVAL; 480 } 481 } 482 483 for (i = 0; i < cnt; i++) { 484 const struct cpumask *mask = NULL; 485 unsigned int flags = 0; 486 487 if (affinity) { 488 if (affinity->is_managed) { 489 flags = IRQD_AFFINITY_MANAGED | 490 IRQD_MANAGED_SHUTDOWN; 491 } 492 mask = &affinity->mask; 493 node = cpu_to_node(cpumask_first(mask)); 494 affinity++; 495 } 496 497 desc = alloc_desc(start + i, node, flags, mask, owner); 498 if (!desc) 499 goto err; 500 irq_insert_desc(start + i, desc); 501 irq_sysfs_add(start + i, desc); 502 irq_add_debugfs_entry(start + i, desc); 503 } 504 bitmap_set(allocated_irqs, start, cnt); 505 return start; 506 507 err: 508 for (i--; i >= 0; i--) 509 free_desc(start + i); 510 return -ENOMEM; 511 } 512 513 static int irq_expand_nr_irqs(unsigned int nr) 514 { 515 if (nr > IRQ_BITMAP_BITS) 516 return -ENOMEM; 517 nr_irqs = nr; 518 return 0; 519 } 520 521 int __init early_irq_init(void) 522 { 523 int i, initcnt, node = first_online_node; 524 struct irq_desc *desc; 525 526 init_irq_default_affinity(); 527 528 /* Let arch update nr_irqs and return the nr of preallocated irqs */ 529 initcnt = arch_probe_nr_irqs(); 530 printk(KERN_INFO "NR_IRQS: %d, nr_irqs: %d, preallocated irqs: %d\n", 531 NR_IRQS, nr_irqs, initcnt); 532 533 if (WARN_ON(nr_irqs > IRQ_BITMAP_BITS)) 534 nr_irqs = IRQ_BITMAP_BITS; 535 536 if (WARN_ON(initcnt > IRQ_BITMAP_BITS)) 537 initcnt = IRQ_BITMAP_BITS; 538 539 if (initcnt > nr_irqs) 540 nr_irqs = initcnt; 541 542 for (i = 0; i < initcnt; i++) { 543 desc = alloc_desc(i, node, 0, NULL, NULL); 544 set_bit(i, allocated_irqs); 545 irq_insert_desc(i, desc); 546 } 547 return arch_early_irq_init(); 548 } 549 550 #else /* !CONFIG_SPARSE_IRQ */ 551 552 struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = { 553 [0 ... NR_IRQS-1] = { 554 .handle_irq = handle_bad_irq, 555 .depth = 1, 556 .lock = __RAW_SPIN_LOCK_UNLOCKED(irq_desc->lock), 557 } 558 }; 559 560 int __init early_irq_init(void) 561 { 562 int count, i, node = first_online_node; 563 struct irq_desc *desc; 564 565 init_irq_default_affinity(); 566 567 printk(KERN_INFO "NR_IRQS: %d\n", NR_IRQS); 568 569 desc = irq_desc; 570 count = ARRAY_SIZE(irq_desc); 571 572 for (i = 0; i < count; i++) { 573 desc[i].kstat_irqs = alloc_percpu(unsigned int); 574 alloc_masks(&desc[i], node); 575 raw_spin_lock_init(&desc[i].lock); 576 lockdep_set_class(&desc[i].lock, &irq_desc_lock_class); 577 mutex_init(&desc[i].request_mutex); 578 desc_set_defaults(i, &desc[i], node, NULL, NULL); 579 } 580 return arch_early_irq_init(); 581 } 582 583 struct irq_desc *irq_to_desc(unsigned int irq) 584 { 585 return (irq < NR_IRQS) ? irq_desc + irq : NULL; 586 } 587 EXPORT_SYMBOL(irq_to_desc); 588 589 static void free_desc(unsigned int irq) 590 { 591 struct irq_desc *desc = irq_to_desc(irq); 592 unsigned long flags; 593 594 raw_spin_lock_irqsave(&desc->lock, flags); 595 desc_set_defaults(irq, desc, irq_desc_get_node(desc), NULL, NULL); 596 raw_spin_unlock_irqrestore(&desc->lock, flags); 597 } 598 599 static inline int alloc_descs(unsigned int start, unsigned int cnt, int node, 600 const struct irq_affinity_desc *affinity, 601 struct module *owner) 602 { 603 u32 i; 604 605 for (i = 0; i < cnt; i++) { 606 struct irq_desc *desc = irq_to_desc(start + i); 607 608 desc->owner = owner; 609 } 610 bitmap_set(allocated_irqs, start, cnt); 611 return start; 612 } 613 614 static int irq_expand_nr_irqs(unsigned int nr) 615 { 616 return -ENOMEM; 617 } 618 619 void irq_mark_irq(unsigned int irq) 620 { 621 mutex_lock(&sparse_irq_lock); 622 bitmap_set(allocated_irqs, irq, 1); 623 mutex_unlock(&sparse_irq_lock); 624 } 625 626 #ifdef CONFIG_GENERIC_IRQ_LEGACY 627 void irq_init_desc(unsigned int irq) 628 { 629 free_desc(irq); 630 } 631 #endif 632 633 #endif /* !CONFIG_SPARSE_IRQ */ 634 635 int handle_irq_desc(struct irq_desc *desc) 636 { 637 struct irq_data *data; 638 639 if (!desc) 640 return -EINVAL; 641 642 data = irq_desc_get_irq_data(desc); 643 if (WARN_ON_ONCE(!in_irq() && handle_enforce_irqctx(data))) 644 return -EPERM; 645 646 generic_handle_irq_desc(desc); 647 return 0; 648 } 649 EXPORT_SYMBOL_GPL(handle_irq_desc); 650 651 /** 652 * generic_handle_irq - Invoke the handler for a particular irq 653 * @irq: The irq number to handle 654 * 655 */ 656 int generic_handle_irq(unsigned int irq) 657 { 658 return handle_irq_desc(irq_to_desc(irq)); 659 } 660 EXPORT_SYMBOL_GPL(generic_handle_irq); 661 662 #ifdef CONFIG_IRQ_DOMAIN 663 /** 664 * generic_handle_domain_irq - Invoke the handler for a HW irq belonging 665 * to a domain, usually for a non-root interrupt 666 * controller 667 * @domain: The domain where to perform the lookup 668 * @hwirq: The HW irq number to convert to a logical one 669 * 670 * Returns: 0 on success, or -EINVAL if conversion has failed 671 * 672 */ 673 int generic_handle_domain_irq(struct irq_domain *domain, unsigned int hwirq) 674 { 675 return handle_irq_desc(irq_resolve_mapping(domain, hwirq)); 676 } 677 EXPORT_SYMBOL_GPL(generic_handle_domain_irq); 678 679 #ifdef CONFIG_HANDLE_DOMAIN_IRQ 680 /** 681 * handle_domain_irq - Invoke the handler for a HW irq belonging to a domain, 682 * usually for a root interrupt controller 683 * @domain: The domain where to perform the lookup 684 * @hwirq: The HW irq number to convert to a logical one 685 * @lookup: Whether to perform the domain lookup or not 686 * @regs: Register file coming from the low-level handling code 687 * 688 * Returns: 0 on success, or -EINVAL if conversion has failed 689 */ 690 int handle_domain_irq(struct irq_domain *domain, 691 unsigned int hwirq, struct pt_regs *regs) 692 { 693 struct pt_regs *old_regs = set_irq_regs(regs); 694 struct irq_desc *desc; 695 int ret = 0; 696 697 irq_enter(); 698 699 /* The irqdomain code provides boundary checks */ 700 desc = irq_resolve_mapping(domain, hwirq); 701 if (likely(desc)) 702 handle_irq_desc(desc); 703 else 704 ret = -EINVAL; 705 706 irq_exit(); 707 set_irq_regs(old_regs); 708 return ret; 709 } 710 711 /** 712 * handle_domain_nmi - Invoke the handler for a HW irq belonging to a domain 713 * @domain: The domain where to perform the lookup 714 * @hwirq: The HW irq number to convert to a logical one 715 * @regs: Register file coming from the low-level handling code 716 * 717 * This function must be called from an NMI context. 718 * 719 * Returns: 0 on success, or -EINVAL if conversion has failed 720 */ 721 int handle_domain_nmi(struct irq_domain *domain, unsigned int hwirq, 722 struct pt_regs *regs) 723 { 724 struct pt_regs *old_regs = set_irq_regs(regs); 725 struct irq_desc *desc; 726 int ret = 0; 727 728 /* 729 * NMI context needs to be setup earlier in order to deal with tracing. 730 */ 731 WARN_ON(!in_nmi()); 732 733 desc = irq_resolve_mapping(domain, hwirq); 734 735 /* 736 * ack_bad_irq is not NMI-safe, just report 737 * an invalid interrupt. 738 */ 739 if (likely(desc)) 740 handle_irq_desc(desc); 741 else 742 ret = -EINVAL; 743 744 set_irq_regs(old_regs); 745 return ret; 746 } 747 #endif 748 #endif 749 750 /* Dynamic interrupt handling */ 751 752 /** 753 * irq_free_descs - free irq descriptors 754 * @from: Start of descriptor range 755 * @cnt: Number of consecutive irqs to free 756 */ 757 void irq_free_descs(unsigned int from, unsigned int cnt) 758 { 759 int i; 760 761 if (from >= nr_irqs || (from + cnt) > nr_irqs) 762 return; 763 764 mutex_lock(&sparse_irq_lock); 765 for (i = 0; i < cnt; i++) 766 free_desc(from + i); 767 768 bitmap_clear(allocated_irqs, from, cnt); 769 mutex_unlock(&sparse_irq_lock); 770 } 771 EXPORT_SYMBOL_GPL(irq_free_descs); 772 773 /** 774 * __irq_alloc_descs - allocate and initialize a range of irq descriptors 775 * @irq: Allocate for specific irq number if irq >= 0 776 * @from: Start the search from this irq number 777 * @cnt: Number of consecutive irqs to allocate. 778 * @node: Preferred node on which the irq descriptor should be allocated 779 * @owner: Owning module (can be NULL) 780 * @affinity: Optional pointer to an affinity mask array of size @cnt which 781 * hints where the irq descriptors should be allocated and which 782 * default affinities to use 783 * 784 * Returns the first irq number or error code 785 */ 786 int __ref 787 __irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int node, 788 struct module *owner, const struct irq_affinity_desc *affinity) 789 { 790 int start, ret; 791 792 if (!cnt) 793 return -EINVAL; 794 795 if (irq >= 0) { 796 if (from > irq) 797 return -EINVAL; 798 from = irq; 799 } else { 800 /* 801 * For interrupts which are freely allocated the 802 * architecture can force a lower bound to the @from 803 * argument. x86 uses this to exclude the GSI space. 804 */ 805 from = arch_dynirq_lower_bound(from); 806 } 807 808 mutex_lock(&sparse_irq_lock); 809 810 start = bitmap_find_next_zero_area(allocated_irqs, IRQ_BITMAP_BITS, 811 from, cnt, 0); 812 ret = -EEXIST; 813 if (irq >=0 && start != irq) 814 goto unlock; 815 816 if (start + cnt > nr_irqs) { 817 ret = irq_expand_nr_irqs(start + cnt); 818 if (ret) 819 goto unlock; 820 } 821 ret = alloc_descs(start, cnt, node, affinity, owner); 822 unlock: 823 mutex_unlock(&sparse_irq_lock); 824 return ret; 825 } 826 EXPORT_SYMBOL_GPL(__irq_alloc_descs); 827 828 /** 829 * irq_get_next_irq - get next allocated irq number 830 * @offset: where to start the search 831 * 832 * Returns next irq number after offset or nr_irqs if none is found. 833 */ 834 unsigned int irq_get_next_irq(unsigned int offset) 835 { 836 return find_next_bit(allocated_irqs, nr_irqs, offset); 837 } 838 839 struct irq_desc * 840 __irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus, 841 unsigned int check) 842 { 843 struct irq_desc *desc = irq_to_desc(irq); 844 845 if (desc) { 846 if (check & _IRQ_DESC_CHECK) { 847 if ((check & _IRQ_DESC_PERCPU) && 848 !irq_settings_is_per_cpu_devid(desc)) 849 return NULL; 850 851 if (!(check & _IRQ_DESC_PERCPU) && 852 irq_settings_is_per_cpu_devid(desc)) 853 return NULL; 854 } 855 856 if (bus) 857 chip_bus_lock(desc); 858 raw_spin_lock_irqsave(&desc->lock, *flags); 859 } 860 return desc; 861 } 862 863 void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus) 864 __releases(&desc->lock) 865 { 866 raw_spin_unlock_irqrestore(&desc->lock, flags); 867 if (bus) 868 chip_bus_sync_unlock(desc); 869 } 870 871 int irq_set_percpu_devid_partition(unsigned int irq, 872 const struct cpumask *affinity) 873 { 874 struct irq_desc *desc = irq_to_desc(irq); 875 876 if (!desc) 877 return -EINVAL; 878 879 if (desc->percpu_enabled) 880 return -EINVAL; 881 882 desc->percpu_enabled = kzalloc(sizeof(*desc->percpu_enabled), GFP_KERNEL); 883 884 if (!desc->percpu_enabled) 885 return -ENOMEM; 886 887 if (affinity) 888 desc->percpu_affinity = affinity; 889 else 890 desc->percpu_affinity = cpu_possible_mask; 891 892 irq_set_percpu_devid_flags(irq); 893 return 0; 894 } 895 896 int irq_set_percpu_devid(unsigned int irq) 897 { 898 return irq_set_percpu_devid_partition(irq, NULL); 899 } 900 901 int irq_get_percpu_devid_partition(unsigned int irq, struct cpumask *affinity) 902 { 903 struct irq_desc *desc = irq_to_desc(irq); 904 905 if (!desc || !desc->percpu_enabled) 906 return -EINVAL; 907 908 if (affinity) 909 cpumask_copy(affinity, desc->percpu_affinity); 910 911 return 0; 912 } 913 EXPORT_SYMBOL_GPL(irq_get_percpu_devid_partition); 914 915 void kstat_incr_irq_this_cpu(unsigned int irq) 916 { 917 kstat_incr_irqs_this_cpu(irq_to_desc(irq)); 918 } 919 920 /** 921 * kstat_irqs_cpu - Get the statistics for an interrupt on a cpu 922 * @irq: The interrupt number 923 * @cpu: The cpu number 924 * 925 * Returns the sum of interrupt counts on @cpu since boot for 926 * @irq. The caller must ensure that the interrupt is not removed 927 * concurrently. 928 */ 929 unsigned int kstat_irqs_cpu(unsigned int irq, int cpu) 930 { 931 struct irq_desc *desc = irq_to_desc(irq); 932 933 return desc && desc->kstat_irqs ? 934 *per_cpu_ptr(desc->kstat_irqs, cpu) : 0; 935 } 936 937 static bool irq_is_nmi(struct irq_desc *desc) 938 { 939 return desc->istate & IRQS_NMI; 940 } 941 942 static unsigned int kstat_irqs(unsigned int irq) 943 { 944 struct irq_desc *desc = irq_to_desc(irq); 945 unsigned int sum = 0; 946 int cpu; 947 948 if (!desc || !desc->kstat_irqs) 949 return 0; 950 if (!irq_settings_is_per_cpu_devid(desc) && 951 !irq_settings_is_per_cpu(desc) && 952 !irq_is_nmi(desc)) 953 return data_race(desc->tot_count); 954 955 for_each_possible_cpu(cpu) 956 sum += data_race(*per_cpu_ptr(desc->kstat_irqs, cpu)); 957 return sum; 958 } 959 960 /** 961 * kstat_irqs_usr - Get the statistics for an interrupt from thread context 962 * @irq: The interrupt number 963 * 964 * Returns the sum of interrupt counts on all cpus since boot for @irq. 965 * 966 * It uses rcu to protect the access since a concurrent removal of an 967 * interrupt descriptor is observing an rcu grace period before 968 * delayed_free_desc()/irq_kobj_release(). 969 */ 970 unsigned int kstat_irqs_usr(unsigned int irq) 971 { 972 unsigned int sum; 973 974 rcu_read_lock(); 975 sum = kstat_irqs(irq); 976 rcu_read_unlock(); 977 return sum; 978 } 979 980 #ifdef CONFIG_LOCKDEP 981 void __irq_set_lockdep_class(unsigned int irq, struct lock_class_key *lock_class, 982 struct lock_class_key *request_class) 983 { 984 struct irq_desc *desc = irq_to_desc(irq); 985 986 if (desc) { 987 lockdep_set_class(&desc->lock, lock_class); 988 lockdep_set_class(&desc->request_mutex, request_class); 989 } 990 } 991 EXPORT_SYMBOL_GPL(__irq_set_lockdep_class); 992 #endif 993