1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Xen event channels 4 * 5 * Xen models interrupts with abstract event channels. Because each 6 * domain gets 1024 event channels, but NR_IRQ is not that large, we 7 * must dynamically map irqs<->event channels. The event channels 8 * interface with the rest of the kernel by defining a xen interrupt 9 * chip. When an event is received, it is mapped to an irq and sent 10 * through the normal interrupt processing path. 11 * 12 * There are four kinds of events which can be mapped to an event 13 * channel: 14 * 15 * 1. Inter-domain notifications. This includes all the virtual 16 * device events, since they're driven by front-ends in another domain 17 * (typically dom0). 18 * 2. VIRQs, typically used for timers. These are per-cpu events. 19 * 3. IPIs. 20 * 4. PIRQs - Hardware interrupts. 21 * 22 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 23 */ 24 25 #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt 26 27 #include <linux/linkage.h> 28 #include <linux/interrupt.h> 29 #include <linux/irq.h> 30 #include <linux/moduleparam.h> 31 #include <linux/string.h> 32 #include <linux/memblock.h> 33 #include <linux/slab.h> 34 #include <linux/irqnr.h> 35 #include <linux/pci.h> 36 #include <linux/spinlock.h> 37 #include <linux/cpuhotplug.h> 38 #include <linux/atomic.h> 39 #include <linux/ktime.h> 40 41 #ifdef CONFIG_X86 42 #include <asm/desc.h> 43 #include <asm/ptrace.h> 44 #include <asm/idtentry.h> 45 #include <asm/irq.h> 46 #include <asm/io_apic.h> 47 #include <asm/i8259.h> 48 #include <asm/xen/pci.h> 49 #endif 50 #include <asm/sync_bitops.h> 51 #include <asm/xen/hypercall.h> 52 #include <asm/xen/hypervisor.h> 53 #include <xen/page.h> 54 55 #include <xen/xen.h> 56 #include <xen/hvm.h> 57 #include <xen/xen-ops.h> 58 #include <xen/events.h> 59 #include <xen/interface/xen.h> 60 #include <xen/interface/event_channel.h> 61 #include <xen/interface/hvm/hvm_op.h> 62 #include <xen/interface/hvm/params.h> 63 #include <xen/interface/physdev.h> 64 #include <xen/interface/sched.h> 65 #include <xen/interface/vcpu.h> 66 #include <asm/hw_irq.h> 67 68 #include "events_internal.h" 69 70 #undef MODULE_PARAM_PREFIX 71 #define MODULE_PARAM_PREFIX "xen." 72 73 static uint __read_mostly event_loop_timeout = 2; 74 module_param(event_loop_timeout, uint, 0644); 75 76 static uint __read_mostly event_eoi_delay = 10; 77 module_param(event_eoi_delay, uint, 0644); 78 79 const struct evtchn_ops *evtchn_ops; 80 81 /* 82 * This lock protects updates to the following mapping and reference-count 83 * arrays. The lock does not need to be acquired to read the mapping tables. 84 */ 85 static DEFINE_MUTEX(irq_mapping_update_lock); 86 87 /* 88 * Lock protecting event handling loop against removing event channels. 89 * Adding of event channels is no issue as the associated IRQ becomes active 90 * only after everything is setup (before request_[threaded_]irq() the handler 91 * can't be entered for an event, as the event channel will be unmasked only 92 * then). 93 */ 94 static DEFINE_RWLOCK(evtchn_rwlock); 95 96 /* 97 * Lock hierarchy: 98 * 99 * irq_mapping_update_lock 100 * evtchn_rwlock 101 * IRQ-desc lock 102 * percpu eoi_list_lock 103 */ 104 105 static LIST_HEAD(xen_irq_list_head); 106 107 /* IRQ <-> VIRQ mapping. */ 108 static DEFINE_PER_CPU(int [NR_VIRQS], virq_to_irq) = {[0 ... NR_VIRQS-1] = -1}; 109 110 /* IRQ <-> IPI mapping */ 111 static DEFINE_PER_CPU(int [XEN_NR_IPIS], ipi_to_irq) = {[0 ... XEN_NR_IPIS-1] = -1}; 112 113 int **evtchn_to_irq; 114 #ifdef CONFIG_X86 115 static unsigned long *pirq_eoi_map; 116 #endif 117 static bool (*pirq_needs_eoi)(unsigned irq); 118 119 #define EVTCHN_ROW(e) (e / (PAGE_SIZE/sizeof(**evtchn_to_irq))) 120 #define EVTCHN_COL(e) (e % (PAGE_SIZE/sizeof(**evtchn_to_irq))) 121 #define EVTCHN_PER_ROW (PAGE_SIZE / sizeof(**evtchn_to_irq)) 122 123 /* Xen will never allocate port zero for any purpose. */ 124 #define VALID_EVTCHN(chn) ((chn) != 0) 125 126 static struct irq_info *legacy_info_ptrs[NR_IRQS_LEGACY]; 127 128 static struct irq_chip xen_dynamic_chip; 129 static struct irq_chip xen_lateeoi_chip; 130 static struct irq_chip xen_percpu_chip; 131 static struct irq_chip xen_pirq_chip; 132 static void enable_dynirq(struct irq_data *data); 133 static void disable_dynirq(struct irq_data *data); 134 135 static DEFINE_PER_CPU(unsigned int, irq_epoch); 136 137 static void clear_evtchn_to_irq_row(unsigned row) 138 { 139 unsigned col; 140 141 for (col = 0; col < EVTCHN_PER_ROW; col++) 142 WRITE_ONCE(evtchn_to_irq[row][col], -1); 143 } 144 145 static void clear_evtchn_to_irq_all(void) 146 { 147 unsigned row; 148 149 for (row = 0; row < EVTCHN_ROW(xen_evtchn_max_channels()); row++) { 150 if (evtchn_to_irq[row] == NULL) 151 continue; 152 clear_evtchn_to_irq_row(row); 153 } 154 } 155 156 static int set_evtchn_to_irq(evtchn_port_t evtchn, unsigned int irq) 157 { 158 unsigned row; 159 unsigned col; 160 161 if (evtchn >= xen_evtchn_max_channels()) 162 return -EINVAL; 163 164 row = EVTCHN_ROW(evtchn); 165 col = EVTCHN_COL(evtchn); 166 167 if (evtchn_to_irq[row] == NULL) { 168 /* Unallocated irq entries return -1 anyway */ 169 if (irq == -1) 170 return 0; 171 172 evtchn_to_irq[row] = (int *)get_zeroed_page(GFP_KERNEL); 173 if (evtchn_to_irq[row] == NULL) 174 return -ENOMEM; 175 176 clear_evtchn_to_irq_row(row); 177 } 178 179 WRITE_ONCE(evtchn_to_irq[row][col], irq); 180 return 0; 181 } 182 183 int get_evtchn_to_irq(evtchn_port_t evtchn) 184 { 185 if (evtchn >= xen_evtchn_max_channels()) 186 return -1; 187 if (evtchn_to_irq[EVTCHN_ROW(evtchn)] == NULL) 188 return -1; 189 return READ_ONCE(evtchn_to_irq[EVTCHN_ROW(evtchn)][EVTCHN_COL(evtchn)]); 190 } 191 192 /* Get info for IRQ */ 193 struct irq_info *info_for_irq(unsigned irq) 194 { 195 if (irq < nr_legacy_irqs()) 196 return legacy_info_ptrs[irq]; 197 else 198 return irq_get_chip_data(irq); 199 } 200 201 static void set_info_for_irq(unsigned int irq, struct irq_info *info) 202 { 203 if (irq < nr_legacy_irqs()) 204 legacy_info_ptrs[irq] = info; 205 else 206 irq_set_chip_data(irq, info); 207 } 208 209 /* Constructors for packed IRQ information. */ 210 static int xen_irq_info_common_setup(struct irq_info *info, 211 unsigned irq, 212 enum xen_irq_type type, 213 evtchn_port_t evtchn, 214 unsigned short cpu) 215 { 216 int ret; 217 218 BUG_ON(info->type != IRQT_UNBOUND && info->type != type); 219 220 info->type = type; 221 info->irq = irq; 222 info->evtchn = evtchn; 223 info->cpu = cpu; 224 225 ret = set_evtchn_to_irq(evtchn, irq); 226 if (ret < 0) 227 return ret; 228 229 irq_clear_status_flags(irq, IRQ_NOREQUEST|IRQ_NOAUTOEN); 230 231 return xen_evtchn_port_setup(info); 232 } 233 234 static int xen_irq_info_evtchn_setup(unsigned irq, 235 evtchn_port_t evtchn) 236 { 237 struct irq_info *info = info_for_irq(irq); 238 239 return xen_irq_info_common_setup(info, irq, IRQT_EVTCHN, evtchn, 0); 240 } 241 242 static int xen_irq_info_ipi_setup(unsigned cpu, 243 unsigned irq, 244 evtchn_port_t evtchn, 245 enum ipi_vector ipi) 246 { 247 struct irq_info *info = info_for_irq(irq); 248 249 info->u.ipi = ipi; 250 251 per_cpu(ipi_to_irq, cpu)[ipi] = irq; 252 253 return xen_irq_info_common_setup(info, irq, IRQT_IPI, evtchn, 0); 254 } 255 256 static int xen_irq_info_virq_setup(unsigned cpu, 257 unsigned irq, 258 evtchn_port_t evtchn, 259 unsigned virq) 260 { 261 struct irq_info *info = info_for_irq(irq); 262 263 info->u.virq = virq; 264 265 per_cpu(virq_to_irq, cpu)[virq] = irq; 266 267 return xen_irq_info_common_setup(info, irq, IRQT_VIRQ, evtchn, 0); 268 } 269 270 static int xen_irq_info_pirq_setup(unsigned irq, 271 evtchn_port_t evtchn, 272 unsigned pirq, 273 unsigned gsi, 274 uint16_t domid, 275 unsigned char flags) 276 { 277 struct irq_info *info = info_for_irq(irq); 278 279 info->u.pirq.pirq = pirq; 280 info->u.pirq.gsi = gsi; 281 info->u.pirq.domid = domid; 282 info->u.pirq.flags = flags; 283 284 return xen_irq_info_common_setup(info, irq, IRQT_PIRQ, evtchn, 0); 285 } 286 287 static void xen_irq_info_cleanup(struct irq_info *info) 288 { 289 set_evtchn_to_irq(info->evtchn, -1); 290 info->evtchn = 0; 291 } 292 293 /* 294 * Accessors for packed IRQ information. 295 */ 296 evtchn_port_t evtchn_from_irq(unsigned irq) 297 { 298 const struct irq_info *info = NULL; 299 300 if (likely(irq < nr_irqs)) 301 info = info_for_irq(irq); 302 if (!info) 303 return 0; 304 305 return info->evtchn; 306 } 307 308 unsigned int irq_from_evtchn(evtchn_port_t evtchn) 309 { 310 return get_evtchn_to_irq(evtchn); 311 } 312 EXPORT_SYMBOL_GPL(irq_from_evtchn); 313 314 int irq_from_virq(unsigned int cpu, unsigned int virq) 315 { 316 return per_cpu(virq_to_irq, cpu)[virq]; 317 } 318 319 static enum ipi_vector ipi_from_irq(unsigned irq) 320 { 321 struct irq_info *info = info_for_irq(irq); 322 323 BUG_ON(info == NULL); 324 BUG_ON(info->type != IRQT_IPI); 325 326 return info->u.ipi; 327 } 328 329 static unsigned virq_from_irq(unsigned irq) 330 { 331 struct irq_info *info = info_for_irq(irq); 332 333 BUG_ON(info == NULL); 334 BUG_ON(info->type != IRQT_VIRQ); 335 336 return info->u.virq; 337 } 338 339 static unsigned pirq_from_irq(unsigned irq) 340 { 341 struct irq_info *info = info_for_irq(irq); 342 343 BUG_ON(info == NULL); 344 BUG_ON(info->type != IRQT_PIRQ); 345 346 return info->u.pirq.pirq; 347 } 348 349 static enum xen_irq_type type_from_irq(unsigned irq) 350 { 351 return info_for_irq(irq)->type; 352 } 353 354 unsigned cpu_from_irq(unsigned irq) 355 { 356 return info_for_irq(irq)->cpu; 357 } 358 359 unsigned int cpu_from_evtchn(evtchn_port_t evtchn) 360 { 361 int irq = get_evtchn_to_irq(evtchn); 362 unsigned ret = 0; 363 364 if (irq != -1) 365 ret = cpu_from_irq(irq); 366 367 return ret; 368 } 369 370 #ifdef CONFIG_X86 371 static bool pirq_check_eoi_map(unsigned irq) 372 { 373 return test_bit(pirq_from_irq(irq), pirq_eoi_map); 374 } 375 #endif 376 377 static bool pirq_needs_eoi_flag(unsigned irq) 378 { 379 struct irq_info *info = info_for_irq(irq); 380 BUG_ON(info->type != IRQT_PIRQ); 381 382 return info->u.pirq.flags & PIRQ_NEEDS_EOI; 383 } 384 385 static void bind_evtchn_to_cpu(evtchn_port_t evtchn, unsigned int cpu) 386 { 387 int irq = get_evtchn_to_irq(evtchn); 388 struct irq_info *info = info_for_irq(irq); 389 390 BUG_ON(irq == -1); 391 #ifdef CONFIG_SMP 392 cpumask_copy(irq_get_affinity_mask(irq), cpumask_of(cpu)); 393 #endif 394 xen_evtchn_port_bind_to_cpu(info, cpu); 395 396 info->cpu = cpu; 397 } 398 399 /** 400 * notify_remote_via_irq - send event to remote end of event channel via irq 401 * @irq: irq of event channel to send event to 402 * 403 * Unlike notify_remote_via_evtchn(), this is safe to use across 404 * save/restore. Notifications on a broken connection are silently 405 * dropped. 406 */ 407 void notify_remote_via_irq(int irq) 408 { 409 evtchn_port_t evtchn = evtchn_from_irq(irq); 410 411 if (VALID_EVTCHN(evtchn)) 412 notify_remote_via_evtchn(evtchn); 413 } 414 EXPORT_SYMBOL_GPL(notify_remote_via_irq); 415 416 struct lateeoi_work { 417 struct delayed_work delayed; 418 spinlock_t eoi_list_lock; 419 struct list_head eoi_list; 420 }; 421 422 static DEFINE_PER_CPU(struct lateeoi_work, lateeoi); 423 424 static void lateeoi_list_del(struct irq_info *info) 425 { 426 struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu); 427 unsigned long flags; 428 429 spin_lock_irqsave(&eoi->eoi_list_lock, flags); 430 list_del_init(&info->eoi_list); 431 spin_unlock_irqrestore(&eoi->eoi_list_lock, flags); 432 } 433 434 static void lateeoi_list_add(struct irq_info *info) 435 { 436 struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu); 437 struct irq_info *elem; 438 u64 now = get_jiffies_64(); 439 unsigned long delay; 440 unsigned long flags; 441 442 if (now < info->eoi_time) 443 delay = info->eoi_time - now; 444 else 445 delay = 1; 446 447 spin_lock_irqsave(&eoi->eoi_list_lock, flags); 448 449 if (list_empty(&eoi->eoi_list)) { 450 list_add(&info->eoi_list, &eoi->eoi_list); 451 mod_delayed_work_on(info->eoi_cpu, system_wq, 452 &eoi->delayed, delay); 453 } else { 454 list_for_each_entry_reverse(elem, &eoi->eoi_list, eoi_list) { 455 if (elem->eoi_time <= info->eoi_time) 456 break; 457 } 458 list_add(&info->eoi_list, &elem->eoi_list); 459 } 460 461 spin_unlock_irqrestore(&eoi->eoi_list_lock, flags); 462 } 463 464 static void xen_irq_lateeoi_locked(struct irq_info *info, bool spurious) 465 { 466 evtchn_port_t evtchn; 467 unsigned int cpu; 468 unsigned int delay = 0; 469 470 evtchn = info->evtchn; 471 if (!VALID_EVTCHN(evtchn) || !list_empty(&info->eoi_list)) 472 return; 473 474 if (spurious) { 475 if ((1 << info->spurious_cnt) < (HZ << 2)) 476 info->spurious_cnt++; 477 if (info->spurious_cnt > 1) { 478 delay = 1 << (info->spurious_cnt - 2); 479 if (delay > HZ) 480 delay = HZ; 481 if (!info->eoi_time) 482 info->eoi_cpu = smp_processor_id(); 483 info->eoi_time = get_jiffies_64() + delay; 484 } 485 } else { 486 info->spurious_cnt = 0; 487 } 488 489 cpu = info->eoi_cpu; 490 if (info->eoi_time && 491 (info->irq_epoch == per_cpu(irq_epoch, cpu) || delay)) { 492 lateeoi_list_add(info); 493 return; 494 } 495 496 info->eoi_time = 0; 497 unmask_evtchn(evtchn); 498 } 499 500 static void xen_irq_lateeoi_worker(struct work_struct *work) 501 { 502 struct lateeoi_work *eoi; 503 struct irq_info *info; 504 u64 now = get_jiffies_64(); 505 unsigned long flags; 506 507 eoi = container_of(to_delayed_work(work), struct lateeoi_work, delayed); 508 509 read_lock_irqsave(&evtchn_rwlock, flags); 510 511 while (true) { 512 spin_lock(&eoi->eoi_list_lock); 513 514 info = list_first_entry_or_null(&eoi->eoi_list, struct irq_info, 515 eoi_list); 516 517 if (info == NULL || now < info->eoi_time) { 518 spin_unlock(&eoi->eoi_list_lock); 519 break; 520 } 521 522 list_del_init(&info->eoi_list); 523 524 spin_unlock(&eoi->eoi_list_lock); 525 526 info->eoi_time = 0; 527 528 xen_irq_lateeoi_locked(info, false); 529 } 530 531 if (info) 532 mod_delayed_work_on(info->eoi_cpu, system_wq, 533 &eoi->delayed, info->eoi_time - now); 534 535 read_unlock_irqrestore(&evtchn_rwlock, flags); 536 } 537 538 static void xen_cpu_init_eoi(unsigned int cpu) 539 { 540 struct lateeoi_work *eoi = &per_cpu(lateeoi, cpu); 541 542 INIT_DELAYED_WORK(&eoi->delayed, xen_irq_lateeoi_worker); 543 spin_lock_init(&eoi->eoi_list_lock); 544 INIT_LIST_HEAD(&eoi->eoi_list); 545 } 546 547 void xen_irq_lateeoi(unsigned int irq, unsigned int eoi_flags) 548 { 549 struct irq_info *info; 550 unsigned long flags; 551 552 read_lock_irqsave(&evtchn_rwlock, flags); 553 554 info = info_for_irq(irq); 555 556 if (info) 557 xen_irq_lateeoi_locked(info, eoi_flags & XEN_EOI_FLAG_SPURIOUS); 558 559 read_unlock_irqrestore(&evtchn_rwlock, flags); 560 } 561 EXPORT_SYMBOL_GPL(xen_irq_lateeoi); 562 563 static void xen_irq_init(unsigned irq) 564 { 565 struct irq_info *info; 566 567 #ifdef CONFIG_SMP 568 /* By default all event channels notify CPU#0. */ 569 cpumask_copy(irq_get_affinity_mask(irq), cpumask_of(0)); 570 #endif 571 572 info = kzalloc(sizeof(*info), GFP_KERNEL); 573 if (info == NULL) 574 panic("Unable to allocate metadata for IRQ%d\n", irq); 575 576 info->type = IRQT_UNBOUND; 577 info->refcnt = -1; 578 579 set_info_for_irq(irq, info); 580 581 INIT_LIST_HEAD(&info->eoi_list); 582 list_add_tail(&info->list, &xen_irq_list_head); 583 } 584 585 static int __must_check xen_allocate_irqs_dynamic(int nvec) 586 { 587 int i, irq = irq_alloc_descs(-1, 0, nvec, -1); 588 589 if (irq >= 0) { 590 for (i = 0; i < nvec; i++) 591 xen_irq_init(irq + i); 592 } 593 594 return irq; 595 } 596 597 static inline int __must_check xen_allocate_irq_dynamic(void) 598 { 599 600 return xen_allocate_irqs_dynamic(1); 601 } 602 603 static int __must_check xen_allocate_irq_gsi(unsigned gsi) 604 { 605 int irq; 606 607 /* 608 * A PV guest has no concept of a GSI (since it has no ACPI 609 * nor access to/knowledge of the physical APICs). Therefore 610 * all IRQs are dynamically allocated from the entire IRQ 611 * space. 612 */ 613 if (xen_pv_domain() && !xen_initial_domain()) 614 return xen_allocate_irq_dynamic(); 615 616 /* Legacy IRQ descriptors are already allocated by the arch. */ 617 if (gsi < nr_legacy_irqs()) 618 irq = gsi; 619 else 620 irq = irq_alloc_desc_at(gsi, -1); 621 622 xen_irq_init(irq); 623 624 return irq; 625 } 626 627 static void xen_free_irq(unsigned irq) 628 { 629 struct irq_info *info = info_for_irq(irq); 630 unsigned long flags; 631 632 if (WARN_ON(!info)) 633 return; 634 635 write_lock_irqsave(&evtchn_rwlock, flags); 636 637 if (!list_empty(&info->eoi_list)) 638 lateeoi_list_del(info); 639 640 list_del(&info->list); 641 642 set_info_for_irq(irq, NULL); 643 644 WARN_ON(info->refcnt > 0); 645 646 write_unlock_irqrestore(&evtchn_rwlock, flags); 647 648 kfree(info); 649 650 /* Legacy IRQ descriptors are managed by the arch. */ 651 if (irq < nr_legacy_irqs()) 652 return; 653 654 irq_free_desc(irq); 655 } 656 657 static void xen_evtchn_close(evtchn_port_t port) 658 { 659 struct evtchn_close close; 660 661 close.port = port; 662 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0) 663 BUG(); 664 } 665 666 static void pirq_query_unmask(int irq) 667 { 668 struct physdev_irq_status_query irq_status; 669 struct irq_info *info = info_for_irq(irq); 670 671 BUG_ON(info->type != IRQT_PIRQ); 672 673 irq_status.irq = pirq_from_irq(irq); 674 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status)) 675 irq_status.flags = 0; 676 677 info->u.pirq.flags &= ~PIRQ_NEEDS_EOI; 678 if (irq_status.flags & XENIRQSTAT_needs_eoi) 679 info->u.pirq.flags |= PIRQ_NEEDS_EOI; 680 } 681 682 static void eoi_pirq(struct irq_data *data) 683 { 684 evtchn_port_t evtchn = evtchn_from_irq(data->irq); 685 struct physdev_eoi eoi = { .irq = pirq_from_irq(data->irq) }; 686 int rc = 0; 687 688 if (!VALID_EVTCHN(evtchn)) 689 return; 690 691 if (unlikely(irqd_is_setaffinity_pending(data)) && 692 likely(!irqd_irq_disabled(data))) { 693 int masked = test_and_set_mask(evtchn); 694 695 clear_evtchn(evtchn); 696 697 irq_move_masked_irq(data); 698 699 if (!masked) 700 unmask_evtchn(evtchn); 701 } else 702 clear_evtchn(evtchn); 703 704 if (pirq_needs_eoi(data->irq)) { 705 rc = HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi); 706 WARN_ON(rc); 707 } 708 } 709 710 static void mask_ack_pirq(struct irq_data *data) 711 { 712 disable_dynirq(data); 713 eoi_pirq(data); 714 } 715 716 static unsigned int __startup_pirq(unsigned int irq) 717 { 718 struct evtchn_bind_pirq bind_pirq; 719 struct irq_info *info = info_for_irq(irq); 720 evtchn_port_t evtchn = evtchn_from_irq(irq); 721 int rc; 722 723 BUG_ON(info->type != IRQT_PIRQ); 724 725 if (VALID_EVTCHN(evtchn)) 726 goto out; 727 728 bind_pirq.pirq = pirq_from_irq(irq); 729 /* NB. We are happy to share unless we are probing. */ 730 bind_pirq.flags = info->u.pirq.flags & PIRQ_SHAREABLE ? 731 BIND_PIRQ__WILL_SHARE : 0; 732 rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq); 733 if (rc != 0) { 734 pr_warn("Failed to obtain physical IRQ %d\n", irq); 735 return 0; 736 } 737 evtchn = bind_pirq.port; 738 739 pirq_query_unmask(irq); 740 741 rc = set_evtchn_to_irq(evtchn, irq); 742 if (rc) 743 goto err; 744 745 info->evtchn = evtchn; 746 bind_evtchn_to_cpu(evtchn, 0); 747 748 rc = xen_evtchn_port_setup(info); 749 if (rc) 750 goto err; 751 752 out: 753 unmask_evtchn(evtchn); 754 eoi_pirq(irq_get_irq_data(irq)); 755 756 return 0; 757 758 err: 759 pr_err("irq%d: Failed to set port to irq mapping (%d)\n", irq, rc); 760 xen_evtchn_close(evtchn); 761 return 0; 762 } 763 764 static unsigned int startup_pirq(struct irq_data *data) 765 { 766 return __startup_pirq(data->irq); 767 } 768 769 static void shutdown_pirq(struct irq_data *data) 770 { 771 unsigned int irq = data->irq; 772 struct irq_info *info = info_for_irq(irq); 773 evtchn_port_t evtchn = evtchn_from_irq(irq); 774 775 BUG_ON(info->type != IRQT_PIRQ); 776 777 if (!VALID_EVTCHN(evtchn)) 778 return; 779 780 mask_evtchn(evtchn); 781 xen_evtchn_close(evtchn); 782 xen_irq_info_cleanup(info); 783 } 784 785 static void enable_pirq(struct irq_data *data) 786 { 787 enable_dynirq(data); 788 } 789 790 static void disable_pirq(struct irq_data *data) 791 { 792 disable_dynirq(data); 793 } 794 795 int xen_irq_from_gsi(unsigned gsi) 796 { 797 struct irq_info *info; 798 799 list_for_each_entry(info, &xen_irq_list_head, list) { 800 if (info->type != IRQT_PIRQ) 801 continue; 802 803 if (info->u.pirq.gsi == gsi) 804 return info->irq; 805 } 806 807 return -1; 808 } 809 EXPORT_SYMBOL_GPL(xen_irq_from_gsi); 810 811 static void __unbind_from_irq(unsigned int irq) 812 { 813 evtchn_port_t evtchn = evtchn_from_irq(irq); 814 struct irq_info *info = info_for_irq(irq); 815 816 if (info->refcnt > 0) { 817 info->refcnt--; 818 if (info->refcnt != 0) 819 return; 820 } 821 822 if (VALID_EVTCHN(evtchn)) { 823 unsigned int cpu = cpu_from_irq(irq); 824 825 xen_evtchn_close(evtchn); 826 827 switch (type_from_irq(irq)) { 828 case IRQT_VIRQ: 829 per_cpu(virq_to_irq, cpu)[virq_from_irq(irq)] = -1; 830 break; 831 case IRQT_IPI: 832 per_cpu(ipi_to_irq, cpu)[ipi_from_irq(irq)] = -1; 833 break; 834 default: 835 break; 836 } 837 838 xen_irq_info_cleanup(info); 839 } 840 841 xen_free_irq(irq); 842 } 843 844 /* 845 * Do not make any assumptions regarding the relationship between the 846 * IRQ number returned here and the Xen pirq argument. 847 * 848 * Note: We don't assign an event channel until the irq actually started 849 * up. Return an existing irq if we've already got one for the gsi. 850 * 851 * Shareable implies level triggered, not shareable implies edge 852 * triggered here. 853 */ 854 int xen_bind_pirq_gsi_to_irq(unsigned gsi, 855 unsigned pirq, int shareable, char *name) 856 { 857 int irq = -1; 858 struct physdev_irq irq_op; 859 int ret; 860 861 mutex_lock(&irq_mapping_update_lock); 862 863 irq = xen_irq_from_gsi(gsi); 864 if (irq != -1) { 865 pr_info("%s: returning irq %d for gsi %u\n", 866 __func__, irq, gsi); 867 goto out; 868 } 869 870 irq = xen_allocate_irq_gsi(gsi); 871 if (irq < 0) 872 goto out; 873 874 irq_op.irq = irq; 875 irq_op.vector = 0; 876 877 /* Only the privileged domain can do this. For non-priv, the pcifront 878 * driver provides a PCI bus that does the call to do exactly 879 * this in the priv domain. */ 880 if (xen_initial_domain() && 881 HYPERVISOR_physdev_op(PHYSDEVOP_alloc_irq_vector, &irq_op)) { 882 xen_free_irq(irq); 883 irq = -ENOSPC; 884 goto out; 885 } 886 887 ret = xen_irq_info_pirq_setup(irq, 0, pirq, gsi, DOMID_SELF, 888 shareable ? PIRQ_SHAREABLE : 0); 889 if (ret < 0) { 890 __unbind_from_irq(irq); 891 irq = ret; 892 goto out; 893 } 894 895 pirq_query_unmask(irq); 896 /* We try to use the handler with the appropriate semantic for the 897 * type of interrupt: if the interrupt is an edge triggered 898 * interrupt we use handle_edge_irq. 899 * 900 * On the other hand if the interrupt is level triggered we use 901 * handle_fasteoi_irq like the native code does for this kind of 902 * interrupts. 903 * 904 * Depending on the Xen version, pirq_needs_eoi might return true 905 * not only for level triggered interrupts but for edge triggered 906 * interrupts too. In any case Xen always honors the eoi mechanism, 907 * not injecting any more pirqs of the same kind if the first one 908 * hasn't received an eoi yet. Therefore using the fasteoi handler 909 * is the right choice either way. 910 */ 911 if (shareable) 912 irq_set_chip_and_handler_name(irq, &xen_pirq_chip, 913 handle_fasteoi_irq, name); 914 else 915 irq_set_chip_and_handler_name(irq, &xen_pirq_chip, 916 handle_edge_irq, name); 917 918 out: 919 mutex_unlock(&irq_mapping_update_lock); 920 921 return irq; 922 } 923 924 #ifdef CONFIG_PCI_MSI 925 int xen_allocate_pirq_msi(struct pci_dev *dev, struct msi_desc *msidesc) 926 { 927 int rc; 928 struct physdev_get_free_pirq op_get_free_pirq; 929 930 op_get_free_pirq.type = MAP_PIRQ_TYPE_MSI; 931 rc = HYPERVISOR_physdev_op(PHYSDEVOP_get_free_pirq, &op_get_free_pirq); 932 933 WARN_ONCE(rc == -ENOSYS, 934 "hypervisor does not support the PHYSDEVOP_get_free_pirq interface\n"); 935 936 return rc ? -1 : op_get_free_pirq.pirq; 937 } 938 939 int xen_bind_pirq_msi_to_irq(struct pci_dev *dev, struct msi_desc *msidesc, 940 int pirq, int nvec, const char *name, domid_t domid) 941 { 942 int i, irq, ret; 943 944 mutex_lock(&irq_mapping_update_lock); 945 946 irq = xen_allocate_irqs_dynamic(nvec); 947 if (irq < 0) 948 goto out; 949 950 for (i = 0; i < nvec; i++) { 951 irq_set_chip_and_handler_name(irq + i, &xen_pirq_chip, handle_edge_irq, name); 952 953 ret = xen_irq_info_pirq_setup(irq + i, 0, pirq + i, 0, domid, 954 i == 0 ? 0 : PIRQ_MSI_GROUP); 955 if (ret < 0) 956 goto error_irq; 957 } 958 959 ret = irq_set_msi_desc(irq, msidesc); 960 if (ret < 0) 961 goto error_irq; 962 out: 963 mutex_unlock(&irq_mapping_update_lock); 964 return irq; 965 error_irq: 966 while (nvec--) 967 __unbind_from_irq(irq + nvec); 968 mutex_unlock(&irq_mapping_update_lock); 969 return ret; 970 } 971 #endif 972 973 int xen_destroy_irq(int irq) 974 { 975 struct physdev_unmap_pirq unmap_irq; 976 struct irq_info *info = info_for_irq(irq); 977 int rc = -ENOENT; 978 979 mutex_lock(&irq_mapping_update_lock); 980 981 /* 982 * If trying to remove a vector in a MSI group different 983 * than the first one skip the PIRQ unmap unless this vector 984 * is the first one in the group. 985 */ 986 if (xen_initial_domain() && !(info->u.pirq.flags & PIRQ_MSI_GROUP)) { 987 unmap_irq.pirq = info->u.pirq.pirq; 988 unmap_irq.domid = info->u.pirq.domid; 989 rc = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap_irq); 990 /* If another domain quits without making the pci_disable_msix 991 * call, the Xen hypervisor takes care of freeing the PIRQs 992 * (free_domain_pirqs). 993 */ 994 if ((rc == -ESRCH && info->u.pirq.domid != DOMID_SELF)) 995 pr_info("domain %d does not have %d anymore\n", 996 info->u.pirq.domid, info->u.pirq.pirq); 997 else if (rc) { 998 pr_warn("unmap irq failed %d\n", rc); 999 goto out; 1000 } 1001 } 1002 1003 xen_free_irq(irq); 1004 1005 out: 1006 mutex_unlock(&irq_mapping_update_lock); 1007 return rc; 1008 } 1009 1010 int xen_irq_from_pirq(unsigned pirq) 1011 { 1012 int irq; 1013 1014 struct irq_info *info; 1015 1016 mutex_lock(&irq_mapping_update_lock); 1017 1018 list_for_each_entry(info, &xen_irq_list_head, list) { 1019 if (info->type != IRQT_PIRQ) 1020 continue; 1021 irq = info->irq; 1022 if (info->u.pirq.pirq == pirq) 1023 goto out; 1024 } 1025 irq = -1; 1026 out: 1027 mutex_unlock(&irq_mapping_update_lock); 1028 1029 return irq; 1030 } 1031 1032 1033 int xen_pirq_from_irq(unsigned irq) 1034 { 1035 return pirq_from_irq(irq); 1036 } 1037 EXPORT_SYMBOL_GPL(xen_pirq_from_irq); 1038 1039 static int bind_evtchn_to_irq_chip(evtchn_port_t evtchn, struct irq_chip *chip) 1040 { 1041 int irq; 1042 int ret; 1043 1044 if (evtchn >= xen_evtchn_max_channels()) 1045 return -ENOMEM; 1046 1047 mutex_lock(&irq_mapping_update_lock); 1048 1049 irq = get_evtchn_to_irq(evtchn); 1050 1051 if (irq == -1) { 1052 irq = xen_allocate_irq_dynamic(); 1053 if (irq < 0) 1054 goto out; 1055 1056 irq_set_chip_and_handler_name(irq, chip, 1057 handle_edge_irq, "event"); 1058 1059 ret = xen_irq_info_evtchn_setup(irq, evtchn); 1060 if (ret < 0) { 1061 __unbind_from_irq(irq); 1062 irq = ret; 1063 goto out; 1064 } 1065 /* New interdomain events are bound to VCPU 0. */ 1066 bind_evtchn_to_cpu(evtchn, 0); 1067 } else { 1068 struct irq_info *info = info_for_irq(irq); 1069 WARN_ON(info == NULL || info->type != IRQT_EVTCHN); 1070 } 1071 1072 out: 1073 mutex_unlock(&irq_mapping_update_lock); 1074 1075 return irq; 1076 } 1077 1078 int bind_evtchn_to_irq(evtchn_port_t evtchn) 1079 { 1080 return bind_evtchn_to_irq_chip(evtchn, &xen_dynamic_chip); 1081 } 1082 EXPORT_SYMBOL_GPL(bind_evtchn_to_irq); 1083 1084 int bind_evtchn_to_irq_lateeoi(evtchn_port_t evtchn) 1085 { 1086 return bind_evtchn_to_irq_chip(evtchn, &xen_lateeoi_chip); 1087 } 1088 EXPORT_SYMBOL_GPL(bind_evtchn_to_irq_lateeoi); 1089 1090 static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu) 1091 { 1092 struct evtchn_bind_ipi bind_ipi; 1093 evtchn_port_t evtchn; 1094 int ret, irq; 1095 1096 mutex_lock(&irq_mapping_update_lock); 1097 1098 irq = per_cpu(ipi_to_irq, cpu)[ipi]; 1099 1100 if (irq == -1) { 1101 irq = xen_allocate_irq_dynamic(); 1102 if (irq < 0) 1103 goto out; 1104 1105 irq_set_chip_and_handler_name(irq, &xen_percpu_chip, 1106 handle_percpu_irq, "ipi"); 1107 1108 bind_ipi.vcpu = xen_vcpu_nr(cpu); 1109 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, 1110 &bind_ipi) != 0) 1111 BUG(); 1112 evtchn = bind_ipi.port; 1113 1114 ret = xen_irq_info_ipi_setup(cpu, irq, evtchn, ipi); 1115 if (ret < 0) { 1116 __unbind_from_irq(irq); 1117 irq = ret; 1118 goto out; 1119 } 1120 bind_evtchn_to_cpu(evtchn, cpu); 1121 } else { 1122 struct irq_info *info = info_for_irq(irq); 1123 WARN_ON(info == NULL || info->type != IRQT_IPI); 1124 } 1125 1126 out: 1127 mutex_unlock(&irq_mapping_update_lock); 1128 return irq; 1129 } 1130 1131 static int bind_interdomain_evtchn_to_irq_chip(unsigned int remote_domain, 1132 evtchn_port_t remote_port, 1133 struct irq_chip *chip) 1134 { 1135 struct evtchn_bind_interdomain bind_interdomain; 1136 int err; 1137 1138 bind_interdomain.remote_dom = remote_domain; 1139 bind_interdomain.remote_port = remote_port; 1140 1141 err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain, 1142 &bind_interdomain); 1143 1144 return err ? : bind_evtchn_to_irq_chip(bind_interdomain.local_port, 1145 chip); 1146 } 1147 1148 int bind_interdomain_evtchn_to_irq(unsigned int remote_domain, 1149 evtchn_port_t remote_port) 1150 { 1151 return bind_interdomain_evtchn_to_irq_chip(remote_domain, remote_port, 1152 &xen_dynamic_chip); 1153 } 1154 EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irq); 1155 1156 int bind_interdomain_evtchn_to_irq_lateeoi(unsigned int remote_domain, 1157 evtchn_port_t remote_port) 1158 { 1159 return bind_interdomain_evtchn_to_irq_chip(remote_domain, remote_port, 1160 &xen_lateeoi_chip); 1161 } 1162 EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irq_lateeoi); 1163 1164 static int find_virq(unsigned int virq, unsigned int cpu, evtchn_port_t *evtchn) 1165 { 1166 struct evtchn_status status; 1167 evtchn_port_t port; 1168 int rc = -ENOENT; 1169 1170 memset(&status, 0, sizeof(status)); 1171 for (port = 0; port < xen_evtchn_max_channels(); port++) { 1172 status.dom = DOMID_SELF; 1173 status.port = port; 1174 rc = HYPERVISOR_event_channel_op(EVTCHNOP_status, &status); 1175 if (rc < 0) 1176 continue; 1177 if (status.status != EVTCHNSTAT_virq) 1178 continue; 1179 if (status.u.virq == virq && status.vcpu == xen_vcpu_nr(cpu)) { 1180 *evtchn = port; 1181 break; 1182 } 1183 } 1184 return rc; 1185 } 1186 1187 /** 1188 * xen_evtchn_nr_channels - number of usable event channel ports 1189 * 1190 * This may be less than the maximum supported by the current 1191 * hypervisor ABI. Use xen_evtchn_max_channels() for the maximum 1192 * supported. 1193 */ 1194 unsigned xen_evtchn_nr_channels(void) 1195 { 1196 return evtchn_ops->nr_channels(); 1197 } 1198 EXPORT_SYMBOL_GPL(xen_evtchn_nr_channels); 1199 1200 int bind_virq_to_irq(unsigned int virq, unsigned int cpu, bool percpu) 1201 { 1202 struct evtchn_bind_virq bind_virq; 1203 evtchn_port_t evtchn = 0; 1204 int irq, ret; 1205 1206 mutex_lock(&irq_mapping_update_lock); 1207 1208 irq = per_cpu(virq_to_irq, cpu)[virq]; 1209 1210 if (irq == -1) { 1211 irq = xen_allocate_irq_dynamic(); 1212 if (irq < 0) 1213 goto out; 1214 1215 if (percpu) 1216 irq_set_chip_and_handler_name(irq, &xen_percpu_chip, 1217 handle_percpu_irq, "virq"); 1218 else 1219 irq_set_chip_and_handler_name(irq, &xen_dynamic_chip, 1220 handle_edge_irq, "virq"); 1221 1222 bind_virq.virq = virq; 1223 bind_virq.vcpu = xen_vcpu_nr(cpu); 1224 ret = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, 1225 &bind_virq); 1226 if (ret == 0) 1227 evtchn = bind_virq.port; 1228 else { 1229 if (ret == -EEXIST) 1230 ret = find_virq(virq, cpu, &evtchn); 1231 BUG_ON(ret < 0); 1232 } 1233 1234 ret = xen_irq_info_virq_setup(cpu, irq, evtchn, virq); 1235 if (ret < 0) { 1236 __unbind_from_irq(irq); 1237 irq = ret; 1238 goto out; 1239 } 1240 1241 bind_evtchn_to_cpu(evtchn, cpu); 1242 } else { 1243 struct irq_info *info = info_for_irq(irq); 1244 WARN_ON(info == NULL || info->type != IRQT_VIRQ); 1245 } 1246 1247 out: 1248 mutex_unlock(&irq_mapping_update_lock); 1249 1250 return irq; 1251 } 1252 1253 static void unbind_from_irq(unsigned int irq) 1254 { 1255 mutex_lock(&irq_mapping_update_lock); 1256 __unbind_from_irq(irq); 1257 mutex_unlock(&irq_mapping_update_lock); 1258 } 1259 1260 static int bind_evtchn_to_irqhandler_chip(evtchn_port_t evtchn, 1261 irq_handler_t handler, 1262 unsigned long irqflags, 1263 const char *devname, void *dev_id, 1264 struct irq_chip *chip) 1265 { 1266 int irq, retval; 1267 1268 irq = bind_evtchn_to_irq_chip(evtchn, chip); 1269 if (irq < 0) 1270 return irq; 1271 retval = request_irq(irq, handler, irqflags, devname, dev_id); 1272 if (retval != 0) { 1273 unbind_from_irq(irq); 1274 return retval; 1275 } 1276 1277 return irq; 1278 } 1279 1280 int bind_evtchn_to_irqhandler(evtchn_port_t evtchn, 1281 irq_handler_t handler, 1282 unsigned long irqflags, 1283 const char *devname, void *dev_id) 1284 { 1285 return bind_evtchn_to_irqhandler_chip(evtchn, handler, irqflags, 1286 devname, dev_id, 1287 &xen_dynamic_chip); 1288 } 1289 EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler); 1290 1291 int bind_evtchn_to_irqhandler_lateeoi(evtchn_port_t evtchn, 1292 irq_handler_t handler, 1293 unsigned long irqflags, 1294 const char *devname, void *dev_id) 1295 { 1296 return bind_evtchn_to_irqhandler_chip(evtchn, handler, irqflags, 1297 devname, dev_id, 1298 &xen_lateeoi_chip); 1299 } 1300 EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler_lateeoi); 1301 1302 static int bind_interdomain_evtchn_to_irqhandler_chip( 1303 unsigned int remote_domain, evtchn_port_t remote_port, 1304 irq_handler_t handler, unsigned long irqflags, 1305 const char *devname, void *dev_id, struct irq_chip *chip) 1306 { 1307 int irq, retval; 1308 1309 irq = bind_interdomain_evtchn_to_irq_chip(remote_domain, remote_port, 1310 chip); 1311 if (irq < 0) 1312 return irq; 1313 1314 retval = request_irq(irq, handler, irqflags, devname, dev_id); 1315 if (retval != 0) { 1316 unbind_from_irq(irq); 1317 return retval; 1318 } 1319 1320 return irq; 1321 } 1322 1323 int bind_interdomain_evtchn_to_irqhandler(unsigned int remote_domain, 1324 evtchn_port_t remote_port, 1325 irq_handler_t handler, 1326 unsigned long irqflags, 1327 const char *devname, 1328 void *dev_id) 1329 { 1330 return bind_interdomain_evtchn_to_irqhandler_chip(remote_domain, 1331 remote_port, handler, irqflags, devname, 1332 dev_id, &xen_dynamic_chip); 1333 } 1334 EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irqhandler); 1335 1336 int bind_interdomain_evtchn_to_irqhandler_lateeoi(unsigned int remote_domain, 1337 evtchn_port_t remote_port, 1338 irq_handler_t handler, 1339 unsigned long irqflags, 1340 const char *devname, 1341 void *dev_id) 1342 { 1343 return bind_interdomain_evtchn_to_irqhandler_chip(remote_domain, 1344 remote_port, handler, irqflags, devname, 1345 dev_id, &xen_lateeoi_chip); 1346 } 1347 EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irqhandler_lateeoi); 1348 1349 int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu, 1350 irq_handler_t handler, 1351 unsigned long irqflags, const char *devname, void *dev_id) 1352 { 1353 int irq, retval; 1354 1355 irq = bind_virq_to_irq(virq, cpu, irqflags & IRQF_PERCPU); 1356 if (irq < 0) 1357 return irq; 1358 retval = request_irq(irq, handler, irqflags, devname, dev_id); 1359 if (retval != 0) { 1360 unbind_from_irq(irq); 1361 return retval; 1362 } 1363 1364 return irq; 1365 } 1366 EXPORT_SYMBOL_GPL(bind_virq_to_irqhandler); 1367 1368 int bind_ipi_to_irqhandler(enum ipi_vector ipi, 1369 unsigned int cpu, 1370 irq_handler_t handler, 1371 unsigned long irqflags, 1372 const char *devname, 1373 void *dev_id) 1374 { 1375 int irq, retval; 1376 1377 irq = bind_ipi_to_irq(ipi, cpu); 1378 if (irq < 0) 1379 return irq; 1380 1381 irqflags |= IRQF_NO_SUSPEND | IRQF_FORCE_RESUME | IRQF_EARLY_RESUME; 1382 retval = request_irq(irq, handler, irqflags, devname, dev_id); 1383 if (retval != 0) { 1384 unbind_from_irq(irq); 1385 return retval; 1386 } 1387 1388 return irq; 1389 } 1390 1391 void unbind_from_irqhandler(unsigned int irq, void *dev_id) 1392 { 1393 struct irq_info *info = info_for_irq(irq); 1394 1395 if (WARN_ON(!info)) 1396 return; 1397 free_irq(irq, dev_id); 1398 unbind_from_irq(irq); 1399 } 1400 EXPORT_SYMBOL_GPL(unbind_from_irqhandler); 1401 1402 /** 1403 * xen_set_irq_priority() - set an event channel priority. 1404 * @irq:irq bound to an event channel. 1405 * @priority: priority between XEN_IRQ_PRIORITY_MAX and XEN_IRQ_PRIORITY_MIN. 1406 */ 1407 int xen_set_irq_priority(unsigned irq, unsigned priority) 1408 { 1409 struct evtchn_set_priority set_priority; 1410 1411 set_priority.port = evtchn_from_irq(irq); 1412 set_priority.priority = priority; 1413 1414 return HYPERVISOR_event_channel_op(EVTCHNOP_set_priority, 1415 &set_priority); 1416 } 1417 EXPORT_SYMBOL_GPL(xen_set_irq_priority); 1418 1419 int evtchn_make_refcounted(evtchn_port_t evtchn) 1420 { 1421 int irq = get_evtchn_to_irq(evtchn); 1422 struct irq_info *info; 1423 1424 if (irq == -1) 1425 return -ENOENT; 1426 1427 info = info_for_irq(irq); 1428 1429 if (!info) 1430 return -ENOENT; 1431 1432 WARN_ON(info->refcnt != -1); 1433 1434 info->refcnt = 1; 1435 1436 return 0; 1437 } 1438 EXPORT_SYMBOL_GPL(evtchn_make_refcounted); 1439 1440 int evtchn_get(evtchn_port_t evtchn) 1441 { 1442 int irq; 1443 struct irq_info *info; 1444 int err = -ENOENT; 1445 1446 if (evtchn >= xen_evtchn_max_channels()) 1447 return -EINVAL; 1448 1449 mutex_lock(&irq_mapping_update_lock); 1450 1451 irq = get_evtchn_to_irq(evtchn); 1452 if (irq == -1) 1453 goto done; 1454 1455 info = info_for_irq(irq); 1456 1457 if (!info) 1458 goto done; 1459 1460 err = -EINVAL; 1461 if (info->refcnt <= 0 || info->refcnt == SHRT_MAX) 1462 goto done; 1463 1464 info->refcnt++; 1465 err = 0; 1466 done: 1467 mutex_unlock(&irq_mapping_update_lock); 1468 1469 return err; 1470 } 1471 EXPORT_SYMBOL_GPL(evtchn_get); 1472 1473 void evtchn_put(evtchn_port_t evtchn) 1474 { 1475 int irq = get_evtchn_to_irq(evtchn); 1476 if (WARN_ON(irq == -1)) 1477 return; 1478 unbind_from_irq(irq); 1479 } 1480 EXPORT_SYMBOL_GPL(evtchn_put); 1481 1482 void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector) 1483 { 1484 int irq; 1485 1486 #ifdef CONFIG_X86 1487 if (unlikely(vector == XEN_NMI_VECTOR)) { 1488 int rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, xen_vcpu_nr(cpu), 1489 NULL); 1490 if (rc < 0) 1491 printk(KERN_WARNING "Sending nmi to CPU%d failed (rc:%d)\n", cpu, rc); 1492 return; 1493 } 1494 #endif 1495 irq = per_cpu(ipi_to_irq, cpu)[vector]; 1496 BUG_ON(irq < 0); 1497 notify_remote_via_irq(irq); 1498 } 1499 1500 struct evtchn_loop_ctrl { 1501 ktime_t timeout; 1502 unsigned count; 1503 bool defer_eoi; 1504 }; 1505 1506 void handle_irq_for_port(evtchn_port_t port, struct evtchn_loop_ctrl *ctrl) 1507 { 1508 int irq; 1509 struct irq_info *info; 1510 1511 irq = get_evtchn_to_irq(port); 1512 if (irq == -1) 1513 return; 1514 1515 /* 1516 * Check for timeout every 256 events. 1517 * We are setting the timeout value only after the first 256 1518 * events in order to not hurt the common case of few loop 1519 * iterations. The 256 is basically an arbitrary value. 1520 * 1521 * In case we are hitting the timeout we need to defer all further 1522 * EOIs in order to ensure to leave the event handling loop rather 1523 * sooner than later. 1524 */ 1525 if (!ctrl->defer_eoi && !(++ctrl->count & 0xff)) { 1526 ktime_t kt = ktime_get(); 1527 1528 if (!ctrl->timeout) { 1529 kt = ktime_add_ms(kt, 1530 jiffies_to_msecs(event_loop_timeout)); 1531 ctrl->timeout = kt; 1532 } else if (kt > ctrl->timeout) { 1533 ctrl->defer_eoi = true; 1534 } 1535 } 1536 1537 info = info_for_irq(irq); 1538 1539 if (ctrl->defer_eoi) { 1540 info->eoi_cpu = smp_processor_id(); 1541 info->irq_epoch = __this_cpu_read(irq_epoch); 1542 info->eoi_time = get_jiffies_64() + event_eoi_delay; 1543 } 1544 1545 generic_handle_irq(irq); 1546 } 1547 1548 static void __xen_evtchn_do_upcall(void) 1549 { 1550 struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu); 1551 int cpu = smp_processor_id(); 1552 struct evtchn_loop_ctrl ctrl = { 0 }; 1553 1554 read_lock(&evtchn_rwlock); 1555 1556 do { 1557 vcpu_info->evtchn_upcall_pending = 0; 1558 1559 xen_evtchn_handle_events(cpu, &ctrl); 1560 1561 BUG_ON(!irqs_disabled()); 1562 1563 virt_rmb(); /* Hypervisor can set upcall pending. */ 1564 1565 } while (vcpu_info->evtchn_upcall_pending); 1566 1567 read_unlock(&evtchn_rwlock); 1568 1569 /* 1570 * Increment irq_epoch only now to defer EOIs only for 1571 * xen_irq_lateeoi() invocations occurring from inside the loop 1572 * above. 1573 */ 1574 __this_cpu_inc(irq_epoch); 1575 } 1576 1577 void xen_evtchn_do_upcall(struct pt_regs *regs) 1578 { 1579 struct pt_regs *old_regs = set_irq_regs(regs); 1580 1581 irq_enter(); 1582 1583 __xen_evtchn_do_upcall(); 1584 1585 irq_exit(); 1586 set_irq_regs(old_regs); 1587 } 1588 1589 void xen_hvm_evtchn_do_upcall(void) 1590 { 1591 __xen_evtchn_do_upcall(); 1592 } 1593 EXPORT_SYMBOL_GPL(xen_hvm_evtchn_do_upcall); 1594 1595 /* Rebind a new event channel to an existing irq. */ 1596 void rebind_evtchn_irq(evtchn_port_t evtchn, int irq) 1597 { 1598 struct irq_info *info = info_for_irq(irq); 1599 1600 if (WARN_ON(!info)) 1601 return; 1602 1603 /* Make sure the irq is masked, since the new event channel 1604 will also be masked. */ 1605 disable_irq(irq); 1606 1607 mutex_lock(&irq_mapping_update_lock); 1608 1609 /* After resume the irq<->evtchn mappings are all cleared out */ 1610 BUG_ON(get_evtchn_to_irq(evtchn) != -1); 1611 /* Expect irq to have been bound before, 1612 so there should be a proper type */ 1613 BUG_ON(info->type == IRQT_UNBOUND); 1614 1615 (void)xen_irq_info_evtchn_setup(irq, evtchn); 1616 1617 mutex_unlock(&irq_mapping_update_lock); 1618 1619 bind_evtchn_to_cpu(evtchn, info->cpu); 1620 /* This will be deferred until interrupt is processed */ 1621 irq_set_affinity(irq, cpumask_of(info->cpu)); 1622 1623 /* Unmask the event channel. */ 1624 enable_irq(irq); 1625 } 1626 1627 /* Rebind an evtchn so that it gets delivered to a specific cpu */ 1628 static int xen_rebind_evtchn_to_cpu(evtchn_port_t evtchn, unsigned int tcpu) 1629 { 1630 struct evtchn_bind_vcpu bind_vcpu; 1631 int masked; 1632 1633 if (!VALID_EVTCHN(evtchn)) 1634 return -1; 1635 1636 if (!xen_support_evtchn_rebind()) 1637 return -1; 1638 1639 /* Send future instances of this interrupt to other vcpu. */ 1640 bind_vcpu.port = evtchn; 1641 bind_vcpu.vcpu = xen_vcpu_nr(tcpu); 1642 1643 /* 1644 * Mask the event while changing the VCPU binding to prevent 1645 * it being delivered on an unexpected VCPU. 1646 */ 1647 masked = test_and_set_mask(evtchn); 1648 1649 /* 1650 * If this fails, it usually just indicates that we're dealing with a 1651 * virq or IPI channel, which don't actually need to be rebound. Ignore 1652 * it, but don't do the xenlinux-level rebind in that case. 1653 */ 1654 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu) >= 0) 1655 bind_evtchn_to_cpu(evtchn, tcpu); 1656 1657 if (!masked) 1658 unmask_evtchn(evtchn); 1659 1660 return 0; 1661 } 1662 1663 static int set_affinity_irq(struct irq_data *data, const struct cpumask *dest, 1664 bool force) 1665 { 1666 unsigned tcpu = cpumask_first_and(dest, cpu_online_mask); 1667 int ret = xen_rebind_evtchn_to_cpu(evtchn_from_irq(data->irq), tcpu); 1668 1669 if (!ret) 1670 irq_data_update_effective_affinity(data, cpumask_of(tcpu)); 1671 1672 return ret; 1673 } 1674 1675 /* To be called with desc->lock held. */ 1676 int xen_set_affinity_evtchn(struct irq_desc *desc, unsigned int tcpu) 1677 { 1678 struct irq_data *d = irq_desc_get_irq_data(desc); 1679 1680 return set_affinity_irq(d, cpumask_of(tcpu), false); 1681 } 1682 EXPORT_SYMBOL_GPL(xen_set_affinity_evtchn); 1683 1684 static void enable_dynirq(struct irq_data *data) 1685 { 1686 evtchn_port_t evtchn = evtchn_from_irq(data->irq); 1687 1688 if (VALID_EVTCHN(evtchn)) 1689 unmask_evtchn(evtchn); 1690 } 1691 1692 static void disable_dynirq(struct irq_data *data) 1693 { 1694 evtchn_port_t evtchn = evtchn_from_irq(data->irq); 1695 1696 if (VALID_EVTCHN(evtchn)) 1697 mask_evtchn(evtchn); 1698 } 1699 1700 static void ack_dynirq(struct irq_data *data) 1701 { 1702 evtchn_port_t evtchn = evtchn_from_irq(data->irq); 1703 1704 if (!VALID_EVTCHN(evtchn)) 1705 return; 1706 1707 if (unlikely(irqd_is_setaffinity_pending(data)) && 1708 likely(!irqd_irq_disabled(data))) { 1709 int masked = test_and_set_mask(evtchn); 1710 1711 clear_evtchn(evtchn); 1712 1713 irq_move_masked_irq(data); 1714 1715 if (!masked) 1716 unmask_evtchn(evtchn); 1717 } else 1718 clear_evtchn(evtchn); 1719 } 1720 1721 static void mask_ack_dynirq(struct irq_data *data) 1722 { 1723 disable_dynirq(data); 1724 ack_dynirq(data); 1725 } 1726 1727 static int retrigger_dynirq(struct irq_data *data) 1728 { 1729 evtchn_port_t evtchn = evtchn_from_irq(data->irq); 1730 int masked; 1731 1732 if (!VALID_EVTCHN(evtchn)) 1733 return 0; 1734 1735 masked = test_and_set_mask(evtchn); 1736 set_evtchn(evtchn); 1737 if (!masked) 1738 unmask_evtchn(evtchn); 1739 1740 return 1; 1741 } 1742 1743 static void restore_pirqs(void) 1744 { 1745 int pirq, rc, irq, gsi; 1746 struct physdev_map_pirq map_irq; 1747 struct irq_info *info; 1748 1749 list_for_each_entry(info, &xen_irq_list_head, list) { 1750 if (info->type != IRQT_PIRQ) 1751 continue; 1752 1753 pirq = info->u.pirq.pirq; 1754 gsi = info->u.pirq.gsi; 1755 irq = info->irq; 1756 1757 /* save/restore of PT devices doesn't work, so at this point the 1758 * only devices present are GSI based emulated devices */ 1759 if (!gsi) 1760 continue; 1761 1762 map_irq.domid = DOMID_SELF; 1763 map_irq.type = MAP_PIRQ_TYPE_GSI; 1764 map_irq.index = gsi; 1765 map_irq.pirq = pirq; 1766 1767 rc = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_irq); 1768 if (rc) { 1769 pr_warn("xen map irq failed gsi=%d irq=%d pirq=%d rc=%d\n", 1770 gsi, irq, pirq, rc); 1771 xen_free_irq(irq); 1772 continue; 1773 } 1774 1775 printk(KERN_DEBUG "xen: --> irq=%d, pirq=%d\n", irq, map_irq.pirq); 1776 1777 __startup_pirq(irq); 1778 } 1779 } 1780 1781 static void restore_cpu_virqs(unsigned int cpu) 1782 { 1783 struct evtchn_bind_virq bind_virq; 1784 evtchn_port_t evtchn; 1785 int virq, irq; 1786 1787 for (virq = 0; virq < NR_VIRQS; virq++) { 1788 if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1) 1789 continue; 1790 1791 BUG_ON(virq_from_irq(irq) != virq); 1792 1793 /* Get a new binding from Xen. */ 1794 bind_virq.virq = virq; 1795 bind_virq.vcpu = xen_vcpu_nr(cpu); 1796 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, 1797 &bind_virq) != 0) 1798 BUG(); 1799 evtchn = bind_virq.port; 1800 1801 /* Record the new mapping. */ 1802 (void)xen_irq_info_virq_setup(cpu, irq, evtchn, virq); 1803 bind_evtchn_to_cpu(evtchn, cpu); 1804 } 1805 } 1806 1807 static void restore_cpu_ipis(unsigned int cpu) 1808 { 1809 struct evtchn_bind_ipi bind_ipi; 1810 evtchn_port_t evtchn; 1811 int ipi, irq; 1812 1813 for (ipi = 0; ipi < XEN_NR_IPIS; ipi++) { 1814 if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1) 1815 continue; 1816 1817 BUG_ON(ipi_from_irq(irq) != ipi); 1818 1819 /* Get a new binding from Xen. */ 1820 bind_ipi.vcpu = xen_vcpu_nr(cpu); 1821 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, 1822 &bind_ipi) != 0) 1823 BUG(); 1824 evtchn = bind_ipi.port; 1825 1826 /* Record the new mapping. */ 1827 (void)xen_irq_info_ipi_setup(cpu, irq, evtchn, ipi); 1828 bind_evtchn_to_cpu(evtchn, cpu); 1829 } 1830 } 1831 1832 /* Clear an irq's pending state, in preparation for polling on it */ 1833 void xen_clear_irq_pending(int irq) 1834 { 1835 evtchn_port_t evtchn = evtchn_from_irq(irq); 1836 1837 if (VALID_EVTCHN(evtchn)) 1838 clear_evtchn(evtchn); 1839 } 1840 EXPORT_SYMBOL(xen_clear_irq_pending); 1841 void xen_set_irq_pending(int irq) 1842 { 1843 evtchn_port_t evtchn = evtchn_from_irq(irq); 1844 1845 if (VALID_EVTCHN(evtchn)) 1846 set_evtchn(evtchn); 1847 } 1848 1849 bool xen_test_irq_pending(int irq) 1850 { 1851 evtchn_port_t evtchn = evtchn_from_irq(irq); 1852 bool ret = false; 1853 1854 if (VALID_EVTCHN(evtchn)) 1855 ret = test_evtchn(evtchn); 1856 1857 return ret; 1858 } 1859 1860 /* Poll waiting for an irq to become pending with timeout. In the usual case, 1861 * the irq will be disabled so it won't deliver an interrupt. */ 1862 void xen_poll_irq_timeout(int irq, u64 timeout) 1863 { 1864 evtchn_port_t evtchn = evtchn_from_irq(irq); 1865 1866 if (VALID_EVTCHN(evtchn)) { 1867 struct sched_poll poll; 1868 1869 poll.nr_ports = 1; 1870 poll.timeout = timeout; 1871 set_xen_guest_handle(poll.ports, &evtchn); 1872 1873 if (HYPERVISOR_sched_op(SCHEDOP_poll, &poll) != 0) 1874 BUG(); 1875 } 1876 } 1877 EXPORT_SYMBOL(xen_poll_irq_timeout); 1878 /* Poll waiting for an irq to become pending. In the usual case, the 1879 * irq will be disabled so it won't deliver an interrupt. */ 1880 void xen_poll_irq(int irq) 1881 { 1882 xen_poll_irq_timeout(irq, 0 /* no timeout */); 1883 } 1884 1885 /* Check whether the IRQ line is shared with other guests. */ 1886 int xen_test_irq_shared(int irq) 1887 { 1888 struct irq_info *info = info_for_irq(irq); 1889 struct physdev_irq_status_query irq_status; 1890 1891 if (WARN_ON(!info)) 1892 return -ENOENT; 1893 1894 irq_status.irq = info->u.pirq.pirq; 1895 1896 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status)) 1897 return 0; 1898 return !(irq_status.flags & XENIRQSTAT_shared); 1899 } 1900 EXPORT_SYMBOL_GPL(xen_test_irq_shared); 1901 1902 void xen_irq_resume(void) 1903 { 1904 unsigned int cpu; 1905 struct irq_info *info; 1906 1907 /* New event-channel space is not 'live' yet. */ 1908 xen_evtchn_resume(); 1909 1910 /* No IRQ <-> event-channel mappings. */ 1911 list_for_each_entry(info, &xen_irq_list_head, list) 1912 info->evtchn = 0; /* zap event-channel binding */ 1913 1914 clear_evtchn_to_irq_all(); 1915 1916 for_each_possible_cpu(cpu) { 1917 restore_cpu_virqs(cpu); 1918 restore_cpu_ipis(cpu); 1919 } 1920 1921 restore_pirqs(); 1922 } 1923 1924 static struct irq_chip xen_dynamic_chip __read_mostly = { 1925 .name = "xen-dyn", 1926 1927 .irq_disable = disable_dynirq, 1928 .irq_mask = disable_dynirq, 1929 .irq_unmask = enable_dynirq, 1930 1931 .irq_ack = ack_dynirq, 1932 .irq_mask_ack = mask_ack_dynirq, 1933 1934 .irq_set_affinity = set_affinity_irq, 1935 .irq_retrigger = retrigger_dynirq, 1936 }; 1937 1938 static struct irq_chip xen_lateeoi_chip __read_mostly = { 1939 /* The chip name needs to contain "xen-dyn" for irqbalance to work. */ 1940 .name = "xen-dyn-lateeoi", 1941 1942 .irq_disable = disable_dynirq, 1943 .irq_mask = disable_dynirq, 1944 .irq_unmask = enable_dynirq, 1945 1946 .irq_ack = mask_ack_dynirq, 1947 .irq_mask_ack = mask_ack_dynirq, 1948 1949 .irq_set_affinity = set_affinity_irq, 1950 .irq_retrigger = retrigger_dynirq, 1951 }; 1952 1953 static struct irq_chip xen_pirq_chip __read_mostly = { 1954 .name = "xen-pirq", 1955 1956 .irq_startup = startup_pirq, 1957 .irq_shutdown = shutdown_pirq, 1958 .irq_enable = enable_pirq, 1959 .irq_disable = disable_pirq, 1960 1961 .irq_mask = disable_dynirq, 1962 .irq_unmask = enable_dynirq, 1963 1964 .irq_ack = eoi_pirq, 1965 .irq_eoi = eoi_pirq, 1966 .irq_mask_ack = mask_ack_pirq, 1967 1968 .irq_set_affinity = set_affinity_irq, 1969 1970 .irq_retrigger = retrigger_dynirq, 1971 }; 1972 1973 static struct irq_chip xen_percpu_chip __read_mostly = { 1974 .name = "xen-percpu", 1975 1976 .irq_disable = disable_dynirq, 1977 .irq_mask = disable_dynirq, 1978 .irq_unmask = enable_dynirq, 1979 1980 .irq_ack = ack_dynirq, 1981 }; 1982 1983 int xen_set_callback_via(uint64_t via) 1984 { 1985 struct xen_hvm_param a; 1986 a.domid = DOMID_SELF; 1987 a.index = HVM_PARAM_CALLBACK_IRQ; 1988 a.value = via; 1989 return HYPERVISOR_hvm_op(HVMOP_set_param, &a); 1990 } 1991 EXPORT_SYMBOL_GPL(xen_set_callback_via); 1992 1993 #ifdef CONFIG_XEN_PVHVM 1994 /* Vector callbacks are better than PCI interrupts to receive event 1995 * channel notifications because we can receive vector callbacks on any 1996 * vcpu and we don't need PCI support or APIC interactions. */ 1997 void xen_setup_callback_vector(void) 1998 { 1999 uint64_t callback_via; 2000 2001 if (xen_have_vector_callback) { 2002 callback_via = HVM_CALLBACK_VECTOR(HYPERVISOR_CALLBACK_VECTOR); 2003 if (xen_set_callback_via(callback_via)) { 2004 pr_err("Request for Xen HVM callback vector failed\n"); 2005 xen_have_vector_callback = 0; 2006 } 2007 } 2008 } 2009 2010 static __init void xen_alloc_callback_vector(void) 2011 { 2012 if (!xen_have_vector_callback) 2013 return; 2014 2015 pr_info("Xen HVM callback vector for event delivery is enabled\n"); 2016 alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, asm_sysvec_xen_hvm_callback); 2017 } 2018 #else 2019 void xen_setup_callback_vector(void) {} 2020 static inline void xen_alloc_callback_vector(void) {} 2021 #endif 2022 2023 static bool fifo_events = true; 2024 module_param(fifo_events, bool, 0); 2025 2026 static int xen_evtchn_cpu_prepare(unsigned int cpu) 2027 { 2028 int ret = 0; 2029 2030 xen_cpu_init_eoi(cpu); 2031 2032 if (evtchn_ops->percpu_init) 2033 ret = evtchn_ops->percpu_init(cpu); 2034 2035 return ret; 2036 } 2037 2038 static int xen_evtchn_cpu_dead(unsigned int cpu) 2039 { 2040 int ret = 0; 2041 2042 if (evtchn_ops->percpu_deinit) 2043 ret = evtchn_ops->percpu_deinit(cpu); 2044 2045 return ret; 2046 } 2047 2048 void __init xen_init_IRQ(void) 2049 { 2050 int ret = -EINVAL; 2051 evtchn_port_t evtchn; 2052 2053 if (fifo_events) 2054 ret = xen_evtchn_fifo_init(); 2055 if (ret < 0) 2056 xen_evtchn_2l_init(); 2057 2058 xen_cpu_init_eoi(smp_processor_id()); 2059 2060 cpuhp_setup_state_nocalls(CPUHP_XEN_EVTCHN_PREPARE, 2061 "xen/evtchn:prepare", 2062 xen_evtchn_cpu_prepare, xen_evtchn_cpu_dead); 2063 2064 evtchn_to_irq = kcalloc(EVTCHN_ROW(xen_evtchn_max_channels()), 2065 sizeof(*evtchn_to_irq), GFP_KERNEL); 2066 BUG_ON(!evtchn_to_irq); 2067 2068 /* No event channels are 'live' right now. */ 2069 for (evtchn = 0; evtchn < xen_evtchn_nr_channels(); evtchn++) 2070 mask_evtchn(evtchn); 2071 2072 pirq_needs_eoi = pirq_needs_eoi_flag; 2073 2074 #ifdef CONFIG_X86 2075 if (xen_pv_domain()) { 2076 if (xen_initial_domain()) 2077 pci_xen_initial_domain(); 2078 } 2079 if (xen_feature(XENFEAT_hvm_callback_vector)) { 2080 xen_setup_callback_vector(); 2081 xen_alloc_callback_vector(); 2082 } 2083 2084 if (xen_hvm_domain()) { 2085 native_init_IRQ(); 2086 /* pci_xen_hvm_init must be called after native_init_IRQ so that 2087 * __acpi_register_gsi can point at the right function */ 2088 pci_xen_hvm_init(); 2089 } else { 2090 int rc; 2091 struct physdev_pirq_eoi_gmfn eoi_gmfn; 2092 2093 pirq_eoi_map = (void *)__get_free_page(GFP_KERNEL|__GFP_ZERO); 2094 eoi_gmfn.gmfn = virt_to_gfn(pirq_eoi_map); 2095 rc = HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn_v2, &eoi_gmfn); 2096 if (rc != 0) { 2097 free_page((unsigned long) pirq_eoi_map); 2098 pirq_eoi_map = NULL; 2099 } else 2100 pirq_needs_eoi = pirq_check_eoi_map; 2101 } 2102 #endif 2103 } 2104