1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2014 Intel Corp. 4 * Author: Jiang Liu <jiang.liu@linux.intel.com> 5 * 6 * This file is licensed under GPLv2. 7 * 8 * This file contains common code to support Message Signaled Interrupts for 9 * PCI compatible and non PCI compatible devices. 10 */ 11 #include <linux/types.h> 12 #include <linux/device.h> 13 #include <linux/irq.h> 14 #include <linux/irqdomain.h> 15 #include <linux/msi.h> 16 #include <linux/slab.h> 17 #include <linux/sysfs.h> 18 #include <linux/pci.h> 19 20 #include "internals.h" 21 22 /** 23 * struct msi_ctrl - MSI internal management control structure 24 * @domid: ID of the domain on which management operations should be done 25 * @first: First (hardware) slot index to operate on 26 * @last: Last (hardware) slot index to operate on 27 * @nirqs: The number of Linux interrupts to allocate. Can be larger 28 * than the range due to PCI/multi-MSI. 29 */ 30 struct msi_ctrl { 31 unsigned int domid; 32 unsigned int first; 33 unsigned int last; 34 unsigned int nirqs; 35 }; 36 37 /* Invalid Xarray index which is outside of any searchable range */ 38 #define MSI_XA_MAX_INDEX (ULONG_MAX - 1) 39 /* The maximum domain size */ 40 #define MSI_XA_DOMAIN_SIZE (MSI_MAX_INDEX + 1) 41 42 static void msi_domain_free_locked(struct device *dev, struct msi_ctrl *ctrl); 43 static unsigned int msi_domain_get_hwsize(struct device *dev, unsigned int domid); 44 static inline int msi_sysfs_create_group(struct device *dev); 45 46 47 /** 48 * msi_alloc_desc - Allocate an initialized msi_desc 49 * @dev: Pointer to the device for which this is allocated 50 * @nvec: The number of vectors used in this entry 51 * @affinity: Optional pointer to an affinity mask array size of @nvec 52 * 53 * If @affinity is not %NULL then an affinity array[@nvec] is allocated 54 * and the affinity masks and flags from @affinity are copied. 55 * 56 * Return: pointer to allocated &msi_desc on success or %NULL on failure 57 */ 58 static struct msi_desc *msi_alloc_desc(struct device *dev, int nvec, 59 const struct irq_affinity_desc *affinity) 60 { 61 struct msi_desc *desc = kzalloc(sizeof(*desc), GFP_KERNEL); 62 63 if (!desc) 64 return NULL; 65 66 desc->dev = dev; 67 desc->nvec_used = nvec; 68 if (affinity) { 69 desc->affinity = kmemdup(affinity, nvec * sizeof(*desc->affinity), GFP_KERNEL); 70 if (!desc->affinity) { 71 kfree(desc); 72 return NULL; 73 } 74 } 75 return desc; 76 } 77 78 static void msi_free_desc(struct msi_desc *desc) 79 { 80 kfree(desc->affinity); 81 kfree(desc); 82 } 83 84 static int msi_insert_desc(struct device *dev, struct msi_desc *desc, 85 unsigned int domid, unsigned int index) 86 { 87 struct msi_device_data *md = dev->msi.data; 88 struct xarray *xa = &md->__domains[domid].store; 89 unsigned int hwsize; 90 int ret; 91 92 hwsize = msi_domain_get_hwsize(dev, domid); 93 if (index >= hwsize) { 94 ret = -ERANGE; 95 goto fail; 96 } 97 98 desc->msi_index = index; 99 ret = xa_insert(xa, index, desc, GFP_KERNEL); 100 if (ret) 101 goto fail; 102 return 0; 103 104 fail: 105 msi_free_desc(desc); 106 return ret; 107 } 108 109 /** 110 * msi_domain_insert_msi_desc - Allocate and initialize a MSI descriptor and 111 * insert it at @init_desc->msi_index 112 * 113 * @dev: Pointer to the device for which the descriptor is allocated 114 * @domid: The id of the interrupt domain to which the desriptor is added 115 * @init_desc: Pointer to an MSI descriptor to initialize the new descriptor 116 * 117 * Return: 0 on success or an appropriate failure code. 118 */ 119 int msi_domain_insert_msi_desc(struct device *dev, unsigned int domid, 120 struct msi_desc *init_desc) 121 { 122 struct msi_desc *desc; 123 124 lockdep_assert_held(&dev->msi.data->mutex); 125 126 desc = msi_alloc_desc(dev, init_desc->nvec_used, init_desc->affinity); 127 if (!desc) 128 return -ENOMEM; 129 130 /* Copy type specific data to the new descriptor. */ 131 desc->pci = init_desc->pci; 132 133 return msi_insert_desc(dev, desc, domid, init_desc->msi_index); 134 } 135 136 static bool msi_desc_match(struct msi_desc *desc, enum msi_desc_filter filter) 137 { 138 switch (filter) { 139 case MSI_DESC_ALL: 140 return true; 141 case MSI_DESC_NOTASSOCIATED: 142 return !desc->irq; 143 case MSI_DESC_ASSOCIATED: 144 return !!desc->irq; 145 } 146 WARN_ON_ONCE(1); 147 return false; 148 } 149 150 static bool msi_ctrl_valid(struct device *dev, struct msi_ctrl *ctrl) 151 { 152 unsigned int hwsize; 153 154 if (WARN_ON_ONCE(ctrl->domid >= MSI_MAX_DEVICE_IRQDOMAINS || 155 !dev->msi.data->__domains[ctrl->domid].domain)) 156 return false; 157 158 hwsize = msi_domain_get_hwsize(dev, ctrl->domid); 159 if (WARN_ON_ONCE(ctrl->first > ctrl->last || 160 ctrl->first >= hwsize || 161 ctrl->last >= hwsize)) 162 return false; 163 return true; 164 } 165 166 static void msi_domain_free_descs(struct device *dev, struct msi_ctrl *ctrl) 167 { 168 struct msi_desc *desc; 169 struct xarray *xa; 170 unsigned long idx; 171 172 lockdep_assert_held(&dev->msi.data->mutex); 173 174 if (!msi_ctrl_valid(dev, ctrl)) 175 return; 176 177 xa = &dev->msi.data->__domains[ctrl->domid].store; 178 xa_for_each_range(xa, idx, desc, ctrl->first, ctrl->last) { 179 xa_erase(xa, idx); 180 181 /* Leak the descriptor when it is still referenced */ 182 if (WARN_ON_ONCE(msi_desc_match(desc, MSI_DESC_ASSOCIATED))) 183 continue; 184 msi_free_desc(desc); 185 } 186 } 187 188 /** 189 * msi_domain_free_msi_descs_range - Free a range of MSI descriptors of a device in an irqdomain 190 * @dev: Device for which to free the descriptors 191 * @domid: Id of the domain to operate on 192 * @first: Index to start freeing from (inclusive) 193 * @last: Last index to be freed (inclusive) 194 */ 195 void msi_domain_free_msi_descs_range(struct device *dev, unsigned int domid, 196 unsigned int first, unsigned int last) 197 { 198 struct msi_ctrl ctrl = { 199 .domid = domid, 200 .first = first, 201 .last = last, 202 }; 203 204 msi_domain_free_descs(dev, &ctrl); 205 } 206 207 /** 208 * msi_domain_add_simple_msi_descs - Allocate and initialize MSI descriptors 209 * @dev: Pointer to the device for which the descriptors are allocated 210 * @ctrl: Allocation control struct 211 * 212 * Return: 0 on success or an appropriate failure code. 213 */ 214 static int msi_domain_add_simple_msi_descs(struct device *dev, struct msi_ctrl *ctrl) 215 { 216 struct msi_desc *desc; 217 unsigned int idx; 218 int ret; 219 220 lockdep_assert_held(&dev->msi.data->mutex); 221 222 if (!msi_ctrl_valid(dev, ctrl)) 223 return -EINVAL; 224 225 for (idx = ctrl->first; idx <= ctrl->last; idx++) { 226 desc = msi_alloc_desc(dev, 1, NULL); 227 if (!desc) 228 goto fail_mem; 229 ret = msi_insert_desc(dev, desc, ctrl->domid, idx); 230 if (ret) 231 goto fail; 232 } 233 return 0; 234 235 fail_mem: 236 ret = -ENOMEM; 237 fail: 238 msi_domain_free_descs(dev, ctrl); 239 return ret; 240 } 241 242 void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg) 243 { 244 *msg = entry->msg; 245 } 246 247 void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg) 248 { 249 struct msi_desc *entry = irq_get_msi_desc(irq); 250 251 __get_cached_msi_msg(entry, msg); 252 } 253 EXPORT_SYMBOL_GPL(get_cached_msi_msg); 254 255 static void msi_device_data_release(struct device *dev, void *res) 256 { 257 struct msi_device_data *md = res; 258 int i; 259 260 for (i = 0; i < MSI_MAX_DEVICE_IRQDOMAINS; i++) { 261 msi_remove_device_irq_domain(dev, i); 262 WARN_ON_ONCE(!xa_empty(&md->__domains[i].store)); 263 xa_destroy(&md->__domains[i].store); 264 } 265 dev->msi.data = NULL; 266 } 267 268 /** 269 * msi_setup_device_data - Setup MSI device data 270 * @dev: Device for which MSI device data should be set up 271 * 272 * Return: 0 on success, appropriate error code otherwise 273 * 274 * This can be called more than once for @dev. If the MSI device data is 275 * already allocated the call succeeds. The allocated memory is 276 * automatically released when the device is destroyed. 277 */ 278 int msi_setup_device_data(struct device *dev) 279 { 280 struct msi_device_data *md; 281 int ret, i; 282 283 if (dev->msi.data) 284 return 0; 285 286 md = devres_alloc(msi_device_data_release, sizeof(*md), GFP_KERNEL); 287 if (!md) 288 return -ENOMEM; 289 290 ret = msi_sysfs_create_group(dev); 291 if (ret) { 292 devres_free(md); 293 return ret; 294 } 295 296 for (i = 0; i < MSI_MAX_DEVICE_IRQDOMAINS; i++) 297 xa_init(&md->__domains[i].store); 298 299 /* 300 * If @dev::msi::domain is set and is a global MSI domain, copy the 301 * pointer into the domain array so all code can operate on domain 302 * ids. The NULL pointer check is required to keep the legacy 303 * architecture specific PCI/MSI support working. 304 */ 305 if (dev->msi.domain && !irq_domain_is_msi_parent(dev->msi.domain)) 306 md->__domains[MSI_DEFAULT_DOMAIN].domain = dev->msi.domain; 307 308 mutex_init(&md->mutex); 309 dev->msi.data = md; 310 devres_add(dev, md); 311 return 0; 312 } 313 314 /** 315 * msi_lock_descs - Lock the MSI descriptor storage of a device 316 * @dev: Device to operate on 317 */ 318 void msi_lock_descs(struct device *dev) 319 { 320 mutex_lock(&dev->msi.data->mutex); 321 } 322 EXPORT_SYMBOL_GPL(msi_lock_descs); 323 324 /** 325 * msi_unlock_descs - Unlock the MSI descriptor storage of a device 326 * @dev: Device to operate on 327 */ 328 void msi_unlock_descs(struct device *dev) 329 { 330 /* Invalidate the index which was cached by the iterator */ 331 dev->msi.data->__iter_idx = MSI_XA_MAX_INDEX; 332 mutex_unlock(&dev->msi.data->mutex); 333 } 334 EXPORT_SYMBOL_GPL(msi_unlock_descs); 335 336 static struct msi_desc *msi_find_desc(struct msi_device_data *md, unsigned int domid, 337 enum msi_desc_filter filter) 338 { 339 struct xarray *xa = &md->__domains[domid].store; 340 struct msi_desc *desc; 341 342 xa_for_each_start(xa, md->__iter_idx, desc, md->__iter_idx) { 343 if (msi_desc_match(desc, filter)) 344 return desc; 345 } 346 md->__iter_idx = MSI_XA_MAX_INDEX; 347 return NULL; 348 } 349 350 /** 351 * msi_domain_first_desc - Get the first MSI descriptor of an irqdomain associated to a device 352 * @dev: Device to operate on 353 * @domid: The id of the interrupt domain which should be walked. 354 * @filter: Descriptor state filter 355 * 356 * Must be called with the MSI descriptor mutex held, i.e. msi_lock_descs() 357 * must be invoked before the call. 358 * 359 * Return: Pointer to the first MSI descriptor matching the search 360 * criteria, NULL if none found. 361 */ 362 struct msi_desc *msi_domain_first_desc(struct device *dev, unsigned int domid, 363 enum msi_desc_filter filter) 364 { 365 struct msi_device_data *md = dev->msi.data; 366 367 if (WARN_ON_ONCE(!md || domid >= MSI_MAX_DEVICE_IRQDOMAINS)) 368 return NULL; 369 370 lockdep_assert_held(&md->mutex); 371 372 md->__iter_idx = 0; 373 return msi_find_desc(md, domid, filter); 374 } 375 EXPORT_SYMBOL_GPL(msi_domain_first_desc); 376 377 /** 378 * msi_next_desc - Get the next MSI descriptor of a device 379 * @dev: Device to operate on 380 * @domid: The id of the interrupt domain which should be walked. 381 * @filter: Descriptor state filter 382 * 383 * The first invocation of msi_next_desc() has to be preceeded by a 384 * successful invocation of __msi_first_desc(). Consecutive invocations are 385 * only valid if the previous one was successful. All these operations have 386 * to be done within the same MSI mutex held region. 387 * 388 * Return: Pointer to the next MSI descriptor matching the search 389 * criteria, NULL if none found. 390 */ 391 struct msi_desc *msi_next_desc(struct device *dev, unsigned int domid, 392 enum msi_desc_filter filter) 393 { 394 struct msi_device_data *md = dev->msi.data; 395 396 if (WARN_ON_ONCE(!md || domid >= MSI_MAX_DEVICE_IRQDOMAINS)) 397 return NULL; 398 399 lockdep_assert_held(&md->mutex); 400 401 if (md->__iter_idx >= (unsigned long)MSI_MAX_INDEX) 402 return NULL; 403 404 md->__iter_idx++; 405 return msi_find_desc(md, domid, filter); 406 } 407 EXPORT_SYMBOL_GPL(msi_next_desc); 408 409 /** 410 * msi_domain_get_virq - Lookup the Linux interrupt number for a MSI index on a interrupt domain 411 * @dev: Device to operate on 412 * @domid: Domain ID of the interrupt domain associated to the device 413 * @index: MSI interrupt index to look for (0-based) 414 * 415 * Return: The Linux interrupt number on success (> 0), 0 if not found 416 */ 417 unsigned int msi_domain_get_virq(struct device *dev, unsigned int domid, unsigned int index) 418 { 419 struct msi_desc *desc; 420 unsigned int ret = 0; 421 bool pcimsi = false; 422 struct xarray *xa; 423 424 if (!dev->msi.data) 425 return 0; 426 427 if (WARN_ON_ONCE(index > MSI_MAX_INDEX || domid >= MSI_MAX_DEVICE_IRQDOMAINS)) 428 return 0; 429 430 /* This check is only valid for the PCI default MSI domain */ 431 if (dev_is_pci(dev) && domid == MSI_DEFAULT_DOMAIN) 432 pcimsi = to_pci_dev(dev)->msi_enabled; 433 434 msi_lock_descs(dev); 435 xa = &dev->msi.data->__domains[domid].store; 436 desc = xa_load(xa, pcimsi ? 0 : index); 437 if (desc && desc->irq) { 438 /* 439 * PCI-MSI has only one descriptor for multiple interrupts. 440 * PCI-MSIX and platform MSI use a descriptor per 441 * interrupt. 442 */ 443 if (pcimsi) { 444 if (index < desc->nvec_used) 445 ret = desc->irq + index; 446 } else { 447 ret = desc->irq; 448 } 449 } 450 451 msi_unlock_descs(dev); 452 return ret; 453 } 454 EXPORT_SYMBOL_GPL(msi_domain_get_virq); 455 456 #ifdef CONFIG_SYSFS 457 static struct attribute *msi_dev_attrs[] = { 458 NULL 459 }; 460 461 static const struct attribute_group msi_irqs_group = { 462 .name = "msi_irqs", 463 .attrs = msi_dev_attrs, 464 }; 465 466 static inline int msi_sysfs_create_group(struct device *dev) 467 { 468 return devm_device_add_group(dev, &msi_irqs_group); 469 } 470 471 static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr, 472 char *buf) 473 { 474 /* MSI vs. MSIX is per device not per interrupt */ 475 bool is_msix = dev_is_pci(dev) ? to_pci_dev(dev)->msix_enabled : false; 476 477 return sysfs_emit(buf, "%s\n", is_msix ? "msix" : "msi"); 478 } 479 480 static void msi_sysfs_remove_desc(struct device *dev, struct msi_desc *desc) 481 { 482 struct device_attribute *attrs = desc->sysfs_attrs; 483 int i; 484 485 if (!attrs) 486 return; 487 488 desc->sysfs_attrs = NULL; 489 for (i = 0; i < desc->nvec_used; i++) { 490 if (attrs[i].show) 491 sysfs_remove_file_from_group(&dev->kobj, &attrs[i].attr, msi_irqs_group.name); 492 kfree(attrs[i].attr.name); 493 } 494 kfree(attrs); 495 } 496 497 static int msi_sysfs_populate_desc(struct device *dev, struct msi_desc *desc) 498 { 499 struct device_attribute *attrs; 500 int ret, i; 501 502 attrs = kcalloc(desc->nvec_used, sizeof(*attrs), GFP_KERNEL); 503 if (!attrs) 504 return -ENOMEM; 505 506 desc->sysfs_attrs = attrs; 507 for (i = 0; i < desc->nvec_used; i++) { 508 sysfs_attr_init(&attrs[i].attr); 509 attrs[i].attr.name = kasprintf(GFP_KERNEL, "%d", desc->irq + i); 510 if (!attrs[i].attr.name) { 511 ret = -ENOMEM; 512 goto fail; 513 } 514 515 attrs[i].attr.mode = 0444; 516 attrs[i].show = msi_mode_show; 517 518 ret = sysfs_add_file_to_group(&dev->kobj, &attrs[i].attr, msi_irqs_group.name); 519 if (ret) { 520 attrs[i].show = NULL; 521 goto fail; 522 } 523 } 524 return 0; 525 526 fail: 527 msi_sysfs_remove_desc(dev, desc); 528 return ret; 529 } 530 531 #ifdef CONFIG_PCI_MSI_ARCH_FALLBACKS 532 /** 533 * msi_device_populate_sysfs - Populate msi_irqs sysfs entries for a device 534 * @dev: The device (PCI, platform etc) which will get sysfs entries 535 */ 536 int msi_device_populate_sysfs(struct device *dev) 537 { 538 struct msi_desc *desc; 539 int ret; 540 541 msi_for_each_desc(desc, dev, MSI_DESC_ASSOCIATED) { 542 if (desc->sysfs_attrs) 543 continue; 544 ret = msi_sysfs_populate_desc(dev, desc); 545 if (ret) 546 return ret; 547 } 548 return 0; 549 } 550 551 /** 552 * msi_device_destroy_sysfs - Destroy msi_irqs sysfs entries for a device 553 * @dev: The device (PCI, platform etc) for which to remove 554 * sysfs entries 555 */ 556 void msi_device_destroy_sysfs(struct device *dev) 557 { 558 struct msi_desc *desc; 559 560 msi_for_each_desc(desc, dev, MSI_DESC_ALL) 561 msi_sysfs_remove_desc(dev, desc); 562 } 563 #endif /* CONFIG_PCI_MSI_ARCH_FALLBACK */ 564 #else /* CONFIG_SYSFS */ 565 static inline int msi_sysfs_create_group(struct device *dev) { return 0; } 566 static inline int msi_sysfs_populate_desc(struct device *dev, struct msi_desc *desc) { return 0; } 567 static inline void msi_sysfs_remove_desc(struct device *dev, struct msi_desc *desc) { } 568 #endif /* !CONFIG_SYSFS */ 569 570 static struct irq_domain *msi_get_device_domain(struct device *dev, unsigned int domid) 571 { 572 struct irq_domain *domain; 573 574 lockdep_assert_held(&dev->msi.data->mutex); 575 576 if (WARN_ON_ONCE(domid >= MSI_MAX_DEVICE_IRQDOMAINS)) 577 return NULL; 578 579 domain = dev->msi.data->__domains[domid].domain; 580 if (!domain) 581 return NULL; 582 583 if (WARN_ON_ONCE(irq_domain_is_msi_parent(domain))) 584 return NULL; 585 586 return domain; 587 } 588 589 static unsigned int msi_domain_get_hwsize(struct device *dev, unsigned int domid) 590 { 591 struct msi_domain_info *info; 592 struct irq_domain *domain; 593 594 domain = msi_get_device_domain(dev, domid); 595 if (domain) { 596 info = domain->host_data; 597 return info->hwsize; 598 } 599 /* No domain, no size... */ 600 return 0; 601 } 602 603 static inline void irq_chip_write_msi_msg(struct irq_data *data, 604 struct msi_msg *msg) 605 { 606 data->chip->irq_write_msi_msg(data, msg); 607 } 608 609 static void msi_check_level(struct irq_domain *domain, struct msi_msg *msg) 610 { 611 struct msi_domain_info *info = domain->host_data; 612 613 /* 614 * If the MSI provider has messed with the second message and 615 * not advertized that it is level-capable, signal the breakage. 616 */ 617 WARN_ON(!((info->flags & MSI_FLAG_LEVEL_CAPABLE) && 618 (info->chip->flags & IRQCHIP_SUPPORTS_LEVEL_MSI)) && 619 (msg[1].address_lo || msg[1].address_hi || msg[1].data)); 620 } 621 622 /** 623 * msi_domain_set_affinity - Generic affinity setter function for MSI domains 624 * @irq_data: The irq data associated to the interrupt 625 * @mask: The affinity mask to set 626 * @force: Flag to enforce setting (disable online checks) 627 * 628 * Intended to be used by MSI interrupt controllers which are 629 * implemented with hierarchical domains. 630 * 631 * Return: IRQ_SET_MASK_* result code 632 */ 633 int msi_domain_set_affinity(struct irq_data *irq_data, 634 const struct cpumask *mask, bool force) 635 { 636 struct irq_data *parent = irq_data->parent_data; 637 struct msi_msg msg[2] = { [1] = { }, }; 638 int ret; 639 640 ret = parent->chip->irq_set_affinity(parent, mask, force); 641 if (ret >= 0 && ret != IRQ_SET_MASK_OK_DONE) { 642 BUG_ON(irq_chip_compose_msi_msg(irq_data, msg)); 643 msi_check_level(irq_data->domain, msg); 644 irq_chip_write_msi_msg(irq_data, msg); 645 } 646 647 return ret; 648 } 649 650 static int msi_domain_activate(struct irq_domain *domain, 651 struct irq_data *irq_data, bool early) 652 { 653 struct msi_msg msg[2] = { [1] = { }, }; 654 655 BUG_ON(irq_chip_compose_msi_msg(irq_data, msg)); 656 msi_check_level(irq_data->domain, msg); 657 irq_chip_write_msi_msg(irq_data, msg); 658 return 0; 659 } 660 661 static void msi_domain_deactivate(struct irq_domain *domain, 662 struct irq_data *irq_data) 663 { 664 struct msi_msg msg[2]; 665 666 memset(msg, 0, sizeof(msg)); 667 irq_chip_write_msi_msg(irq_data, msg); 668 } 669 670 static int msi_domain_alloc(struct irq_domain *domain, unsigned int virq, 671 unsigned int nr_irqs, void *arg) 672 { 673 struct msi_domain_info *info = domain->host_data; 674 struct msi_domain_ops *ops = info->ops; 675 irq_hw_number_t hwirq = ops->get_hwirq(info, arg); 676 int i, ret; 677 678 if (irq_find_mapping(domain, hwirq) > 0) 679 return -EEXIST; 680 681 if (domain->parent) { 682 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg); 683 if (ret < 0) 684 return ret; 685 } 686 687 for (i = 0; i < nr_irqs; i++) { 688 ret = ops->msi_init(domain, info, virq + i, hwirq + i, arg); 689 if (ret < 0) { 690 if (ops->msi_free) { 691 for (i--; i > 0; i--) 692 ops->msi_free(domain, info, virq + i); 693 } 694 irq_domain_free_irqs_top(domain, virq, nr_irqs); 695 return ret; 696 } 697 } 698 699 return 0; 700 } 701 702 static void msi_domain_free(struct irq_domain *domain, unsigned int virq, 703 unsigned int nr_irqs) 704 { 705 struct msi_domain_info *info = domain->host_data; 706 int i; 707 708 if (info->ops->msi_free) { 709 for (i = 0; i < nr_irqs; i++) 710 info->ops->msi_free(domain, info, virq + i); 711 } 712 irq_domain_free_irqs_top(domain, virq, nr_irqs); 713 } 714 715 static const struct irq_domain_ops msi_domain_ops = { 716 .alloc = msi_domain_alloc, 717 .free = msi_domain_free, 718 .activate = msi_domain_activate, 719 .deactivate = msi_domain_deactivate, 720 }; 721 722 static irq_hw_number_t msi_domain_ops_get_hwirq(struct msi_domain_info *info, 723 msi_alloc_info_t *arg) 724 { 725 return arg->hwirq; 726 } 727 728 static int msi_domain_ops_prepare(struct irq_domain *domain, struct device *dev, 729 int nvec, msi_alloc_info_t *arg) 730 { 731 memset(arg, 0, sizeof(*arg)); 732 return 0; 733 } 734 735 static void msi_domain_ops_set_desc(msi_alloc_info_t *arg, 736 struct msi_desc *desc) 737 { 738 arg->desc = desc; 739 } 740 741 static int msi_domain_ops_init(struct irq_domain *domain, 742 struct msi_domain_info *info, 743 unsigned int virq, irq_hw_number_t hwirq, 744 msi_alloc_info_t *arg) 745 { 746 irq_domain_set_hwirq_and_chip(domain, virq, hwirq, info->chip, 747 info->chip_data); 748 if (info->handler && info->handler_name) { 749 __irq_set_handler(virq, info->handler, 0, info->handler_name); 750 if (info->handler_data) 751 irq_set_handler_data(virq, info->handler_data); 752 } 753 return 0; 754 } 755 756 static struct msi_domain_ops msi_domain_ops_default = { 757 .get_hwirq = msi_domain_ops_get_hwirq, 758 .msi_init = msi_domain_ops_init, 759 .msi_prepare = msi_domain_ops_prepare, 760 .set_desc = msi_domain_ops_set_desc, 761 }; 762 763 static void msi_domain_update_dom_ops(struct msi_domain_info *info) 764 { 765 struct msi_domain_ops *ops = info->ops; 766 767 if (ops == NULL) { 768 info->ops = &msi_domain_ops_default; 769 return; 770 } 771 772 if (!(info->flags & MSI_FLAG_USE_DEF_DOM_OPS)) 773 return; 774 775 if (ops->get_hwirq == NULL) 776 ops->get_hwirq = msi_domain_ops_default.get_hwirq; 777 if (ops->msi_init == NULL) 778 ops->msi_init = msi_domain_ops_default.msi_init; 779 if (ops->msi_prepare == NULL) 780 ops->msi_prepare = msi_domain_ops_default.msi_prepare; 781 if (ops->set_desc == NULL) 782 ops->set_desc = msi_domain_ops_default.set_desc; 783 } 784 785 static void msi_domain_update_chip_ops(struct msi_domain_info *info) 786 { 787 struct irq_chip *chip = info->chip; 788 789 BUG_ON(!chip || !chip->irq_mask || !chip->irq_unmask); 790 if (!chip->irq_set_affinity) 791 chip->irq_set_affinity = msi_domain_set_affinity; 792 } 793 794 static struct irq_domain *__msi_create_irq_domain(struct fwnode_handle *fwnode, 795 struct msi_domain_info *info, 796 unsigned int flags, 797 struct irq_domain *parent) 798 { 799 struct irq_domain *domain; 800 801 if (info->hwsize > MSI_XA_DOMAIN_SIZE) 802 return NULL; 803 804 /* 805 * Hardware size 0 is valid for backwards compatibility and for 806 * domains which are not backed by a hardware table. Grant the 807 * maximum index space. 808 */ 809 if (!info->hwsize) 810 info->hwsize = MSI_XA_DOMAIN_SIZE; 811 812 msi_domain_update_dom_ops(info); 813 if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS) 814 msi_domain_update_chip_ops(info); 815 816 domain = irq_domain_create_hierarchy(parent, flags | IRQ_DOMAIN_FLAG_MSI, 0, 817 fwnode, &msi_domain_ops, info); 818 819 if (domain) { 820 if (!domain->name && info->chip) 821 domain->name = info->chip->name; 822 irq_domain_update_bus_token(domain, info->bus_token); 823 } 824 825 return domain; 826 } 827 828 /** 829 * msi_create_irq_domain - Create an MSI interrupt domain 830 * @fwnode: Optional fwnode of the interrupt controller 831 * @info: MSI domain info 832 * @parent: Parent irq domain 833 * 834 * Return: pointer to the created &struct irq_domain or %NULL on failure 835 */ 836 struct irq_domain *msi_create_irq_domain(struct fwnode_handle *fwnode, 837 struct msi_domain_info *info, 838 struct irq_domain *parent) 839 { 840 return __msi_create_irq_domain(fwnode, info, 0, parent); 841 } 842 843 /** 844 * msi_parent_init_dev_msi_info - Delegate initialization of device MSI info down 845 * in the domain hierarchy 846 * @dev: The device for which the domain should be created 847 * @domain: The domain in the hierarchy this op is being called on 848 * @msi_parent_domain: The IRQ_DOMAIN_FLAG_MSI_PARENT domain for the child to 849 * be created 850 * @msi_child_info: The MSI domain info of the IRQ_DOMAIN_FLAG_MSI_DEVICE 851 * domain to be created 852 * 853 * Return: true on success, false otherwise 854 * 855 * This is the most complex problem of per device MSI domains and the 856 * underlying interrupt domain hierarchy: 857 * 858 * The device domain to be initialized requests the broadest feature set 859 * possible and the underlying domain hierarchy puts restrictions on it. 860 * 861 * That's trivial for a simple parent->child relationship, but it gets 862 * interesting with an intermediate domain: root->parent->child. The 863 * intermediate 'parent' can expand the capabilities which the 'root' 864 * domain is providing. So that creates a classic hen and egg problem: 865 * Which entity is doing the restrictions/expansions? 866 * 867 * One solution is to let the root domain handle the initialization that's 868 * why there is the @domain and the @msi_parent_domain pointer. 869 */ 870 bool msi_parent_init_dev_msi_info(struct device *dev, struct irq_domain *domain, 871 struct irq_domain *msi_parent_domain, 872 struct msi_domain_info *msi_child_info) 873 { 874 struct irq_domain *parent = domain->parent; 875 876 if (WARN_ON_ONCE(!parent || !parent->msi_parent_ops || 877 !parent->msi_parent_ops->init_dev_msi_info)) 878 return false; 879 880 return parent->msi_parent_ops->init_dev_msi_info(dev, parent, msi_parent_domain, 881 msi_child_info); 882 } 883 884 /** 885 * msi_create_device_irq_domain - Create a device MSI interrupt domain 886 * @dev: Pointer to the device 887 * @domid: Domain id 888 * @template: MSI domain info bundle used as template 889 * @hwsize: Maximum number of MSI table entries (0 if unknown or unlimited) 890 * @domain_data: Optional pointer to domain specific data which is set in 891 * msi_domain_info::data 892 * @chip_data: Optional pointer to chip specific data which is set in 893 * msi_domain_info::chip_data 894 * 895 * Return: True on success, false otherwise 896 * 897 * There is no firmware node required for this interface because the per 898 * device domains are software constructs which are actually closer to the 899 * hardware reality than any firmware can describe them. 900 * 901 * The domain name and the irq chip name for a MSI device domain are 902 * composed by: "$(PREFIX)$(CHIPNAME)-$(DEVNAME)" 903 * 904 * $PREFIX: Optional prefix provided by the underlying MSI parent domain 905 * via msi_parent_ops::prefix. If that pointer is NULL the prefix 906 * is empty. 907 * $CHIPNAME: The name of the irq_chip in @template 908 * $DEVNAME: The name of the device 909 * 910 * This results in understandable chip names and hardware interrupt numbers 911 * in e.g. /proc/interrupts 912 * 913 * PCI-MSI-0000:00:1c.0 0-edge Parent domain has no prefix 914 * IR-PCI-MSI-0000:00:1c.4 0-edge Same with interrupt remapping prefix 'IR-' 915 * 916 * IR-PCI-MSIX-0000:3d:00.0 0-edge Hardware interrupt numbers reflect 917 * IR-PCI-MSIX-0000:3d:00.0 1-edge the real MSI-X index on that device 918 * IR-PCI-MSIX-0000:3d:00.0 2-edge 919 * 920 * On IMS domains the hardware interrupt number is either a table entry 921 * index or a purely software managed index but it is guaranteed to be 922 * unique. 923 * 924 * The domain pointer is stored in @dev::msi::data::__irqdomains[]. All 925 * subsequent operations on the domain depend on the domain id. 926 * 927 * The domain is automatically freed when the device is removed via devres 928 * in the context of @dev::msi::data freeing, but it can also be 929 * independently removed via @msi_remove_device_irq_domain(). 930 */ 931 bool msi_create_device_irq_domain(struct device *dev, unsigned int domid, 932 const struct msi_domain_template *template, 933 unsigned int hwsize, void *domain_data, 934 void *chip_data) 935 { 936 struct irq_domain *domain, *parent = dev->msi.domain; 937 const struct msi_parent_ops *pops; 938 struct msi_domain_template *bundle; 939 struct fwnode_handle *fwnode; 940 941 if (!irq_domain_is_msi_parent(parent)) 942 return false; 943 944 if (domid >= MSI_MAX_DEVICE_IRQDOMAINS) 945 return false; 946 947 bundle = kmemdup(template, sizeof(*bundle), GFP_KERNEL); 948 if (!bundle) 949 return false; 950 951 bundle->info.hwsize = hwsize; 952 bundle->info.chip = &bundle->chip; 953 bundle->info.ops = &bundle->ops; 954 bundle->info.data = domain_data; 955 bundle->info.chip_data = chip_data; 956 957 pops = parent->msi_parent_ops; 958 snprintf(bundle->name, sizeof(bundle->name), "%s%s-%s", 959 pops->prefix ? : "", bundle->chip.name, dev_name(dev)); 960 bundle->chip.name = bundle->name; 961 962 fwnode = irq_domain_alloc_named_fwnode(bundle->name); 963 if (!fwnode) 964 goto free_bundle; 965 966 if (msi_setup_device_data(dev)) 967 goto free_fwnode; 968 969 msi_lock_descs(dev); 970 971 if (WARN_ON_ONCE(msi_get_device_domain(dev, domid))) 972 goto fail; 973 974 if (!pops->init_dev_msi_info(dev, parent, parent, &bundle->info)) 975 goto fail; 976 977 domain = __msi_create_irq_domain(fwnode, &bundle->info, IRQ_DOMAIN_FLAG_MSI_DEVICE, parent); 978 if (!domain) 979 goto fail; 980 981 domain->dev = dev; 982 dev->msi.data->__domains[domid].domain = domain; 983 msi_unlock_descs(dev); 984 return true; 985 986 fail: 987 msi_unlock_descs(dev); 988 free_fwnode: 989 kfree(fwnode); 990 free_bundle: 991 kfree(bundle); 992 return false; 993 } 994 995 /** 996 * msi_remove_device_irq_domain - Free a device MSI interrupt domain 997 * @dev: Pointer to the device 998 * @domid: Domain id 999 */ 1000 void msi_remove_device_irq_domain(struct device *dev, unsigned int domid) 1001 { 1002 struct msi_domain_info *info; 1003 struct irq_domain *domain; 1004 1005 msi_lock_descs(dev); 1006 1007 domain = msi_get_device_domain(dev, domid); 1008 1009 if (!domain || !irq_domain_is_msi_device(domain)) 1010 goto unlock; 1011 1012 dev->msi.data->__domains[domid].domain = NULL; 1013 info = domain->host_data; 1014 irq_domain_remove(domain); 1015 kfree(container_of(info, struct msi_domain_template, info)); 1016 1017 unlock: 1018 msi_unlock_descs(dev); 1019 } 1020 1021 /** 1022 * msi_match_device_irq_domain - Match a device irq domain against a bus token 1023 * @dev: Pointer to the device 1024 * @domid: Domain id 1025 * @bus_token: Bus token to match against the domain bus token 1026 * 1027 * Return: True if device domain exists and bus tokens match. 1028 */ 1029 bool msi_match_device_irq_domain(struct device *dev, unsigned int domid, 1030 enum irq_domain_bus_token bus_token) 1031 { 1032 struct msi_domain_info *info; 1033 struct irq_domain *domain; 1034 bool ret = false; 1035 1036 msi_lock_descs(dev); 1037 domain = msi_get_device_domain(dev, domid); 1038 if (domain && irq_domain_is_msi_device(domain)) { 1039 info = domain->host_data; 1040 ret = info->bus_token == bus_token; 1041 } 1042 msi_unlock_descs(dev); 1043 return ret; 1044 } 1045 1046 int msi_domain_prepare_irqs(struct irq_domain *domain, struct device *dev, 1047 int nvec, msi_alloc_info_t *arg) 1048 { 1049 struct msi_domain_info *info = domain->host_data; 1050 struct msi_domain_ops *ops = info->ops; 1051 1052 return ops->msi_prepare(domain, dev, nvec, arg); 1053 } 1054 1055 int msi_domain_populate_irqs(struct irq_domain *domain, struct device *dev, 1056 int virq_base, int nvec, msi_alloc_info_t *arg) 1057 { 1058 struct msi_domain_info *info = domain->host_data; 1059 struct msi_domain_ops *ops = info->ops; 1060 struct msi_ctrl ctrl = { 1061 .domid = MSI_DEFAULT_DOMAIN, 1062 .first = virq_base, 1063 .last = virq_base + nvec - 1, 1064 }; 1065 struct msi_desc *desc; 1066 struct xarray *xa; 1067 int ret, virq; 1068 1069 if (!msi_ctrl_valid(dev, &ctrl)) 1070 return -EINVAL; 1071 1072 msi_lock_descs(dev); 1073 ret = msi_domain_add_simple_msi_descs(dev, &ctrl); 1074 if (ret) 1075 goto unlock; 1076 1077 xa = &dev->msi.data->__domains[ctrl.domid].store; 1078 1079 for (virq = virq_base; virq < virq_base + nvec; virq++) { 1080 desc = xa_load(xa, virq); 1081 desc->irq = virq; 1082 1083 ops->set_desc(arg, desc); 1084 ret = irq_domain_alloc_irqs_hierarchy(domain, virq, 1, arg); 1085 if (ret) 1086 goto fail; 1087 1088 irq_set_msi_desc(virq, desc); 1089 } 1090 msi_unlock_descs(dev); 1091 return 0; 1092 1093 fail: 1094 for (--virq; virq >= virq_base; virq--) 1095 irq_domain_free_irqs_common(domain, virq, 1); 1096 msi_domain_free_descs(dev, &ctrl); 1097 unlock: 1098 msi_unlock_descs(dev); 1099 return ret; 1100 } 1101 1102 /* 1103 * Carefully check whether the device can use reservation mode. If 1104 * reservation mode is enabled then the early activation will assign a 1105 * dummy vector to the device. If the PCI/MSI device does not support 1106 * masking of the entry then this can result in spurious interrupts when 1107 * the device driver is not absolutely careful. But even then a malfunction 1108 * of the hardware could result in a spurious interrupt on the dummy vector 1109 * and render the device unusable. If the entry can be masked then the core 1110 * logic will prevent the spurious interrupt and reservation mode can be 1111 * used. For now reservation mode is restricted to PCI/MSI. 1112 */ 1113 static bool msi_check_reservation_mode(struct irq_domain *domain, 1114 struct msi_domain_info *info, 1115 struct device *dev) 1116 { 1117 struct msi_desc *desc; 1118 1119 switch(domain->bus_token) { 1120 case DOMAIN_BUS_PCI_MSI: 1121 case DOMAIN_BUS_PCI_DEVICE_MSI: 1122 case DOMAIN_BUS_PCI_DEVICE_MSIX: 1123 case DOMAIN_BUS_VMD_MSI: 1124 break; 1125 default: 1126 return false; 1127 } 1128 1129 if (!(info->flags & MSI_FLAG_MUST_REACTIVATE)) 1130 return false; 1131 1132 if (IS_ENABLED(CONFIG_PCI_MSI) && pci_msi_ignore_mask) 1133 return false; 1134 1135 /* 1136 * Checking the first MSI descriptor is sufficient. MSIX supports 1137 * masking and MSI does so when the can_mask attribute is set. 1138 */ 1139 desc = msi_first_desc(dev, MSI_DESC_ALL); 1140 return desc->pci.msi_attrib.is_msix || desc->pci.msi_attrib.can_mask; 1141 } 1142 1143 static int msi_handle_pci_fail(struct irq_domain *domain, struct msi_desc *desc, 1144 int allocated) 1145 { 1146 switch(domain->bus_token) { 1147 case DOMAIN_BUS_PCI_MSI: 1148 case DOMAIN_BUS_PCI_DEVICE_MSI: 1149 case DOMAIN_BUS_PCI_DEVICE_MSIX: 1150 case DOMAIN_BUS_VMD_MSI: 1151 if (IS_ENABLED(CONFIG_PCI_MSI)) 1152 break; 1153 fallthrough; 1154 default: 1155 return -ENOSPC; 1156 } 1157 1158 /* Let a failed PCI multi MSI allocation retry */ 1159 if (desc->nvec_used > 1) 1160 return 1; 1161 1162 /* If there was a successful allocation let the caller know */ 1163 return allocated ? allocated : -ENOSPC; 1164 } 1165 1166 #define VIRQ_CAN_RESERVE 0x01 1167 #define VIRQ_ACTIVATE 0x02 1168 #define VIRQ_NOMASK_QUIRK 0x04 1169 1170 static int msi_init_virq(struct irq_domain *domain, int virq, unsigned int vflags) 1171 { 1172 struct irq_data *irqd = irq_domain_get_irq_data(domain, virq); 1173 int ret; 1174 1175 if (!(vflags & VIRQ_CAN_RESERVE)) { 1176 irqd_clr_can_reserve(irqd); 1177 if (vflags & VIRQ_NOMASK_QUIRK) 1178 irqd_set_msi_nomask_quirk(irqd); 1179 1180 /* 1181 * If the interrupt is managed but no CPU is available to 1182 * service it, shut it down until better times. Note that 1183 * we only do this on the !RESERVE path as x86 (the only 1184 * architecture using this flag) deals with this in a 1185 * different way by using a catch-all vector. 1186 */ 1187 if ((vflags & VIRQ_ACTIVATE) && 1188 irqd_affinity_is_managed(irqd) && 1189 !cpumask_intersects(irq_data_get_affinity_mask(irqd), 1190 cpu_online_mask)) { 1191 irqd_set_managed_shutdown(irqd); 1192 return 0; 1193 } 1194 } 1195 1196 if (!(vflags & VIRQ_ACTIVATE)) 1197 return 0; 1198 1199 ret = irq_domain_activate_irq(irqd, vflags & VIRQ_CAN_RESERVE); 1200 if (ret) 1201 return ret; 1202 /* 1203 * If the interrupt uses reservation mode, clear the activated bit 1204 * so request_irq() will assign the final vector. 1205 */ 1206 if (vflags & VIRQ_CAN_RESERVE) 1207 irqd_clr_activated(irqd); 1208 return 0; 1209 } 1210 1211 static int __msi_domain_alloc_irqs(struct device *dev, struct irq_domain *domain, 1212 struct msi_ctrl *ctrl) 1213 { 1214 struct xarray *xa = &dev->msi.data->__domains[ctrl->domid].store; 1215 struct msi_domain_info *info = domain->host_data; 1216 struct msi_domain_ops *ops = info->ops; 1217 unsigned int vflags = 0, allocated = 0; 1218 msi_alloc_info_t arg = { }; 1219 struct msi_desc *desc; 1220 unsigned long idx; 1221 int i, ret, virq; 1222 1223 ret = msi_domain_prepare_irqs(domain, dev, ctrl->nirqs, &arg); 1224 if (ret) 1225 return ret; 1226 1227 /* 1228 * This flag is set by the PCI layer as we need to activate 1229 * the MSI entries before the PCI layer enables MSI in the 1230 * card. Otherwise the card latches a random msi message. 1231 */ 1232 if (info->flags & MSI_FLAG_ACTIVATE_EARLY) 1233 vflags |= VIRQ_ACTIVATE; 1234 1235 /* 1236 * Interrupt can use a reserved vector and will not occupy 1237 * a real device vector until the interrupt is requested. 1238 */ 1239 if (msi_check_reservation_mode(domain, info, dev)) { 1240 vflags |= VIRQ_CAN_RESERVE; 1241 /* 1242 * MSI affinity setting requires a special quirk (X86) when 1243 * reservation mode is active. 1244 */ 1245 if (info->flags & MSI_FLAG_NOMASK_QUIRK) 1246 vflags |= VIRQ_NOMASK_QUIRK; 1247 } 1248 1249 xa_for_each_range(xa, idx, desc, ctrl->first, ctrl->last) { 1250 if (!msi_desc_match(desc, MSI_DESC_NOTASSOCIATED)) 1251 continue; 1252 1253 /* This should return -ECONFUSED... */ 1254 if (WARN_ON_ONCE(allocated >= ctrl->nirqs)) 1255 return -EINVAL; 1256 1257 ops->set_desc(&arg, desc); 1258 1259 virq = __irq_domain_alloc_irqs(domain, -1, desc->nvec_used, 1260 dev_to_node(dev), &arg, false, 1261 desc->affinity); 1262 if (virq < 0) 1263 return msi_handle_pci_fail(domain, desc, allocated); 1264 1265 for (i = 0; i < desc->nvec_used; i++) { 1266 irq_set_msi_desc_off(virq, i, desc); 1267 irq_debugfs_copy_devname(virq + i, dev); 1268 ret = msi_init_virq(domain, virq + i, vflags); 1269 if (ret) 1270 return ret; 1271 } 1272 if (info->flags & MSI_FLAG_DEV_SYSFS) { 1273 ret = msi_sysfs_populate_desc(dev, desc); 1274 if (ret) 1275 return ret; 1276 } 1277 allocated++; 1278 } 1279 return 0; 1280 } 1281 1282 static int msi_domain_alloc_simple_msi_descs(struct device *dev, 1283 struct msi_domain_info *info, 1284 struct msi_ctrl *ctrl) 1285 { 1286 if (!(info->flags & MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS)) 1287 return 0; 1288 1289 return msi_domain_add_simple_msi_descs(dev, ctrl); 1290 } 1291 1292 static int __msi_domain_alloc_locked(struct device *dev, struct msi_ctrl *ctrl) 1293 { 1294 struct msi_domain_info *info; 1295 struct msi_domain_ops *ops; 1296 struct irq_domain *domain; 1297 int ret; 1298 1299 if (!msi_ctrl_valid(dev, ctrl)) 1300 return -EINVAL; 1301 1302 domain = msi_get_device_domain(dev, ctrl->domid); 1303 if (!domain) 1304 return -ENODEV; 1305 1306 info = domain->host_data; 1307 1308 ret = msi_domain_alloc_simple_msi_descs(dev, info, ctrl); 1309 if (ret) 1310 return ret; 1311 1312 ops = info->ops; 1313 if (ops->domain_alloc_irqs) 1314 return ops->domain_alloc_irqs(domain, dev, ctrl->nirqs); 1315 1316 return __msi_domain_alloc_irqs(dev, domain, ctrl); 1317 } 1318 1319 static int msi_domain_alloc_locked(struct device *dev, struct msi_ctrl *ctrl) 1320 { 1321 int ret = __msi_domain_alloc_locked(dev, ctrl); 1322 1323 if (ret) 1324 msi_domain_free_locked(dev, ctrl); 1325 return ret; 1326 } 1327 1328 /** 1329 * msi_domain_alloc_irqs_range_locked - Allocate interrupts from a MSI interrupt domain 1330 * @dev: Pointer to device struct of the device for which the interrupts 1331 * are allocated 1332 * @domid: Id of the interrupt domain to operate on 1333 * @first: First index to allocate (inclusive) 1334 * @last: Last index to allocate (inclusive) 1335 * 1336 * Must be invoked from within a msi_lock_descs() / msi_unlock_descs() 1337 * pair. Use this for MSI irqdomains which implement their own descriptor 1338 * allocation/free. 1339 * 1340 * Return: %0 on success or an error code. 1341 */ 1342 int msi_domain_alloc_irqs_range_locked(struct device *dev, unsigned int domid, 1343 unsigned int first, unsigned int last) 1344 { 1345 struct msi_ctrl ctrl = { 1346 .domid = domid, 1347 .first = first, 1348 .last = last, 1349 .nirqs = last + 1 - first, 1350 }; 1351 1352 return msi_domain_alloc_locked(dev, &ctrl); 1353 } 1354 1355 /** 1356 * msi_domain_alloc_irqs_range - Allocate interrupts from a MSI interrupt domain 1357 * @dev: Pointer to device struct of the device for which the interrupts 1358 * are allocated 1359 * @domid: Id of the interrupt domain to operate on 1360 * @first: First index to allocate (inclusive) 1361 * @last: Last index to allocate (inclusive) 1362 * 1363 * Return: %0 on success or an error code. 1364 */ 1365 int msi_domain_alloc_irqs_range(struct device *dev, unsigned int domid, 1366 unsigned int first, unsigned int last) 1367 { 1368 int ret; 1369 1370 msi_lock_descs(dev); 1371 ret = msi_domain_alloc_irqs_range_locked(dev, domid, first, last); 1372 msi_unlock_descs(dev); 1373 return ret; 1374 } 1375 1376 /** 1377 * msi_domain_alloc_irqs_all_locked - Allocate all interrupts from a MSI interrupt domain 1378 * 1379 * @dev: Pointer to device struct of the device for which the interrupts 1380 * are allocated 1381 * @domid: Id of the interrupt domain to operate on 1382 * @nirqs: The number of interrupts to allocate 1383 * 1384 * This function scans all MSI descriptors of the MSI domain and allocates interrupts 1385 * for all unassigned ones. That function is to be used for MSI domain usage where 1386 * the descriptor allocation is handled at the call site, e.g. PCI/MSI[X]. 1387 * 1388 * Return: %0 on success or an error code. 1389 */ 1390 int msi_domain_alloc_irqs_all_locked(struct device *dev, unsigned int domid, int nirqs) 1391 { 1392 struct msi_ctrl ctrl = { 1393 .domid = domid, 1394 .first = 0, 1395 .last = msi_domain_get_hwsize(dev, domid) - 1, 1396 .nirqs = nirqs, 1397 }; 1398 1399 return msi_domain_alloc_locked(dev, &ctrl); 1400 } 1401 1402 static void __msi_domain_free_irqs(struct device *dev, struct irq_domain *domain, 1403 struct msi_ctrl *ctrl) 1404 { 1405 struct xarray *xa = &dev->msi.data->__domains[ctrl->domid].store; 1406 struct msi_domain_info *info = domain->host_data; 1407 struct irq_data *irqd; 1408 struct msi_desc *desc; 1409 unsigned long idx; 1410 int i; 1411 1412 xa_for_each_range(xa, idx, desc, ctrl->first, ctrl->last) { 1413 /* Only handle MSI entries which have an interrupt associated */ 1414 if (!msi_desc_match(desc, MSI_DESC_ASSOCIATED)) 1415 continue; 1416 1417 /* Make sure all interrupts are deactivated */ 1418 for (i = 0; i < desc->nvec_used; i++) { 1419 irqd = irq_domain_get_irq_data(domain, desc->irq + i); 1420 if (irqd && irqd_is_activated(irqd)) 1421 irq_domain_deactivate_irq(irqd); 1422 } 1423 1424 irq_domain_free_irqs(desc->irq, desc->nvec_used); 1425 if (info->flags & MSI_FLAG_DEV_SYSFS) 1426 msi_sysfs_remove_desc(dev, desc); 1427 desc->irq = 0; 1428 } 1429 } 1430 1431 static void msi_domain_free_locked(struct device *dev, struct msi_ctrl *ctrl) 1432 { 1433 struct msi_domain_info *info; 1434 struct msi_domain_ops *ops; 1435 struct irq_domain *domain; 1436 1437 if (!msi_ctrl_valid(dev, ctrl)) 1438 return; 1439 1440 domain = msi_get_device_domain(dev, ctrl->domid); 1441 if (!domain) 1442 return; 1443 1444 info = domain->host_data; 1445 ops = info->ops; 1446 1447 if (ops->domain_free_irqs) 1448 ops->domain_free_irqs(domain, dev); 1449 else 1450 __msi_domain_free_irqs(dev, domain, ctrl); 1451 1452 if (ops->msi_post_free) 1453 ops->msi_post_free(domain, dev); 1454 1455 if (info->flags & MSI_FLAG_FREE_MSI_DESCS) 1456 msi_domain_free_descs(dev, ctrl); 1457 } 1458 1459 /** 1460 * msi_domain_free_irqs_range_locked - Free a range of interrupts from a MSI interrupt domain 1461 * associated to @dev with msi_lock held 1462 * @dev: Pointer to device struct of the device for which the interrupts 1463 * are freed 1464 * @domid: Id of the interrupt domain to operate on 1465 * @first: First index to free (inclusive) 1466 * @last: Last index to free (inclusive) 1467 */ 1468 void msi_domain_free_irqs_range_locked(struct device *dev, unsigned int domid, 1469 unsigned int first, unsigned int last) 1470 { 1471 struct msi_ctrl ctrl = { 1472 .domid = domid, 1473 .first = first, 1474 .last = last, 1475 }; 1476 msi_domain_free_locked(dev, &ctrl); 1477 } 1478 1479 /** 1480 * msi_domain_free_irqs_range - Free a range of interrupts from a MSI interrupt domain 1481 * associated to @dev 1482 * @dev: Pointer to device struct of the device for which the interrupts 1483 * are freed 1484 * @domid: Id of the interrupt domain to operate on 1485 * @first: First index to free (inclusive) 1486 * @last: Last index to free (inclusive) 1487 */ 1488 void msi_domain_free_irqs_range(struct device *dev, unsigned int domid, 1489 unsigned int first, unsigned int last) 1490 { 1491 msi_lock_descs(dev); 1492 msi_domain_free_irqs_range_locked(dev, domid, first, last); 1493 msi_unlock_descs(dev); 1494 } 1495 1496 /** 1497 * msi_domain_free_irqs_all_locked - Free all interrupts from a MSI interrupt domain 1498 * associated to a device 1499 * @dev: Pointer to device struct of the device for which the interrupts 1500 * are freed 1501 * @domid: The id of the domain to operate on 1502 * 1503 * Must be invoked from within a msi_lock_descs() / msi_unlock_descs() 1504 * pair. Use this for MSI irqdomains which implement their own vector 1505 * allocation. 1506 */ 1507 void msi_domain_free_irqs_all_locked(struct device *dev, unsigned int domid) 1508 { 1509 msi_domain_free_irqs_range_locked(dev, domid, 0, 1510 msi_domain_get_hwsize(dev, domid) - 1); 1511 } 1512 1513 /** 1514 * msi_domain_free_irqs_all - Free all interrupts from a MSI interrupt domain 1515 * associated to a device 1516 * @dev: Pointer to device struct of the device for which the interrupts 1517 * are freed 1518 * @domid: The id of the domain to operate on 1519 */ 1520 void msi_domain_free_irqs_all(struct device *dev, unsigned int domid) 1521 { 1522 msi_lock_descs(dev); 1523 msi_domain_free_irqs_all_locked(dev, domid); 1524 msi_unlock_descs(dev); 1525 } 1526 1527 /** 1528 * msi_get_domain_info - Get the MSI interrupt domain info for @domain 1529 * @domain: The interrupt domain to retrieve data from 1530 * 1531 * Return: the pointer to the msi_domain_info stored in @domain->host_data. 1532 */ 1533 struct msi_domain_info *msi_get_domain_info(struct irq_domain *domain) 1534 { 1535 return (struct msi_domain_info *)domain->host_data; 1536 } 1537