1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2014 Intel Corp. 4 * Author: Jiang Liu <jiang.liu@linux.intel.com> 5 * 6 * This file is licensed under GPLv2. 7 * 8 * This file contains common code to support Message Signaled Interrupts for 9 * PCI compatible and non PCI compatible devices. 10 */ 11 #include <linux/types.h> 12 #include <linux/device.h> 13 #include <linux/irq.h> 14 #include <linux/irqdomain.h> 15 #include <linux/msi.h> 16 #include <linux/slab.h> 17 #include <linux/sysfs.h> 18 #include <linux/pci.h> 19 20 #include "internals.h" 21 22 static inline int msi_sysfs_create_group(struct device *dev); 23 24 /** 25 * msi_alloc_desc - Allocate an initialized msi_desc 26 * @dev: Pointer to the device for which this is allocated 27 * @nvec: The number of vectors used in this entry 28 * @affinity: Optional pointer to an affinity mask array size of @nvec 29 * 30 * If @affinity is not %NULL then an affinity array[@nvec] is allocated 31 * and the affinity masks and flags from @affinity are copied. 32 * 33 * Return: pointer to allocated &msi_desc on success or %NULL on failure 34 */ 35 static struct msi_desc *msi_alloc_desc(struct device *dev, int nvec, 36 const struct irq_affinity_desc *affinity) 37 { 38 struct msi_desc *desc = kzalloc(sizeof(*desc), GFP_KERNEL); 39 40 if (!desc) 41 return NULL; 42 43 desc->dev = dev; 44 desc->nvec_used = nvec; 45 if (affinity) { 46 desc->affinity = kmemdup(affinity, nvec * sizeof(*desc->affinity), GFP_KERNEL); 47 if (!desc->affinity) { 48 kfree(desc); 49 return NULL; 50 } 51 } 52 return desc; 53 } 54 55 static void msi_free_desc(struct msi_desc *desc) 56 { 57 kfree(desc->affinity); 58 kfree(desc); 59 } 60 61 static int msi_insert_desc(struct msi_device_data *md, struct msi_desc *desc, unsigned int index) 62 { 63 int ret; 64 65 desc->msi_index = index; 66 ret = xa_insert(&md->__store, index, desc, GFP_KERNEL); 67 if (ret) 68 msi_free_desc(desc); 69 return ret; 70 } 71 72 /** 73 * msi_add_msi_desc - Allocate and initialize a MSI descriptor 74 * @dev: Pointer to the device for which the descriptor is allocated 75 * @init_desc: Pointer to an MSI descriptor to initialize the new descriptor 76 * 77 * Return: 0 on success or an appropriate failure code. 78 */ 79 int msi_add_msi_desc(struct device *dev, struct msi_desc *init_desc) 80 { 81 struct msi_desc *desc; 82 83 lockdep_assert_held(&dev->msi.data->mutex); 84 85 desc = msi_alloc_desc(dev, init_desc->nvec_used, init_desc->affinity); 86 if (!desc) 87 return -ENOMEM; 88 89 /* Copy type specific data to the new descriptor. */ 90 desc->pci = init_desc->pci; 91 return msi_insert_desc(dev->msi.data, desc, init_desc->msi_index); 92 } 93 94 /** 95 * msi_add_simple_msi_descs - Allocate and initialize MSI descriptors 96 * @dev: Pointer to the device for which the descriptors are allocated 97 * @index: Index for the first MSI descriptor 98 * @ndesc: Number of descriptors to allocate 99 * 100 * Return: 0 on success or an appropriate failure code. 101 */ 102 static int msi_add_simple_msi_descs(struct device *dev, unsigned int index, unsigned int ndesc) 103 { 104 unsigned int idx, last = index + ndesc - 1; 105 struct msi_desc *desc; 106 int ret; 107 108 lockdep_assert_held(&dev->msi.data->mutex); 109 110 for (idx = index; idx <= last; idx++) { 111 desc = msi_alloc_desc(dev, 1, NULL); 112 if (!desc) 113 goto fail_mem; 114 ret = msi_insert_desc(dev->msi.data, desc, idx); 115 if (ret) 116 goto fail; 117 } 118 return 0; 119 120 fail_mem: 121 ret = -ENOMEM; 122 fail: 123 msi_free_msi_descs_range(dev, index, last); 124 return ret; 125 } 126 127 static bool msi_desc_match(struct msi_desc *desc, enum msi_desc_filter filter) 128 { 129 switch (filter) { 130 case MSI_DESC_ALL: 131 return true; 132 case MSI_DESC_NOTASSOCIATED: 133 return !desc->irq; 134 case MSI_DESC_ASSOCIATED: 135 return !!desc->irq; 136 } 137 WARN_ON_ONCE(1); 138 return false; 139 } 140 141 /** 142 * msi_free_msi_descs_range - Free MSI descriptors of a device 143 * @dev: Device to free the descriptors 144 * @first_index: Index to start freeing from 145 * @last_index: Last index to be freed 146 */ 147 void msi_free_msi_descs_range(struct device *dev, unsigned int first_index, 148 unsigned int last_index) 149 { 150 struct xarray *xa = &dev->msi.data->__store; 151 struct msi_desc *desc; 152 unsigned long idx; 153 154 lockdep_assert_held(&dev->msi.data->mutex); 155 156 xa_for_each_range(xa, idx, desc, first_index, last_index) { 157 xa_erase(xa, idx); 158 159 /* Leak the descriptor when it is still referenced */ 160 if (WARN_ON_ONCE(msi_desc_match(desc, MSI_DESC_ASSOCIATED))) 161 continue; 162 msi_free_desc(desc); 163 } 164 } 165 166 void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg) 167 { 168 *msg = entry->msg; 169 } 170 171 void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg) 172 { 173 struct msi_desc *entry = irq_get_msi_desc(irq); 174 175 __get_cached_msi_msg(entry, msg); 176 } 177 EXPORT_SYMBOL_GPL(get_cached_msi_msg); 178 179 static void msi_device_data_release(struct device *dev, void *res) 180 { 181 struct msi_device_data *md = res; 182 183 WARN_ON_ONCE(!xa_empty(&md->__store)); 184 xa_destroy(&md->__store); 185 dev->msi.data = NULL; 186 } 187 188 /** 189 * msi_setup_device_data - Setup MSI device data 190 * @dev: Device for which MSI device data should be set up 191 * 192 * Return: 0 on success, appropriate error code otherwise 193 * 194 * This can be called more than once for @dev. If the MSI device data is 195 * already allocated the call succeeds. The allocated memory is 196 * automatically released when the device is destroyed. 197 */ 198 int msi_setup_device_data(struct device *dev) 199 { 200 struct msi_device_data *md; 201 int ret; 202 203 if (dev->msi.data) 204 return 0; 205 206 md = devres_alloc(msi_device_data_release, sizeof(*md), GFP_KERNEL); 207 if (!md) 208 return -ENOMEM; 209 210 ret = msi_sysfs_create_group(dev); 211 if (ret) { 212 devres_free(md); 213 return ret; 214 } 215 216 xa_init(&md->__store); 217 mutex_init(&md->mutex); 218 dev->msi.data = md; 219 devres_add(dev, md); 220 return 0; 221 } 222 223 /** 224 * msi_lock_descs - Lock the MSI descriptor storage of a device 225 * @dev: Device to operate on 226 */ 227 void msi_lock_descs(struct device *dev) 228 { 229 mutex_lock(&dev->msi.data->mutex); 230 } 231 EXPORT_SYMBOL_GPL(msi_lock_descs); 232 233 /** 234 * msi_unlock_descs - Unlock the MSI descriptor storage of a device 235 * @dev: Device to operate on 236 */ 237 void msi_unlock_descs(struct device *dev) 238 { 239 /* Invalidate the index wich was cached by the iterator */ 240 dev->msi.data->__iter_idx = MSI_MAX_INDEX; 241 mutex_unlock(&dev->msi.data->mutex); 242 } 243 EXPORT_SYMBOL_GPL(msi_unlock_descs); 244 245 static struct msi_desc *msi_find_desc(struct msi_device_data *md, enum msi_desc_filter filter) 246 { 247 struct msi_desc *desc; 248 249 xa_for_each_start(&md->__store, md->__iter_idx, desc, md->__iter_idx) { 250 if (msi_desc_match(desc, filter)) 251 return desc; 252 } 253 md->__iter_idx = MSI_MAX_INDEX; 254 return NULL; 255 } 256 257 /** 258 * msi_first_desc - Get the first MSI descriptor of a device 259 * @dev: Device to operate on 260 * @filter: Descriptor state filter 261 * 262 * Must be called with the MSI descriptor mutex held, i.e. msi_lock_descs() 263 * must be invoked before the call. 264 * 265 * Return: Pointer to the first MSI descriptor matching the search 266 * criteria, NULL if none found. 267 */ 268 struct msi_desc *msi_first_desc(struct device *dev, enum msi_desc_filter filter) 269 { 270 struct msi_device_data *md = dev->msi.data; 271 272 if (WARN_ON_ONCE(!md)) 273 return NULL; 274 275 lockdep_assert_held(&md->mutex); 276 277 md->__iter_idx = 0; 278 return msi_find_desc(md, filter); 279 } 280 EXPORT_SYMBOL_GPL(msi_first_desc); 281 282 /** 283 * msi_next_desc - Get the next MSI descriptor of a device 284 * @dev: Device to operate on 285 * @filter: Descriptor state filter 286 * 287 * The first invocation of msi_next_desc() has to be preceeded by a 288 * successful invocation of __msi_first_desc(). Consecutive invocations are 289 * only valid if the previous one was successful. All these operations have 290 * to be done within the same MSI mutex held region. 291 * 292 * Return: Pointer to the next MSI descriptor matching the search 293 * criteria, NULL if none found. 294 */ 295 struct msi_desc *msi_next_desc(struct device *dev, enum msi_desc_filter filter) 296 { 297 struct msi_device_data *md = dev->msi.data; 298 299 if (WARN_ON_ONCE(!md)) 300 return NULL; 301 302 lockdep_assert_held(&md->mutex); 303 304 if (md->__iter_idx >= (unsigned long)MSI_MAX_INDEX) 305 return NULL; 306 307 md->__iter_idx++; 308 return msi_find_desc(md, filter); 309 } 310 EXPORT_SYMBOL_GPL(msi_next_desc); 311 312 /** 313 * msi_get_virq - Return Linux interrupt number of a MSI interrupt 314 * @dev: Device to operate on 315 * @index: MSI interrupt index to look for (0-based) 316 * 317 * Return: The Linux interrupt number on success (> 0), 0 if not found 318 */ 319 unsigned int msi_get_virq(struct device *dev, unsigned int index) 320 { 321 struct msi_desc *desc; 322 unsigned int ret = 0; 323 bool pcimsi; 324 325 if (!dev->msi.data) 326 return 0; 327 328 pcimsi = dev_is_pci(dev) ? to_pci_dev(dev)->msi_enabled : false; 329 330 msi_lock_descs(dev); 331 desc = xa_load(&dev->msi.data->__store, pcimsi ? 0 : index); 332 if (desc && desc->irq) { 333 /* 334 * PCI-MSI has only one descriptor for multiple interrupts. 335 * PCI-MSIX and platform MSI use a descriptor per 336 * interrupt. 337 */ 338 if (pcimsi) { 339 if (index < desc->nvec_used) 340 ret = desc->irq + index; 341 } else { 342 ret = desc->irq; 343 } 344 } 345 msi_unlock_descs(dev); 346 return ret; 347 } 348 EXPORT_SYMBOL_GPL(msi_get_virq); 349 350 #ifdef CONFIG_SYSFS 351 static struct attribute *msi_dev_attrs[] = { 352 NULL 353 }; 354 355 static const struct attribute_group msi_irqs_group = { 356 .name = "msi_irqs", 357 .attrs = msi_dev_attrs, 358 }; 359 360 static inline int msi_sysfs_create_group(struct device *dev) 361 { 362 return devm_device_add_group(dev, &msi_irqs_group); 363 } 364 365 static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr, 366 char *buf) 367 { 368 /* MSI vs. MSIX is per device not per interrupt */ 369 bool is_msix = dev_is_pci(dev) ? to_pci_dev(dev)->msix_enabled : false; 370 371 return sysfs_emit(buf, "%s\n", is_msix ? "msix" : "msi"); 372 } 373 374 static void msi_sysfs_remove_desc(struct device *dev, struct msi_desc *desc) 375 { 376 struct device_attribute *attrs = desc->sysfs_attrs; 377 int i; 378 379 if (!attrs) 380 return; 381 382 desc->sysfs_attrs = NULL; 383 for (i = 0; i < desc->nvec_used; i++) { 384 if (attrs[i].show) 385 sysfs_remove_file_from_group(&dev->kobj, &attrs[i].attr, msi_irqs_group.name); 386 kfree(attrs[i].attr.name); 387 } 388 kfree(attrs); 389 } 390 391 static int msi_sysfs_populate_desc(struct device *dev, struct msi_desc *desc) 392 { 393 struct device_attribute *attrs; 394 int ret, i; 395 396 attrs = kcalloc(desc->nvec_used, sizeof(*attrs), GFP_KERNEL); 397 if (!attrs) 398 return -ENOMEM; 399 400 desc->sysfs_attrs = attrs; 401 for (i = 0; i < desc->nvec_used; i++) { 402 sysfs_attr_init(&attrs[i].attr); 403 attrs[i].attr.name = kasprintf(GFP_KERNEL, "%d", desc->irq + i); 404 if (!attrs[i].attr.name) { 405 ret = -ENOMEM; 406 goto fail; 407 } 408 409 attrs[i].attr.mode = 0444; 410 attrs[i].show = msi_mode_show; 411 412 ret = sysfs_add_file_to_group(&dev->kobj, &attrs[i].attr, msi_irqs_group.name); 413 if (ret) { 414 attrs[i].show = NULL; 415 goto fail; 416 } 417 } 418 return 0; 419 420 fail: 421 msi_sysfs_remove_desc(dev, desc); 422 return ret; 423 } 424 425 #ifdef CONFIG_PCI_MSI_ARCH_FALLBACKS 426 /** 427 * msi_device_populate_sysfs - Populate msi_irqs sysfs entries for a device 428 * @dev: The device (PCI, platform etc) which will get sysfs entries 429 */ 430 int msi_device_populate_sysfs(struct device *dev) 431 { 432 struct msi_desc *desc; 433 int ret; 434 435 msi_for_each_desc(desc, dev, MSI_DESC_ASSOCIATED) { 436 if (desc->sysfs_attrs) 437 continue; 438 ret = msi_sysfs_populate_desc(dev, desc); 439 if (ret) 440 return ret; 441 } 442 return 0; 443 } 444 445 /** 446 * msi_device_destroy_sysfs - Destroy msi_irqs sysfs entries for a device 447 * @dev: The device (PCI, platform etc) for which to remove 448 * sysfs entries 449 */ 450 void msi_device_destroy_sysfs(struct device *dev) 451 { 452 struct msi_desc *desc; 453 454 msi_for_each_desc(desc, dev, MSI_DESC_ALL) 455 msi_sysfs_remove_desc(dev, desc); 456 } 457 #endif /* CONFIG_PCI_MSI_ARCH_FALLBACK */ 458 #else /* CONFIG_SYSFS */ 459 static inline int msi_sysfs_create_group(struct device *dev) { return 0; } 460 static inline int msi_sysfs_populate_desc(struct device *dev, struct msi_desc *desc) { return 0; } 461 static inline void msi_sysfs_remove_desc(struct device *dev, struct msi_desc *desc) { } 462 #endif /* !CONFIG_SYSFS */ 463 464 static int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, int nvec); 465 static void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev); 466 467 static inline void irq_chip_write_msi_msg(struct irq_data *data, 468 struct msi_msg *msg) 469 { 470 data->chip->irq_write_msi_msg(data, msg); 471 } 472 473 static void msi_check_level(struct irq_domain *domain, struct msi_msg *msg) 474 { 475 struct msi_domain_info *info = domain->host_data; 476 477 /* 478 * If the MSI provider has messed with the second message and 479 * not advertized that it is level-capable, signal the breakage. 480 */ 481 WARN_ON(!((info->flags & MSI_FLAG_LEVEL_CAPABLE) && 482 (info->chip->flags & IRQCHIP_SUPPORTS_LEVEL_MSI)) && 483 (msg[1].address_lo || msg[1].address_hi || msg[1].data)); 484 } 485 486 /** 487 * msi_domain_set_affinity - Generic affinity setter function for MSI domains 488 * @irq_data: The irq data associated to the interrupt 489 * @mask: The affinity mask to set 490 * @force: Flag to enforce setting (disable online checks) 491 * 492 * Intended to be used by MSI interrupt controllers which are 493 * implemented with hierarchical domains. 494 * 495 * Return: IRQ_SET_MASK_* result code 496 */ 497 int msi_domain_set_affinity(struct irq_data *irq_data, 498 const struct cpumask *mask, bool force) 499 { 500 struct irq_data *parent = irq_data->parent_data; 501 struct msi_msg msg[2] = { [1] = { }, }; 502 int ret; 503 504 ret = parent->chip->irq_set_affinity(parent, mask, force); 505 if (ret >= 0 && ret != IRQ_SET_MASK_OK_DONE) { 506 BUG_ON(irq_chip_compose_msi_msg(irq_data, msg)); 507 msi_check_level(irq_data->domain, msg); 508 irq_chip_write_msi_msg(irq_data, msg); 509 } 510 511 return ret; 512 } 513 514 static int msi_domain_activate(struct irq_domain *domain, 515 struct irq_data *irq_data, bool early) 516 { 517 struct msi_msg msg[2] = { [1] = { }, }; 518 519 BUG_ON(irq_chip_compose_msi_msg(irq_data, msg)); 520 msi_check_level(irq_data->domain, msg); 521 irq_chip_write_msi_msg(irq_data, msg); 522 return 0; 523 } 524 525 static void msi_domain_deactivate(struct irq_domain *domain, 526 struct irq_data *irq_data) 527 { 528 struct msi_msg msg[2]; 529 530 memset(msg, 0, sizeof(msg)); 531 irq_chip_write_msi_msg(irq_data, msg); 532 } 533 534 static int msi_domain_alloc(struct irq_domain *domain, unsigned int virq, 535 unsigned int nr_irqs, void *arg) 536 { 537 struct msi_domain_info *info = domain->host_data; 538 struct msi_domain_ops *ops = info->ops; 539 irq_hw_number_t hwirq = ops->get_hwirq(info, arg); 540 int i, ret; 541 542 if (irq_find_mapping(domain, hwirq) > 0) 543 return -EEXIST; 544 545 if (domain->parent) { 546 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg); 547 if (ret < 0) 548 return ret; 549 } 550 551 for (i = 0; i < nr_irqs; i++) { 552 ret = ops->msi_init(domain, info, virq + i, hwirq + i, arg); 553 if (ret < 0) { 554 if (ops->msi_free) { 555 for (i--; i > 0; i--) 556 ops->msi_free(domain, info, virq + i); 557 } 558 irq_domain_free_irqs_top(domain, virq, nr_irqs); 559 return ret; 560 } 561 } 562 563 return 0; 564 } 565 566 static void msi_domain_free(struct irq_domain *domain, unsigned int virq, 567 unsigned int nr_irqs) 568 { 569 struct msi_domain_info *info = domain->host_data; 570 int i; 571 572 if (info->ops->msi_free) { 573 for (i = 0; i < nr_irqs; i++) 574 info->ops->msi_free(domain, info, virq + i); 575 } 576 irq_domain_free_irqs_top(domain, virq, nr_irqs); 577 } 578 579 static const struct irq_domain_ops msi_domain_ops = { 580 .alloc = msi_domain_alloc, 581 .free = msi_domain_free, 582 .activate = msi_domain_activate, 583 .deactivate = msi_domain_deactivate, 584 }; 585 586 static irq_hw_number_t msi_domain_ops_get_hwirq(struct msi_domain_info *info, 587 msi_alloc_info_t *arg) 588 { 589 return arg->hwirq; 590 } 591 592 static int msi_domain_ops_prepare(struct irq_domain *domain, struct device *dev, 593 int nvec, msi_alloc_info_t *arg) 594 { 595 memset(arg, 0, sizeof(*arg)); 596 return 0; 597 } 598 599 static void msi_domain_ops_set_desc(msi_alloc_info_t *arg, 600 struct msi_desc *desc) 601 { 602 arg->desc = desc; 603 } 604 605 static int msi_domain_ops_init(struct irq_domain *domain, 606 struct msi_domain_info *info, 607 unsigned int virq, irq_hw_number_t hwirq, 608 msi_alloc_info_t *arg) 609 { 610 irq_domain_set_hwirq_and_chip(domain, virq, hwirq, info->chip, 611 info->chip_data); 612 if (info->handler && info->handler_name) { 613 __irq_set_handler(virq, info->handler, 0, info->handler_name); 614 if (info->handler_data) 615 irq_set_handler_data(virq, info->handler_data); 616 } 617 return 0; 618 } 619 620 static int msi_domain_ops_check(struct irq_domain *domain, 621 struct msi_domain_info *info, 622 struct device *dev) 623 { 624 return 0; 625 } 626 627 static struct msi_domain_ops msi_domain_ops_default = { 628 .get_hwirq = msi_domain_ops_get_hwirq, 629 .msi_init = msi_domain_ops_init, 630 .msi_check = msi_domain_ops_check, 631 .msi_prepare = msi_domain_ops_prepare, 632 .set_desc = msi_domain_ops_set_desc, 633 .domain_alloc_irqs = __msi_domain_alloc_irqs, 634 .domain_free_irqs = __msi_domain_free_irqs, 635 }; 636 637 static void msi_domain_update_dom_ops(struct msi_domain_info *info) 638 { 639 struct msi_domain_ops *ops = info->ops; 640 641 if (ops == NULL) { 642 info->ops = &msi_domain_ops_default; 643 return; 644 } 645 646 if (ops->domain_alloc_irqs == NULL) 647 ops->domain_alloc_irqs = msi_domain_ops_default.domain_alloc_irqs; 648 if (ops->domain_free_irqs == NULL) 649 ops->domain_free_irqs = msi_domain_ops_default.domain_free_irqs; 650 651 if (!(info->flags & MSI_FLAG_USE_DEF_DOM_OPS)) 652 return; 653 654 if (ops->get_hwirq == NULL) 655 ops->get_hwirq = msi_domain_ops_default.get_hwirq; 656 if (ops->msi_init == NULL) 657 ops->msi_init = msi_domain_ops_default.msi_init; 658 if (ops->msi_check == NULL) 659 ops->msi_check = msi_domain_ops_default.msi_check; 660 if (ops->msi_prepare == NULL) 661 ops->msi_prepare = msi_domain_ops_default.msi_prepare; 662 if (ops->set_desc == NULL) 663 ops->set_desc = msi_domain_ops_default.set_desc; 664 } 665 666 static void msi_domain_update_chip_ops(struct msi_domain_info *info) 667 { 668 struct irq_chip *chip = info->chip; 669 670 BUG_ON(!chip || !chip->irq_mask || !chip->irq_unmask); 671 if (!chip->irq_set_affinity) 672 chip->irq_set_affinity = msi_domain_set_affinity; 673 } 674 675 /** 676 * msi_create_irq_domain - Create an MSI interrupt domain 677 * @fwnode: Optional fwnode of the interrupt controller 678 * @info: MSI domain info 679 * @parent: Parent irq domain 680 * 681 * Return: pointer to the created &struct irq_domain or %NULL on failure 682 */ 683 struct irq_domain *msi_create_irq_domain(struct fwnode_handle *fwnode, 684 struct msi_domain_info *info, 685 struct irq_domain *parent) 686 { 687 struct irq_domain *domain; 688 689 msi_domain_update_dom_ops(info); 690 if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS) 691 msi_domain_update_chip_ops(info); 692 693 domain = irq_domain_create_hierarchy(parent, IRQ_DOMAIN_FLAG_MSI, 0, 694 fwnode, &msi_domain_ops, info); 695 696 if (domain) { 697 if (!domain->name && info->chip) 698 domain->name = info->chip->name; 699 irq_domain_update_bus_token(domain, info->bus_token); 700 } 701 702 return domain; 703 } 704 705 int msi_domain_prepare_irqs(struct irq_domain *domain, struct device *dev, 706 int nvec, msi_alloc_info_t *arg) 707 { 708 struct msi_domain_info *info = domain->host_data; 709 struct msi_domain_ops *ops = info->ops; 710 int ret; 711 712 ret = ops->msi_check(domain, info, dev); 713 if (ret == 0) 714 ret = ops->msi_prepare(domain, dev, nvec, arg); 715 716 return ret; 717 } 718 719 int msi_domain_populate_irqs(struct irq_domain *domain, struct device *dev, 720 int virq_base, int nvec, msi_alloc_info_t *arg) 721 { 722 struct msi_domain_info *info = domain->host_data; 723 struct msi_domain_ops *ops = info->ops; 724 struct msi_desc *desc; 725 int ret, virq; 726 727 msi_lock_descs(dev); 728 ret = msi_add_simple_msi_descs(dev, virq_base, nvec); 729 if (ret) 730 goto unlock; 731 732 for (virq = virq_base; virq < virq_base + nvec; virq++) { 733 desc = xa_load(&dev->msi.data->__store, virq); 734 desc->irq = virq; 735 736 ops->set_desc(arg, desc); 737 ret = irq_domain_alloc_irqs_hierarchy(domain, virq, 1, arg); 738 if (ret) 739 goto fail; 740 741 irq_set_msi_desc(virq, desc); 742 } 743 msi_unlock_descs(dev); 744 return 0; 745 746 fail: 747 for (--virq; virq >= virq_base; virq--) 748 irq_domain_free_irqs_common(domain, virq, 1); 749 msi_free_msi_descs_range(dev, virq_base, virq_base + nvec - 1); 750 unlock: 751 msi_unlock_descs(dev); 752 return ret; 753 } 754 755 /* 756 * Carefully check whether the device can use reservation mode. If 757 * reservation mode is enabled then the early activation will assign a 758 * dummy vector to the device. If the PCI/MSI device does not support 759 * masking of the entry then this can result in spurious interrupts when 760 * the device driver is not absolutely careful. But even then a malfunction 761 * of the hardware could result in a spurious interrupt on the dummy vector 762 * and render the device unusable. If the entry can be masked then the core 763 * logic will prevent the spurious interrupt and reservation mode can be 764 * used. For now reservation mode is restricted to PCI/MSI. 765 */ 766 static bool msi_check_reservation_mode(struct irq_domain *domain, 767 struct msi_domain_info *info, 768 struct device *dev) 769 { 770 struct msi_desc *desc; 771 772 switch(domain->bus_token) { 773 case DOMAIN_BUS_PCI_MSI: 774 case DOMAIN_BUS_VMD_MSI: 775 break; 776 default: 777 return false; 778 } 779 780 if (!(info->flags & MSI_FLAG_MUST_REACTIVATE)) 781 return false; 782 783 if (IS_ENABLED(CONFIG_PCI_MSI) && pci_msi_ignore_mask) 784 return false; 785 786 /* 787 * Checking the first MSI descriptor is sufficient. MSIX supports 788 * masking and MSI does so when the can_mask attribute is set. 789 */ 790 desc = msi_first_desc(dev, MSI_DESC_ALL); 791 return desc->pci.msi_attrib.is_msix || desc->pci.msi_attrib.can_mask; 792 } 793 794 static int msi_handle_pci_fail(struct irq_domain *domain, struct msi_desc *desc, 795 int allocated) 796 { 797 switch(domain->bus_token) { 798 case DOMAIN_BUS_PCI_MSI: 799 case DOMAIN_BUS_VMD_MSI: 800 if (IS_ENABLED(CONFIG_PCI_MSI)) 801 break; 802 fallthrough; 803 default: 804 return -ENOSPC; 805 } 806 807 /* Let a failed PCI multi MSI allocation retry */ 808 if (desc->nvec_used > 1) 809 return 1; 810 811 /* If there was a successful allocation let the caller know */ 812 return allocated ? allocated : -ENOSPC; 813 } 814 815 #define VIRQ_CAN_RESERVE 0x01 816 #define VIRQ_ACTIVATE 0x02 817 #define VIRQ_NOMASK_QUIRK 0x04 818 819 static int msi_init_virq(struct irq_domain *domain, int virq, unsigned int vflags) 820 { 821 struct irq_data *irqd = irq_domain_get_irq_data(domain, virq); 822 int ret; 823 824 if (!(vflags & VIRQ_CAN_RESERVE)) { 825 irqd_clr_can_reserve(irqd); 826 if (vflags & VIRQ_NOMASK_QUIRK) 827 irqd_set_msi_nomask_quirk(irqd); 828 829 /* 830 * If the interrupt is managed but no CPU is available to 831 * service it, shut it down until better times. Note that 832 * we only do this on the !RESERVE path as x86 (the only 833 * architecture using this flag) deals with this in a 834 * different way by using a catch-all vector. 835 */ 836 if ((vflags & VIRQ_ACTIVATE) && 837 irqd_affinity_is_managed(irqd) && 838 !cpumask_intersects(irq_data_get_affinity_mask(irqd), 839 cpu_online_mask)) { 840 irqd_set_managed_shutdown(irqd); 841 return 0; 842 } 843 } 844 845 if (!(vflags & VIRQ_ACTIVATE)) 846 return 0; 847 848 ret = irq_domain_activate_irq(irqd, vflags & VIRQ_CAN_RESERVE); 849 if (ret) 850 return ret; 851 /* 852 * If the interrupt uses reservation mode, clear the activated bit 853 * so request_irq() will assign the final vector. 854 */ 855 if (vflags & VIRQ_CAN_RESERVE) 856 irqd_clr_activated(irqd); 857 return 0; 858 } 859 860 static int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, 861 int nvec) 862 { 863 struct msi_domain_info *info = domain->host_data; 864 struct msi_domain_ops *ops = info->ops; 865 msi_alloc_info_t arg = { }; 866 unsigned int vflags = 0; 867 struct msi_desc *desc; 868 int allocated = 0; 869 int i, ret, virq; 870 871 ret = msi_domain_prepare_irqs(domain, dev, nvec, &arg); 872 if (ret) 873 return ret; 874 875 /* 876 * This flag is set by the PCI layer as we need to activate 877 * the MSI entries before the PCI layer enables MSI in the 878 * card. Otherwise the card latches a random msi message. 879 */ 880 if (info->flags & MSI_FLAG_ACTIVATE_EARLY) 881 vflags |= VIRQ_ACTIVATE; 882 883 /* 884 * Interrupt can use a reserved vector and will not occupy 885 * a real device vector until the interrupt is requested. 886 */ 887 if (msi_check_reservation_mode(domain, info, dev)) { 888 vflags |= VIRQ_CAN_RESERVE; 889 /* 890 * MSI affinity setting requires a special quirk (X86) when 891 * reservation mode is active. 892 */ 893 if (domain->flags & IRQ_DOMAIN_MSI_NOMASK_QUIRK) 894 vflags |= VIRQ_NOMASK_QUIRK; 895 } 896 897 msi_for_each_desc(desc, dev, MSI_DESC_NOTASSOCIATED) { 898 ops->set_desc(&arg, desc); 899 900 virq = __irq_domain_alloc_irqs(domain, -1, desc->nvec_used, 901 dev_to_node(dev), &arg, false, 902 desc->affinity); 903 if (virq < 0) 904 return msi_handle_pci_fail(domain, desc, allocated); 905 906 for (i = 0; i < desc->nvec_used; i++) { 907 irq_set_msi_desc_off(virq, i, desc); 908 irq_debugfs_copy_devname(virq + i, dev); 909 ret = msi_init_virq(domain, virq + i, vflags); 910 if (ret) 911 return ret; 912 } 913 if (info->flags & MSI_FLAG_DEV_SYSFS) { 914 ret = msi_sysfs_populate_desc(dev, desc); 915 if (ret) 916 return ret; 917 } 918 allocated++; 919 } 920 return 0; 921 } 922 923 static int msi_domain_add_simple_msi_descs(struct msi_domain_info *info, 924 struct device *dev, 925 unsigned int num_descs) 926 { 927 if (!(info->flags & MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS)) 928 return 0; 929 930 return msi_add_simple_msi_descs(dev, 0, num_descs); 931 } 932 933 /** 934 * msi_domain_alloc_irqs_descs_locked - Allocate interrupts from a MSI interrupt domain 935 * @domain: The domain to allocate from 936 * @dev: Pointer to device struct of the device for which the interrupts 937 * are allocated 938 * @nvec: The number of interrupts to allocate 939 * 940 * Must be invoked from within a msi_lock_descs() / msi_unlock_descs() 941 * pair. Use this for MSI irqdomains which implement their own vector 942 * allocation/free. 943 * 944 * Return: %0 on success or an error code. 945 */ 946 int msi_domain_alloc_irqs_descs_locked(struct irq_domain *domain, struct device *dev, 947 int nvec) 948 { 949 struct msi_domain_info *info = domain->host_data; 950 struct msi_domain_ops *ops = info->ops; 951 int ret; 952 953 lockdep_assert_held(&dev->msi.data->mutex); 954 955 ret = msi_domain_add_simple_msi_descs(info, dev, nvec); 956 if (ret) 957 return ret; 958 959 ret = ops->domain_alloc_irqs(domain, dev, nvec); 960 if (ret) 961 msi_domain_free_irqs_descs_locked(domain, dev); 962 return ret; 963 } 964 965 /** 966 * msi_domain_alloc_irqs - Allocate interrupts from a MSI interrupt domain 967 * @domain: The domain to allocate from 968 * @dev: Pointer to device struct of the device for which the interrupts 969 * are allocated 970 * @nvec: The number of interrupts to allocate 971 * 972 * Return: %0 on success or an error code. 973 */ 974 int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, int nvec) 975 { 976 int ret; 977 978 msi_lock_descs(dev); 979 ret = msi_domain_alloc_irqs_descs_locked(domain, dev, nvec); 980 msi_unlock_descs(dev); 981 return ret; 982 } 983 984 static void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev) 985 { 986 struct msi_domain_info *info = domain->host_data; 987 struct irq_data *irqd; 988 struct msi_desc *desc; 989 int i; 990 991 /* Only handle MSI entries which have an interrupt associated */ 992 msi_for_each_desc(desc, dev, MSI_DESC_ASSOCIATED) { 993 /* Make sure all interrupts are deactivated */ 994 for (i = 0; i < desc->nvec_used; i++) { 995 irqd = irq_domain_get_irq_data(domain, desc->irq + i); 996 if (irqd && irqd_is_activated(irqd)) 997 irq_domain_deactivate_irq(irqd); 998 } 999 1000 irq_domain_free_irqs(desc->irq, desc->nvec_used); 1001 if (info->flags & MSI_FLAG_DEV_SYSFS) 1002 msi_sysfs_remove_desc(dev, desc); 1003 desc->irq = 0; 1004 } 1005 } 1006 1007 static void msi_domain_free_msi_descs(struct msi_domain_info *info, 1008 struct device *dev) 1009 { 1010 if (info->flags & MSI_FLAG_FREE_MSI_DESCS) 1011 msi_free_msi_descs(dev); 1012 } 1013 1014 /** 1015 * msi_domain_free_irqs_descs_locked - Free interrupts from a MSI interrupt @domain associated to @dev 1016 * @domain: The domain to managing the interrupts 1017 * @dev: Pointer to device struct of the device for which the interrupts 1018 * are free 1019 * 1020 * Must be invoked from within a msi_lock_descs() / msi_unlock_descs() 1021 * pair. Use this for MSI irqdomains which implement their own vector 1022 * allocation. 1023 */ 1024 void msi_domain_free_irqs_descs_locked(struct irq_domain *domain, struct device *dev) 1025 { 1026 struct msi_domain_info *info = domain->host_data; 1027 struct msi_domain_ops *ops = info->ops; 1028 1029 lockdep_assert_held(&dev->msi.data->mutex); 1030 1031 ops->domain_free_irqs(domain, dev); 1032 if (ops->msi_post_free) 1033 ops->msi_post_free(domain, dev); 1034 msi_domain_free_msi_descs(info, dev); 1035 } 1036 1037 /** 1038 * msi_domain_free_irqs - Free interrupts from a MSI interrupt @domain associated to @dev 1039 * @domain: The domain to managing the interrupts 1040 * @dev: Pointer to device struct of the device for which the interrupts 1041 * are free 1042 */ 1043 void msi_domain_free_irqs(struct irq_domain *domain, struct device *dev) 1044 { 1045 msi_lock_descs(dev); 1046 msi_domain_free_irqs_descs_locked(domain, dev); 1047 msi_unlock_descs(dev); 1048 } 1049 1050 /** 1051 * msi_get_domain_info - Get the MSI interrupt domain info for @domain 1052 * @domain: The interrupt domain to retrieve data from 1053 * 1054 * Return: the pointer to the msi_domain_info stored in @domain->host_data. 1055 */ 1056 struct msi_domain_info *msi_get_domain_info(struct irq_domain *domain) 1057 { 1058 return (struct msi_domain_info *)domain->host_data; 1059 } 1060