152a65ff5SThomas Gleixner // SPDX-License-Identifier: GPL-2.0 2f3cf8bb0SJiang Liu /* 3f3cf8bb0SJiang Liu * Copyright (C) 2014 Intel Corp. 4f3cf8bb0SJiang Liu * Author: Jiang Liu <jiang.liu@linux.intel.com> 5f3cf8bb0SJiang Liu * 6f3cf8bb0SJiang Liu * This file is licensed under GPLv2. 7f3cf8bb0SJiang Liu * 8a359f757SIngo Molnar * This file contains common code to support Message Signaled Interrupts for 9f3cf8bb0SJiang Liu * PCI compatible and non PCI compatible devices. 10f3cf8bb0SJiang Liu */ 11aeeb5965SJiang Liu #include <linux/types.h> 12aeeb5965SJiang Liu #include <linux/device.h> 13f3cf8bb0SJiang Liu #include <linux/irq.h> 14f3cf8bb0SJiang Liu #include <linux/irqdomain.h> 15f3cf8bb0SJiang Liu #include <linux/msi.h> 164e201566SMarc Zyngier #include <linux/slab.h> 173ba1f050SThomas Gleixner #include <linux/sysfs.h> 182f170814SBarry Song #include <linux/pci.h> 19d9109698SJiang Liu 2007557ccbSThomas Gleixner #include "internals.h" 2107557ccbSThomas Gleixner 2228f4b041SThomas Gleixner /** 233b35e7e6SRandy Dunlap * alloc_msi_entry - Allocate an initialized msi_desc 2428f4b041SThomas Gleixner * @dev: Pointer to the device for which this is allocated 2528f4b041SThomas Gleixner * @nvec: The number of vectors used in this entry 2628f4b041SThomas Gleixner * @affinity: Optional pointer to an affinity mask array size of @nvec 2728f4b041SThomas Gleixner * 283b35e7e6SRandy Dunlap * If @affinity is not %NULL then an affinity array[@nvec] is allocated 29bec04037SDou Liyang * and the affinity masks and flags from @affinity are copied. 303b35e7e6SRandy Dunlap * 313b35e7e6SRandy Dunlap * Return: pointer to allocated &msi_desc on success or %NULL on failure 3228f4b041SThomas Gleixner */ 33bec04037SDou Liyang struct msi_desc *alloc_msi_entry(struct device *dev, int nvec, 34bec04037SDou Liyang const struct irq_affinity_desc *affinity) 35aa48b6f7SJiang Liu { 3628f4b041SThomas Gleixner struct msi_desc *desc; 3728f4b041SThomas Gleixner 3828f4b041SThomas Gleixner desc = kzalloc(sizeof(*desc), GFP_KERNEL); 39aa48b6f7SJiang Liu if (!desc) 40aa48b6f7SJiang Liu return NULL; 41aa48b6f7SJiang Liu 42aa48b6f7SJiang Liu INIT_LIST_HEAD(&desc->list); 43aa48b6f7SJiang Liu desc->dev = dev; 4428f4b041SThomas Gleixner desc->nvec_used = nvec; 4528f4b041SThomas Gleixner if (affinity) { 4628f4b041SThomas Gleixner desc->affinity = kmemdup(affinity, 4728f4b041SThomas Gleixner nvec * sizeof(*desc->affinity), GFP_KERNEL); 4828f4b041SThomas Gleixner if (!desc->affinity) { 4928f4b041SThomas Gleixner kfree(desc); 5028f4b041SThomas Gleixner return NULL; 5128f4b041SThomas Gleixner } 5228f4b041SThomas Gleixner } 53aa48b6f7SJiang Liu 54aa48b6f7SJiang Liu return desc; 55aa48b6f7SJiang Liu } 56aa48b6f7SJiang Liu 57aa48b6f7SJiang Liu void free_msi_entry(struct msi_desc *entry) 58aa48b6f7SJiang Liu { 5928f4b041SThomas Gleixner kfree(entry->affinity); 60aa48b6f7SJiang Liu kfree(entry); 61aa48b6f7SJiang Liu } 62aa48b6f7SJiang Liu 6338b6a1cfSJiang Liu void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg) 6438b6a1cfSJiang Liu { 6538b6a1cfSJiang Liu *msg = entry->msg; 6638b6a1cfSJiang Liu } 6738b6a1cfSJiang Liu 6838b6a1cfSJiang Liu void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg) 6938b6a1cfSJiang Liu { 7038b6a1cfSJiang Liu struct msi_desc *entry = irq_get_msi_desc(irq); 7138b6a1cfSJiang Liu 7238b6a1cfSJiang Liu __get_cached_msi_msg(entry, msg); 7338b6a1cfSJiang Liu } 7438b6a1cfSJiang Liu EXPORT_SYMBOL_GPL(get_cached_msi_msg); 7538b6a1cfSJiang Liu 761197528aSThomas Gleixner #ifdef CONFIG_SYSFS 772f170814SBarry Song static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr, 782f170814SBarry Song char *buf) 792f170814SBarry Song { 802f170814SBarry Song struct msi_desc *entry; 812f170814SBarry Song bool is_msix = false; 822f170814SBarry Song unsigned long irq; 832f170814SBarry Song int retval; 842f170814SBarry Song 852f170814SBarry Song retval = kstrtoul(attr->attr.name, 10, &irq); 862f170814SBarry Song if (retval) 872f170814SBarry Song return retval; 882f170814SBarry Song 892f170814SBarry Song entry = irq_get_msi_desc(irq); 902f170814SBarry Song if (!entry) 912f170814SBarry Song return -ENODEV; 922f170814SBarry Song 932f170814SBarry Song if (dev_is_pci(dev)) 94*e58f2259SThomas Gleixner is_msix = entry->pci.msi_attrib.is_msix; 952f170814SBarry Song 962f170814SBarry Song return sysfs_emit(buf, "%s\n", is_msix ? "msix" : "msi"); 972f170814SBarry Song } 982f170814SBarry Song 992f170814SBarry Song /** 1002f170814SBarry Song * msi_populate_sysfs - Populate msi_irqs sysfs entries for devices 1012f170814SBarry Song * @dev: The device(PCI, platform etc) who will get sysfs entries 1022f170814SBarry Song * 1032f170814SBarry Song * Return attribute_group ** so that specific bus MSI can save it to 1042f170814SBarry Song * somewhere during initilizing msi irqs. If devices has no MSI irq, 1052f170814SBarry Song * return NULL; if it fails to populate sysfs, return ERR_PTR 1062f170814SBarry Song */ 1072f170814SBarry Song const struct attribute_group **msi_populate_sysfs(struct device *dev) 1082f170814SBarry Song { 1092f170814SBarry Song const struct attribute_group **msi_irq_groups; 1102f170814SBarry Song struct attribute **msi_attrs, *msi_attr; 1112f170814SBarry Song struct device_attribute *msi_dev_attr; 1122f170814SBarry Song struct attribute_group *msi_irq_group; 1132f170814SBarry Song struct msi_desc *entry; 1142f170814SBarry Song int ret = -ENOMEM; 1152f170814SBarry Song int num_msi = 0; 1162f170814SBarry Song int count = 0; 1172f170814SBarry Song int i; 1182f170814SBarry Song 1192f170814SBarry Song /* Determine how many msi entries we have */ 1202f170814SBarry Song for_each_msi_entry(entry, dev) 1212f170814SBarry Song num_msi += entry->nvec_used; 1222f170814SBarry Song if (!num_msi) 1232f170814SBarry Song return NULL; 1242f170814SBarry Song 1252f170814SBarry Song /* Dynamically create the MSI attributes for the device */ 1262f170814SBarry Song msi_attrs = kcalloc(num_msi + 1, sizeof(void *), GFP_KERNEL); 1272f170814SBarry Song if (!msi_attrs) 1282f170814SBarry Song return ERR_PTR(-ENOMEM); 1292f170814SBarry Song 1302f170814SBarry Song for_each_msi_entry(entry, dev) { 1312f170814SBarry Song for (i = 0; i < entry->nvec_used; i++) { 1322f170814SBarry Song msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL); 1332f170814SBarry Song if (!msi_dev_attr) 1342f170814SBarry Song goto error_attrs; 1352f170814SBarry Song msi_attrs[count] = &msi_dev_attr->attr; 1362f170814SBarry Song 1372f170814SBarry Song sysfs_attr_init(&msi_dev_attr->attr); 1382f170814SBarry Song msi_dev_attr->attr.name = kasprintf(GFP_KERNEL, "%d", 1392f170814SBarry Song entry->irq + i); 1402f170814SBarry Song if (!msi_dev_attr->attr.name) 1412f170814SBarry Song goto error_attrs; 1422f170814SBarry Song msi_dev_attr->attr.mode = 0444; 1432f170814SBarry Song msi_dev_attr->show = msi_mode_show; 1442f170814SBarry Song ++count; 1452f170814SBarry Song } 1462f170814SBarry Song } 1472f170814SBarry Song 1482f170814SBarry Song msi_irq_group = kzalloc(sizeof(*msi_irq_group), GFP_KERNEL); 1492f170814SBarry Song if (!msi_irq_group) 1502f170814SBarry Song goto error_attrs; 1512f170814SBarry Song msi_irq_group->name = "msi_irqs"; 1522f170814SBarry Song msi_irq_group->attrs = msi_attrs; 1532f170814SBarry Song 1542f170814SBarry Song msi_irq_groups = kcalloc(2, sizeof(void *), GFP_KERNEL); 1552f170814SBarry Song if (!msi_irq_groups) 1562f170814SBarry Song goto error_irq_group; 1572f170814SBarry Song msi_irq_groups[0] = msi_irq_group; 1582f170814SBarry Song 1592f170814SBarry Song ret = sysfs_create_groups(&dev->kobj, msi_irq_groups); 1602f170814SBarry Song if (ret) 1612f170814SBarry Song goto error_irq_groups; 1622f170814SBarry Song 1632f170814SBarry Song return msi_irq_groups; 1642f170814SBarry Song 1652f170814SBarry Song error_irq_groups: 1662f170814SBarry Song kfree(msi_irq_groups); 1672f170814SBarry Song error_irq_group: 1682f170814SBarry Song kfree(msi_irq_group); 1692f170814SBarry Song error_attrs: 1702f170814SBarry Song count = 0; 1712f170814SBarry Song msi_attr = msi_attrs[count]; 1722f170814SBarry Song while (msi_attr) { 1732f170814SBarry Song msi_dev_attr = container_of(msi_attr, struct device_attribute, attr); 1742f170814SBarry Song kfree(msi_attr->name); 1752f170814SBarry Song kfree(msi_dev_attr); 1762f170814SBarry Song ++count; 1772f170814SBarry Song msi_attr = msi_attrs[count]; 1782f170814SBarry Song } 1792f170814SBarry Song kfree(msi_attrs); 1802f170814SBarry Song return ERR_PTR(ret); 1812f170814SBarry Song } 1822f170814SBarry Song 1832f170814SBarry Song /** 1842f170814SBarry Song * msi_destroy_sysfs - Destroy msi_irqs sysfs entries for devices 1852f170814SBarry Song * @dev: The device(PCI, platform etc) who will remove sysfs entries 1862f170814SBarry Song * @msi_irq_groups: attribute_group for device msi_irqs entries 1872f170814SBarry Song */ 1882f170814SBarry Song void msi_destroy_sysfs(struct device *dev, const struct attribute_group **msi_irq_groups) 1892f170814SBarry Song { 1902f170814SBarry Song struct device_attribute *dev_attr; 1912f170814SBarry Song struct attribute **msi_attrs; 1922f170814SBarry Song int count = 0; 1932f170814SBarry Song 1942f170814SBarry Song if (msi_irq_groups) { 1952f170814SBarry Song sysfs_remove_groups(&dev->kobj, msi_irq_groups); 1962f170814SBarry Song msi_attrs = msi_irq_groups[0]->attrs; 1972f170814SBarry Song while (msi_attrs[count]) { 1982f170814SBarry Song dev_attr = container_of(msi_attrs[count], 1992f170814SBarry Song struct device_attribute, attr); 2002f170814SBarry Song kfree(dev_attr->attr.name); 2012f170814SBarry Song kfree(dev_attr); 2022f170814SBarry Song ++count; 2032f170814SBarry Song } 2042f170814SBarry Song kfree(msi_attrs); 2052f170814SBarry Song kfree(msi_irq_groups[0]); 2062f170814SBarry Song kfree(msi_irq_groups); 2072f170814SBarry Song } 2082f170814SBarry Song } 2091197528aSThomas Gleixner #endif 2102f170814SBarry Song 211f3cf8bb0SJiang Liu #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN 21274faaf7aSThomas Gleixner static inline void irq_chip_write_msi_msg(struct irq_data *data, 21374faaf7aSThomas Gleixner struct msi_msg *msg) 21474faaf7aSThomas Gleixner { 21574faaf7aSThomas Gleixner data->chip->irq_write_msi_msg(data, msg); 21674faaf7aSThomas Gleixner } 21774faaf7aSThomas Gleixner 2180be8153cSMarc Zyngier static void msi_check_level(struct irq_domain *domain, struct msi_msg *msg) 2190be8153cSMarc Zyngier { 2200be8153cSMarc Zyngier struct msi_domain_info *info = domain->host_data; 2210be8153cSMarc Zyngier 2220be8153cSMarc Zyngier /* 2230be8153cSMarc Zyngier * If the MSI provider has messed with the second message and 2240be8153cSMarc Zyngier * not advertized that it is level-capable, signal the breakage. 2250be8153cSMarc Zyngier */ 2260be8153cSMarc Zyngier WARN_ON(!((info->flags & MSI_FLAG_LEVEL_CAPABLE) && 2270be8153cSMarc Zyngier (info->chip->flags & IRQCHIP_SUPPORTS_LEVEL_MSI)) && 2280be8153cSMarc Zyngier (msg[1].address_lo || msg[1].address_hi || msg[1].data)); 2290be8153cSMarc Zyngier } 2300be8153cSMarc Zyngier 231f3cf8bb0SJiang Liu /** 232f3cf8bb0SJiang Liu * msi_domain_set_affinity - Generic affinity setter function for MSI domains 233f3cf8bb0SJiang Liu * @irq_data: The irq data associated to the interrupt 234f3cf8bb0SJiang Liu * @mask: The affinity mask to set 235f3cf8bb0SJiang Liu * @force: Flag to enforce setting (disable online checks) 236f3cf8bb0SJiang Liu * 237f3cf8bb0SJiang Liu * Intended to be used by MSI interrupt controllers which are 238f3cf8bb0SJiang Liu * implemented with hierarchical domains. 2393b35e7e6SRandy Dunlap * 2403b35e7e6SRandy Dunlap * Return: IRQ_SET_MASK_* result code 241f3cf8bb0SJiang Liu */ 242f3cf8bb0SJiang Liu int msi_domain_set_affinity(struct irq_data *irq_data, 243f3cf8bb0SJiang Liu const struct cpumask *mask, bool force) 244f3cf8bb0SJiang Liu { 245f3cf8bb0SJiang Liu struct irq_data *parent = irq_data->parent_data; 2460be8153cSMarc Zyngier struct msi_msg msg[2] = { [1] = { }, }; 247f3cf8bb0SJiang Liu int ret; 248f3cf8bb0SJiang Liu 249f3cf8bb0SJiang Liu ret = parent->chip->irq_set_affinity(parent, mask, force); 250f3cf8bb0SJiang Liu if (ret >= 0 && ret != IRQ_SET_MASK_OK_DONE) { 2510be8153cSMarc Zyngier BUG_ON(irq_chip_compose_msi_msg(irq_data, msg)); 2520be8153cSMarc Zyngier msi_check_level(irq_data->domain, msg); 2530be8153cSMarc Zyngier irq_chip_write_msi_msg(irq_data, msg); 254f3cf8bb0SJiang Liu } 255f3cf8bb0SJiang Liu 256f3cf8bb0SJiang Liu return ret; 257f3cf8bb0SJiang Liu } 258f3cf8bb0SJiang Liu 25972491643SThomas Gleixner static int msi_domain_activate(struct irq_domain *domain, 26072491643SThomas Gleixner struct irq_data *irq_data, bool early) 261f3cf8bb0SJiang Liu { 2620be8153cSMarc Zyngier struct msi_msg msg[2] = { [1] = { }, }; 263f3cf8bb0SJiang Liu 2640be8153cSMarc Zyngier BUG_ON(irq_chip_compose_msi_msg(irq_data, msg)); 2650be8153cSMarc Zyngier msi_check_level(irq_data->domain, msg); 2660be8153cSMarc Zyngier irq_chip_write_msi_msg(irq_data, msg); 26772491643SThomas Gleixner return 0; 268f3cf8bb0SJiang Liu } 269f3cf8bb0SJiang Liu 270f3cf8bb0SJiang Liu static void msi_domain_deactivate(struct irq_domain *domain, 271f3cf8bb0SJiang Liu struct irq_data *irq_data) 272f3cf8bb0SJiang Liu { 2730be8153cSMarc Zyngier struct msi_msg msg[2]; 274f3cf8bb0SJiang Liu 2750be8153cSMarc Zyngier memset(msg, 0, sizeof(msg)); 2760be8153cSMarc Zyngier irq_chip_write_msi_msg(irq_data, msg); 277f3cf8bb0SJiang Liu } 278f3cf8bb0SJiang Liu 279f3cf8bb0SJiang Liu static int msi_domain_alloc(struct irq_domain *domain, unsigned int virq, 280f3cf8bb0SJiang Liu unsigned int nr_irqs, void *arg) 281f3cf8bb0SJiang Liu { 282f3cf8bb0SJiang Liu struct msi_domain_info *info = domain->host_data; 283f3cf8bb0SJiang Liu struct msi_domain_ops *ops = info->ops; 284f3cf8bb0SJiang Liu irq_hw_number_t hwirq = ops->get_hwirq(info, arg); 285f3cf8bb0SJiang Liu int i, ret; 286f3cf8bb0SJiang Liu 287f3cf8bb0SJiang Liu if (irq_find_mapping(domain, hwirq) > 0) 288f3cf8bb0SJiang Liu return -EEXIST; 289f3cf8bb0SJiang Liu 290bf6f869fSLiu Jiang if (domain->parent) { 291f3cf8bb0SJiang Liu ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg); 292f3cf8bb0SJiang Liu if (ret < 0) 293f3cf8bb0SJiang Liu return ret; 294bf6f869fSLiu Jiang } 295f3cf8bb0SJiang Liu 296f3cf8bb0SJiang Liu for (i = 0; i < nr_irqs; i++) { 297f3cf8bb0SJiang Liu ret = ops->msi_init(domain, info, virq + i, hwirq + i, arg); 298f3cf8bb0SJiang Liu if (ret < 0) { 299f3cf8bb0SJiang Liu if (ops->msi_free) { 300f3cf8bb0SJiang Liu for (i--; i > 0; i--) 301f3cf8bb0SJiang Liu ops->msi_free(domain, info, virq + i); 302f3cf8bb0SJiang Liu } 303f3cf8bb0SJiang Liu irq_domain_free_irqs_top(domain, virq, nr_irqs); 304f3cf8bb0SJiang Liu return ret; 305f3cf8bb0SJiang Liu } 306f3cf8bb0SJiang Liu } 307f3cf8bb0SJiang Liu 308f3cf8bb0SJiang Liu return 0; 309f3cf8bb0SJiang Liu } 310f3cf8bb0SJiang Liu 311f3cf8bb0SJiang Liu static void msi_domain_free(struct irq_domain *domain, unsigned int virq, 312f3cf8bb0SJiang Liu unsigned int nr_irqs) 313f3cf8bb0SJiang Liu { 314f3cf8bb0SJiang Liu struct msi_domain_info *info = domain->host_data; 315f3cf8bb0SJiang Liu int i; 316f3cf8bb0SJiang Liu 317f3cf8bb0SJiang Liu if (info->ops->msi_free) { 318f3cf8bb0SJiang Liu for (i = 0; i < nr_irqs; i++) 319f3cf8bb0SJiang Liu info->ops->msi_free(domain, info, virq + i); 320f3cf8bb0SJiang Liu } 321f3cf8bb0SJiang Liu irq_domain_free_irqs_top(domain, virq, nr_irqs); 322f3cf8bb0SJiang Liu } 323f3cf8bb0SJiang Liu 32401364028SKrzysztof Kozlowski static const struct irq_domain_ops msi_domain_ops = { 325f3cf8bb0SJiang Liu .alloc = msi_domain_alloc, 326f3cf8bb0SJiang Liu .free = msi_domain_free, 327f3cf8bb0SJiang Liu .activate = msi_domain_activate, 328f3cf8bb0SJiang Liu .deactivate = msi_domain_deactivate, 329f3cf8bb0SJiang Liu }; 330f3cf8bb0SJiang Liu 331aeeb5965SJiang Liu static irq_hw_number_t msi_domain_ops_get_hwirq(struct msi_domain_info *info, 332aeeb5965SJiang Liu msi_alloc_info_t *arg) 333aeeb5965SJiang Liu { 334aeeb5965SJiang Liu return arg->hwirq; 335aeeb5965SJiang Liu } 336aeeb5965SJiang Liu 337aeeb5965SJiang Liu static int msi_domain_ops_prepare(struct irq_domain *domain, struct device *dev, 338aeeb5965SJiang Liu int nvec, msi_alloc_info_t *arg) 339aeeb5965SJiang Liu { 340aeeb5965SJiang Liu memset(arg, 0, sizeof(*arg)); 341aeeb5965SJiang Liu return 0; 342aeeb5965SJiang Liu } 343aeeb5965SJiang Liu 344aeeb5965SJiang Liu static void msi_domain_ops_set_desc(msi_alloc_info_t *arg, 345aeeb5965SJiang Liu struct msi_desc *desc) 346aeeb5965SJiang Liu { 347aeeb5965SJiang Liu arg->desc = desc; 348aeeb5965SJiang Liu } 349aeeb5965SJiang Liu 350aeeb5965SJiang Liu static int msi_domain_ops_init(struct irq_domain *domain, 351aeeb5965SJiang Liu struct msi_domain_info *info, 352aeeb5965SJiang Liu unsigned int virq, irq_hw_number_t hwirq, 353aeeb5965SJiang Liu msi_alloc_info_t *arg) 354aeeb5965SJiang Liu { 355aeeb5965SJiang Liu irq_domain_set_hwirq_and_chip(domain, virq, hwirq, info->chip, 356aeeb5965SJiang Liu info->chip_data); 357aeeb5965SJiang Liu if (info->handler && info->handler_name) { 358aeeb5965SJiang Liu __irq_set_handler(virq, info->handler, 0, info->handler_name); 359aeeb5965SJiang Liu if (info->handler_data) 360aeeb5965SJiang Liu irq_set_handler_data(virq, info->handler_data); 361aeeb5965SJiang Liu } 362aeeb5965SJiang Liu return 0; 363aeeb5965SJiang Liu } 364aeeb5965SJiang Liu 365aeeb5965SJiang Liu static int msi_domain_ops_check(struct irq_domain *domain, 366aeeb5965SJiang Liu struct msi_domain_info *info, 367aeeb5965SJiang Liu struct device *dev) 368aeeb5965SJiang Liu { 369aeeb5965SJiang Liu return 0; 370aeeb5965SJiang Liu } 371aeeb5965SJiang Liu 372aeeb5965SJiang Liu static struct msi_domain_ops msi_domain_ops_default = { 373aeeb5965SJiang Liu .get_hwirq = msi_domain_ops_get_hwirq, 374aeeb5965SJiang Liu .msi_init = msi_domain_ops_init, 375aeeb5965SJiang Liu .msi_check = msi_domain_ops_check, 376aeeb5965SJiang Liu .msi_prepare = msi_domain_ops_prepare, 377aeeb5965SJiang Liu .set_desc = msi_domain_ops_set_desc, 37843e9e705SThomas Gleixner .domain_alloc_irqs = __msi_domain_alloc_irqs, 37943e9e705SThomas Gleixner .domain_free_irqs = __msi_domain_free_irqs, 380aeeb5965SJiang Liu }; 381aeeb5965SJiang Liu 382aeeb5965SJiang Liu static void msi_domain_update_dom_ops(struct msi_domain_info *info) 383aeeb5965SJiang Liu { 384aeeb5965SJiang Liu struct msi_domain_ops *ops = info->ops; 385aeeb5965SJiang Liu 386aeeb5965SJiang Liu if (ops == NULL) { 387aeeb5965SJiang Liu info->ops = &msi_domain_ops_default; 388aeeb5965SJiang Liu return; 389aeeb5965SJiang Liu } 390aeeb5965SJiang Liu 39143e9e705SThomas Gleixner if (ops->domain_alloc_irqs == NULL) 39243e9e705SThomas Gleixner ops->domain_alloc_irqs = msi_domain_ops_default.domain_alloc_irqs; 39343e9e705SThomas Gleixner if (ops->domain_free_irqs == NULL) 39443e9e705SThomas Gleixner ops->domain_free_irqs = msi_domain_ops_default.domain_free_irqs; 39543e9e705SThomas Gleixner 39643e9e705SThomas Gleixner if (!(info->flags & MSI_FLAG_USE_DEF_DOM_OPS)) 39743e9e705SThomas Gleixner return; 39843e9e705SThomas Gleixner 399aeeb5965SJiang Liu if (ops->get_hwirq == NULL) 400aeeb5965SJiang Liu ops->get_hwirq = msi_domain_ops_default.get_hwirq; 401aeeb5965SJiang Liu if (ops->msi_init == NULL) 402aeeb5965SJiang Liu ops->msi_init = msi_domain_ops_default.msi_init; 403aeeb5965SJiang Liu if (ops->msi_check == NULL) 404aeeb5965SJiang Liu ops->msi_check = msi_domain_ops_default.msi_check; 405aeeb5965SJiang Liu if (ops->msi_prepare == NULL) 406aeeb5965SJiang Liu ops->msi_prepare = msi_domain_ops_default.msi_prepare; 407aeeb5965SJiang Liu if (ops->set_desc == NULL) 408aeeb5965SJiang Liu ops->set_desc = msi_domain_ops_default.set_desc; 409aeeb5965SJiang Liu } 410aeeb5965SJiang Liu 411aeeb5965SJiang Liu static void msi_domain_update_chip_ops(struct msi_domain_info *info) 412aeeb5965SJiang Liu { 413aeeb5965SJiang Liu struct irq_chip *chip = info->chip; 414aeeb5965SJiang Liu 4150701c53eSMarc Zyngier BUG_ON(!chip || !chip->irq_mask || !chip->irq_unmask); 416aeeb5965SJiang Liu if (!chip->irq_set_affinity) 417aeeb5965SJiang Liu chip->irq_set_affinity = msi_domain_set_affinity; 418aeeb5965SJiang Liu } 419aeeb5965SJiang Liu 420f3cf8bb0SJiang Liu /** 4213b35e7e6SRandy Dunlap * msi_create_irq_domain - Create an MSI interrupt domain 422be5436c8SMarc Zyngier * @fwnode: Optional fwnode of the interrupt controller 423f3cf8bb0SJiang Liu * @info: MSI domain info 424f3cf8bb0SJiang Liu * @parent: Parent irq domain 4253b35e7e6SRandy Dunlap * 4263b35e7e6SRandy Dunlap * Return: pointer to the created &struct irq_domain or %NULL on failure 427f3cf8bb0SJiang Liu */ 428be5436c8SMarc Zyngier struct irq_domain *msi_create_irq_domain(struct fwnode_handle *fwnode, 429f3cf8bb0SJiang Liu struct msi_domain_info *info, 430f3cf8bb0SJiang Liu struct irq_domain *parent) 431f3cf8bb0SJiang Liu { 432a97b852bSMarc Zyngier struct irq_domain *domain; 433a97b852bSMarc Zyngier 434aeeb5965SJiang Liu msi_domain_update_dom_ops(info); 435aeeb5965SJiang Liu if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS) 436aeeb5965SJiang Liu msi_domain_update_chip_ops(info); 437f3cf8bb0SJiang Liu 438a97b852bSMarc Zyngier domain = irq_domain_create_hierarchy(parent, IRQ_DOMAIN_FLAG_MSI, 0, 43988156f00SEric Auger fwnode, &msi_domain_ops, info); 4400165308aSThomas Gleixner 4410165308aSThomas Gleixner if (domain && !domain->name && info->chip) 442a97b852bSMarc Zyngier domain->name = info->chip->name; 443a97b852bSMarc Zyngier 444a97b852bSMarc Zyngier return domain; 445f3cf8bb0SJiang Liu } 446f3cf8bb0SJiang Liu 447b2eba39bSMarc Zyngier int msi_domain_prepare_irqs(struct irq_domain *domain, struct device *dev, 448b2eba39bSMarc Zyngier int nvec, msi_alloc_info_t *arg) 449b2eba39bSMarc Zyngier { 450b2eba39bSMarc Zyngier struct msi_domain_info *info = domain->host_data; 451b2eba39bSMarc Zyngier struct msi_domain_ops *ops = info->ops; 452b2eba39bSMarc Zyngier int ret; 453b2eba39bSMarc Zyngier 454b2eba39bSMarc Zyngier ret = ops->msi_check(domain, info, dev); 455b2eba39bSMarc Zyngier if (ret == 0) 456b2eba39bSMarc Zyngier ret = ops->msi_prepare(domain, dev, nvec, arg); 457b2eba39bSMarc Zyngier 458b2eba39bSMarc Zyngier return ret; 459b2eba39bSMarc Zyngier } 460b2eba39bSMarc Zyngier 4612145ac93SMarc Zyngier int msi_domain_populate_irqs(struct irq_domain *domain, struct device *dev, 4622145ac93SMarc Zyngier int virq, int nvec, msi_alloc_info_t *arg) 4632145ac93SMarc Zyngier { 4642145ac93SMarc Zyngier struct msi_domain_info *info = domain->host_data; 4652145ac93SMarc Zyngier struct msi_domain_ops *ops = info->ops; 4662145ac93SMarc Zyngier struct msi_desc *desc; 4672145ac93SMarc Zyngier int ret = 0; 4682145ac93SMarc Zyngier 4692145ac93SMarc Zyngier for_each_msi_entry(desc, dev) { 4702145ac93SMarc Zyngier /* Don't even try the multi-MSI brain damage. */ 4712145ac93SMarc Zyngier if (WARN_ON(!desc->irq || desc->nvec_used != 1)) { 4722145ac93SMarc Zyngier ret = -EINVAL; 4732145ac93SMarc Zyngier break; 4742145ac93SMarc Zyngier } 4752145ac93SMarc Zyngier 4762145ac93SMarc Zyngier if (!(desc->irq >= virq && desc->irq < (virq + nvec))) 4772145ac93SMarc Zyngier continue; 4782145ac93SMarc Zyngier 4792145ac93SMarc Zyngier ops->set_desc(arg, desc); 4802145ac93SMarc Zyngier /* Assumes the domain mutex is held! */ 481596a7a1dSJohn Keeping ret = irq_domain_alloc_irqs_hierarchy(domain, desc->irq, 1, 482596a7a1dSJohn Keeping arg); 4832145ac93SMarc Zyngier if (ret) 4842145ac93SMarc Zyngier break; 4852145ac93SMarc Zyngier 486596a7a1dSJohn Keeping irq_set_msi_desc_off(desc->irq, 0, desc); 4872145ac93SMarc Zyngier } 4882145ac93SMarc Zyngier 4892145ac93SMarc Zyngier if (ret) { 4902145ac93SMarc Zyngier /* Mop up the damage */ 4912145ac93SMarc Zyngier for_each_msi_entry(desc, dev) { 4922145ac93SMarc Zyngier if (!(desc->irq >= virq && desc->irq < (virq + nvec))) 4932145ac93SMarc Zyngier continue; 4942145ac93SMarc Zyngier 4952145ac93SMarc Zyngier irq_domain_free_irqs_common(domain, desc->irq, 1); 4962145ac93SMarc Zyngier } 4972145ac93SMarc Zyngier } 4982145ac93SMarc Zyngier 4992145ac93SMarc Zyngier return ret; 5002145ac93SMarc Zyngier } 5012145ac93SMarc Zyngier 502bc976233SThomas Gleixner /* 503bc976233SThomas Gleixner * Carefully check whether the device can use reservation mode. If 504bc976233SThomas Gleixner * reservation mode is enabled then the early activation will assign a 505bc976233SThomas Gleixner * dummy vector to the device. If the PCI/MSI device does not support 506bc976233SThomas Gleixner * masking of the entry then this can result in spurious interrupts when 507bc976233SThomas Gleixner * the device driver is not absolutely careful. But even then a malfunction 508bc976233SThomas Gleixner * of the hardware could result in a spurious interrupt on the dummy vector 509bc976233SThomas Gleixner * and render the device unusable. If the entry can be masked then the core 510bc976233SThomas Gleixner * logic will prevent the spurious interrupt and reservation mode can be 511bc976233SThomas Gleixner * used. For now reservation mode is restricted to PCI/MSI. 512bc976233SThomas Gleixner */ 513bc976233SThomas Gleixner static bool msi_check_reservation_mode(struct irq_domain *domain, 514bc976233SThomas Gleixner struct msi_domain_info *info, 515bc976233SThomas Gleixner struct device *dev) 516da5dd9e8SThomas Gleixner { 517bc976233SThomas Gleixner struct msi_desc *desc; 518bc976233SThomas Gleixner 519c6c9e283SThomas Gleixner switch(domain->bus_token) { 520c6c9e283SThomas Gleixner case DOMAIN_BUS_PCI_MSI: 521c6c9e283SThomas Gleixner case DOMAIN_BUS_VMD_MSI: 522c6c9e283SThomas Gleixner break; 523c6c9e283SThomas Gleixner default: 524bc976233SThomas Gleixner return false; 525c6c9e283SThomas Gleixner } 526bc976233SThomas Gleixner 527da5dd9e8SThomas Gleixner if (!(info->flags & MSI_FLAG_MUST_REACTIVATE)) 528da5dd9e8SThomas Gleixner return false; 529bc976233SThomas Gleixner 530bc976233SThomas Gleixner if (IS_ENABLED(CONFIG_PCI_MSI) && pci_msi_ignore_mask) 531bc976233SThomas Gleixner return false; 532bc976233SThomas Gleixner 533bc976233SThomas Gleixner /* 534bc976233SThomas Gleixner * Checking the first MSI descriptor is sufficient. MSIX supports 5359c8e9c96SThomas Gleixner * masking and MSI does so when the can_mask attribute is set. 536bc976233SThomas Gleixner */ 537bc976233SThomas Gleixner desc = first_msi_entry(dev); 538*e58f2259SThomas Gleixner return desc->pci.msi_attrib.is_msix || desc->pci.msi_attrib.can_mask; 539da5dd9e8SThomas Gleixner } 540da5dd9e8SThomas Gleixner 54143e9e705SThomas Gleixner int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, 542d9109698SJiang Liu int nvec) 543d9109698SJiang Liu { 544d9109698SJiang Liu struct msi_domain_info *info = domain->host_data; 545d9109698SJiang Liu struct msi_domain_ops *ops = info->ops; 546da5dd9e8SThomas Gleixner struct irq_data *irq_data; 547d9109698SJiang Liu struct msi_desc *desc; 54806fde695SZenghui Yu msi_alloc_info_t arg = { }; 549b6140914SThomas Gleixner int i, ret, virq; 550da5dd9e8SThomas Gleixner bool can_reserve; 551d9109698SJiang Liu 552b2eba39bSMarc Zyngier ret = msi_domain_prepare_irqs(domain, dev, nvec, &arg); 553d9109698SJiang Liu if (ret) 554d9109698SJiang Liu return ret; 555d9109698SJiang Liu 556d9109698SJiang Liu for_each_msi_entry(desc, dev) { 557d9109698SJiang Liu ops->set_desc(&arg, desc); 558d9109698SJiang Liu 559b6140914SThomas Gleixner virq = __irq_domain_alloc_irqs(domain, -1, desc->nvec_used, 56006ee6d57SThomas Gleixner dev_to_node(dev), &arg, false, 5610972fa57SThomas Gleixner desc->affinity); 562d9109698SJiang Liu if (virq < 0) { 563d9109698SJiang Liu ret = -ENOSPC; 564d9109698SJiang Liu if (ops->handle_error) 565d9109698SJiang Liu ret = ops->handle_error(domain, desc, ret); 566d9109698SJiang Liu return ret; 567d9109698SJiang Liu } 568d9109698SJiang Liu 56907557ccbSThomas Gleixner for (i = 0; i < desc->nvec_used; i++) { 570d9109698SJiang Liu irq_set_msi_desc_off(virq, i, desc); 57107557ccbSThomas Gleixner irq_debugfs_copy_devname(virq + i, dev); 57207557ccbSThomas Gleixner } 573d9109698SJiang Liu } 574d9109698SJiang Liu 575bc976233SThomas Gleixner can_reserve = msi_check_reservation_mode(domain, info, dev); 576da5dd9e8SThomas Gleixner 577f3b0946dSMarc Zyngier /* 578f3b0946dSMarc Zyngier * This flag is set by the PCI layer as we need to activate 579f3b0946dSMarc Zyngier * the MSI entries before the PCI layer enables MSI in the 580f3b0946dSMarc Zyngier * card. Otherwise the card latches a random msi message. 581f3b0946dSMarc Zyngier */ 582da5dd9e8SThomas Gleixner if (!(info->flags & MSI_FLAG_ACTIVATE_EARLY)) 5834c457e8cSMarc Zyngier goto skip_activate; 584f3b0946dSMarc Zyngier 5854c457e8cSMarc Zyngier for_each_msi_vector(desc, i, dev) { 5864c457e8cSMarc Zyngier if (desc->irq == i) { 5874c457e8cSMarc Zyngier virq = desc->irq; 5884c457e8cSMarc Zyngier dev_dbg(dev, "irq [%d-%d] for MSI\n", 5894c457e8cSMarc Zyngier virq, virq + desc->nvec_used - 1); 5904c457e8cSMarc Zyngier } 5914c457e8cSMarc Zyngier 5924c457e8cSMarc Zyngier irq_data = irq_domain_get_irq_data(domain, i); 5936f1a4891SThomas Gleixner if (!can_reserve) { 594bc976233SThomas Gleixner irqd_clr_can_reserve(irq_data); 5956f1a4891SThomas Gleixner if (domain->flags & IRQ_DOMAIN_MSI_NOMASK_QUIRK) 5966f1a4891SThomas Gleixner irqd_set_msi_nomask_quirk(irq_data); 5976f1a4891SThomas Gleixner } 598bc976233SThomas Gleixner ret = irq_domain_activate_irq(irq_data, can_reserve); 599bb9b428aSThomas Gleixner if (ret) 600bb9b428aSThomas Gleixner goto cleanup; 601da5dd9e8SThomas Gleixner } 602da5dd9e8SThomas Gleixner 6034c457e8cSMarc Zyngier skip_activate: 604da5dd9e8SThomas Gleixner /* 605da5dd9e8SThomas Gleixner * If these interrupts use reservation mode, clear the activated bit 606da5dd9e8SThomas Gleixner * so request_irq() will assign the final vector. 607da5dd9e8SThomas Gleixner */ 608da5dd9e8SThomas Gleixner if (can_reserve) { 6094c457e8cSMarc Zyngier for_each_msi_vector(desc, i, dev) { 6104c457e8cSMarc Zyngier irq_data = irq_domain_get_irq_data(domain, i); 61122d0b12fSThomas Gleixner irqd_clr_activated(irq_data); 612f3b0946dSMarc Zyngier } 613d9109698SJiang Liu } 614d9109698SJiang Liu return 0; 615bb9b428aSThomas Gleixner 616bb9b428aSThomas Gleixner cleanup: 617bb9b428aSThomas Gleixner msi_domain_free_irqs(domain, dev); 618bb9b428aSThomas Gleixner return ret; 619d9109698SJiang Liu } 620d9109698SJiang Liu 621d9109698SJiang Liu /** 62243e9e705SThomas Gleixner * msi_domain_alloc_irqs - Allocate interrupts from a MSI interrupt domain 62343e9e705SThomas Gleixner * @domain: The domain to allocate from 624d9109698SJiang Liu * @dev: Pointer to device struct of the device for which the interrupts 62543e9e705SThomas Gleixner * are allocated 62643e9e705SThomas Gleixner * @nvec: The number of interrupts to allocate 62743e9e705SThomas Gleixner * 6283b35e7e6SRandy Dunlap * Return: %0 on success or an error code. 629d9109698SJiang Liu */ 63043e9e705SThomas Gleixner int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, 63143e9e705SThomas Gleixner int nvec) 63243e9e705SThomas Gleixner { 63343e9e705SThomas Gleixner struct msi_domain_info *info = domain->host_data; 63443e9e705SThomas Gleixner struct msi_domain_ops *ops = info->ops; 63543e9e705SThomas Gleixner 63643e9e705SThomas Gleixner return ops->domain_alloc_irqs(domain, dev, nvec); 63743e9e705SThomas Gleixner } 63843e9e705SThomas Gleixner 63943e9e705SThomas Gleixner void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev) 640d9109698SJiang Liu { 641dbbc9357SBixuan Cui struct irq_data *irq_data; 642d9109698SJiang Liu struct msi_desc *desc; 643dbbc9357SBixuan Cui int i; 644dbbc9357SBixuan Cui 645dbbc9357SBixuan Cui for_each_msi_vector(desc, i, dev) { 646dbbc9357SBixuan Cui irq_data = irq_domain_get_irq_data(domain, i); 647dbbc9357SBixuan Cui if (irqd_is_activated(irq_data)) 648dbbc9357SBixuan Cui irq_domain_deactivate_irq(irq_data); 649dbbc9357SBixuan Cui } 650d9109698SJiang Liu 651d9109698SJiang Liu for_each_msi_entry(desc, dev) { 652fe0c52fcSMarc Zyngier /* 653fe0c52fcSMarc Zyngier * We might have failed to allocate an MSI early 654fe0c52fcSMarc Zyngier * enough that there is no IRQ associated to this 655fe0c52fcSMarc Zyngier * entry. If that's the case, don't do anything. 656fe0c52fcSMarc Zyngier */ 657fe0c52fcSMarc Zyngier if (desc->irq) { 658d9109698SJiang Liu irq_domain_free_irqs(desc->irq, desc->nvec_used); 659d9109698SJiang Liu desc->irq = 0; 660d9109698SJiang Liu } 661d9109698SJiang Liu } 662fe0c52fcSMarc Zyngier } 663d9109698SJiang Liu 664d9109698SJiang Liu /** 6653b35e7e6SRandy Dunlap * msi_domain_free_irqs - Free interrupts from a MSI interrupt @domain associated to @dev 66643e9e705SThomas Gleixner * @domain: The domain to managing the interrupts 66743e9e705SThomas Gleixner * @dev: Pointer to device struct of the device for which the interrupts 66843e9e705SThomas Gleixner * are free 66943e9e705SThomas Gleixner */ 67043e9e705SThomas Gleixner void msi_domain_free_irqs(struct irq_domain *domain, struct device *dev) 67143e9e705SThomas Gleixner { 67243e9e705SThomas Gleixner struct msi_domain_info *info = domain->host_data; 67343e9e705SThomas Gleixner struct msi_domain_ops *ops = info->ops; 67443e9e705SThomas Gleixner 67543e9e705SThomas Gleixner return ops->domain_free_irqs(domain, dev); 67643e9e705SThomas Gleixner } 67743e9e705SThomas Gleixner 67843e9e705SThomas Gleixner /** 679f3cf8bb0SJiang Liu * msi_get_domain_info - Get the MSI interrupt domain info for @domain 680f3cf8bb0SJiang Liu * @domain: The interrupt domain to retrieve data from 681f3cf8bb0SJiang Liu * 6823b35e7e6SRandy Dunlap * Return: the pointer to the msi_domain_info stored in @domain->host_data. 683f3cf8bb0SJiang Liu */ 684f3cf8bb0SJiang Liu struct msi_domain_info *msi_get_domain_info(struct irq_domain *domain) 685f3cf8bb0SJiang Liu { 686f3cf8bb0SJiang Liu return (struct msi_domain_info *)domain->host_data; 687f3cf8bb0SJiang Liu } 688f3cf8bb0SJiang Liu 689f3cf8bb0SJiang Liu #endif /* CONFIG_GENERIC_MSI_IRQ_DOMAIN */ 690