1 /* 2 * linux/kernel/irq/msi.c 3 * 4 * Copyright (C) 2014 Intel Corp. 5 * Author: Jiang Liu <jiang.liu@linux.intel.com> 6 * 7 * This file is licensed under GPLv2. 8 * 9 * This file contains common code to support Message Signalled Interrupt for 10 * PCI compatible and non PCI compatible devices. 11 */ 12 #include <linux/types.h> 13 #include <linux/device.h> 14 #include <linux/irq.h> 15 #include <linux/irqdomain.h> 16 #include <linux/msi.h> 17 #include <linux/slab.h> 18 19 /** 20 * alloc_msi_entry - Allocate an initialize msi_entry 21 * @dev: Pointer to the device for which this is allocated 22 * @nvec: The number of vectors used in this entry 23 * @affinity: Optional pointer to an affinity mask array size of @nvec 24 * 25 * If @affinity is not NULL then a an affinity array[@nvec] is allocated 26 * and the affinity masks from @affinity are copied. 27 */ 28 struct msi_desc * 29 alloc_msi_entry(struct device *dev, int nvec, const struct cpumask *affinity) 30 { 31 struct msi_desc *desc; 32 33 desc = kzalloc(sizeof(*desc), GFP_KERNEL); 34 if (!desc) 35 return NULL; 36 37 INIT_LIST_HEAD(&desc->list); 38 desc->dev = dev; 39 desc->nvec_used = nvec; 40 if (affinity) { 41 desc->affinity = kmemdup(affinity, 42 nvec * sizeof(*desc->affinity), GFP_KERNEL); 43 if (!desc->affinity) { 44 kfree(desc); 45 return NULL; 46 } 47 } 48 49 return desc; 50 } 51 52 void free_msi_entry(struct msi_desc *entry) 53 { 54 kfree(entry->affinity); 55 kfree(entry); 56 } 57 58 void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg) 59 { 60 *msg = entry->msg; 61 } 62 63 void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg) 64 { 65 struct msi_desc *entry = irq_get_msi_desc(irq); 66 67 __get_cached_msi_msg(entry, msg); 68 } 69 EXPORT_SYMBOL_GPL(get_cached_msi_msg); 70 71 #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN 72 static inline void irq_chip_write_msi_msg(struct irq_data *data, 73 struct msi_msg *msg) 74 { 75 data->chip->irq_write_msi_msg(data, msg); 76 } 77 78 /** 79 * msi_domain_set_affinity - Generic affinity setter function for MSI domains 80 * @irq_data: The irq data associated to the interrupt 81 * @mask: The affinity mask to set 82 * @force: Flag to enforce setting (disable online checks) 83 * 84 * Intended to be used by MSI interrupt controllers which are 85 * implemented with hierarchical domains. 86 */ 87 int msi_domain_set_affinity(struct irq_data *irq_data, 88 const struct cpumask *mask, bool force) 89 { 90 struct irq_data *parent = irq_data->parent_data; 91 struct msi_msg msg; 92 int ret; 93 94 ret = parent->chip->irq_set_affinity(parent, mask, force); 95 if (ret >= 0 && ret != IRQ_SET_MASK_OK_DONE) { 96 BUG_ON(irq_chip_compose_msi_msg(irq_data, &msg)); 97 irq_chip_write_msi_msg(irq_data, &msg); 98 } 99 100 return ret; 101 } 102 103 static void msi_domain_activate(struct irq_domain *domain, 104 struct irq_data *irq_data) 105 { 106 struct msi_msg msg; 107 108 BUG_ON(irq_chip_compose_msi_msg(irq_data, &msg)); 109 irq_chip_write_msi_msg(irq_data, &msg); 110 } 111 112 static void msi_domain_deactivate(struct irq_domain *domain, 113 struct irq_data *irq_data) 114 { 115 struct msi_msg msg; 116 117 memset(&msg, 0, sizeof(msg)); 118 irq_chip_write_msi_msg(irq_data, &msg); 119 } 120 121 static int msi_domain_alloc(struct irq_domain *domain, unsigned int virq, 122 unsigned int nr_irqs, void *arg) 123 { 124 struct msi_domain_info *info = domain->host_data; 125 struct msi_domain_ops *ops = info->ops; 126 irq_hw_number_t hwirq = ops->get_hwirq(info, arg); 127 int i, ret; 128 129 if (irq_find_mapping(domain, hwirq) > 0) 130 return -EEXIST; 131 132 if (domain->parent) { 133 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg); 134 if (ret < 0) 135 return ret; 136 } 137 138 for (i = 0; i < nr_irqs; i++) { 139 ret = ops->msi_init(domain, info, virq + i, hwirq + i, arg); 140 if (ret < 0) { 141 if (ops->msi_free) { 142 for (i--; i > 0; i--) 143 ops->msi_free(domain, info, virq + i); 144 } 145 irq_domain_free_irqs_top(domain, virq, nr_irqs); 146 return ret; 147 } 148 } 149 150 return 0; 151 } 152 153 static void msi_domain_free(struct irq_domain *domain, unsigned int virq, 154 unsigned int nr_irqs) 155 { 156 struct msi_domain_info *info = domain->host_data; 157 int i; 158 159 if (info->ops->msi_free) { 160 for (i = 0; i < nr_irqs; i++) 161 info->ops->msi_free(domain, info, virq + i); 162 } 163 irq_domain_free_irqs_top(domain, virq, nr_irqs); 164 } 165 166 static const struct irq_domain_ops msi_domain_ops = { 167 .alloc = msi_domain_alloc, 168 .free = msi_domain_free, 169 .activate = msi_domain_activate, 170 .deactivate = msi_domain_deactivate, 171 }; 172 173 #ifdef GENERIC_MSI_DOMAIN_OPS 174 static irq_hw_number_t msi_domain_ops_get_hwirq(struct msi_domain_info *info, 175 msi_alloc_info_t *arg) 176 { 177 return arg->hwirq; 178 } 179 180 static int msi_domain_ops_prepare(struct irq_domain *domain, struct device *dev, 181 int nvec, msi_alloc_info_t *arg) 182 { 183 memset(arg, 0, sizeof(*arg)); 184 return 0; 185 } 186 187 static void msi_domain_ops_set_desc(msi_alloc_info_t *arg, 188 struct msi_desc *desc) 189 { 190 arg->desc = desc; 191 } 192 #else 193 #define msi_domain_ops_get_hwirq NULL 194 #define msi_domain_ops_prepare NULL 195 #define msi_domain_ops_set_desc NULL 196 #endif /* !GENERIC_MSI_DOMAIN_OPS */ 197 198 static int msi_domain_ops_init(struct irq_domain *domain, 199 struct msi_domain_info *info, 200 unsigned int virq, irq_hw_number_t hwirq, 201 msi_alloc_info_t *arg) 202 { 203 irq_domain_set_hwirq_and_chip(domain, virq, hwirq, info->chip, 204 info->chip_data); 205 if (info->handler && info->handler_name) { 206 __irq_set_handler(virq, info->handler, 0, info->handler_name); 207 if (info->handler_data) 208 irq_set_handler_data(virq, info->handler_data); 209 } 210 return 0; 211 } 212 213 static int msi_domain_ops_check(struct irq_domain *domain, 214 struct msi_domain_info *info, 215 struct device *dev) 216 { 217 return 0; 218 } 219 220 static struct msi_domain_ops msi_domain_ops_default = { 221 .get_hwirq = msi_domain_ops_get_hwirq, 222 .msi_init = msi_domain_ops_init, 223 .msi_check = msi_domain_ops_check, 224 .msi_prepare = msi_domain_ops_prepare, 225 .set_desc = msi_domain_ops_set_desc, 226 }; 227 228 static void msi_domain_update_dom_ops(struct msi_domain_info *info) 229 { 230 struct msi_domain_ops *ops = info->ops; 231 232 if (ops == NULL) { 233 info->ops = &msi_domain_ops_default; 234 return; 235 } 236 237 if (ops->get_hwirq == NULL) 238 ops->get_hwirq = msi_domain_ops_default.get_hwirq; 239 if (ops->msi_init == NULL) 240 ops->msi_init = msi_domain_ops_default.msi_init; 241 if (ops->msi_check == NULL) 242 ops->msi_check = msi_domain_ops_default.msi_check; 243 if (ops->msi_prepare == NULL) 244 ops->msi_prepare = msi_domain_ops_default.msi_prepare; 245 if (ops->set_desc == NULL) 246 ops->set_desc = msi_domain_ops_default.set_desc; 247 } 248 249 static void msi_domain_update_chip_ops(struct msi_domain_info *info) 250 { 251 struct irq_chip *chip = info->chip; 252 253 BUG_ON(!chip || !chip->irq_mask || !chip->irq_unmask); 254 if (!chip->irq_set_affinity) 255 chip->irq_set_affinity = msi_domain_set_affinity; 256 } 257 258 /** 259 * msi_create_irq_domain - Create a MSI interrupt domain 260 * @fwnode: Optional fwnode of the interrupt controller 261 * @info: MSI domain info 262 * @parent: Parent irq domain 263 */ 264 struct irq_domain *msi_create_irq_domain(struct fwnode_handle *fwnode, 265 struct msi_domain_info *info, 266 struct irq_domain *parent) 267 { 268 if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS) 269 msi_domain_update_dom_ops(info); 270 if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS) 271 msi_domain_update_chip_ops(info); 272 273 return irq_domain_create_hierarchy(parent, IRQ_DOMAIN_FLAG_MSI, 0, 274 fwnode, &msi_domain_ops, info); 275 } 276 277 int msi_domain_prepare_irqs(struct irq_domain *domain, struct device *dev, 278 int nvec, msi_alloc_info_t *arg) 279 { 280 struct msi_domain_info *info = domain->host_data; 281 struct msi_domain_ops *ops = info->ops; 282 int ret; 283 284 ret = ops->msi_check(domain, info, dev); 285 if (ret == 0) 286 ret = ops->msi_prepare(domain, dev, nvec, arg); 287 288 return ret; 289 } 290 291 int msi_domain_populate_irqs(struct irq_domain *domain, struct device *dev, 292 int virq, int nvec, msi_alloc_info_t *arg) 293 { 294 struct msi_domain_info *info = domain->host_data; 295 struct msi_domain_ops *ops = info->ops; 296 struct msi_desc *desc; 297 int ret = 0; 298 299 for_each_msi_entry(desc, dev) { 300 /* Don't even try the multi-MSI brain damage. */ 301 if (WARN_ON(!desc->irq || desc->nvec_used != 1)) { 302 ret = -EINVAL; 303 break; 304 } 305 306 if (!(desc->irq >= virq && desc->irq < (virq + nvec))) 307 continue; 308 309 ops->set_desc(arg, desc); 310 /* Assumes the domain mutex is held! */ 311 ret = irq_domain_alloc_irqs_recursive(domain, virq, 1, arg); 312 if (ret) 313 break; 314 315 irq_set_msi_desc_off(virq, 0, desc); 316 } 317 318 if (ret) { 319 /* Mop up the damage */ 320 for_each_msi_entry(desc, dev) { 321 if (!(desc->irq >= virq && desc->irq < (virq + nvec))) 322 continue; 323 324 irq_domain_free_irqs_common(domain, desc->irq, 1); 325 } 326 } 327 328 return ret; 329 } 330 331 /** 332 * msi_domain_alloc_irqs - Allocate interrupts from a MSI interrupt domain 333 * @domain: The domain to allocate from 334 * @dev: Pointer to device struct of the device for which the interrupts 335 * are allocated 336 * @nvec: The number of interrupts to allocate 337 * 338 * Returns 0 on success or an error code. 339 */ 340 int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, 341 int nvec) 342 { 343 struct msi_domain_info *info = domain->host_data; 344 struct msi_domain_ops *ops = info->ops; 345 msi_alloc_info_t arg; 346 struct msi_desc *desc; 347 int i, ret, virq; 348 349 ret = msi_domain_prepare_irqs(domain, dev, nvec, &arg); 350 if (ret) 351 return ret; 352 353 for_each_msi_entry(desc, dev) { 354 ops->set_desc(&arg, desc); 355 356 virq = __irq_domain_alloc_irqs(domain, -1, desc->nvec_used, 357 dev_to_node(dev), &arg, false, 358 desc->affinity); 359 if (virq < 0) { 360 ret = -ENOSPC; 361 if (ops->handle_error) 362 ret = ops->handle_error(domain, desc, ret); 363 if (ops->msi_finish) 364 ops->msi_finish(&arg, ret); 365 return ret; 366 } 367 368 for (i = 0; i < desc->nvec_used; i++) 369 irq_set_msi_desc_off(virq, i, desc); 370 } 371 372 if (ops->msi_finish) 373 ops->msi_finish(&arg, 0); 374 375 for_each_msi_entry(desc, dev) { 376 virq = desc->irq; 377 if (desc->nvec_used == 1) 378 dev_dbg(dev, "irq %d for MSI\n", virq); 379 else 380 dev_dbg(dev, "irq [%d-%d] for MSI\n", 381 virq, virq + desc->nvec_used - 1); 382 /* 383 * This flag is set by the PCI layer as we need to activate 384 * the MSI entries before the PCI layer enables MSI in the 385 * card. Otherwise the card latches a random msi message. 386 */ 387 if (info->flags & MSI_FLAG_ACTIVATE_EARLY) { 388 struct irq_data *irq_data; 389 390 irq_data = irq_domain_get_irq_data(domain, desc->irq); 391 irq_domain_activate_irq(irq_data); 392 } 393 } 394 395 return 0; 396 } 397 398 /** 399 * msi_domain_free_irqs - Free interrupts from a MSI interrupt @domain associated tp @dev 400 * @domain: The domain to managing the interrupts 401 * @dev: Pointer to device struct of the device for which the interrupts 402 * are free 403 */ 404 void msi_domain_free_irqs(struct irq_domain *domain, struct device *dev) 405 { 406 struct msi_desc *desc; 407 408 for_each_msi_entry(desc, dev) { 409 /* 410 * We might have failed to allocate an MSI early 411 * enough that there is no IRQ associated to this 412 * entry. If that's the case, don't do anything. 413 */ 414 if (desc->irq) { 415 irq_domain_free_irqs(desc->irq, desc->nvec_used); 416 desc->irq = 0; 417 } 418 } 419 } 420 421 /** 422 * msi_get_domain_info - Get the MSI interrupt domain info for @domain 423 * @domain: The interrupt domain to retrieve data from 424 * 425 * Returns the pointer to the msi_domain_info stored in 426 * @domain->host_data. 427 */ 428 struct msi_domain_info *msi_get_domain_info(struct irq_domain *domain) 429 { 430 return (struct msi_domain_info *)domain->host_data; 431 } 432 433 #endif /* CONFIG_GENERIC_MSI_IRQ_DOMAIN */ 434