xref: /openbmc/linux/kernel/irq/msi.c (revision 13e7accb81d6c07993385af8342238ff22b41ac8)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2014 Intel Corp.
4  * Author: Jiang Liu <jiang.liu@linux.intel.com>
5  *
6  * This file is licensed under GPLv2.
7  *
8  * This file contains common code to support Message Signaled Interrupts for
9  * PCI compatible and non PCI compatible devices.
10  */
11 #include <linux/types.h>
12 #include <linux/device.h>
13 #include <linux/irq.h>
14 #include <linux/irqdomain.h>
15 #include <linux/msi.h>
16 #include <linux/slab.h>
17 #include <linux/sysfs.h>
18 #include <linux/pci.h>
19 
20 #include "internals.h"
21 
22 static inline int msi_sysfs_create_group(struct device *dev);
23 
24 /**
25  * msi_alloc_desc - Allocate an initialized msi_desc
26  * @dev:	Pointer to the device for which this is allocated
27  * @nvec:	The number of vectors used in this entry
28  * @affinity:	Optional pointer to an affinity mask array size of @nvec
29  *
30  * If @affinity is not %NULL then an affinity array[@nvec] is allocated
31  * and the affinity masks and flags from @affinity are copied.
32  *
33  * Return: pointer to allocated &msi_desc on success or %NULL on failure
34  */
35 static struct msi_desc *msi_alloc_desc(struct device *dev, int nvec,
36 					const struct irq_affinity_desc *affinity)
37 {
38 	struct msi_desc *desc = kzalloc(sizeof(*desc), GFP_KERNEL);
39 
40 	if (!desc)
41 		return NULL;
42 
43 	desc->dev = dev;
44 	desc->nvec_used = nvec;
45 	if (affinity) {
46 		desc->affinity = kmemdup(affinity, nvec * sizeof(*desc->affinity), GFP_KERNEL);
47 		if (!desc->affinity) {
48 			kfree(desc);
49 			return NULL;
50 		}
51 	}
52 	return desc;
53 }
54 
55 static void msi_free_desc(struct msi_desc *desc)
56 {
57 	kfree(desc->affinity);
58 	kfree(desc);
59 }
60 
61 static int msi_insert_desc(struct msi_device_data *md, struct msi_desc *desc, unsigned int index)
62 {
63 	int ret;
64 
65 	desc->msi_index = index;
66 	ret = xa_insert(&md->__store, index, desc, GFP_KERNEL);
67 	if (ret)
68 		msi_free_desc(desc);
69 	return ret;
70 }
71 
72 /**
73  * msi_add_msi_desc - Allocate and initialize a MSI descriptor
74  * @dev:	Pointer to the device for which the descriptor is allocated
75  * @init_desc:	Pointer to an MSI descriptor to initialize the new descriptor
76  *
77  * Return: 0 on success or an appropriate failure code.
78  */
79 int msi_add_msi_desc(struct device *dev, struct msi_desc *init_desc)
80 {
81 	struct msi_desc *desc;
82 
83 	lockdep_assert_held(&dev->msi.data->mutex);
84 
85 	desc = msi_alloc_desc(dev, init_desc->nvec_used, init_desc->affinity);
86 	if (!desc)
87 		return -ENOMEM;
88 
89 	/* Copy type specific data to the new descriptor. */
90 	desc->pci = init_desc->pci;
91 	return msi_insert_desc(dev->msi.data, desc, init_desc->msi_index);
92 }
93 
94 /**
95  * msi_add_simple_msi_descs - Allocate and initialize MSI descriptors
96  * @dev:	Pointer to the device for which the descriptors are allocated
97  * @index:	Index for the first MSI descriptor
98  * @ndesc:	Number of descriptors to allocate
99  *
100  * Return: 0 on success or an appropriate failure code.
101  */
102 static int msi_add_simple_msi_descs(struct device *dev, unsigned int index, unsigned int ndesc)
103 {
104 	unsigned int idx, last = index + ndesc - 1;
105 	struct msi_desc *desc;
106 	int ret;
107 
108 	lockdep_assert_held(&dev->msi.data->mutex);
109 
110 	for (idx = index; idx <= last; idx++) {
111 		desc = msi_alloc_desc(dev, 1, NULL);
112 		if (!desc)
113 			goto fail_mem;
114 		ret = msi_insert_desc(dev->msi.data, desc, idx);
115 		if (ret)
116 			goto fail;
117 	}
118 	return 0;
119 
120 fail_mem:
121 	ret = -ENOMEM;
122 fail:
123 	msi_free_msi_descs_range(dev, index, last);
124 	return ret;
125 }
126 
127 static bool msi_desc_match(struct msi_desc *desc, enum msi_desc_filter filter)
128 {
129 	switch (filter) {
130 	case MSI_DESC_ALL:
131 		return true;
132 	case MSI_DESC_NOTASSOCIATED:
133 		return !desc->irq;
134 	case MSI_DESC_ASSOCIATED:
135 		return !!desc->irq;
136 	}
137 	WARN_ON_ONCE(1);
138 	return false;
139 }
140 
141 /**
142  * msi_free_msi_descs_range - Free MSI descriptors of a device
143  * @dev:		Device to free the descriptors
144  * @first_index:	Index to start freeing from
145  * @last_index:		Last index to be freed
146  */
147 void msi_free_msi_descs_range(struct device *dev, unsigned int first_index,
148 			      unsigned int last_index)
149 {
150 	struct xarray *xa = &dev->msi.data->__store;
151 	struct msi_desc *desc;
152 	unsigned long idx;
153 
154 	lockdep_assert_held(&dev->msi.data->mutex);
155 
156 	xa_for_each_range(xa, idx, desc, first_index, last_index) {
157 		xa_erase(xa, idx);
158 
159 		/* Leak the descriptor when it is still referenced */
160 		if (WARN_ON_ONCE(msi_desc_match(desc, MSI_DESC_ASSOCIATED)))
161 			continue;
162 		msi_free_desc(desc);
163 	}
164 }
165 
166 void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
167 {
168 	*msg = entry->msg;
169 }
170 
171 void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg)
172 {
173 	struct msi_desc *entry = irq_get_msi_desc(irq);
174 
175 	__get_cached_msi_msg(entry, msg);
176 }
177 EXPORT_SYMBOL_GPL(get_cached_msi_msg);
178 
179 static void msi_device_data_release(struct device *dev, void *res)
180 {
181 	struct msi_device_data *md = res;
182 
183 	WARN_ON_ONCE(!xa_empty(&md->__store));
184 	xa_destroy(&md->__store);
185 	dev->msi.data = NULL;
186 }
187 
188 /**
189  * msi_setup_device_data - Setup MSI device data
190  * @dev:	Device for which MSI device data should be set up
191  *
192  * Return: 0 on success, appropriate error code otherwise
193  *
194  * This can be called more than once for @dev. If the MSI device data is
195  * already allocated the call succeeds. The allocated memory is
196  * automatically released when the device is destroyed.
197  */
198 int msi_setup_device_data(struct device *dev)
199 {
200 	struct msi_device_data *md;
201 	int ret;
202 
203 	if (dev->msi.data)
204 		return 0;
205 
206 	md = devres_alloc(msi_device_data_release, sizeof(*md), GFP_KERNEL);
207 	if (!md)
208 		return -ENOMEM;
209 
210 	ret = msi_sysfs_create_group(dev);
211 	if (ret) {
212 		devres_free(md);
213 		return ret;
214 	}
215 
216 	xa_init(&md->__store);
217 	mutex_init(&md->mutex);
218 	dev->msi.data = md;
219 	devres_add(dev, md);
220 	return 0;
221 }
222 
223 /**
224  * msi_lock_descs - Lock the MSI descriptor storage of a device
225  * @dev:	Device to operate on
226  */
227 void msi_lock_descs(struct device *dev)
228 {
229 	mutex_lock(&dev->msi.data->mutex);
230 }
231 EXPORT_SYMBOL_GPL(msi_lock_descs);
232 
233 /**
234  * msi_unlock_descs - Unlock the MSI descriptor storage of a device
235  * @dev:	Device to operate on
236  */
237 void msi_unlock_descs(struct device *dev)
238 {
239 	/* Invalidate the index wich was cached by the iterator */
240 	dev->msi.data->__iter_idx = MSI_MAX_INDEX;
241 	mutex_unlock(&dev->msi.data->mutex);
242 }
243 EXPORT_SYMBOL_GPL(msi_unlock_descs);
244 
245 static struct msi_desc *msi_find_desc(struct msi_device_data *md, enum msi_desc_filter filter)
246 {
247 	struct msi_desc *desc;
248 
249 	xa_for_each_start(&md->__store, md->__iter_idx, desc, md->__iter_idx) {
250 		if (msi_desc_match(desc, filter))
251 			return desc;
252 	}
253 	md->__iter_idx = MSI_MAX_INDEX;
254 	return NULL;
255 }
256 
257 /**
258  * msi_first_desc - Get the first MSI descriptor of a device
259  * @dev:	Device to operate on
260  * @filter:	Descriptor state filter
261  *
262  * Must be called with the MSI descriptor mutex held, i.e. msi_lock_descs()
263  * must be invoked before the call.
264  *
265  * Return: Pointer to the first MSI descriptor matching the search
266  *	   criteria, NULL if none found.
267  */
268 struct msi_desc *msi_first_desc(struct device *dev, enum msi_desc_filter filter)
269 {
270 	struct msi_device_data *md = dev->msi.data;
271 
272 	if (WARN_ON_ONCE(!md))
273 		return NULL;
274 
275 	lockdep_assert_held(&md->mutex);
276 
277 	md->__iter_idx = 0;
278 	return msi_find_desc(md, filter);
279 }
280 EXPORT_SYMBOL_GPL(msi_first_desc);
281 
282 /**
283  * msi_next_desc - Get the next MSI descriptor of a device
284  * @dev:	Device to operate on
285  * @filter:	Descriptor state filter
286  *
287  * The first invocation of msi_next_desc() has to be preceeded by a
288  * successful invocation of __msi_first_desc(). Consecutive invocations are
289  * only valid if the previous one was successful. All these operations have
290  * to be done within the same MSI mutex held region.
291  *
292  * Return: Pointer to the next MSI descriptor matching the search
293  *	   criteria, NULL if none found.
294  */
295 struct msi_desc *msi_next_desc(struct device *dev, enum msi_desc_filter filter)
296 {
297 	struct msi_device_data *md = dev->msi.data;
298 
299 	if (WARN_ON_ONCE(!md))
300 		return NULL;
301 
302 	lockdep_assert_held(&md->mutex);
303 
304 	if (md->__iter_idx >= (unsigned long)MSI_MAX_INDEX)
305 		return NULL;
306 
307 	md->__iter_idx++;
308 	return msi_find_desc(md, filter);
309 }
310 EXPORT_SYMBOL_GPL(msi_next_desc);
311 
312 /**
313  * msi_get_virq - Return Linux interrupt number of a MSI interrupt
314  * @dev:	Device to operate on
315  * @index:	MSI interrupt index to look for (0-based)
316  *
317  * Return: The Linux interrupt number on success (> 0), 0 if not found
318  */
319 unsigned int msi_get_virq(struct device *dev, unsigned int index)
320 {
321 	struct msi_desc *desc;
322 	unsigned int ret = 0;
323 	bool pcimsi;
324 
325 	if (!dev->msi.data)
326 		return 0;
327 
328 	pcimsi = dev_is_pci(dev) ? to_pci_dev(dev)->msi_enabled : false;
329 
330 	msi_lock_descs(dev);
331 	desc = xa_load(&dev->msi.data->__store, pcimsi ? 0 : index);
332 	if (desc && desc->irq) {
333 		/*
334 		 * PCI-MSI has only one descriptor for multiple interrupts.
335 		 * PCI-MSIX and platform MSI use a descriptor per
336 		 * interrupt.
337 		 */
338 		if (pcimsi) {
339 			if (index < desc->nvec_used)
340 				ret = desc->irq + index;
341 		} else {
342 			ret = desc->irq;
343 		}
344 	}
345 	msi_unlock_descs(dev);
346 	return ret;
347 }
348 EXPORT_SYMBOL_GPL(msi_get_virq);
349 
350 #ifdef CONFIG_SYSFS
351 static struct attribute *msi_dev_attrs[] = {
352 	NULL
353 };
354 
355 static const struct attribute_group msi_irqs_group = {
356 	.name	= "msi_irqs",
357 	.attrs	= msi_dev_attrs,
358 };
359 
360 static inline int msi_sysfs_create_group(struct device *dev)
361 {
362 	return devm_device_add_group(dev, &msi_irqs_group);
363 }
364 
365 static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr,
366 			     char *buf)
367 {
368 	/* MSI vs. MSIX is per device not per interrupt */
369 	bool is_msix = dev_is_pci(dev) ? to_pci_dev(dev)->msix_enabled : false;
370 
371 	return sysfs_emit(buf, "%s\n", is_msix ? "msix" : "msi");
372 }
373 
374 static void msi_sysfs_remove_desc(struct device *dev, struct msi_desc *desc)
375 {
376 	struct device_attribute *attrs = desc->sysfs_attrs;
377 	int i;
378 
379 	if (!attrs)
380 		return;
381 
382 	desc->sysfs_attrs = NULL;
383 	for (i = 0; i < desc->nvec_used; i++) {
384 		if (attrs[i].show)
385 			sysfs_remove_file_from_group(&dev->kobj, &attrs[i].attr, msi_irqs_group.name);
386 		kfree(attrs[i].attr.name);
387 	}
388 	kfree(attrs);
389 }
390 
391 static int msi_sysfs_populate_desc(struct device *dev, struct msi_desc *desc)
392 {
393 	struct device_attribute *attrs;
394 	int ret, i;
395 
396 	attrs = kcalloc(desc->nvec_used, sizeof(*attrs), GFP_KERNEL);
397 	if (!attrs)
398 		return -ENOMEM;
399 
400 	desc->sysfs_attrs = attrs;
401 	for (i = 0; i < desc->nvec_used; i++) {
402 		sysfs_attr_init(&attrs[i].attr);
403 		attrs[i].attr.name = kasprintf(GFP_KERNEL, "%d", desc->irq + i);
404 		if (!attrs[i].attr.name) {
405 			ret = -ENOMEM;
406 			goto fail;
407 		}
408 
409 		attrs[i].attr.mode = 0444;
410 		attrs[i].show = msi_mode_show;
411 
412 		ret = sysfs_add_file_to_group(&dev->kobj, &attrs[i].attr, msi_irqs_group.name);
413 		if (ret) {
414 			attrs[i].show = NULL;
415 			goto fail;
416 		}
417 	}
418 	return 0;
419 
420 fail:
421 	msi_sysfs_remove_desc(dev, desc);
422 	return ret;
423 }
424 
425 #ifdef CONFIG_PCI_MSI_ARCH_FALLBACKS
426 /**
427  * msi_device_populate_sysfs - Populate msi_irqs sysfs entries for a device
428  * @dev:	The device (PCI, platform etc) which will get sysfs entries
429  */
430 int msi_device_populate_sysfs(struct device *dev)
431 {
432 	struct msi_desc *desc;
433 	int ret;
434 
435 	msi_for_each_desc(desc, dev, MSI_DESC_ASSOCIATED) {
436 		if (desc->sysfs_attrs)
437 			continue;
438 		ret = msi_sysfs_populate_desc(dev, desc);
439 		if (ret)
440 			return ret;
441 	}
442 	return 0;
443 }
444 
445 /**
446  * msi_device_destroy_sysfs - Destroy msi_irqs sysfs entries for a device
447  * @dev:		The device (PCI, platform etc) for which to remove
448  *			sysfs entries
449  */
450 void msi_device_destroy_sysfs(struct device *dev)
451 {
452 	struct msi_desc *desc;
453 
454 	msi_for_each_desc(desc, dev, MSI_DESC_ALL)
455 		msi_sysfs_remove_desc(dev, desc);
456 }
457 #endif /* CONFIG_PCI_MSI_ARCH_FALLBACK */
458 #else /* CONFIG_SYSFS */
459 static inline int msi_sysfs_create_group(struct device *dev) { return 0; }
460 static inline int msi_sysfs_populate_desc(struct device *dev, struct msi_desc *desc) { return 0; }
461 static inline void msi_sysfs_remove_desc(struct device *dev, struct msi_desc *desc) { }
462 #endif /* !CONFIG_SYSFS */
463 
464 static int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, int nvec);
465 static void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev);
466 
467 static inline void irq_chip_write_msi_msg(struct irq_data *data,
468 					  struct msi_msg *msg)
469 {
470 	data->chip->irq_write_msi_msg(data, msg);
471 }
472 
473 static void msi_check_level(struct irq_domain *domain, struct msi_msg *msg)
474 {
475 	struct msi_domain_info *info = domain->host_data;
476 
477 	/*
478 	 * If the MSI provider has messed with the second message and
479 	 * not advertized that it is level-capable, signal the breakage.
480 	 */
481 	WARN_ON(!((info->flags & MSI_FLAG_LEVEL_CAPABLE) &&
482 		  (info->chip->flags & IRQCHIP_SUPPORTS_LEVEL_MSI)) &&
483 		(msg[1].address_lo || msg[1].address_hi || msg[1].data));
484 }
485 
486 /**
487  * msi_domain_set_affinity - Generic affinity setter function for MSI domains
488  * @irq_data:	The irq data associated to the interrupt
489  * @mask:	The affinity mask to set
490  * @force:	Flag to enforce setting (disable online checks)
491  *
492  * Intended to be used by MSI interrupt controllers which are
493  * implemented with hierarchical domains.
494  *
495  * Return: IRQ_SET_MASK_* result code
496  */
497 int msi_domain_set_affinity(struct irq_data *irq_data,
498 			    const struct cpumask *mask, bool force)
499 {
500 	struct irq_data *parent = irq_data->parent_data;
501 	struct msi_msg msg[2] = { [1] = { }, };
502 	int ret;
503 
504 	ret = parent->chip->irq_set_affinity(parent, mask, force);
505 	if (ret >= 0 && ret != IRQ_SET_MASK_OK_DONE) {
506 		BUG_ON(irq_chip_compose_msi_msg(irq_data, msg));
507 		msi_check_level(irq_data->domain, msg);
508 		irq_chip_write_msi_msg(irq_data, msg);
509 	}
510 
511 	return ret;
512 }
513 
514 static int msi_domain_activate(struct irq_domain *domain,
515 			       struct irq_data *irq_data, bool early)
516 {
517 	struct msi_msg msg[2] = { [1] = { }, };
518 
519 	BUG_ON(irq_chip_compose_msi_msg(irq_data, msg));
520 	msi_check_level(irq_data->domain, msg);
521 	irq_chip_write_msi_msg(irq_data, msg);
522 	return 0;
523 }
524 
525 static void msi_domain_deactivate(struct irq_domain *domain,
526 				  struct irq_data *irq_data)
527 {
528 	struct msi_msg msg[2];
529 
530 	memset(msg, 0, sizeof(msg));
531 	irq_chip_write_msi_msg(irq_data, msg);
532 }
533 
534 static int msi_domain_alloc(struct irq_domain *domain, unsigned int virq,
535 			    unsigned int nr_irqs, void *arg)
536 {
537 	struct msi_domain_info *info = domain->host_data;
538 	struct msi_domain_ops *ops = info->ops;
539 	irq_hw_number_t hwirq = ops->get_hwirq(info, arg);
540 	int i, ret;
541 
542 	if (irq_find_mapping(domain, hwirq) > 0)
543 		return -EEXIST;
544 
545 	if (domain->parent) {
546 		ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);
547 		if (ret < 0)
548 			return ret;
549 	}
550 
551 	for (i = 0; i < nr_irqs; i++) {
552 		ret = ops->msi_init(domain, info, virq + i, hwirq + i, arg);
553 		if (ret < 0) {
554 			if (ops->msi_free) {
555 				for (i--; i > 0; i--)
556 					ops->msi_free(domain, info, virq + i);
557 			}
558 			irq_domain_free_irqs_top(domain, virq, nr_irqs);
559 			return ret;
560 		}
561 	}
562 
563 	return 0;
564 }
565 
566 static void msi_domain_free(struct irq_domain *domain, unsigned int virq,
567 			    unsigned int nr_irqs)
568 {
569 	struct msi_domain_info *info = domain->host_data;
570 	int i;
571 
572 	if (info->ops->msi_free) {
573 		for (i = 0; i < nr_irqs; i++)
574 			info->ops->msi_free(domain, info, virq + i);
575 	}
576 	irq_domain_free_irqs_top(domain, virq, nr_irqs);
577 }
578 
579 static const struct irq_domain_ops msi_domain_ops = {
580 	.alloc		= msi_domain_alloc,
581 	.free		= msi_domain_free,
582 	.activate	= msi_domain_activate,
583 	.deactivate	= msi_domain_deactivate,
584 };
585 
586 static irq_hw_number_t msi_domain_ops_get_hwirq(struct msi_domain_info *info,
587 						msi_alloc_info_t *arg)
588 {
589 	return arg->hwirq;
590 }
591 
592 static int msi_domain_ops_prepare(struct irq_domain *domain, struct device *dev,
593 				  int nvec, msi_alloc_info_t *arg)
594 {
595 	memset(arg, 0, sizeof(*arg));
596 	return 0;
597 }
598 
599 static void msi_domain_ops_set_desc(msi_alloc_info_t *arg,
600 				    struct msi_desc *desc)
601 {
602 	arg->desc = desc;
603 }
604 
605 static int msi_domain_ops_init(struct irq_domain *domain,
606 			       struct msi_domain_info *info,
607 			       unsigned int virq, irq_hw_number_t hwirq,
608 			       msi_alloc_info_t *arg)
609 {
610 	irq_domain_set_hwirq_and_chip(domain, virq, hwirq, info->chip,
611 				      info->chip_data);
612 	if (info->handler && info->handler_name) {
613 		__irq_set_handler(virq, info->handler, 0, info->handler_name);
614 		if (info->handler_data)
615 			irq_set_handler_data(virq, info->handler_data);
616 	}
617 	return 0;
618 }
619 
620 static int msi_domain_ops_check(struct irq_domain *domain,
621 				struct msi_domain_info *info,
622 				struct device *dev)
623 {
624 	return 0;
625 }
626 
627 static struct msi_domain_ops msi_domain_ops_default = {
628 	.get_hwirq		= msi_domain_ops_get_hwirq,
629 	.msi_init		= msi_domain_ops_init,
630 	.msi_check		= msi_domain_ops_check,
631 	.msi_prepare		= msi_domain_ops_prepare,
632 	.set_desc		= msi_domain_ops_set_desc,
633 	.domain_alloc_irqs	= __msi_domain_alloc_irqs,
634 	.domain_free_irqs	= __msi_domain_free_irqs,
635 };
636 
637 static void msi_domain_update_dom_ops(struct msi_domain_info *info)
638 {
639 	struct msi_domain_ops *ops = info->ops;
640 
641 	if (ops == NULL) {
642 		info->ops = &msi_domain_ops_default;
643 		return;
644 	}
645 
646 	if (ops->domain_alloc_irqs == NULL)
647 		ops->domain_alloc_irqs = msi_domain_ops_default.domain_alloc_irqs;
648 	if (ops->domain_free_irqs == NULL)
649 		ops->domain_free_irqs = msi_domain_ops_default.domain_free_irqs;
650 
651 	if (!(info->flags & MSI_FLAG_USE_DEF_DOM_OPS))
652 		return;
653 
654 	if (ops->get_hwirq == NULL)
655 		ops->get_hwirq = msi_domain_ops_default.get_hwirq;
656 	if (ops->msi_init == NULL)
657 		ops->msi_init = msi_domain_ops_default.msi_init;
658 	if (ops->msi_check == NULL)
659 		ops->msi_check = msi_domain_ops_default.msi_check;
660 	if (ops->msi_prepare == NULL)
661 		ops->msi_prepare = msi_domain_ops_default.msi_prepare;
662 	if (ops->set_desc == NULL)
663 		ops->set_desc = msi_domain_ops_default.set_desc;
664 }
665 
666 static void msi_domain_update_chip_ops(struct msi_domain_info *info)
667 {
668 	struct irq_chip *chip = info->chip;
669 
670 	BUG_ON(!chip || !chip->irq_mask || !chip->irq_unmask);
671 	if (!chip->irq_set_affinity)
672 		chip->irq_set_affinity = msi_domain_set_affinity;
673 }
674 
675 /**
676  * msi_create_irq_domain - Create an MSI interrupt domain
677  * @fwnode:	Optional fwnode of the interrupt controller
678  * @info:	MSI domain info
679  * @parent:	Parent irq domain
680  *
681  * Return: pointer to the created &struct irq_domain or %NULL on failure
682  */
683 struct irq_domain *msi_create_irq_domain(struct fwnode_handle *fwnode,
684 					 struct msi_domain_info *info,
685 					 struct irq_domain *parent)
686 {
687 	struct irq_domain *domain;
688 
689 	msi_domain_update_dom_ops(info);
690 	if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)
691 		msi_domain_update_chip_ops(info);
692 
693 	domain = irq_domain_create_hierarchy(parent, IRQ_DOMAIN_FLAG_MSI, 0,
694 					     fwnode, &msi_domain_ops, info);
695 
696 	if (domain) {
697 		if (!domain->name && info->chip)
698 			domain->name = info->chip->name;
699 		irq_domain_update_bus_token(domain, info->bus_token);
700 	}
701 
702 	return domain;
703 }
704 
705 int msi_domain_prepare_irqs(struct irq_domain *domain, struct device *dev,
706 			    int nvec, msi_alloc_info_t *arg)
707 {
708 	struct msi_domain_info *info = domain->host_data;
709 	struct msi_domain_ops *ops = info->ops;
710 	int ret;
711 
712 	ret = ops->msi_check(domain, info, dev);
713 	if (ret == 0)
714 		ret = ops->msi_prepare(domain, dev, nvec, arg);
715 
716 	return ret;
717 }
718 
719 int msi_domain_populate_irqs(struct irq_domain *domain, struct device *dev,
720 			     int virq_base, int nvec, msi_alloc_info_t *arg)
721 {
722 	struct msi_domain_info *info = domain->host_data;
723 	struct msi_domain_ops *ops = info->ops;
724 	struct msi_desc *desc;
725 	int ret, virq;
726 
727 	msi_lock_descs(dev);
728 	ret = msi_add_simple_msi_descs(dev, virq_base, nvec);
729 	if (ret)
730 		goto unlock;
731 
732 	for (virq = virq_base; virq < virq_base + nvec; virq++) {
733 		desc = xa_load(&dev->msi.data->__store, virq);
734 		desc->irq = virq;
735 
736 		ops->set_desc(arg, desc);
737 		ret = irq_domain_alloc_irqs_hierarchy(domain, virq, 1, arg);
738 		if (ret)
739 			goto fail;
740 
741 		irq_set_msi_desc(virq, desc);
742 	}
743 	msi_unlock_descs(dev);
744 	return 0;
745 
746 fail:
747 	for (--virq; virq >= virq_base; virq--)
748 		irq_domain_free_irqs_common(domain, virq, 1);
749 	msi_free_msi_descs_range(dev, virq_base, virq_base + nvec - 1);
750 unlock:
751 	msi_unlock_descs(dev);
752 	return ret;
753 }
754 
755 /*
756  * Carefully check whether the device can use reservation mode. If
757  * reservation mode is enabled then the early activation will assign a
758  * dummy vector to the device. If the PCI/MSI device does not support
759  * masking of the entry then this can result in spurious interrupts when
760  * the device driver is not absolutely careful. But even then a malfunction
761  * of the hardware could result in a spurious interrupt on the dummy vector
762  * and render the device unusable. If the entry can be masked then the core
763  * logic will prevent the spurious interrupt and reservation mode can be
764  * used. For now reservation mode is restricted to PCI/MSI.
765  */
766 static bool msi_check_reservation_mode(struct irq_domain *domain,
767 				       struct msi_domain_info *info,
768 				       struct device *dev)
769 {
770 	struct msi_desc *desc;
771 
772 	switch(domain->bus_token) {
773 	case DOMAIN_BUS_PCI_MSI:
774 	case DOMAIN_BUS_VMD_MSI:
775 		break;
776 	default:
777 		return false;
778 	}
779 
780 	if (!(info->flags & MSI_FLAG_MUST_REACTIVATE))
781 		return false;
782 
783 	if (IS_ENABLED(CONFIG_PCI_MSI) && pci_msi_ignore_mask)
784 		return false;
785 
786 	/*
787 	 * Checking the first MSI descriptor is sufficient. MSIX supports
788 	 * masking and MSI does so when the can_mask attribute is set.
789 	 */
790 	desc = msi_first_desc(dev, MSI_DESC_ALL);
791 	return desc->pci.msi_attrib.is_msix || desc->pci.msi_attrib.can_mask;
792 }
793 
794 static int msi_handle_pci_fail(struct irq_domain *domain, struct msi_desc *desc,
795 			       int allocated)
796 {
797 	switch(domain->bus_token) {
798 	case DOMAIN_BUS_PCI_MSI:
799 	case DOMAIN_BUS_VMD_MSI:
800 		if (IS_ENABLED(CONFIG_PCI_MSI))
801 			break;
802 		fallthrough;
803 	default:
804 		return -ENOSPC;
805 	}
806 
807 	/* Let a failed PCI multi MSI allocation retry */
808 	if (desc->nvec_used > 1)
809 		return 1;
810 
811 	/* If there was a successful allocation let the caller know */
812 	return allocated ? allocated : -ENOSPC;
813 }
814 
815 #define VIRQ_CAN_RESERVE	0x01
816 #define VIRQ_ACTIVATE		0x02
817 #define VIRQ_NOMASK_QUIRK	0x04
818 
819 static int msi_init_virq(struct irq_domain *domain, int virq, unsigned int vflags)
820 {
821 	struct irq_data *irqd = irq_domain_get_irq_data(domain, virq);
822 	int ret;
823 
824 	if (!(vflags & VIRQ_CAN_RESERVE)) {
825 		irqd_clr_can_reserve(irqd);
826 		if (vflags & VIRQ_NOMASK_QUIRK)
827 			irqd_set_msi_nomask_quirk(irqd);
828 
829 		/*
830 		 * If the interrupt is managed but no CPU is available to
831 		 * service it, shut it down until better times. Note that
832 		 * we only do this on the !RESERVE path as x86 (the only
833 		 * architecture using this flag) deals with this in a
834 		 * different way by using a catch-all vector.
835 		 */
836 		if ((vflags & VIRQ_ACTIVATE) &&
837 		    irqd_affinity_is_managed(irqd) &&
838 		    !cpumask_intersects(irq_data_get_affinity_mask(irqd),
839 					cpu_online_mask)) {
840 			    irqd_set_managed_shutdown(irqd);
841 			    return 0;
842 		    }
843 	}
844 
845 	if (!(vflags & VIRQ_ACTIVATE))
846 		return 0;
847 
848 	ret = irq_domain_activate_irq(irqd, vflags & VIRQ_CAN_RESERVE);
849 	if (ret)
850 		return ret;
851 	/*
852 	 * If the interrupt uses reservation mode, clear the activated bit
853 	 * so request_irq() will assign the final vector.
854 	 */
855 	if (vflags & VIRQ_CAN_RESERVE)
856 		irqd_clr_activated(irqd);
857 	return 0;
858 }
859 
860 static int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev,
861 				   int nvec)
862 {
863 	struct msi_domain_info *info = domain->host_data;
864 	struct msi_domain_ops *ops = info->ops;
865 	msi_alloc_info_t arg = { };
866 	unsigned int vflags = 0;
867 	struct msi_desc *desc;
868 	int allocated = 0;
869 	int i, ret, virq;
870 
871 	ret = msi_domain_prepare_irqs(domain, dev, nvec, &arg);
872 	if (ret)
873 		return ret;
874 
875 	/*
876 	 * This flag is set by the PCI layer as we need to activate
877 	 * the MSI entries before the PCI layer enables MSI in the
878 	 * card. Otherwise the card latches a random msi message.
879 	 */
880 	if (info->flags & MSI_FLAG_ACTIVATE_EARLY)
881 		vflags |= VIRQ_ACTIVATE;
882 
883 	/*
884 	 * Interrupt can use a reserved vector and will not occupy
885 	 * a real device vector until the interrupt is requested.
886 	 */
887 	if (msi_check_reservation_mode(domain, info, dev)) {
888 		vflags |= VIRQ_CAN_RESERVE;
889 		/*
890 		 * MSI affinity setting requires a special quirk (X86) when
891 		 * reservation mode is active.
892 		 */
893 		if (domain->flags & IRQ_DOMAIN_MSI_NOMASK_QUIRK)
894 			vflags |= VIRQ_NOMASK_QUIRK;
895 	}
896 
897 	msi_for_each_desc(desc, dev, MSI_DESC_NOTASSOCIATED) {
898 		ops->set_desc(&arg, desc);
899 
900 		virq = __irq_domain_alloc_irqs(domain, -1, desc->nvec_used,
901 					       dev_to_node(dev), &arg, false,
902 					       desc->affinity);
903 		if (virq < 0)
904 			return msi_handle_pci_fail(domain, desc, allocated);
905 
906 		for (i = 0; i < desc->nvec_used; i++) {
907 			irq_set_msi_desc_off(virq, i, desc);
908 			irq_debugfs_copy_devname(virq + i, dev);
909 			ret = msi_init_virq(domain, virq + i, vflags);
910 			if (ret)
911 				return ret;
912 		}
913 		if (info->flags & MSI_FLAG_DEV_SYSFS) {
914 			ret = msi_sysfs_populate_desc(dev, desc);
915 			if (ret)
916 				return ret;
917 		}
918 		allocated++;
919 	}
920 	return 0;
921 }
922 
923 static int msi_domain_add_simple_msi_descs(struct msi_domain_info *info,
924 					   struct device *dev,
925 					   unsigned int num_descs)
926 {
927 	if (!(info->flags & MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS))
928 		return 0;
929 
930 	return msi_add_simple_msi_descs(dev, 0, num_descs);
931 }
932 
933 /**
934  * msi_domain_alloc_irqs_descs_locked - Allocate interrupts from a MSI interrupt domain
935  * @domain:	The domain to allocate from
936  * @dev:	Pointer to device struct of the device for which the interrupts
937  *		are allocated
938  * @nvec:	The number of interrupts to allocate
939  *
940  * Must be invoked from within a msi_lock_descs() / msi_unlock_descs()
941  * pair. Use this for MSI irqdomains which implement their own vector
942  * allocation/free.
943  *
944  * Return: %0 on success or an error code.
945  */
946 int msi_domain_alloc_irqs_descs_locked(struct irq_domain *domain, struct device *dev,
947 				       int nvec)
948 {
949 	struct msi_domain_info *info = domain->host_data;
950 	struct msi_domain_ops *ops = info->ops;
951 	int ret;
952 
953 	lockdep_assert_held(&dev->msi.data->mutex);
954 
955 	ret = msi_domain_add_simple_msi_descs(info, dev, nvec);
956 	if (ret)
957 		return ret;
958 
959 	ret = ops->domain_alloc_irqs(domain, dev, nvec);
960 	if (ret)
961 		msi_domain_free_irqs_descs_locked(domain, dev);
962 	return ret;
963 }
964 
965 /**
966  * msi_domain_alloc_irqs - Allocate interrupts from a MSI interrupt domain
967  * @domain:	The domain to allocate from
968  * @dev:	Pointer to device struct of the device for which the interrupts
969  *		are allocated
970  * @nvec:	The number of interrupts to allocate
971  *
972  * Return: %0 on success or an error code.
973  */
974 int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, int nvec)
975 {
976 	int ret;
977 
978 	msi_lock_descs(dev);
979 	ret = msi_domain_alloc_irqs_descs_locked(domain, dev, nvec);
980 	msi_unlock_descs(dev);
981 	return ret;
982 }
983 
984 static void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev)
985 {
986 	struct msi_domain_info *info = domain->host_data;
987 	struct irq_data *irqd;
988 	struct msi_desc *desc;
989 	int i;
990 
991 	/* Only handle MSI entries which have an interrupt associated */
992 	msi_for_each_desc(desc, dev, MSI_DESC_ASSOCIATED) {
993 		/* Make sure all interrupts are deactivated */
994 		for (i = 0; i < desc->nvec_used; i++) {
995 			irqd = irq_domain_get_irq_data(domain, desc->irq + i);
996 			if (irqd && irqd_is_activated(irqd))
997 				irq_domain_deactivate_irq(irqd);
998 		}
999 
1000 		irq_domain_free_irqs(desc->irq, desc->nvec_used);
1001 		if (info->flags & MSI_FLAG_DEV_SYSFS)
1002 			msi_sysfs_remove_desc(dev, desc);
1003 		desc->irq = 0;
1004 	}
1005 }
1006 
1007 static void msi_domain_free_msi_descs(struct msi_domain_info *info,
1008 				      struct device *dev)
1009 {
1010 	if (info->flags & MSI_FLAG_FREE_MSI_DESCS)
1011 		msi_free_msi_descs(dev);
1012 }
1013 
1014 /**
1015  * msi_domain_free_irqs_descs_locked - Free interrupts from a MSI interrupt @domain associated to @dev
1016  * @domain:	The domain to managing the interrupts
1017  * @dev:	Pointer to device struct of the device for which the interrupts
1018  *		are free
1019  *
1020  * Must be invoked from within a msi_lock_descs() / msi_unlock_descs()
1021  * pair. Use this for MSI irqdomains which implement their own vector
1022  * allocation.
1023  */
1024 void msi_domain_free_irqs_descs_locked(struct irq_domain *domain, struct device *dev)
1025 {
1026 	struct msi_domain_info *info = domain->host_data;
1027 	struct msi_domain_ops *ops = info->ops;
1028 
1029 	lockdep_assert_held(&dev->msi.data->mutex);
1030 
1031 	ops->domain_free_irqs(domain, dev);
1032 	if (ops->msi_post_free)
1033 		ops->msi_post_free(domain, dev);
1034 	msi_domain_free_msi_descs(info, dev);
1035 }
1036 
1037 /**
1038  * msi_domain_free_irqs - Free interrupts from a MSI interrupt @domain associated to @dev
1039  * @domain:	The domain to managing the interrupts
1040  * @dev:	Pointer to device struct of the device for which the interrupts
1041  *		are free
1042  */
1043 void msi_domain_free_irqs(struct irq_domain *domain, struct device *dev)
1044 {
1045 	msi_lock_descs(dev);
1046 	msi_domain_free_irqs_descs_locked(domain, dev);
1047 	msi_unlock_descs(dev);
1048 }
1049 
1050 /**
1051  * msi_get_domain_info - Get the MSI interrupt domain info for @domain
1052  * @domain:	The interrupt domain to retrieve data from
1053  *
1054  * Return: the pointer to the msi_domain_info stored in @domain->host_data.
1055  */
1056 struct msi_domain_info *msi_get_domain_info(struct irq_domain *domain)
1057 {
1058 	return (struct msi_domain_info *)domain->host_data;
1059 }
1060