xref: /openbmc/linux/drivers/pci/msi/api.c (revision e6cc6f17)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * PCI MSI/MSI-X — Exported APIs for device drivers
4  *
5  * Copyright (C) 2003-2004 Intel
6  * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
7  * Copyright (C) 2016 Christoph Hellwig.
8  * Copyright (C) 2022 Linutronix GmbH
9  */
10 
11 #include <linux/export.h>
12 #include <linux/irq.h>
13 
14 #include "msi.h"
15 
16 /**
17  * pci_enable_msi() - Enable MSI interrupt mode on device
18  * @dev: the PCI device to operate on
19  *
20  * Legacy device driver API to enable MSI interrupts mode on device and
21  * allocate a single interrupt vector. On success, the allocated vector
22  * Linux IRQ will be saved at @dev->irq. The driver must invoke
23  * pci_disable_msi() on cleanup.
24  *
25  * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API
26  * pair should, in general, be used instead.
27  *
28  * Return: 0 on success, errno otherwise
29  */
pci_enable_msi(struct pci_dev * dev)30 int pci_enable_msi(struct pci_dev *dev)
31 {
32 	int rc = __pci_enable_msi_range(dev, 1, 1, NULL);
33 	if (rc < 0)
34 		return rc;
35 	return 0;
36 }
37 EXPORT_SYMBOL(pci_enable_msi);
38 
39 /**
40  * pci_disable_msi() - Disable MSI interrupt mode on device
41  * @dev: the PCI device to operate on
42  *
43  * Legacy device driver API to disable MSI interrupt mode on device,
44  * free earlier allocated interrupt vectors, and restore INTx emulation.
45  * The PCI device Linux IRQ (@dev->irq) is restored to its default
46  * pin-assertion IRQ. This is the cleanup pair of pci_enable_msi().
47  *
48  * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API
49  * pair should, in general, be used instead.
50  */
pci_disable_msi(struct pci_dev * dev)51 void pci_disable_msi(struct pci_dev *dev)
52 {
53 	if (!pci_msi_enabled() || !dev || !dev->msi_enabled)
54 		return;
55 
56 	msi_lock_descs(&dev->dev);
57 	pci_msi_shutdown(dev);
58 	pci_free_msi_irqs(dev);
59 	msi_unlock_descs(&dev->dev);
60 }
61 EXPORT_SYMBOL(pci_disable_msi);
62 
63 /**
64  * pci_msix_vec_count() - Get number of MSI-X interrupt vectors on device
65  * @dev: the PCI device to operate on
66  *
67  * Return: number of MSI-X interrupt vectors available on this device
68  * (i.e., the device's MSI-X capability structure "table size"), -EINVAL
69  * if the device is not MSI-X capable, other errnos otherwise.
70  */
pci_msix_vec_count(struct pci_dev * dev)71 int pci_msix_vec_count(struct pci_dev *dev)
72 {
73 	u16 control;
74 
75 	if (!dev->msix_cap)
76 		return -EINVAL;
77 
78 	pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control);
79 	return msix_table_size(control);
80 }
81 EXPORT_SYMBOL(pci_msix_vec_count);
82 
83 /**
84  * pci_enable_msix_range() - Enable MSI-X interrupt mode on device
85  * @dev:     the PCI device to operate on
86  * @entries: input/output parameter, array of MSI-X configuration entries
87  * @minvec:  minimum required number of MSI-X vectors
88  * @maxvec:  maximum desired number of MSI-X vectors
89  *
90  * Legacy device driver API to enable MSI-X interrupt mode on device and
91  * configure its MSI-X capability structure as appropriate.  The passed
92  * @entries array must have each of its members "entry" field set to a
93  * desired (valid) MSI-X vector number, where the range of valid MSI-X
94  * vector numbers can be queried through pci_msix_vec_count().  If
95  * successful, the driver must invoke pci_disable_msix() on cleanup.
96  *
97  * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API
98  * pair should, in general, be used instead.
99  *
100  * Return: number of MSI-X vectors allocated (which might be smaller
101  * than @maxvecs), where Linux IRQ numbers for such allocated vectors
102  * are saved back in the @entries array elements' "vector" field. Return
103  * -ENOSPC if less than @minvecs interrupt vectors are available.
104  * Return -EINVAL if one of the passed @entries members "entry" field
105  * was invalid or a duplicate, or if plain MSI interrupts mode was
106  * earlier enabled on device. Return other errnos otherwise.
107  */
pci_enable_msix_range(struct pci_dev * dev,struct msix_entry * entries,int minvec,int maxvec)108 int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries,
109 			  int minvec, int maxvec)
110 {
111 	return __pci_enable_msix_range(dev, entries, minvec, maxvec, NULL, 0);
112 }
113 EXPORT_SYMBOL(pci_enable_msix_range);
114 
115 /**
116  * pci_msix_can_alloc_dyn - Query whether dynamic allocation after enabling
117  *			    MSI-X is supported
118  *
119  * @dev:	PCI device to operate on
120  *
121  * Return: True if supported, false otherwise
122  */
pci_msix_can_alloc_dyn(struct pci_dev * dev)123 bool pci_msix_can_alloc_dyn(struct pci_dev *dev)
124 {
125 	if (!dev->msix_cap)
126 		return false;
127 
128 	return pci_msi_domain_supports(dev, MSI_FLAG_PCI_MSIX_ALLOC_DYN, DENY_LEGACY);
129 }
130 EXPORT_SYMBOL_GPL(pci_msix_can_alloc_dyn);
131 
132 /**
133  * pci_msix_alloc_irq_at - Allocate an MSI-X interrupt after enabling MSI-X
134  *			   at a given MSI-X vector index or any free vector index
135  *
136  * @dev:	PCI device to operate on
137  * @index:	Index to allocate. If @index == MSI_ANY_INDEX this allocates
138  *		the next free index in the MSI-X table
139  * @affdesc:	Optional pointer to an affinity descriptor structure. NULL otherwise
140  *
141  * Return: A struct msi_map
142  *
143  *	On success msi_map::index contains the allocated index (>= 0) and
144  *	msi_map::virq contains the allocated Linux interrupt number (> 0).
145  *
146  *	On fail msi_map::index contains the error code and msi_map::virq
147  *	is set to 0.
148  */
pci_msix_alloc_irq_at(struct pci_dev * dev,unsigned int index,const struct irq_affinity_desc * affdesc)149 struct msi_map pci_msix_alloc_irq_at(struct pci_dev *dev, unsigned int index,
150 				     const struct irq_affinity_desc *affdesc)
151 {
152 	struct msi_map map = { .index = -ENOTSUPP };
153 
154 	if (!dev->msix_enabled)
155 		return map;
156 
157 	if (!pci_msix_can_alloc_dyn(dev))
158 		return map;
159 
160 	return msi_domain_alloc_irq_at(&dev->dev, MSI_DEFAULT_DOMAIN, index, affdesc, NULL);
161 }
162 EXPORT_SYMBOL_GPL(pci_msix_alloc_irq_at);
163 
164 /**
165  * pci_msix_free_irq - Free an interrupt on a PCI/MSIX interrupt domain
166  *
167  * @dev:	The PCI device to operate on
168  * @map:	A struct msi_map describing the interrupt to free
169  *
170  * Undo an interrupt vector allocation. Does not disable MSI-X.
171  */
pci_msix_free_irq(struct pci_dev * dev,struct msi_map map)172 void pci_msix_free_irq(struct pci_dev *dev, struct msi_map map)
173 {
174 	if (WARN_ON_ONCE(map.index < 0 || map.virq <= 0))
175 		return;
176 	if (WARN_ON_ONCE(!pci_msix_can_alloc_dyn(dev)))
177 		return;
178 	msi_domain_free_irqs_range(&dev->dev, MSI_DEFAULT_DOMAIN, map.index, map.index);
179 }
180 EXPORT_SYMBOL_GPL(pci_msix_free_irq);
181 
182 /**
183  * pci_disable_msix() - Disable MSI-X interrupt mode on device
184  * @dev: the PCI device to operate on
185  *
186  * Legacy device driver API to disable MSI-X interrupt mode on device,
187  * free earlier-allocated interrupt vectors, and restore INTx.
188  * The PCI device Linux IRQ (@dev->irq) is restored to its default pin
189  * assertion IRQ. This is the cleanup pair of pci_enable_msix_range().
190  *
191  * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API
192  * pair should, in general, be used instead.
193  */
pci_disable_msix(struct pci_dev * dev)194 void pci_disable_msix(struct pci_dev *dev)
195 {
196 	if (!pci_msi_enabled() || !dev || !dev->msix_enabled)
197 		return;
198 
199 	msi_lock_descs(&dev->dev);
200 	pci_msix_shutdown(dev);
201 	pci_free_msi_irqs(dev);
202 	msi_unlock_descs(&dev->dev);
203 }
204 EXPORT_SYMBOL(pci_disable_msix);
205 
206 /**
207  * pci_alloc_irq_vectors() - Allocate multiple device interrupt vectors
208  * @dev:      the PCI device to operate on
209  * @min_vecs: minimum required number of vectors (must be >= 1)
210  * @max_vecs: maximum desired number of vectors
211  * @flags:    One or more of:
212  *
213  *            * %PCI_IRQ_MSIX      Allow trying MSI-X vector allocations
214  *            * %PCI_IRQ_MSI       Allow trying MSI vector allocations
215  *
216  *            * %PCI_IRQ_LEGACY    Allow trying legacy INTx interrupts, if
217  *              and only if @min_vecs == 1
218  *
219  *            * %PCI_IRQ_AFFINITY  Auto-manage IRQs affinity by spreading
220  *              the vectors around available CPUs
221  *
222  * Allocate up to @max_vecs interrupt vectors on device. MSI-X irq
223  * vector allocation has a higher precedence over plain MSI, which has a
224  * higher precedence over legacy INTx emulation.
225  *
226  * Upon a successful allocation, the caller should use pci_irq_vector()
227  * to get the Linux IRQ number to be passed to request_threaded_irq().
228  * The driver must call pci_free_irq_vectors() on cleanup.
229  *
230  * Return: number of allocated vectors (which might be smaller than
231  * @max_vecs), -ENOSPC if less than @min_vecs interrupt vectors are
232  * available, other errnos otherwise.
233  */
pci_alloc_irq_vectors(struct pci_dev * dev,unsigned int min_vecs,unsigned int max_vecs,unsigned int flags)234 int pci_alloc_irq_vectors(struct pci_dev *dev, unsigned int min_vecs,
235 			  unsigned int max_vecs, unsigned int flags)
236 {
237 	return pci_alloc_irq_vectors_affinity(dev, min_vecs, max_vecs,
238 					      flags, NULL);
239 }
240 EXPORT_SYMBOL(pci_alloc_irq_vectors);
241 
242 /**
243  * pci_alloc_irq_vectors_affinity() - Allocate multiple device interrupt
244  *                                    vectors with affinity requirements
245  * @dev:      the PCI device to operate on
246  * @min_vecs: minimum required number of vectors (must be >= 1)
247  * @max_vecs: maximum desired number of vectors
248  * @flags:    allocation flags, as in pci_alloc_irq_vectors()
249  * @affd:     affinity requirements (can be %NULL).
250  *
251  * Same as pci_alloc_irq_vectors(), but with the extra @affd parameter.
252  * Check that function docs, and &struct irq_affinity, for more details.
253  */
pci_alloc_irq_vectors_affinity(struct pci_dev * dev,unsigned int min_vecs,unsigned int max_vecs,unsigned int flags,struct irq_affinity * affd)254 int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
255 				   unsigned int max_vecs, unsigned int flags,
256 				   struct irq_affinity *affd)
257 {
258 	struct irq_affinity msi_default_affd = {0};
259 	int nvecs = -ENOSPC;
260 
261 	if (flags & PCI_IRQ_AFFINITY) {
262 		if (!affd)
263 			affd = &msi_default_affd;
264 	} else {
265 		if (WARN_ON(affd))
266 			affd = NULL;
267 	}
268 
269 	if (flags & PCI_IRQ_MSIX) {
270 		nvecs = __pci_enable_msix_range(dev, NULL, min_vecs, max_vecs,
271 						affd, flags);
272 		if (nvecs > 0)
273 			return nvecs;
274 	}
275 
276 	if (flags & PCI_IRQ_MSI) {
277 		nvecs = __pci_enable_msi_range(dev, min_vecs, max_vecs, affd);
278 		if (nvecs > 0)
279 			return nvecs;
280 	}
281 
282 	/* use legacy IRQ if allowed */
283 	if (flags & PCI_IRQ_LEGACY) {
284 		if (min_vecs == 1 && dev->irq) {
285 			/*
286 			 * Invoke the affinity spreading logic to ensure that
287 			 * the device driver can adjust queue configuration
288 			 * for the single interrupt case.
289 			 */
290 			if (affd)
291 				irq_create_affinity_masks(1, affd);
292 			pci_intx(dev, 1);
293 			return 1;
294 		}
295 	}
296 
297 	return nvecs;
298 }
299 EXPORT_SYMBOL(pci_alloc_irq_vectors_affinity);
300 
301 /**
302  * pci_irq_vector() - Get Linux IRQ number of a device interrupt vector
303  * @dev: the PCI device to operate on
304  * @nr:  device-relative interrupt vector index (0-based); has different
305  *       meanings, depending on interrupt mode:
306  *
307  *         * MSI-X     the index in the MSI-X vector table
308  *         * MSI       the index of the enabled MSI vectors
309  *         * INTx      must be 0
310  *
311  * Return: the Linux IRQ number, or -EINVAL if @nr is out of range
312  */
pci_irq_vector(struct pci_dev * dev,unsigned int nr)313 int pci_irq_vector(struct pci_dev *dev, unsigned int nr)
314 {
315 	unsigned int irq;
316 
317 	if (!dev->msi_enabled && !dev->msix_enabled)
318 		return !nr ? dev->irq : -EINVAL;
319 
320 	irq = msi_get_virq(&dev->dev, nr);
321 	return irq ? irq : -EINVAL;
322 }
323 EXPORT_SYMBOL(pci_irq_vector);
324 
325 /**
326  * pci_irq_get_affinity() - Get a device interrupt vector affinity
327  * @dev: the PCI device to operate on
328  * @nr:  device-relative interrupt vector index (0-based); has different
329  *       meanings, depending on interrupt mode:
330  *
331  *         * MSI-X     the index in the MSI-X vector table
332  *         * MSI       the index of the enabled MSI vectors
333  *         * INTx      must be 0
334  *
335  * Return: MSI/MSI-X vector affinity, NULL if @nr is out of range or if
336  * the MSI(-X) vector was allocated without explicit affinity
337  * requirements (e.g., by pci_enable_msi(), pci_enable_msix_range(), or
338  * pci_alloc_irq_vectors() without the %PCI_IRQ_AFFINITY flag). Return a
339  * generic set of CPU IDs representing all possible CPUs available
340  * during system boot if the device is in legacy INTx mode.
341  */
pci_irq_get_affinity(struct pci_dev * dev,int nr)342 const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr)
343 {
344 	int idx, irq = pci_irq_vector(dev, nr);
345 	struct msi_desc *desc;
346 
347 	if (WARN_ON_ONCE(irq <= 0))
348 		return NULL;
349 
350 	desc = irq_get_msi_desc(irq);
351 	/* Non-MSI does not have the information handy */
352 	if (!desc)
353 		return cpu_possible_mask;
354 
355 	/* MSI[X] interrupts can be allocated without affinity descriptor */
356 	if (!desc->affinity)
357 		return NULL;
358 
359 	/*
360 	 * MSI has a mask array in the descriptor.
361 	 * MSI-X has a single mask.
362 	 */
363 	idx = dev->msi_enabled ? nr : 0;
364 	return &desc->affinity[idx].mask;
365 }
366 EXPORT_SYMBOL(pci_irq_get_affinity);
367 
368 /**
369  * pci_ims_alloc_irq - Allocate an interrupt on a PCI/IMS interrupt domain
370  * @dev:	The PCI device to operate on
371  * @icookie:	Pointer to an IMS implementation specific cookie for this
372  *		IMS instance (PASID, queue ID, pointer...).
373  *		The cookie content is copied into the MSI descriptor for the
374  *		interrupt chip callbacks or domain specific setup functions.
375  * @affdesc:	Optional pointer to an interrupt affinity descriptor
376  *
377  * There is no index for IMS allocations as IMS is an implementation
378  * specific storage and does not have any direct associations between
379  * index, which might be a pure software construct, and device
380  * functionality. This association is established by the driver either via
381  * the index - if there is a hardware table - or in case of purely software
382  * managed IMS implementation the association happens via the
383  * irq_write_msi_msg() callback of the implementation specific interrupt
384  * chip, which utilizes the provided @icookie to store the MSI message in
385  * the appropriate place.
386  *
387  * Return: A struct msi_map
388  *
389  *	On success msi_map::index contains the allocated index (>= 0) and
390  *	msi_map::virq the allocated Linux interrupt number (> 0).
391  *
392  *	On fail msi_map::index contains the error code and msi_map::virq
393  *	is set to 0.
394  */
pci_ims_alloc_irq(struct pci_dev * dev,union msi_instance_cookie * icookie,const struct irq_affinity_desc * affdesc)395 struct msi_map pci_ims_alloc_irq(struct pci_dev *dev, union msi_instance_cookie *icookie,
396 				 const struct irq_affinity_desc *affdesc)
397 {
398 	return msi_domain_alloc_irq_at(&dev->dev, MSI_SECONDARY_DOMAIN, MSI_ANY_INDEX,
399 				       affdesc, icookie);
400 }
401 EXPORT_SYMBOL_GPL(pci_ims_alloc_irq);
402 
403 /**
404  * pci_ims_free_irq - Allocate an interrupt on a PCI/IMS interrupt domain
405  *		      which was allocated via pci_ims_alloc_irq()
406  * @dev:	The PCI device to operate on
407  * @map:	A struct msi_map describing the interrupt to free as
408  *		returned from pci_ims_alloc_irq()
409  */
pci_ims_free_irq(struct pci_dev * dev,struct msi_map map)410 void pci_ims_free_irq(struct pci_dev *dev, struct msi_map map)
411 {
412 	if (WARN_ON_ONCE(map.index < 0 || map.virq <= 0))
413 		return;
414 	msi_domain_free_irqs_range(&dev->dev, MSI_SECONDARY_DOMAIN, map.index, map.index);
415 }
416 EXPORT_SYMBOL_GPL(pci_ims_free_irq);
417 
418 /**
419  * pci_free_irq_vectors() - Free previously allocated IRQs for a device
420  * @dev: the PCI device to operate on
421  *
422  * Undo the interrupt vector allocations and possible device MSI/MSI-X
423  * enablement earlier done through pci_alloc_irq_vectors_affinity() or
424  * pci_alloc_irq_vectors().
425  */
pci_free_irq_vectors(struct pci_dev * dev)426 void pci_free_irq_vectors(struct pci_dev *dev)
427 {
428 	pci_disable_msix(dev);
429 	pci_disable_msi(dev);
430 }
431 EXPORT_SYMBOL(pci_free_irq_vectors);
432 
433 /**
434  * pci_restore_msi_state() - Restore cached MSI(-X) state on device
435  * @dev: the PCI device to operate on
436  *
437  * Write the Linux-cached MSI(-X) state back on device. This is
438  * typically useful upon system resume, or after an error-recovery PCI
439  * adapter reset.
440  */
pci_restore_msi_state(struct pci_dev * dev)441 void pci_restore_msi_state(struct pci_dev *dev)
442 {
443 	__pci_restore_msi_state(dev);
444 	__pci_restore_msix_state(dev);
445 }
446 EXPORT_SYMBOL_GPL(pci_restore_msi_state);
447 
448 /**
449  * pci_msi_enabled() - Are MSI(-X) interrupts enabled system-wide?
450  *
451  * Return: true if MSI has not been globally disabled through ACPI FADT,
452  * PCI bridge quirks, or the "pci=nomsi" kernel command-line option.
453  */
pci_msi_enabled(void)454 int pci_msi_enabled(void)
455 {
456 	return pci_msi_enable;
457 }
458 EXPORT_SYMBOL(pci_msi_enabled);
459