1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2023, Intel Corporation. */
3 
4 #include "ice.h"
5 #include "ice_lib.h"
6 #include "ice_irq.h"
7 
8 /**
9  * ice_init_irq_tracker - initialize interrupt tracker
10  * @pf: board private structure
11  * @max_vectors: maximum number of vectors that tracker can hold
12  * @num_static: number of preallocated interrupts
13  */
14 static void
15 ice_init_irq_tracker(struct ice_pf *pf, unsigned int max_vectors,
16 		     unsigned int num_static)
17 {
18 	pf->irq_tracker.num_entries = max_vectors;
19 	pf->irq_tracker.num_static = num_static;
20 	xa_init_flags(&pf->irq_tracker.entries, XA_FLAGS_ALLOC);
21 }
22 
23 /**
24  * ice_deinit_irq_tracker - free xarray tracker
25  * @pf: board private structure
26  */
27 static void ice_deinit_irq_tracker(struct ice_pf *pf)
28 {
29 	xa_destroy(&pf->irq_tracker.entries);
30 }
31 
32 /**
33  * ice_free_irq_res - free a block of resources
34  * @pf: board private structure
35  * @index: starting index previously returned by ice_get_res
36  */
37 static void ice_free_irq_res(struct ice_pf *pf, u16 index)
38 {
39 	struct ice_irq_entry *entry;
40 
41 	entry = xa_erase(&pf->irq_tracker.entries, index);
42 	kfree(entry);
43 }
44 
45 /**
46  * ice_get_irq_res - get an interrupt resource
47  * @pf: board private structure
48  * @dyn_only: force entry to be dynamically allocated
49  *
50  * Allocate new irq entry in the free slot of the tracker. Since xarray
51  * is used, always allocate new entry at the lowest possible index. Set
52  * proper allocation limit for maximum tracker entries.
53  *
54  * Returns allocated irq entry or NULL on failure.
55  */
56 static struct ice_irq_entry *ice_get_irq_res(struct ice_pf *pf, bool dyn_only)
57 {
58 	struct xa_limit limit = { .max = pf->irq_tracker.num_entries,
59 				  .min = 0 };
60 	unsigned int num_static = pf->irq_tracker.num_static;
61 	struct ice_irq_entry *entry;
62 	unsigned int index;
63 	int ret;
64 
65 	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
66 	if (!entry)
67 		return NULL;
68 
69 	/* skip preallocated entries if the caller says so */
70 	if (dyn_only)
71 		limit.min = num_static;
72 
73 	ret = xa_alloc(&pf->irq_tracker.entries, &index, entry, limit,
74 		       GFP_KERNEL);
75 
76 	if (ret) {
77 		kfree(entry);
78 		entry = NULL;
79 	} else {
80 		entry->index = index;
81 		entry->dynamic = index >= num_static;
82 	}
83 
84 	return entry;
85 }
86 
87 /**
88  * ice_reduce_msix_usage - Reduce usage of MSI-X vectors
89  * @pf: board private structure
90  * @v_remain: number of remaining MSI-X vectors to be distributed
91  *
92  * Reduce the usage of MSI-X vectors when entire request cannot be fulfilled.
93  * pf->num_lan_msix and pf->num_rdma_msix values are set based on number of
94  * remaining vectors.
95  */
96 static void ice_reduce_msix_usage(struct ice_pf *pf, int v_remain)
97 {
98 	int v_rdma;
99 
100 	if (!ice_is_rdma_ena(pf)) {
101 		pf->num_lan_msix = v_remain;
102 		return;
103 	}
104 
105 	/* RDMA needs at least 1 interrupt in addition to AEQ MSIX */
106 	v_rdma = ICE_RDMA_NUM_AEQ_MSIX + 1;
107 
108 	if (v_remain < ICE_MIN_LAN_TXRX_MSIX + ICE_MIN_RDMA_MSIX) {
109 		dev_warn(ice_pf_to_dev(pf), "Not enough MSI-X vectors to support RDMA.\n");
110 		clear_bit(ICE_FLAG_RDMA_ENA, pf->flags);
111 
112 		pf->num_rdma_msix = 0;
113 		pf->num_lan_msix = ICE_MIN_LAN_TXRX_MSIX;
114 	} else if ((v_remain < ICE_MIN_LAN_TXRX_MSIX + v_rdma) ||
115 		   (v_remain - v_rdma < v_rdma)) {
116 		/* Support minimum RDMA and give remaining vectors to LAN MSIX
117 		 */
118 		pf->num_rdma_msix = ICE_MIN_RDMA_MSIX;
119 		pf->num_lan_msix = v_remain - ICE_MIN_RDMA_MSIX;
120 	} else {
121 		/* Split remaining MSIX with RDMA after accounting for AEQ MSIX
122 		 */
123 		pf->num_rdma_msix = (v_remain - ICE_RDMA_NUM_AEQ_MSIX) / 2 +
124 				    ICE_RDMA_NUM_AEQ_MSIX;
125 		pf->num_lan_msix = v_remain - pf->num_rdma_msix;
126 	}
127 }
128 
129 /**
130  * ice_ena_msix_range - Request a range of MSIX vectors from the OS
131  * @pf: board private structure
132  *
133  * Compute the number of MSIX vectors wanted and request from the OS. Adjust
134  * device usage if there are not enough vectors. Return the number of vectors
135  * reserved or negative on failure.
136  */
137 static int ice_ena_msix_range(struct ice_pf *pf)
138 {
139 	int num_cpus, hw_num_msix, v_other, v_wanted, v_actual;
140 	struct device *dev = ice_pf_to_dev(pf);
141 	int err;
142 
143 	hw_num_msix = pf->hw.func_caps.common_cap.num_msix_vectors;
144 	num_cpus = num_online_cpus();
145 
146 	/* LAN miscellaneous handler */
147 	v_other = ICE_MIN_LAN_OICR_MSIX;
148 
149 	/* Flow Director */
150 	if (test_bit(ICE_FLAG_FD_ENA, pf->flags))
151 		v_other += ICE_FDIR_MSIX;
152 
153 	/* switchdev */
154 	v_other += ICE_ESWITCH_MSIX;
155 
156 	v_wanted = v_other;
157 
158 	/* LAN traffic */
159 	pf->num_lan_msix = num_cpus;
160 	v_wanted += pf->num_lan_msix;
161 
162 	/* RDMA auxiliary driver */
163 	if (ice_is_rdma_ena(pf)) {
164 		pf->num_rdma_msix = num_cpus + ICE_RDMA_NUM_AEQ_MSIX;
165 		v_wanted += pf->num_rdma_msix;
166 	}
167 
168 	if (v_wanted > hw_num_msix) {
169 		int v_remain;
170 
171 		dev_warn(dev, "not enough device MSI-X vectors. wanted = %d, available = %d\n",
172 			 v_wanted, hw_num_msix);
173 
174 		if (hw_num_msix < ICE_MIN_MSIX) {
175 			err = -ERANGE;
176 			goto exit_err;
177 		}
178 
179 		v_remain = hw_num_msix - v_other;
180 		if (v_remain < ICE_MIN_LAN_TXRX_MSIX) {
181 			v_other = ICE_MIN_MSIX - ICE_MIN_LAN_TXRX_MSIX;
182 			v_remain = ICE_MIN_LAN_TXRX_MSIX;
183 		}
184 
185 		ice_reduce_msix_usage(pf, v_remain);
186 		v_wanted = pf->num_lan_msix + pf->num_rdma_msix + v_other;
187 
188 		dev_notice(dev, "Reducing request to %d MSI-X vectors for LAN traffic.\n",
189 			   pf->num_lan_msix);
190 		if (ice_is_rdma_ena(pf))
191 			dev_notice(dev, "Reducing request to %d MSI-X vectors for RDMA.\n",
192 				   pf->num_rdma_msix);
193 	}
194 
195 	/* actually reserve the vectors */
196 	v_actual = pci_alloc_irq_vectors(pf->pdev, ICE_MIN_MSIX, v_wanted,
197 					 PCI_IRQ_MSIX);
198 	if (v_actual < 0) {
199 		dev_err(dev, "unable to reserve MSI-X vectors\n");
200 		err = v_actual;
201 		goto exit_err;
202 	}
203 
204 	if (v_actual < v_wanted) {
205 		dev_warn(dev, "not enough OS MSI-X vectors. requested = %d, obtained = %d\n",
206 			 v_wanted, v_actual);
207 
208 		if (v_actual < ICE_MIN_MSIX) {
209 			/* error if we can't get minimum vectors */
210 			pci_free_irq_vectors(pf->pdev);
211 			err = -ERANGE;
212 			goto exit_err;
213 		} else {
214 			int v_remain = v_actual - v_other;
215 
216 			if (v_remain < ICE_MIN_LAN_TXRX_MSIX)
217 				v_remain = ICE_MIN_LAN_TXRX_MSIX;
218 
219 			ice_reduce_msix_usage(pf, v_remain);
220 
221 			dev_notice(dev, "Enabled %d MSI-X vectors for LAN traffic.\n",
222 				   pf->num_lan_msix);
223 
224 			if (ice_is_rdma_ena(pf))
225 				dev_notice(dev, "Enabled %d MSI-X vectors for RDMA.\n",
226 					   pf->num_rdma_msix);
227 		}
228 	}
229 
230 	return v_actual;
231 
232 exit_err:
233 	pf->num_rdma_msix = 0;
234 	pf->num_lan_msix = 0;
235 	return err;
236 }
237 
238 /**
239  * ice_clear_interrupt_scheme - Undo things done by ice_init_interrupt_scheme
240  * @pf: board private structure
241  */
242 void ice_clear_interrupt_scheme(struct ice_pf *pf)
243 {
244 	pci_free_irq_vectors(pf->pdev);
245 	ice_deinit_irq_tracker(pf);
246 }
247 
248 /**
249  * ice_init_interrupt_scheme - Determine proper interrupt scheme
250  * @pf: board private structure to initialize
251  */
252 int ice_init_interrupt_scheme(struct ice_pf *pf)
253 {
254 	int total_vectors = pf->hw.func_caps.common_cap.num_msix_vectors;
255 	int vectors, max_vectors;
256 
257 	vectors = ice_ena_msix_range(pf);
258 
259 	if (vectors < 0)
260 		return -ENOMEM;
261 
262 	if (pci_msix_can_alloc_dyn(pf->pdev))
263 		max_vectors = total_vectors;
264 	else
265 		max_vectors = vectors;
266 
267 	ice_init_irq_tracker(pf, max_vectors, vectors);
268 
269 	return 0;
270 }
271 
272 /**
273  * ice_alloc_irq - Allocate new interrupt vector
274  * @pf: board private structure
275  * @dyn_only: force dynamic allocation of the interrupt
276  *
277  * Allocate new interrupt vector for a given owner id.
278  * return struct msi_map with interrupt details and track
279  * allocated interrupt appropriately.
280  *
281  * This function reserves new irq entry from the irq_tracker.
282  * if according to the tracker information all interrupts that
283  * were allocated with ice_pci_alloc_irq_vectors are already used
284  * and dynamically allocated interrupts are supported then new
285  * interrupt will be allocated with pci_msix_alloc_irq_at.
286  *
287  * Some callers may only support dynamically allocated interrupts.
288  * This is indicated with dyn_only flag.
289  *
290  * On failure, return map with negative .index. The caller
291  * is expected to check returned map index.
292  *
293  */
294 struct msi_map ice_alloc_irq(struct ice_pf *pf, bool dyn_only)
295 {
296 	int sriov_base_vector = pf->sriov_base_vector;
297 	struct msi_map map = { .index = -ENOENT };
298 	struct device *dev = ice_pf_to_dev(pf);
299 	struct ice_irq_entry *entry;
300 
301 	entry = ice_get_irq_res(pf, dyn_only);
302 	if (!entry)
303 		return map;
304 
305 	/* fail if we're about to violate SRIOV vectors space */
306 	if (sriov_base_vector && entry->index >= sriov_base_vector)
307 		goto exit_free_res;
308 
309 	if (pci_msix_can_alloc_dyn(pf->pdev) && entry->dynamic) {
310 		map = pci_msix_alloc_irq_at(pf->pdev, entry->index, NULL);
311 		if (map.index < 0)
312 			goto exit_free_res;
313 		dev_dbg(dev, "allocated new irq at index %d\n", map.index);
314 	} else {
315 		map.index = entry->index;
316 		map.virq = pci_irq_vector(pf->pdev, map.index);
317 	}
318 
319 	return map;
320 
321 exit_free_res:
322 	dev_err(dev, "Could not allocate irq at idx %d\n", entry->index);
323 	ice_free_irq_res(pf, entry->index);
324 	return map;
325 }
326 
327 /**
328  * ice_free_irq - Free interrupt vector
329  * @pf: board private structure
330  * @map: map with interrupt details
331  *
332  * Remove allocated interrupt from the interrupt tracker. If interrupt was
333  * allocated dynamically, free respective interrupt vector.
334  */
335 void ice_free_irq(struct ice_pf *pf, struct msi_map map)
336 {
337 	struct ice_irq_entry *entry;
338 
339 	entry = xa_load(&pf->irq_tracker.entries, map.index);
340 
341 	if (!entry) {
342 		dev_err(ice_pf_to_dev(pf), "Failed to get MSIX interrupt entry at index %d",
343 			map.index);
344 		return;
345 	}
346 
347 	dev_dbg(ice_pf_to_dev(pf), "Free irq at index %d\n", map.index);
348 
349 	if (entry->dynamic)
350 		pci_msix_free_irq(pf->pdev, map);
351 
352 	ice_free_irq_res(pf, map.index);
353 }
354 
355 /**
356  * ice_get_max_used_msix_vector - Get the max used interrupt vector
357  * @pf: board private structure
358  *
359  * Return index of maximum used interrupt vectors with respect to the
360  * beginning of the MSIX table. Take into account that some interrupts
361  * may have been dynamically allocated after MSIX was initially enabled.
362  */
363 int ice_get_max_used_msix_vector(struct ice_pf *pf)
364 {
365 	unsigned long start, index, max_idx;
366 	void *entry;
367 
368 	/* Treat all preallocated interrupts as used */
369 	start = pf->irq_tracker.num_static;
370 	max_idx = start - 1;
371 
372 	xa_for_each_start(&pf->irq_tracker.entries, index, entry, start) {
373 		if (index > max_idx)
374 			max_idx = index;
375 	}
376 
377 	return max_idx;
378 }
379