1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2023, Intel Corporation. */
3
4 #include "ice.h"
5 #include "ice_lib.h"
6 #include "ice_irq.h"
7
8 /**
9 * ice_init_irq_tracker - initialize interrupt tracker
10 * @pf: board private structure
11 * @max_vectors: maximum number of vectors that tracker can hold
12 * @num_static: number of preallocated interrupts
13 */
14 static void
ice_init_irq_tracker(struct ice_pf * pf,unsigned int max_vectors,unsigned int num_static)15 ice_init_irq_tracker(struct ice_pf *pf, unsigned int max_vectors,
16 unsigned int num_static)
17 {
18 pf->irq_tracker.num_entries = max_vectors;
19 pf->irq_tracker.num_static = num_static;
20 xa_init_flags(&pf->irq_tracker.entries, XA_FLAGS_ALLOC);
21 }
22
23 /**
24 * ice_deinit_irq_tracker - free xarray tracker
25 * @pf: board private structure
26 */
ice_deinit_irq_tracker(struct ice_pf * pf)27 static void ice_deinit_irq_tracker(struct ice_pf *pf)
28 {
29 xa_destroy(&pf->irq_tracker.entries);
30 }
31
32 /**
33 * ice_free_irq_res - free a block of resources
34 * @pf: board private structure
35 * @index: starting index previously returned by ice_get_res
36 */
ice_free_irq_res(struct ice_pf * pf,u16 index)37 static void ice_free_irq_res(struct ice_pf *pf, u16 index)
38 {
39 struct ice_irq_entry *entry;
40
41 entry = xa_erase(&pf->irq_tracker.entries, index);
42 kfree(entry);
43 }
44
45 /**
46 * ice_get_irq_res - get an interrupt resource
47 * @pf: board private structure
48 * @dyn_only: force entry to be dynamically allocated
49 *
50 * Allocate new irq entry in the free slot of the tracker. Since xarray
51 * is used, always allocate new entry at the lowest possible index. Set
52 * proper allocation limit for maximum tracker entries.
53 *
54 * Returns allocated irq entry or NULL on failure.
55 */
ice_get_irq_res(struct ice_pf * pf,bool dyn_only)56 static struct ice_irq_entry *ice_get_irq_res(struct ice_pf *pf, bool dyn_only)
57 {
58 struct xa_limit limit = { .max = pf->irq_tracker.num_entries,
59 .min = 0 };
60 unsigned int num_static = pf->irq_tracker.num_static;
61 struct ice_irq_entry *entry;
62 unsigned int index;
63 int ret;
64
65 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
66 if (!entry)
67 return NULL;
68
69 /* skip preallocated entries if the caller says so */
70 if (dyn_only)
71 limit.min = num_static;
72
73 ret = xa_alloc(&pf->irq_tracker.entries, &index, entry, limit,
74 GFP_KERNEL);
75
76 if (ret) {
77 kfree(entry);
78 entry = NULL;
79 } else {
80 entry->index = index;
81 entry->dynamic = index >= num_static;
82 }
83
84 return entry;
85 }
86
87 /**
88 * ice_reduce_msix_usage - Reduce usage of MSI-X vectors
89 * @pf: board private structure
90 * @v_remain: number of remaining MSI-X vectors to be distributed
91 *
92 * Reduce the usage of MSI-X vectors when entire request cannot be fulfilled.
93 * pf->num_lan_msix and pf->num_rdma_msix values are set based on number of
94 * remaining vectors.
95 */
ice_reduce_msix_usage(struct ice_pf * pf,int v_remain)96 static void ice_reduce_msix_usage(struct ice_pf *pf, int v_remain)
97 {
98 int v_rdma;
99
100 if (!ice_is_rdma_ena(pf)) {
101 pf->num_lan_msix = v_remain;
102 return;
103 }
104
105 /* RDMA needs at least 1 interrupt in addition to AEQ MSIX */
106 v_rdma = ICE_RDMA_NUM_AEQ_MSIX + 1;
107
108 if (v_remain < ICE_MIN_LAN_TXRX_MSIX + ICE_MIN_RDMA_MSIX) {
109 dev_warn(ice_pf_to_dev(pf), "Not enough MSI-X vectors to support RDMA.\n");
110 clear_bit(ICE_FLAG_RDMA_ENA, pf->flags);
111
112 pf->num_rdma_msix = 0;
113 pf->num_lan_msix = ICE_MIN_LAN_TXRX_MSIX;
114 } else if ((v_remain < ICE_MIN_LAN_TXRX_MSIX + v_rdma) ||
115 (v_remain - v_rdma < v_rdma)) {
116 /* Support minimum RDMA and give remaining vectors to LAN MSIX
117 */
118 pf->num_rdma_msix = ICE_MIN_RDMA_MSIX;
119 pf->num_lan_msix = v_remain - ICE_MIN_RDMA_MSIX;
120 } else {
121 /* Split remaining MSIX with RDMA after accounting for AEQ MSIX
122 */
123 pf->num_rdma_msix = (v_remain - ICE_RDMA_NUM_AEQ_MSIX) / 2 +
124 ICE_RDMA_NUM_AEQ_MSIX;
125 pf->num_lan_msix = v_remain - pf->num_rdma_msix;
126 }
127 }
128
129 /**
130 * ice_ena_msix_range - Request a range of MSIX vectors from the OS
131 * @pf: board private structure
132 *
133 * Compute the number of MSIX vectors wanted and request from the OS. Adjust
134 * device usage if there are not enough vectors. Return the number of vectors
135 * reserved or negative on failure.
136 */
ice_ena_msix_range(struct ice_pf * pf)137 static int ice_ena_msix_range(struct ice_pf *pf)
138 {
139 int num_cpus, hw_num_msix, v_other, v_wanted, v_actual;
140 struct device *dev = ice_pf_to_dev(pf);
141 int err;
142
143 hw_num_msix = pf->hw.func_caps.common_cap.num_msix_vectors;
144 num_cpus = num_online_cpus();
145
146 /* LAN miscellaneous handler */
147 v_other = ICE_MIN_LAN_OICR_MSIX;
148
149 /* Flow Director */
150 if (test_bit(ICE_FLAG_FD_ENA, pf->flags))
151 v_other += ICE_FDIR_MSIX;
152
153 /* switchdev */
154 v_other += ICE_ESWITCH_MSIX;
155
156 v_wanted = v_other;
157
158 /* LAN traffic */
159 pf->num_lan_msix = num_cpus;
160 v_wanted += pf->num_lan_msix;
161
162 /* RDMA auxiliary driver */
163 if (ice_is_rdma_ena(pf)) {
164 pf->num_rdma_msix = num_cpus + ICE_RDMA_NUM_AEQ_MSIX;
165 v_wanted += pf->num_rdma_msix;
166 }
167
168 if (v_wanted > hw_num_msix) {
169 int v_remain;
170
171 dev_warn(dev, "not enough device MSI-X vectors. wanted = %d, available = %d\n",
172 v_wanted, hw_num_msix);
173
174 if (hw_num_msix < ICE_MIN_MSIX) {
175 err = -ERANGE;
176 goto exit_err;
177 }
178
179 v_remain = hw_num_msix - v_other;
180 if (v_remain < ICE_MIN_LAN_TXRX_MSIX) {
181 v_other = ICE_MIN_MSIX - ICE_MIN_LAN_TXRX_MSIX;
182 v_remain = ICE_MIN_LAN_TXRX_MSIX;
183 }
184
185 ice_reduce_msix_usage(pf, v_remain);
186 v_wanted = pf->num_lan_msix + pf->num_rdma_msix + v_other;
187
188 dev_notice(dev, "Reducing request to %d MSI-X vectors for LAN traffic.\n",
189 pf->num_lan_msix);
190 if (ice_is_rdma_ena(pf))
191 dev_notice(dev, "Reducing request to %d MSI-X vectors for RDMA.\n",
192 pf->num_rdma_msix);
193 }
194
195 /* actually reserve the vectors */
196 v_actual = pci_alloc_irq_vectors(pf->pdev, ICE_MIN_MSIX, v_wanted,
197 PCI_IRQ_MSIX);
198 if (v_actual < 0) {
199 dev_err(dev, "unable to reserve MSI-X vectors\n");
200 err = v_actual;
201 goto exit_err;
202 }
203
204 if (v_actual < v_wanted) {
205 dev_warn(dev, "not enough OS MSI-X vectors. requested = %d, obtained = %d\n",
206 v_wanted, v_actual);
207
208 if (v_actual < ICE_MIN_MSIX) {
209 /* error if we can't get minimum vectors */
210 pci_free_irq_vectors(pf->pdev);
211 err = -ERANGE;
212 goto exit_err;
213 } else {
214 int v_remain = v_actual - v_other;
215
216 if (v_remain < ICE_MIN_LAN_TXRX_MSIX)
217 v_remain = ICE_MIN_LAN_TXRX_MSIX;
218
219 ice_reduce_msix_usage(pf, v_remain);
220
221 dev_notice(dev, "Enabled %d MSI-X vectors for LAN traffic.\n",
222 pf->num_lan_msix);
223
224 if (ice_is_rdma_ena(pf))
225 dev_notice(dev, "Enabled %d MSI-X vectors for RDMA.\n",
226 pf->num_rdma_msix);
227 }
228 }
229
230 return v_actual;
231
232 exit_err:
233 pf->num_rdma_msix = 0;
234 pf->num_lan_msix = 0;
235 return err;
236 }
237
238 /**
239 * ice_clear_interrupt_scheme - Undo things done by ice_init_interrupt_scheme
240 * @pf: board private structure
241 */
ice_clear_interrupt_scheme(struct ice_pf * pf)242 void ice_clear_interrupt_scheme(struct ice_pf *pf)
243 {
244 pci_free_irq_vectors(pf->pdev);
245 ice_deinit_irq_tracker(pf);
246 }
247
248 /**
249 * ice_init_interrupt_scheme - Determine proper interrupt scheme
250 * @pf: board private structure to initialize
251 */
ice_init_interrupt_scheme(struct ice_pf * pf)252 int ice_init_interrupt_scheme(struct ice_pf *pf)
253 {
254 int total_vectors = pf->hw.func_caps.common_cap.num_msix_vectors;
255 int vectors, max_vectors;
256
257 vectors = ice_ena_msix_range(pf);
258
259 if (vectors < 0)
260 return -ENOMEM;
261
262 if (pci_msix_can_alloc_dyn(pf->pdev))
263 max_vectors = total_vectors;
264 else
265 max_vectors = vectors;
266
267 ice_init_irq_tracker(pf, max_vectors, vectors);
268
269 return 0;
270 }
271
272 /**
273 * ice_alloc_irq - Allocate new interrupt vector
274 * @pf: board private structure
275 * @dyn_only: force dynamic allocation of the interrupt
276 *
277 * Allocate new interrupt vector for a given owner id.
278 * return struct msi_map with interrupt details and track
279 * allocated interrupt appropriately.
280 *
281 * This function reserves new irq entry from the irq_tracker.
282 * if according to the tracker information all interrupts that
283 * were allocated with ice_pci_alloc_irq_vectors are already used
284 * and dynamically allocated interrupts are supported then new
285 * interrupt will be allocated with pci_msix_alloc_irq_at.
286 *
287 * Some callers may only support dynamically allocated interrupts.
288 * This is indicated with dyn_only flag.
289 *
290 * On failure, return map with negative .index. The caller
291 * is expected to check returned map index.
292 *
293 */
ice_alloc_irq(struct ice_pf * pf,bool dyn_only)294 struct msi_map ice_alloc_irq(struct ice_pf *pf, bool dyn_only)
295 {
296 int sriov_base_vector = pf->sriov_base_vector;
297 struct msi_map map = { .index = -ENOENT };
298 struct device *dev = ice_pf_to_dev(pf);
299 struct ice_irq_entry *entry;
300
301 entry = ice_get_irq_res(pf, dyn_only);
302 if (!entry)
303 return map;
304
305 /* fail if we're about to violate SRIOV vectors space */
306 if (sriov_base_vector && entry->index >= sriov_base_vector)
307 goto exit_free_res;
308
309 if (pci_msix_can_alloc_dyn(pf->pdev) && entry->dynamic) {
310 map = pci_msix_alloc_irq_at(pf->pdev, entry->index, NULL);
311 if (map.index < 0)
312 goto exit_free_res;
313 dev_dbg(dev, "allocated new irq at index %d\n", map.index);
314 } else {
315 map.index = entry->index;
316 map.virq = pci_irq_vector(pf->pdev, map.index);
317 }
318
319 return map;
320
321 exit_free_res:
322 dev_err(dev, "Could not allocate irq at idx %d\n", entry->index);
323 ice_free_irq_res(pf, entry->index);
324 return map;
325 }
326
327 /**
328 * ice_free_irq - Free interrupt vector
329 * @pf: board private structure
330 * @map: map with interrupt details
331 *
332 * Remove allocated interrupt from the interrupt tracker. If interrupt was
333 * allocated dynamically, free respective interrupt vector.
334 */
ice_free_irq(struct ice_pf * pf,struct msi_map map)335 void ice_free_irq(struct ice_pf *pf, struct msi_map map)
336 {
337 struct ice_irq_entry *entry;
338
339 entry = xa_load(&pf->irq_tracker.entries, map.index);
340
341 if (!entry) {
342 dev_err(ice_pf_to_dev(pf), "Failed to get MSIX interrupt entry at index %d",
343 map.index);
344 return;
345 }
346
347 dev_dbg(ice_pf_to_dev(pf), "Free irq at index %d\n", map.index);
348
349 if (entry->dynamic)
350 pci_msix_free_irq(pf->pdev, map);
351
352 ice_free_irq_res(pf, map.index);
353 }
354
355 /**
356 * ice_get_max_used_msix_vector - Get the max used interrupt vector
357 * @pf: board private structure
358 *
359 * Return index of maximum used interrupt vectors with respect to the
360 * beginning of the MSIX table. Take into account that some interrupts
361 * may have been dynamically allocated after MSIX was initially enabled.
362 */
ice_get_max_used_msix_vector(struct ice_pf * pf)363 int ice_get_max_used_msix_vector(struct ice_pf *pf)
364 {
365 unsigned long start, index, max_idx;
366 void *entry;
367
368 /* Treat all preallocated interrupts as used */
369 start = pf->irq_tracker.num_static;
370 max_idx = start - 1;
371
372 xa_for_each_start(&pf->irq_tracker.entries, index, entry, start) {
373 if (index > max_idx)
374 max_idx = index;
375 }
376
377 return max_idx;
378 }
379