1424544dfSShay Drory // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2424544dfSShay Drory /* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
3424544dfSShay Drory
4424544dfSShay Drory #include "mlx5_core.h"
5424544dfSShay Drory #include "mlx5_irq.h"
6424544dfSShay Drory #include "pci_irq.h"
7424544dfSShay Drory
cpu_put(struct mlx5_irq_pool * pool,int cpu)8061f5b23SShay Drory static void cpu_put(struct mlx5_irq_pool *pool, int cpu)
9061f5b23SShay Drory {
10061f5b23SShay Drory pool->irqs_per_cpu[cpu]--;
11061f5b23SShay Drory }
12061f5b23SShay Drory
cpu_get(struct mlx5_irq_pool * pool,int cpu)13061f5b23SShay Drory static void cpu_get(struct mlx5_irq_pool *pool, int cpu)
14061f5b23SShay Drory {
15061f5b23SShay Drory pool->irqs_per_cpu[cpu]++;
16061f5b23SShay Drory }
17061f5b23SShay Drory
18061f5b23SShay Drory /* Gets the least loaded CPU. e.g.: the CPU with least IRQs bound to it */
cpu_get_least_loaded(struct mlx5_irq_pool * pool,const struct cpumask * req_mask)19061f5b23SShay Drory static int cpu_get_least_loaded(struct mlx5_irq_pool *pool,
20061f5b23SShay Drory const struct cpumask *req_mask)
21061f5b23SShay Drory {
22061f5b23SShay Drory int best_cpu = -1;
23061f5b23SShay Drory int cpu;
24061f5b23SShay Drory
25061f5b23SShay Drory for_each_cpu_and(cpu, req_mask, cpu_online_mask) {
26061f5b23SShay Drory /* CPU has zero IRQs on it. No need to search any more CPUs. */
27061f5b23SShay Drory if (!pool->irqs_per_cpu[cpu]) {
28061f5b23SShay Drory best_cpu = cpu;
29061f5b23SShay Drory break;
30061f5b23SShay Drory }
31061f5b23SShay Drory if (best_cpu < 0)
32061f5b23SShay Drory best_cpu = cpu;
33061f5b23SShay Drory if (pool->irqs_per_cpu[cpu] < pool->irqs_per_cpu[best_cpu])
34061f5b23SShay Drory best_cpu = cpu;
35061f5b23SShay Drory }
36061f5b23SShay Drory if (best_cpu == -1) {
37061f5b23SShay Drory /* There isn't online CPUs in req_mask */
38061f5b23SShay Drory mlx5_core_err(pool->dev, "NO online CPUs in req_mask (%*pbl)\n",
39061f5b23SShay Drory cpumask_pr_args(req_mask));
40061f5b23SShay Drory best_cpu = cpumask_first(cpu_online_mask);
41061f5b23SShay Drory }
42061f5b23SShay Drory pool->irqs_per_cpu[best_cpu]++;
43061f5b23SShay Drory return best_cpu;
44061f5b23SShay Drory }
45061f5b23SShay Drory
46424544dfSShay Drory /* Creating an IRQ from irq_pool */
47424544dfSShay Drory static struct mlx5_irq *
irq_pool_request_irq(struct mlx5_irq_pool * pool,struct irq_affinity_desc * af_desc)48bbac70c7SEli Cohen irq_pool_request_irq(struct mlx5_irq_pool *pool, struct irq_affinity_desc *af_desc)
49424544dfSShay Drory {
50bbac70c7SEli Cohen struct irq_affinity_desc auto_desc = {};
51424544dfSShay Drory u32 irq_index;
52424544dfSShay Drory int err;
53424544dfSShay Drory
54061f5b23SShay Drory err = xa_alloc(&pool->irqs, &irq_index, NULL, pool->xa_num_irqs, GFP_KERNEL);
55424544dfSShay Drory if (err)
56424544dfSShay Drory return ERR_PTR(err);
57061f5b23SShay Drory if (pool->irqs_per_cpu) {
58bbac70c7SEli Cohen if (cpumask_weight(&af_desc->mask) > 1)
59061f5b23SShay Drory /* if req_mask contain more then one CPU, set the least loadad CPU
60061f5b23SShay Drory * of req_mask
61061f5b23SShay Drory */
62bbac70c7SEli Cohen cpumask_set_cpu(cpu_get_least_loaded(pool, &af_desc->mask),
63bbac70c7SEli Cohen &auto_desc.mask);
64061f5b23SShay Drory else
65bbac70c7SEli Cohen cpu_get(pool, cpumask_first(&af_desc->mask));
66061f5b23SShay Drory }
67bbac70c7SEli Cohen return mlx5_irq_alloc(pool, irq_index,
683354822cSEli Cohen cpumask_empty(&auto_desc.mask) ? af_desc : &auto_desc,
693354822cSEli Cohen NULL);
70424544dfSShay Drory }
71424544dfSShay Drory
72061f5b23SShay Drory /* Looking for the IRQ with the smallest refcount that fits req_mask.
73061f5b23SShay Drory * If pool is sf_comp_pool, then we are looking for an IRQ with any of the
74061f5b23SShay Drory * requested CPUs in req_mask.
75061f5b23SShay Drory * for example: req_mask = 0xf, irq0_mask = 0x10, irq1_mask = 0x1. irq0_mask
76061f5b23SShay Drory * isn't subset of req_mask, so we will skip it. irq1_mask is subset of req_mask,
77061f5b23SShay Drory * we don't skip it.
78061f5b23SShay Drory * If pool is sf_ctrl_pool, then all IRQs have the same mask, so any IRQ will
79061f5b23SShay Drory * fit. And since mask is subset of itself, we will pass the first if bellow.
80061f5b23SShay Drory */
81424544dfSShay Drory static struct mlx5_irq *
irq_pool_find_least_loaded(struct mlx5_irq_pool * pool,const struct cpumask * req_mask)82424544dfSShay Drory irq_pool_find_least_loaded(struct mlx5_irq_pool *pool, const struct cpumask *req_mask)
83424544dfSShay Drory {
84424544dfSShay Drory int start = pool->xa_num_irqs.min;
85424544dfSShay Drory int end = pool->xa_num_irqs.max;
86424544dfSShay Drory struct mlx5_irq *irq = NULL;
87424544dfSShay Drory struct mlx5_irq *iter;
88424544dfSShay Drory int irq_refcount = 0;
89424544dfSShay Drory unsigned long index;
90424544dfSShay Drory
91424544dfSShay Drory lockdep_assert_held(&pool->lock);
92424544dfSShay Drory xa_for_each_range(&pool->irqs, index, iter, start, end) {
93424544dfSShay Drory struct cpumask *iter_mask = mlx5_irq_get_affinity_mask(iter);
94424544dfSShay Drory int iter_refcount = mlx5_irq_read_locked(iter);
95424544dfSShay Drory
96061f5b23SShay Drory if (!cpumask_subset(iter_mask, req_mask))
97061f5b23SShay Drory /* skip IRQs with a mask which is not subset of req_mask */
98424544dfSShay Drory continue;
99424544dfSShay Drory if (iter_refcount < pool->min_threshold)
100424544dfSShay Drory /* If we found an IRQ with less than min_thres, return it */
101424544dfSShay Drory return iter;
102424544dfSShay Drory if (!irq || iter_refcount < irq_refcount) {
103424544dfSShay Drory /* In case we won't find an IRQ with less than min_thres,
104424544dfSShay Drory * keep a pointer to the least used IRQ
105424544dfSShay Drory */
106424544dfSShay Drory irq_refcount = iter_refcount;
107424544dfSShay Drory irq = iter;
108424544dfSShay Drory }
109424544dfSShay Drory }
110424544dfSShay Drory return irq;
111424544dfSShay Drory }
112424544dfSShay Drory
113424544dfSShay Drory /**
114424544dfSShay Drory * mlx5_irq_affinity_request - request an IRQ according to the given mask.
115424544dfSShay Drory * @pool: IRQ pool to request from.
116bbac70c7SEli Cohen * @af_desc: affinity descriptor for this IRQ.
117424544dfSShay Drory *
118424544dfSShay Drory * This function returns a pointer to IRQ, or ERR_PTR in case of error.
119424544dfSShay Drory */
120424544dfSShay Drory struct mlx5_irq *
mlx5_irq_affinity_request(struct mlx5_irq_pool * pool,struct irq_affinity_desc * af_desc)121bbac70c7SEli Cohen mlx5_irq_affinity_request(struct mlx5_irq_pool *pool, struct irq_affinity_desc *af_desc)
122424544dfSShay Drory {
123424544dfSShay Drory struct mlx5_irq *least_loaded_irq, *new_irq;
124424544dfSShay Drory
125424544dfSShay Drory mutex_lock(&pool->lock);
126bbac70c7SEli Cohen least_loaded_irq = irq_pool_find_least_loaded(pool, &af_desc->mask);
127424544dfSShay Drory if (least_loaded_irq &&
128424544dfSShay Drory mlx5_irq_read_locked(least_loaded_irq) < pool->min_threshold)
129424544dfSShay Drory goto out;
130424544dfSShay Drory /* We didn't find an IRQ with less than min_thres, try to allocate a new IRQ */
131bbac70c7SEli Cohen new_irq = irq_pool_request_irq(pool, af_desc);
132424544dfSShay Drory if (IS_ERR(new_irq)) {
133424544dfSShay Drory if (!least_loaded_irq) {
134424544dfSShay Drory /* We failed to create an IRQ and we didn't find an IRQ */
135424544dfSShay Drory mlx5_core_err(pool->dev, "Didn't find a matching IRQ. err = %ld\n",
136424544dfSShay Drory PTR_ERR(new_irq));
137424544dfSShay Drory mutex_unlock(&pool->lock);
138424544dfSShay Drory return new_irq;
139424544dfSShay Drory }
140424544dfSShay Drory /* We failed to create a new IRQ for the requested affinity,
141424544dfSShay Drory * sharing existing IRQ.
142424544dfSShay Drory */
143424544dfSShay Drory goto out;
144424544dfSShay Drory }
145424544dfSShay Drory least_loaded_irq = new_irq;
146424544dfSShay Drory goto unlock;
147424544dfSShay Drory out:
148424544dfSShay Drory mlx5_irq_get_locked(least_loaded_irq);
149424544dfSShay Drory if (mlx5_irq_read_locked(least_loaded_irq) > pool->max_threshold)
150424544dfSShay Drory mlx5_core_dbg(pool->dev, "IRQ %u overloaded, pool_name: %s, %u EQs on this irq\n",
151424544dfSShay Drory pci_irq_vector(pool->dev->pdev,
152424544dfSShay Drory mlx5_irq_get_index(least_loaded_irq)), pool->name,
153424544dfSShay Drory mlx5_irq_read_locked(least_loaded_irq) / MLX5_EQ_REFS_PER_IRQ);
154424544dfSShay Drory unlock:
155424544dfSShay Drory mutex_unlock(&pool->lock);
156424544dfSShay Drory return least_loaded_irq;
157424544dfSShay Drory }
158061f5b23SShay Drory
mlx5_irq_affinity_irq_release(struct mlx5_core_dev * dev,struct mlx5_irq * irq)159*a1772de7SMaher Sanalla void mlx5_irq_affinity_irq_release(struct mlx5_core_dev *dev, struct mlx5_irq *irq)
160061f5b23SShay Drory {
161061f5b23SShay Drory struct mlx5_irq_pool *pool = mlx5_irq_pool_get(dev);
162*a1772de7SMaher Sanalla int cpu;
163061f5b23SShay Drory
164*a1772de7SMaher Sanalla cpu = cpumask_first(mlx5_irq_get_affinity_mask(irq));
165061f5b23SShay Drory synchronize_irq(pci_irq_vector(pool->dev->pdev,
166*a1772de7SMaher Sanalla mlx5_irq_get_index(irq)));
167*a1772de7SMaher Sanalla if (mlx5_irq_put(irq))
168061f5b23SShay Drory if (pool->irqs_per_cpu)
169061f5b23SShay Drory cpu_put(pool, cpu);
170061f5b23SShay Drory }
171