1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2 /* Copyright(c) 2014 - 2020 Intel Corporation */
3 #include <linux/kernel.h>
4 #include <linux/init.h>
5 #include <linux/types.h>
6 #include <linux/pci.h>
7 #include <linux/slab.h>
8 #include <linux/errno.h>
9 #include <linux/interrupt.h>
10 #include "adf_accel_devices.h"
11 #include "adf_common_drv.h"
12 #include "adf_cfg.h"
13 #include "adf_cfg_strings.h"
14 #include "adf_cfg_common.h"
15 #include "adf_transport_access_macros.h"
16 #include "adf_transport_internal.h"
17 
18 #define ADF_MAX_NUM_VFS	32
19 static struct workqueue_struct *adf_misc_wq;
20 
adf_enable_msix(struct adf_accel_dev * accel_dev)21 static int adf_enable_msix(struct adf_accel_dev *accel_dev)
22 {
23 	struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
24 	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
25 	u32 msix_num_entries = hw_data->num_banks + 1;
26 	int ret;
27 
28 	if (hw_data->set_msix_rttable)
29 		hw_data->set_msix_rttable(accel_dev);
30 
31 	ret = pci_alloc_irq_vectors(pci_dev_info->pci_dev, msix_num_entries,
32 				    msix_num_entries, PCI_IRQ_MSIX);
33 	if (unlikely(ret < 0)) {
34 		dev_err(&GET_DEV(accel_dev),
35 			"Failed to allocate %d MSI-X vectors\n",
36 			msix_num_entries);
37 		return ret;
38 	}
39 	return 0;
40 }
41 
adf_disable_msix(struct adf_accel_pci * pci_dev_info)42 static void adf_disable_msix(struct adf_accel_pci *pci_dev_info)
43 {
44 	pci_free_irq_vectors(pci_dev_info->pci_dev);
45 }
46 
adf_msix_isr_bundle(int irq,void * bank_ptr)47 static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr)
48 {
49 	struct adf_etr_bank_data *bank = bank_ptr;
50 	struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);
51 
52 	csr_ops->write_csr_int_flag_and_col(bank->csr_addr, bank->bank_number,
53 					    0);
54 	tasklet_hi_schedule(&bank->resp_handler);
55 	return IRQ_HANDLED;
56 }
57 
58 #ifdef CONFIG_PCI_IOV
adf_enable_vf2pf_interrupts(struct adf_accel_dev * accel_dev,u32 vf_mask)59 void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask)
60 {
61 	void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
62 	unsigned long flags;
63 
64 	spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags);
65 	GET_PFVF_OPS(accel_dev)->enable_vf2pf_interrupts(pmisc_addr, vf_mask);
66 	spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags);
67 }
68 
adf_disable_all_vf2pf_interrupts(struct adf_accel_dev * accel_dev)69 void adf_disable_all_vf2pf_interrupts(struct adf_accel_dev *accel_dev)
70 {
71 	void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
72 	unsigned long flags;
73 
74 	spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags);
75 	GET_PFVF_OPS(accel_dev)->disable_all_vf2pf_interrupts(pmisc_addr);
76 	spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags);
77 }
78 
adf_disable_pending_vf2pf_interrupts(struct adf_accel_dev * accel_dev)79 static u32 adf_disable_pending_vf2pf_interrupts(struct adf_accel_dev *accel_dev)
80 {
81 	void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
82 	u32 pending;
83 
84 	spin_lock(&accel_dev->pf.vf2pf_ints_lock);
85 	pending = GET_PFVF_OPS(accel_dev)->disable_pending_vf2pf_interrupts(pmisc_addr);
86 	spin_unlock(&accel_dev->pf.vf2pf_ints_lock);
87 
88 	return pending;
89 }
90 
adf_handle_vf2pf_int(struct adf_accel_dev * accel_dev)91 static bool adf_handle_vf2pf_int(struct adf_accel_dev *accel_dev)
92 {
93 	bool irq_handled = false;
94 	unsigned long vf_mask;
95 
96 	/* Get the interrupt sources triggered by VFs, except for those already disabled */
97 	vf_mask = adf_disable_pending_vf2pf_interrupts(accel_dev);
98 	if (vf_mask) {
99 		struct adf_accel_vf_info *vf_info;
100 		int i;
101 
102 		/*
103 		 * Handle VF2PF interrupt unless the VF is malicious and
104 		 * is attempting to flood the host OS with VF2PF interrupts.
105 		 */
106 		for_each_set_bit(i, &vf_mask, ADF_MAX_NUM_VFS) {
107 			vf_info = accel_dev->pf.vf_info + i;
108 
109 			if (!__ratelimit(&vf_info->vf2pf_ratelimit)) {
110 				dev_info(&GET_DEV(accel_dev),
111 					 "Too many ints from VF%d\n",
112 					  vf_info->vf_nr);
113 				continue;
114 			}
115 
116 			adf_schedule_vf2pf_handler(vf_info);
117 			irq_handled = true;
118 		}
119 	}
120 	return irq_handled;
121 }
122 #endif /* CONFIG_PCI_IOV */
123 
adf_handle_pm_int(struct adf_accel_dev * accel_dev)124 static bool adf_handle_pm_int(struct adf_accel_dev *accel_dev)
125 {
126 	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
127 
128 	if (hw_data->handle_pm_interrupt &&
129 	    hw_data->handle_pm_interrupt(accel_dev))
130 		return true;
131 
132 	return false;
133 }
134 
adf_msix_isr_ae(int irq,void * dev_ptr)135 static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr)
136 {
137 	struct adf_accel_dev *accel_dev = dev_ptr;
138 
139 #ifdef CONFIG_PCI_IOV
140 	/* If SR-IOV is enabled (vf_info is non-NULL), check for VF->PF ints */
141 	if (accel_dev->pf.vf_info && adf_handle_vf2pf_int(accel_dev))
142 		return IRQ_HANDLED;
143 #endif /* CONFIG_PCI_IOV */
144 
145 	if (adf_handle_pm_int(accel_dev))
146 		return IRQ_HANDLED;
147 
148 	dev_dbg(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n",
149 		accel_dev->accel_id);
150 
151 	return IRQ_NONE;
152 }
153 
adf_free_irqs(struct adf_accel_dev * accel_dev)154 static void adf_free_irqs(struct adf_accel_dev *accel_dev)
155 {
156 	struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
157 	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
158 	struct adf_irq *irqs = pci_dev_info->msix_entries.irqs;
159 	struct adf_etr_data *etr_data = accel_dev->transport;
160 	int clust_irq = hw_data->num_banks;
161 	int irq, i = 0;
162 
163 	if (pci_dev_info->msix_entries.num_entries > 1) {
164 		for (i = 0; i < hw_data->num_banks; i++) {
165 			if (irqs[i].enabled) {
166 				irq = pci_irq_vector(pci_dev_info->pci_dev, i);
167 				irq_set_affinity_hint(irq, NULL);
168 				free_irq(irq, &etr_data->banks[i]);
169 			}
170 		}
171 	}
172 
173 	if (irqs[i].enabled) {
174 		irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq);
175 		free_irq(irq, accel_dev);
176 	}
177 }
178 
adf_request_irqs(struct adf_accel_dev * accel_dev)179 static int adf_request_irqs(struct adf_accel_dev *accel_dev)
180 {
181 	struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
182 	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
183 	struct adf_irq *irqs = pci_dev_info->msix_entries.irqs;
184 	struct adf_etr_data *etr_data = accel_dev->transport;
185 	int clust_irq = hw_data->num_banks;
186 	int ret, irq, i = 0;
187 	char *name;
188 
189 	/* Request msix irq for all banks unless SR-IOV enabled */
190 	if (!accel_dev->pf.vf_info) {
191 		for (i = 0; i < hw_data->num_banks; i++) {
192 			struct adf_etr_bank_data *bank = &etr_data->banks[i];
193 			unsigned int cpu, cpus = num_online_cpus();
194 
195 			name = irqs[i].name;
196 			snprintf(name, ADF_MAX_MSIX_VECTOR_NAME,
197 				 "qat%d-bundle%d", accel_dev->accel_id, i);
198 			irq = pci_irq_vector(pci_dev_info->pci_dev, i);
199 			if (unlikely(irq < 0)) {
200 				dev_err(&GET_DEV(accel_dev),
201 					"Failed to get IRQ number of device vector %d - %s\n",
202 					i, name);
203 				ret = irq;
204 				goto err;
205 			}
206 			ret = request_irq(irq, adf_msix_isr_bundle, 0,
207 					  &name[0], bank);
208 			if (ret) {
209 				dev_err(&GET_DEV(accel_dev),
210 					"Failed to allocate IRQ %d for %s\n",
211 					irq, name);
212 				goto err;
213 			}
214 
215 			cpu = ((accel_dev->accel_id * hw_data->num_banks) +
216 			       i) % cpus;
217 			irq_set_affinity_hint(irq, get_cpu_mask(cpu));
218 			irqs[i].enabled = true;
219 		}
220 	}
221 
222 	/* Request msix irq for AE */
223 	name = irqs[i].name;
224 	snprintf(name, ADF_MAX_MSIX_VECTOR_NAME,
225 		 "qat%d-ae-cluster", accel_dev->accel_id);
226 	irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq);
227 	if (unlikely(irq < 0)) {
228 		dev_err(&GET_DEV(accel_dev),
229 			"Failed to get IRQ number of device vector %d - %s\n",
230 			i, name);
231 		ret = irq;
232 		goto err;
233 	}
234 	ret = request_irq(irq, adf_msix_isr_ae, 0, &name[0], accel_dev);
235 	if (ret) {
236 		dev_err(&GET_DEV(accel_dev),
237 			"Failed to allocate IRQ %d for %s\n", irq, name);
238 		goto err;
239 	}
240 	irqs[i].enabled = true;
241 	return ret;
242 err:
243 	adf_free_irqs(accel_dev);
244 	return ret;
245 }
246 
adf_isr_alloc_msix_vectors_data(struct adf_accel_dev * accel_dev)247 static int adf_isr_alloc_msix_vectors_data(struct adf_accel_dev *accel_dev)
248 {
249 	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
250 	u32 msix_num_entries = 1;
251 	struct adf_irq *irqs;
252 
253 	/* If SR-IOV is disabled (vf_info is NULL), add entries for each bank */
254 	if (!accel_dev->pf.vf_info)
255 		msix_num_entries += hw_data->num_banks;
256 
257 	irqs = kzalloc_node(msix_num_entries * sizeof(*irqs),
258 			    GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev)));
259 	if (!irqs)
260 		return -ENOMEM;
261 
262 	accel_dev->accel_pci_dev.msix_entries.num_entries = msix_num_entries;
263 	accel_dev->accel_pci_dev.msix_entries.irqs = irqs;
264 	return 0;
265 }
266 
adf_isr_free_msix_vectors_data(struct adf_accel_dev * accel_dev)267 static void adf_isr_free_msix_vectors_data(struct adf_accel_dev *accel_dev)
268 {
269 	kfree(accel_dev->accel_pci_dev.msix_entries.irqs);
270 	accel_dev->accel_pci_dev.msix_entries.irqs = NULL;
271 }
272 
adf_setup_bh(struct adf_accel_dev * accel_dev)273 static int adf_setup_bh(struct adf_accel_dev *accel_dev)
274 {
275 	struct adf_etr_data *priv_data = accel_dev->transport;
276 	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
277 	int i;
278 
279 	for (i = 0; i < hw_data->num_banks; i++)
280 		tasklet_init(&priv_data->banks[i].resp_handler,
281 			     adf_response_handler,
282 			     (unsigned long)&priv_data->banks[i]);
283 	return 0;
284 }
285 
adf_cleanup_bh(struct adf_accel_dev * accel_dev)286 static void adf_cleanup_bh(struct adf_accel_dev *accel_dev)
287 {
288 	struct adf_etr_data *priv_data = accel_dev->transport;
289 	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
290 	int i;
291 
292 	for (i = 0; i < hw_data->num_banks; i++) {
293 		tasklet_disable(&priv_data->banks[i].resp_handler);
294 		tasklet_kill(&priv_data->banks[i].resp_handler);
295 	}
296 }
297 
298 /**
299  * adf_isr_resource_free() - Free IRQ for acceleration device
300  * @accel_dev:  Pointer to acceleration device.
301  *
302  * Function frees interrupts for acceleration device.
303  */
adf_isr_resource_free(struct adf_accel_dev * accel_dev)304 void adf_isr_resource_free(struct adf_accel_dev *accel_dev)
305 {
306 	adf_free_irqs(accel_dev);
307 	adf_cleanup_bh(accel_dev);
308 	adf_disable_msix(&accel_dev->accel_pci_dev);
309 	adf_isr_free_msix_vectors_data(accel_dev);
310 }
311 EXPORT_SYMBOL_GPL(adf_isr_resource_free);
312 
313 /**
314  * adf_isr_resource_alloc() - Allocate IRQ for acceleration device
315  * @accel_dev:  Pointer to acceleration device.
316  *
317  * Function allocates interrupts for acceleration device.
318  *
319  * Return: 0 on success, error code otherwise.
320  */
adf_isr_resource_alloc(struct adf_accel_dev * accel_dev)321 int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev)
322 {
323 	int ret;
324 
325 	ret = adf_isr_alloc_msix_vectors_data(accel_dev);
326 	if (ret)
327 		goto err_out;
328 
329 	ret = adf_enable_msix(accel_dev);
330 	if (ret)
331 		goto err_free_msix_table;
332 
333 	ret = adf_setup_bh(accel_dev);
334 	if (ret)
335 		goto err_disable_msix;
336 
337 	ret = adf_request_irqs(accel_dev);
338 	if (ret)
339 		goto err_cleanup_bh;
340 
341 	return 0;
342 
343 err_cleanup_bh:
344 	adf_cleanup_bh(accel_dev);
345 
346 err_disable_msix:
347 	adf_disable_msix(&accel_dev->accel_pci_dev);
348 
349 err_free_msix_table:
350 	adf_isr_free_msix_vectors_data(accel_dev);
351 
352 err_out:
353 	return ret;
354 }
355 EXPORT_SYMBOL_GPL(adf_isr_resource_alloc);
356 
357 /**
358  * adf_init_misc_wq() - Init misc workqueue
359  *
360  * Function init workqueue 'qat_misc_wq' for general purpose.
361  *
362  * Return: 0 on success, error code otherwise.
363  */
adf_init_misc_wq(void)364 int __init adf_init_misc_wq(void)
365 {
366 	adf_misc_wq = alloc_workqueue("qat_misc_wq", WQ_MEM_RECLAIM, 0);
367 
368 	return !adf_misc_wq ? -ENOMEM : 0;
369 }
370 
adf_exit_misc_wq(void)371 void adf_exit_misc_wq(void)
372 {
373 	if (adf_misc_wq)
374 		destroy_workqueue(adf_misc_wq);
375 
376 	adf_misc_wq = NULL;
377 }
378 
adf_misc_wq_queue_work(struct work_struct * work)379 bool adf_misc_wq_queue_work(struct work_struct *work)
380 {
381 	return queue_work(adf_misc_wq, work);
382 }
383 
adf_misc_wq_queue_delayed_work(struct delayed_work * work,unsigned long delay)384 bool adf_misc_wq_queue_delayed_work(struct delayed_work *work,
385 				    unsigned long delay)
386 {
387 	return queue_delayed_work(adf_misc_wq, work, delay);
388 }
389