xref: /openbmc/linux/drivers/vdpa/ifcvf/ifcvf_main.c (revision 4cf8b6d0)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Intel IFC VF NIC driver for virtio dataplane offloading
4  *
5  * Copyright (C) 2020 Intel Corporation.
6  *
7  * Author: Zhu Lingshan <lingshan.zhu@intel.com>
8  *
9  */
10 
11 #include <linux/interrupt.h>
12 #include <linux/module.h>
13 #include <linux/pci.h>
14 #include <linux/sysfs.h>
15 #include "ifcvf_base.h"
16 
17 #define DRIVER_AUTHOR   "Intel Corporation"
18 #define IFCVF_DRIVER_NAME       "ifcvf"
19 
ifcvf_config_changed(int irq,void * arg)20 static irqreturn_t ifcvf_config_changed(int irq, void *arg)
21 {
22 	struct ifcvf_hw *vf = arg;
23 
24 	if (vf->config_cb.callback)
25 		return vf->config_cb.callback(vf->config_cb.private);
26 
27 	return IRQ_HANDLED;
28 }
29 
ifcvf_vq_intr_handler(int irq,void * arg)30 static irqreturn_t ifcvf_vq_intr_handler(int irq, void *arg)
31 {
32 	struct vring_info *vring = arg;
33 
34 	if (vring->cb.callback)
35 		return vring->cb.callback(vring->cb.private);
36 
37 	return IRQ_HANDLED;
38 }
39 
ifcvf_vqs_reused_intr_handler(int irq,void * arg)40 static irqreturn_t ifcvf_vqs_reused_intr_handler(int irq, void *arg)
41 {
42 	struct ifcvf_hw *vf = arg;
43 	struct vring_info *vring;
44 	int i;
45 
46 	for (i = 0; i < vf->nr_vring; i++) {
47 		vring = &vf->vring[i];
48 		if (vring->cb.callback)
49 			vring->cb.callback(vring->cb.private);
50 	}
51 
52 	return IRQ_HANDLED;
53 }
54 
ifcvf_dev_intr_handler(int irq,void * arg)55 static irqreturn_t ifcvf_dev_intr_handler(int irq, void *arg)
56 {
57 	struct ifcvf_hw *vf = arg;
58 	u8 isr;
59 
60 	isr = vp_ioread8(vf->isr);
61 	if (isr & VIRTIO_PCI_ISR_CONFIG)
62 		ifcvf_config_changed(irq, arg);
63 
64 	return ifcvf_vqs_reused_intr_handler(irq, arg);
65 }
66 
ifcvf_free_irq_vectors(void * data)67 static void ifcvf_free_irq_vectors(void *data)
68 {
69 	pci_free_irq_vectors(data);
70 }
71 
ifcvf_free_per_vq_irq(struct ifcvf_hw * vf)72 static void ifcvf_free_per_vq_irq(struct ifcvf_hw *vf)
73 {
74 	struct pci_dev *pdev = vf->pdev;
75 	int i;
76 
77 	for (i = 0; i < vf->nr_vring; i++) {
78 		if (vf->vring[i].irq != -EINVAL) {
79 			devm_free_irq(&pdev->dev, vf->vring[i].irq, &vf->vring[i]);
80 			vf->vring[i].irq = -EINVAL;
81 		}
82 	}
83 }
84 
ifcvf_free_vqs_reused_irq(struct ifcvf_hw * vf)85 static void ifcvf_free_vqs_reused_irq(struct ifcvf_hw *vf)
86 {
87 	struct pci_dev *pdev = vf->pdev;
88 
89 	if (vf->vqs_reused_irq != -EINVAL) {
90 		devm_free_irq(&pdev->dev, vf->vqs_reused_irq, vf);
91 		vf->vqs_reused_irq = -EINVAL;
92 	}
93 
94 }
95 
ifcvf_free_vq_irq(struct ifcvf_hw * vf)96 static void ifcvf_free_vq_irq(struct ifcvf_hw *vf)
97 {
98 	if (vf->msix_vector_status == MSIX_VECTOR_PER_VQ_AND_CONFIG)
99 		ifcvf_free_per_vq_irq(vf);
100 	else
101 		ifcvf_free_vqs_reused_irq(vf);
102 }
103 
ifcvf_free_config_irq(struct ifcvf_hw * vf)104 static void ifcvf_free_config_irq(struct ifcvf_hw *vf)
105 {
106 	struct pci_dev *pdev = vf->pdev;
107 
108 	if (vf->config_irq == -EINVAL)
109 		return;
110 
111 	/* If the irq is shared by all vqs and the config interrupt,
112 	 * it is already freed in ifcvf_free_vq_irq, so here only
113 	 * need to free config irq when msix_vector_status != MSIX_VECTOR_DEV_SHARED
114 	 */
115 	if (vf->msix_vector_status != MSIX_VECTOR_DEV_SHARED) {
116 		devm_free_irq(&pdev->dev, vf->config_irq, vf);
117 		vf->config_irq = -EINVAL;
118 	}
119 }
120 
ifcvf_free_irq(struct ifcvf_hw * vf)121 static void ifcvf_free_irq(struct ifcvf_hw *vf)
122 {
123 	struct pci_dev *pdev = vf->pdev;
124 
125 	ifcvf_free_vq_irq(vf);
126 	ifcvf_free_config_irq(vf);
127 	ifcvf_free_irq_vectors(pdev);
128 	vf->num_msix_vectors = 0;
129 }
130 
131 /* ifcvf MSIX vectors allocator, this helper tries to allocate
132  * vectors for all virtqueues and the config interrupt.
133  * It returns the number of allocated vectors, negative
134  * return value when fails.
135  */
ifcvf_alloc_vectors(struct ifcvf_hw * vf)136 static int ifcvf_alloc_vectors(struct ifcvf_hw *vf)
137 {
138 	struct pci_dev *pdev = vf->pdev;
139 	int max_intr, ret;
140 
141 	/* all queues and config interrupt  */
142 	max_intr = vf->nr_vring + 1;
143 	ret = pci_alloc_irq_vectors(pdev, 1, max_intr, PCI_IRQ_MSIX | PCI_IRQ_AFFINITY);
144 
145 	if (ret < 0) {
146 		IFCVF_ERR(pdev, "Failed to alloc IRQ vectors\n");
147 		return ret;
148 	}
149 
150 	if (ret < max_intr)
151 		IFCVF_INFO(pdev,
152 			   "Requested %u vectors, however only %u allocated, lower performance\n",
153 			   max_intr, ret);
154 
155 	return ret;
156 }
157 
ifcvf_request_per_vq_irq(struct ifcvf_hw * vf)158 static int ifcvf_request_per_vq_irq(struct ifcvf_hw *vf)
159 {
160 	struct pci_dev *pdev = vf->pdev;
161 	int i, vector, ret, irq;
162 
163 	vf->vqs_reused_irq = -EINVAL;
164 	for (i = 0; i < vf->nr_vring; i++) {
165 		snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n", pci_name(pdev), i);
166 		vector = i;
167 		irq = pci_irq_vector(pdev, vector);
168 		ret = devm_request_irq(&pdev->dev, irq,
169 				       ifcvf_vq_intr_handler, 0,
170 				       vf->vring[i].msix_name,
171 				       &vf->vring[i]);
172 		if (ret) {
173 			IFCVF_ERR(pdev, "Failed to request irq for vq %d\n", i);
174 			goto err;
175 		}
176 
177 		vf->vring[i].irq = irq;
178 		ret = ifcvf_set_vq_vector(vf, i, vector);
179 		if (ret == VIRTIO_MSI_NO_VECTOR) {
180 			IFCVF_ERR(pdev, "No msix vector for vq %u\n", i);
181 			goto err;
182 		}
183 	}
184 
185 	return 0;
186 err:
187 	ifcvf_free_irq(vf);
188 
189 	return -EFAULT;
190 }
191 
ifcvf_request_vqs_reused_irq(struct ifcvf_hw * vf)192 static int ifcvf_request_vqs_reused_irq(struct ifcvf_hw *vf)
193 {
194 	struct pci_dev *pdev = vf->pdev;
195 	int i, vector, ret, irq;
196 
197 	vector = 0;
198 	snprintf(vf->vring[0].msix_name, 256, "ifcvf[%s]-vqs-reused-irq\n", pci_name(pdev));
199 	irq = pci_irq_vector(pdev, vector);
200 	ret = devm_request_irq(&pdev->dev, irq,
201 			       ifcvf_vqs_reused_intr_handler, 0,
202 			       vf->vring[0].msix_name, vf);
203 	if (ret) {
204 		IFCVF_ERR(pdev, "Failed to request reused irq for the device\n");
205 		goto err;
206 	}
207 
208 	vf->vqs_reused_irq = irq;
209 	for (i = 0; i < vf->nr_vring; i++) {
210 		vf->vring[i].irq = -EINVAL;
211 		ret = ifcvf_set_vq_vector(vf, i, vector);
212 		if (ret == VIRTIO_MSI_NO_VECTOR) {
213 			IFCVF_ERR(pdev, "No msix vector for vq %u\n", i);
214 			goto err;
215 		}
216 	}
217 
218 	return 0;
219 err:
220 	ifcvf_free_irq(vf);
221 
222 	return -EFAULT;
223 }
224 
ifcvf_request_dev_irq(struct ifcvf_hw * vf)225 static int ifcvf_request_dev_irq(struct ifcvf_hw *vf)
226 {
227 	struct pci_dev *pdev = vf->pdev;
228 	int i, vector, ret, irq;
229 
230 	vector = 0;
231 	snprintf(vf->vring[0].msix_name, 256, "ifcvf[%s]-dev-irq\n", pci_name(pdev));
232 	irq = pci_irq_vector(pdev, vector);
233 	ret = devm_request_irq(&pdev->dev, irq,
234 			       ifcvf_dev_intr_handler, 0,
235 			       vf->vring[0].msix_name, vf);
236 	if (ret) {
237 		IFCVF_ERR(pdev, "Failed to request irq for the device\n");
238 		goto err;
239 	}
240 
241 	vf->vqs_reused_irq = irq;
242 	for (i = 0; i < vf->nr_vring; i++) {
243 		vf->vring[i].irq = -EINVAL;
244 		ret = ifcvf_set_vq_vector(vf, i, vector);
245 		if (ret == VIRTIO_MSI_NO_VECTOR) {
246 			IFCVF_ERR(pdev, "No msix vector for vq %u\n", i);
247 			goto err;
248 		}
249 	}
250 
251 	vf->config_irq = irq;
252 	ret = ifcvf_set_config_vector(vf, vector);
253 	if (ret == VIRTIO_MSI_NO_VECTOR) {
254 		IFCVF_ERR(pdev, "No msix vector for device config\n");
255 		goto err;
256 	}
257 
258 	return 0;
259 err:
260 	ifcvf_free_irq(vf);
261 
262 	return -EFAULT;
263 
264 }
265 
ifcvf_request_vq_irq(struct ifcvf_hw * vf)266 static int ifcvf_request_vq_irq(struct ifcvf_hw *vf)
267 {
268 	int ret;
269 
270 	if (vf->msix_vector_status == MSIX_VECTOR_PER_VQ_AND_CONFIG)
271 		ret = ifcvf_request_per_vq_irq(vf);
272 	else
273 		ret = ifcvf_request_vqs_reused_irq(vf);
274 
275 	return ret;
276 }
277 
ifcvf_request_config_irq(struct ifcvf_hw * vf)278 static int ifcvf_request_config_irq(struct ifcvf_hw *vf)
279 {
280 	struct pci_dev *pdev = vf->pdev;
281 	int config_vector, ret;
282 
283 	if (vf->msix_vector_status == MSIX_VECTOR_PER_VQ_AND_CONFIG)
284 		config_vector = vf->nr_vring;
285 	else if (vf->msix_vector_status ==  MSIX_VECTOR_SHARED_VQ_AND_CONFIG)
286 		/* vector 0 for vqs and 1 for config interrupt */
287 		config_vector = 1;
288 	else if (vf->msix_vector_status == MSIX_VECTOR_DEV_SHARED)
289 		/* re-use the vqs vector */
290 		return 0;
291 	else
292 		return -EINVAL;
293 
294 	snprintf(vf->config_msix_name, 256, "ifcvf[%s]-config\n",
295 		 pci_name(pdev));
296 	vf->config_irq = pci_irq_vector(pdev, config_vector);
297 	ret = devm_request_irq(&pdev->dev, vf->config_irq,
298 			       ifcvf_config_changed, 0,
299 			       vf->config_msix_name, vf);
300 	if (ret) {
301 		IFCVF_ERR(pdev, "Failed to request config irq\n");
302 		goto err;
303 	}
304 
305 	ret = ifcvf_set_config_vector(vf, config_vector);
306 	if (ret == VIRTIO_MSI_NO_VECTOR) {
307 		IFCVF_ERR(pdev, "No msix vector for device config\n");
308 		goto err;
309 	}
310 
311 	return 0;
312 err:
313 	ifcvf_free_irq(vf);
314 
315 	return -EFAULT;
316 }
317 
ifcvf_request_irq(struct ifcvf_hw * vf)318 static int ifcvf_request_irq(struct ifcvf_hw *vf)
319 {
320 	int nvectors, ret, max_intr;
321 
322 	nvectors = ifcvf_alloc_vectors(vf);
323 	if (nvectors <= 0)
324 		return -EFAULT;
325 
326 	vf->msix_vector_status = MSIX_VECTOR_PER_VQ_AND_CONFIG;
327 	max_intr = vf->nr_vring + 1;
328 	if (nvectors < max_intr)
329 		vf->msix_vector_status = MSIX_VECTOR_SHARED_VQ_AND_CONFIG;
330 
331 	if (nvectors == 1) {
332 		vf->msix_vector_status = MSIX_VECTOR_DEV_SHARED;
333 		ret = ifcvf_request_dev_irq(vf);
334 
335 		return ret;
336 	}
337 
338 	ret = ifcvf_request_vq_irq(vf);
339 	if (ret)
340 		return ret;
341 
342 	ret = ifcvf_request_config_irq(vf);
343 
344 	if (ret)
345 		return ret;
346 
347 	vf->num_msix_vectors = nvectors;
348 
349 	return 0;
350 }
351 
vdpa_to_adapter(struct vdpa_device * vdpa_dev)352 static struct ifcvf_adapter *vdpa_to_adapter(struct vdpa_device *vdpa_dev)
353 {
354 	return container_of(vdpa_dev, struct ifcvf_adapter, vdpa);
355 }
356 
vdpa_to_vf(struct vdpa_device * vdpa_dev)357 static struct ifcvf_hw *vdpa_to_vf(struct vdpa_device *vdpa_dev)
358 {
359 	struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
360 
361 	return adapter->vf;
362 }
363 
ifcvf_vdpa_get_device_features(struct vdpa_device * vdpa_dev)364 static u64 ifcvf_vdpa_get_device_features(struct vdpa_device *vdpa_dev)
365 {
366 	struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
367 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
368 	struct pci_dev *pdev = adapter->pdev;
369 	u32 type = vf->dev_type;
370 	u64 features;
371 
372 	if (type == VIRTIO_ID_NET || type == VIRTIO_ID_BLOCK)
373 		features = ifcvf_get_dev_features(vf);
374 	else {
375 		features = 0;
376 		IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", vf->dev_type);
377 	}
378 
379 	return features;
380 }
381 
ifcvf_vdpa_set_driver_features(struct vdpa_device * vdpa_dev,u64 features)382 static int ifcvf_vdpa_set_driver_features(struct vdpa_device *vdpa_dev, u64 features)
383 {
384 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
385 	int ret;
386 
387 	ret = ifcvf_verify_min_features(vf, features);
388 	if (ret)
389 		return ret;
390 
391 	ifcvf_set_driver_features(vf, features);
392 
393 	return 0;
394 }
395 
ifcvf_vdpa_get_driver_features(struct vdpa_device * vdpa_dev)396 static u64 ifcvf_vdpa_get_driver_features(struct vdpa_device *vdpa_dev)
397 {
398 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
399 	u64 features;
400 
401 	features = ifcvf_get_driver_features(vf);
402 
403 	return features;
404 }
405 
ifcvf_vdpa_get_status(struct vdpa_device * vdpa_dev)406 static u8 ifcvf_vdpa_get_status(struct vdpa_device *vdpa_dev)
407 {
408 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
409 
410 	return ifcvf_get_status(vf);
411 }
412 
ifcvf_vdpa_set_status(struct vdpa_device * vdpa_dev,u8 status)413 static void ifcvf_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status)
414 {
415 	struct ifcvf_hw *vf;
416 	u8 status_old;
417 	int ret;
418 
419 	vf  = vdpa_to_vf(vdpa_dev);
420 	status_old = ifcvf_get_status(vf);
421 
422 	if (status_old == status)
423 		return;
424 
425 	if ((status & VIRTIO_CONFIG_S_DRIVER_OK) &&
426 	    !(status_old & VIRTIO_CONFIG_S_DRIVER_OK)) {
427 		ret = ifcvf_request_irq(vf);
428 		if (ret) {
429 			IFCVF_ERR(vf->pdev, "failed to request irq with error %d\n", ret);
430 			return;
431 		}
432 	}
433 
434 	ifcvf_set_status(vf, status);
435 }
436 
ifcvf_vdpa_reset(struct vdpa_device * vdpa_dev)437 static int ifcvf_vdpa_reset(struct vdpa_device *vdpa_dev)
438 {
439 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
440 	u8 status = ifcvf_get_status(vf);
441 
442 	ifcvf_stop(vf);
443 
444 	if (status & VIRTIO_CONFIG_S_DRIVER_OK)
445 		ifcvf_free_irq(vf);
446 
447 	ifcvf_reset(vf);
448 
449 	return 0;
450 }
451 
ifcvf_vdpa_get_vq_num_max(struct vdpa_device * vdpa_dev)452 static u16 ifcvf_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev)
453 {
454 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
455 
456 	return ifcvf_get_max_vq_size(vf);
457 }
458 
ifcvf_vdpa_get_vq_state(struct vdpa_device * vdpa_dev,u16 qid,struct vdpa_vq_state * state)459 static int ifcvf_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
460 				   struct vdpa_vq_state *state)
461 {
462 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
463 
464 	state->split.avail_index = ifcvf_get_vq_state(vf, qid);
465 	return 0;
466 }
467 
ifcvf_vdpa_set_vq_state(struct vdpa_device * vdpa_dev,u16 qid,const struct vdpa_vq_state * state)468 static int ifcvf_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
469 				   const struct vdpa_vq_state *state)
470 {
471 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
472 
473 	return ifcvf_set_vq_state(vf, qid, state->split.avail_index);
474 }
475 
ifcvf_vdpa_set_vq_cb(struct vdpa_device * vdpa_dev,u16 qid,struct vdpa_callback * cb)476 static void ifcvf_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid,
477 				 struct vdpa_callback *cb)
478 {
479 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
480 
481 	vf->vring[qid].cb = *cb;
482 }
483 
ifcvf_vdpa_set_vq_ready(struct vdpa_device * vdpa_dev,u16 qid,bool ready)484 static void ifcvf_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev,
485 				    u16 qid, bool ready)
486 {
487 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
488 
489 	ifcvf_set_vq_ready(vf, qid, ready);
490 }
491 
ifcvf_vdpa_get_vq_ready(struct vdpa_device * vdpa_dev,u16 qid)492 static bool ifcvf_vdpa_get_vq_ready(struct vdpa_device *vdpa_dev, u16 qid)
493 {
494 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
495 
496 	return ifcvf_get_vq_ready(vf, qid);
497 }
498 
ifcvf_vdpa_set_vq_num(struct vdpa_device * vdpa_dev,u16 qid,u32 num)499 static void ifcvf_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid,
500 				  u32 num)
501 {
502 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
503 
504 	ifcvf_set_vq_num(vf, qid, num);
505 }
506 
ifcvf_vdpa_set_vq_address(struct vdpa_device * vdpa_dev,u16 qid,u64 desc_area,u64 driver_area,u64 device_area)507 static int ifcvf_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid,
508 				     u64 desc_area, u64 driver_area,
509 				     u64 device_area)
510 {
511 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
512 
513 	return ifcvf_set_vq_address(vf, qid, desc_area, driver_area, device_area);
514 }
515 
ifcvf_vdpa_kick_vq(struct vdpa_device * vdpa_dev,u16 qid)516 static void ifcvf_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid)
517 {
518 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
519 
520 	ifcvf_notify_queue(vf, qid);
521 }
522 
ifcvf_vdpa_get_generation(struct vdpa_device * vdpa_dev)523 static u32 ifcvf_vdpa_get_generation(struct vdpa_device *vdpa_dev)
524 {
525 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
526 
527 	return vp_ioread8(&vf->common_cfg->config_generation);
528 }
529 
ifcvf_vdpa_get_device_id(struct vdpa_device * vdpa_dev)530 static u32 ifcvf_vdpa_get_device_id(struct vdpa_device *vdpa_dev)
531 {
532 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
533 
534 	return vf->dev_type;
535 }
536 
ifcvf_vdpa_get_vendor_id(struct vdpa_device * vdpa_dev)537 static u32 ifcvf_vdpa_get_vendor_id(struct vdpa_device *vdpa_dev)
538 {
539 	struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
540 	struct pci_dev *pdev = adapter->pdev;
541 
542 	return pdev->subsystem_vendor;
543 }
544 
ifcvf_vdpa_get_vq_align(struct vdpa_device * vdpa_dev)545 static u32 ifcvf_vdpa_get_vq_align(struct vdpa_device *vdpa_dev)
546 {
547 	return IFCVF_QUEUE_ALIGNMENT;
548 }
549 
ifcvf_vdpa_get_config_size(struct vdpa_device * vdpa_dev)550 static size_t ifcvf_vdpa_get_config_size(struct vdpa_device *vdpa_dev)
551 {
552 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
553 
554 	return  vf->config_size;
555 }
556 
ifcvf_vdpa_get_vq_group(struct vdpa_device * vdpa,u16 idx)557 static u32 ifcvf_vdpa_get_vq_group(struct vdpa_device *vdpa, u16 idx)
558 {
559 	return 0;
560 }
561 
ifcvf_vdpa_get_config(struct vdpa_device * vdpa_dev,unsigned int offset,void * buf,unsigned int len)562 static void ifcvf_vdpa_get_config(struct vdpa_device *vdpa_dev,
563 				  unsigned int offset,
564 				  void *buf, unsigned int len)
565 {
566 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
567 
568 	ifcvf_read_dev_config(vf, offset, buf, len);
569 }
570 
ifcvf_vdpa_set_config(struct vdpa_device * vdpa_dev,unsigned int offset,const void * buf,unsigned int len)571 static void ifcvf_vdpa_set_config(struct vdpa_device *vdpa_dev,
572 				  unsigned int offset, const void *buf,
573 				  unsigned int len)
574 {
575 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
576 
577 	ifcvf_write_dev_config(vf, offset, buf, len);
578 }
579 
ifcvf_vdpa_set_config_cb(struct vdpa_device * vdpa_dev,struct vdpa_callback * cb)580 static void ifcvf_vdpa_set_config_cb(struct vdpa_device *vdpa_dev,
581 				     struct vdpa_callback *cb)
582 {
583 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
584 
585 	vf->config_cb.callback = cb->callback;
586 	vf->config_cb.private = cb->private;
587 }
588 
ifcvf_vdpa_get_vq_irq(struct vdpa_device * vdpa_dev,u16 qid)589 static int ifcvf_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev,
590 				 u16 qid)
591 {
592 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
593 
594 	if (vf->vqs_reused_irq < 0)
595 		return vf->vring[qid].irq;
596 	else
597 		return -EINVAL;
598 }
599 
ifcvf_get_vq_notification(struct vdpa_device * vdpa_dev,u16 idx)600 static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,
601 							       u16 idx)
602 {
603 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
604 	struct vdpa_notification_area area;
605 
606 	area.addr = vf->vring[idx].notify_pa;
607 	if (!vf->notify_off_multiplier)
608 		area.size = PAGE_SIZE;
609 	else
610 		area.size = vf->notify_off_multiplier;
611 
612 	return area;
613 }
614 
615 /*
616  * IFCVF currently doesn't have on-chip IOMMU, so not
617  * implemented set_map()/dma_map()/dma_unmap()
618  */
619 static const struct vdpa_config_ops ifc_vdpa_ops = {
620 	.get_device_features = ifcvf_vdpa_get_device_features,
621 	.set_driver_features = ifcvf_vdpa_set_driver_features,
622 	.get_driver_features = ifcvf_vdpa_get_driver_features,
623 	.get_status	= ifcvf_vdpa_get_status,
624 	.set_status	= ifcvf_vdpa_set_status,
625 	.reset		= ifcvf_vdpa_reset,
626 	.get_vq_num_max	= ifcvf_vdpa_get_vq_num_max,
627 	.get_vq_state	= ifcvf_vdpa_get_vq_state,
628 	.set_vq_state	= ifcvf_vdpa_set_vq_state,
629 	.set_vq_cb	= ifcvf_vdpa_set_vq_cb,
630 	.set_vq_ready	= ifcvf_vdpa_set_vq_ready,
631 	.get_vq_ready	= ifcvf_vdpa_get_vq_ready,
632 	.set_vq_num	= ifcvf_vdpa_set_vq_num,
633 	.set_vq_address	= ifcvf_vdpa_set_vq_address,
634 	.get_vq_irq	= ifcvf_vdpa_get_vq_irq,
635 	.kick_vq	= ifcvf_vdpa_kick_vq,
636 	.get_generation	= ifcvf_vdpa_get_generation,
637 	.get_device_id	= ifcvf_vdpa_get_device_id,
638 	.get_vendor_id	= ifcvf_vdpa_get_vendor_id,
639 	.get_vq_align	= ifcvf_vdpa_get_vq_align,
640 	.get_vq_group	= ifcvf_vdpa_get_vq_group,
641 	.get_config_size	= ifcvf_vdpa_get_config_size,
642 	.get_config	= ifcvf_vdpa_get_config,
643 	.set_config	= ifcvf_vdpa_set_config,
644 	.set_config_cb  = ifcvf_vdpa_set_config_cb,
645 	.get_vq_notification = ifcvf_get_vq_notification,
646 };
647 
648 static struct virtio_device_id id_table_net[] = {
649 	{VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID},
650 	{0},
651 };
652 
653 static struct virtio_device_id id_table_blk[] = {
654 	{VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID},
655 	{0},
656 };
657 
get_dev_type(struct pci_dev * pdev)658 static u32 get_dev_type(struct pci_dev *pdev)
659 {
660 	u32 dev_type;
661 
662 	/* This drirver drives both modern virtio devices and transitional
663 	 * devices in modern mode.
664 	 * vDPA requires feature bit VIRTIO_F_ACCESS_PLATFORM,
665 	 * so legacy devices and transitional devices in legacy
666 	 * mode will not work for vDPA, this driver will not
667 	 * drive devices with legacy interface.
668 	 */
669 
670 	if (pdev->device < 0x1040)
671 		dev_type =  pdev->subsystem_device;
672 	else
673 		dev_type =  pdev->device - 0x1040;
674 
675 	return dev_type;
676 }
677 
ifcvf_vdpa_dev_add(struct vdpa_mgmt_dev * mdev,const char * name,const struct vdpa_dev_set_config * config)678 static int ifcvf_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
679 			      const struct vdpa_dev_set_config *config)
680 {
681 	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
682 	struct ifcvf_adapter *adapter;
683 	struct vdpa_device *vdpa_dev;
684 	struct pci_dev *pdev;
685 	struct ifcvf_hw *vf;
686 	u64 device_features;
687 	int ret;
688 
689 	ifcvf_mgmt_dev = container_of(mdev, struct ifcvf_vdpa_mgmt_dev, mdev);
690 	vf = &ifcvf_mgmt_dev->vf;
691 	pdev = vf->pdev;
692 	adapter = vdpa_alloc_device(struct ifcvf_adapter, vdpa,
693 				    &pdev->dev, &ifc_vdpa_ops, 1, 1, NULL, false);
694 	if (IS_ERR(adapter)) {
695 		IFCVF_ERR(pdev, "Failed to allocate vDPA structure");
696 		return PTR_ERR(adapter);
697 	}
698 
699 	ifcvf_mgmt_dev->adapter = adapter;
700 	adapter->pdev = pdev;
701 	adapter->vdpa.dma_dev = &pdev->dev;
702 	adapter->vdpa.mdev = mdev;
703 	adapter->vf = vf;
704 	vdpa_dev = &adapter->vdpa;
705 
706 	device_features = vf->hw_features;
707 	if (config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {
708 		if (config->device_features & ~device_features) {
709 			IFCVF_ERR(pdev, "The provisioned features 0x%llx are not supported by this device with features 0x%llx\n",
710 				  config->device_features, device_features);
711 			return -EINVAL;
712 		}
713 		device_features &= config->device_features;
714 	}
715 	vf->dev_features = device_features;
716 
717 	if (name)
718 		ret = dev_set_name(&vdpa_dev->dev, "%s", name);
719 	else
720 		ret = dev_set_name(&vdpa_dev->dev, "vdpa%u", vdpa_dev->index);
721 
722 	ret = _vdpa_register_device(&adapter->vdpa, vf->nr_vring);
723 	if (ret) {
724 		put_device(&adapter->vdpa.dev);
725 		IFCVF_ERR(pdev, "Failed to register to vDPA bus");
726 		return ret;
727 	}
728 
729 	return 0;
730 }
731 
ifcvf_vdpa_dev_del(struct vdpa_mgmt_dev * mdev,struct vdpa_device * dev)732 static void ifcvf_vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev)
733 {
734 	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
735 
736 	ifcvf_mgmt_dev = container_of(mdev, struct ifcvf_vdpa_mgmt_dev, mdev);
737 	_vdpa_unregister_device(dev);
738 	ifcvf_mgmt_dev->adapter = NULL;
739 }
740 
741 static const struct vdpa_mgmtdev_ops ifcvf_vdpa_mgmt_dev_ops = {
742 	.dev_add = ifcvf_vdpa_dev_add,
743 	.dev_del = ifcvf_vdpa_dev_del
744 };
745 
ifcvf_probe(struct pci_dev * pdev,const struct pci_device_id * id)746 static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
747 {
748 	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
749 	struct device *dev = &pdev->dev;
750 	struct ifcvf_hw *vf;
751 	u32 dev_type;
752 	int ret, i;
753 
754 	ret = pcim_enable_device(pdev);
755 	if (ret) {
756 		IFCVF_ERR(pdev, "Failed to enable device\n");
757 		return ret;
758 	}
759 	ret = pcim_iomap_regions(pdev, BIT(0) | BIT(2) | BIT(4),
760 				 IFCVF_DRIVER_NAME);
761 	if (ret) {
762 		IFCVF_ERR(pdev, "Failed to request MMIO region\n");
763 		return ret;
764 	}
765 
766 	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
767 	if (ret) {
768 		IFCVF_ERR(pdev, "No usable DMA configuration\n");
769 		return ret;
770 	}
771 
772 	ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev);
773 	if (ret) {
774 		IFCVF_ERR(pdev,
775 			  "Failed for adding devres for freeing irq vectors\n");
776 		return ret;
777 	}
778 
779 	pci_set_master(pdev);
780 	ifcvf_mgmt_dev = kzalloc(sizeof(struct ifcvf_vdpa_mgmt_dev), GFP_KERNEL);
781 	if (!ifcvf_mgmt_dev) {
782 		IFCVF_ERR(pdev, "Failed to alloc memory for the vDPA management device\n");
783 		return -ENOMEM;
784 	}
785 
786 	vf = &ifcvf_mgmt_dev->vf;
787 	vf->dev_type = get_dev_type(pdev);
788 	vf->base = pcim_iomap_table(pdev);
789 	vf->pdev = pdev;
790 
791 	ret = ifcvf_init_hw(vf, pdev);
792 	if (ret) {
793 		IFCVF_ERR(pdev, "Failed to init IFCVF hw\n");
794 		goto err;
795 	}
796 
797 	for (i = 0; i < vf->nr_vring; i++)
798 		vf->vring[i].irq = -EINVAL;
799 
800 	vf->hw_features = ifcvf_get_hw_features(vf);
801 	vf->config_size = ifcvf_get_config_size(vf);
802 
803 	dev_type = get_dev_type(pdev);
804 	switch (dev_type) {
805 	case VIRTIO_ID_NET:
806 		ifcvf_mgmt_dev->mdev.id_table = id_table_net;
807 		break;
808 	case VIRTIO_ID_BLOCK:
809 		ifcvf_mgmt_dev->mdev.id_table = id_table_blk;
810 		break;
811 	default:
812 		IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", dev_type);
813 		ret = -EOPNOTSUPP;
814 		goto err;
815 	}
816 
817 	ifcvf_mgmt_dev->mdev.ops = &ifcvf_vdpa_mgmt_dev_ops;
818 	ifcvf_mgmt_dev->mdev.device = dev;
819 	ifcvf_mgmt_dev->mdev.max_supported_vqs = vf->nr_vring;
820 	ifcvf_mgmt_dev->mdev.supported_features = vf->hw_features;
821 	ifcvf_mgmt_dev->mdev.config_attr_mask = (1 << VDPA_ATTR_DEV_FEATURES);
822 
823 	ret = vdpa_mgmtdev_register(&ifcvf_mgmt_dev->mdev);
824 	if (ret) {
825 		IFCVF_ERR(pdev,
826 			  "Failed to initialize the management interfaces\n");
827 		goto err;
828 	}
829 
830 	pci_set_drvdata(pdev, ifcvf_mgmt_dev);
831 
832 	return 0;
833 
834 err:
835 	kfree(ifcvf_mgmt_dev->vf.vring);
836 	kfree(ifcvf_mgmt_dev);
837 	return ret;
838 }
839 
ifcvf_remove(struct pci_dev * pdev)840 static void ifcvf_remove(struct pci_dev *pdev)
841 {
842 	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
843 
844 	ifcvf_mgmt_dev = pci_get_drvdata(pdev);
845 	vdpa_mgmtdev_unregister(&ifcvf_mgmt_dev->mdev);
846 	kfree(ifcvf_mgmt_dev->vf.vring);
847 	kfree(ifcvf_mgmt_dev);
848 }
849 
850 static struct pci_device_id ifcvf_pci_ids[] = {
851 	/* N3000 network device */
852 	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_REDHAT_QUMRANET,
853 			 N3000_DEVICE_ID,
854 			 PCI_VENDOR_ID_INTEL,
855 			 N3000_SUBSYS_DEVICE_ID) },
856 	/* C5000X-PL network device
857 	 * F2000X-PL network device
858 	 */
859 	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_REDHAT_QUMRANET,
860 			 VIRTIO_TRANS_ID_NET,
861 			 PCI_VENDOR_ID_INTEL,
862 			 VIRTIO_ID_NET) },
863 	/* C5000X-PL block device */
864 	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_REDHAT_QUMRANET,
865 			 VIRTIO_TRANS_ID_BLOCK,
866 			 PCI_VENDOR_ID_INTEL,
867 			 VIRTIO_ID_BLOCK) },
868 
869 	{ 0 },
870 };
871 MODULE_DEVICE_TABLE(pci, ifcvf_pci_ids);
872 
873 static struct pci_driver ifcvf_driver = {
874 	.name     = IFCVF_DRIVER_NAME,
875 	.id_table = ifcvf_pci_ids,
876 	.probe    = ifcvf_probe,
877 	.remove   = ifcvf_remove,
878 };
879 
880 module_pci_driver(ifcvf_driver);
881 
882 MODULE_LICENSE("GPL v2");
883