xref: /openbmc/linux/drivers/vhost/vdpa.c (revision bd4af432) 
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2018-2020 Intel Corporation.
4  * Copyright (C) 2020 Red Hat, Inc.
5  *
6  * Author: Tiwei Bie <tiwei.bie@intel.com>
7  *         Jason Wang <jasowang@redhat.com>
8  *
9  * Thanks Michael S. Tsirkin for the valuable comments and
10  * suggestions.  And thanks to Cunming Liang and Zhihong Wang for all
11  * their supports.
12  */
13 
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/cdev.h>
17 #include <linux/device.h>
18 #include <linux/iommu.h>
19 #include <linux/uuid.h>
20 #include <linux/vdpa.h>
21 #include <linux/nospec.h>
22 #include <linux/vhost.h>
23 #include <linux/virtio_net.h>
24 
25 #include "vhost.h"
26 
27 enum {
28 	VHOST_VDPA_FEATURES =
29 		(1ULL << VIRTIO_F_NOTIFY_ON_EMPTY) |
30 		(1ULL << VIRTIO_F_ANY_LAYOUT) |
31 		(1ULL << VIRTIO_F_VERSION_1) |
32 		(1ULL << VIRTIO_F_IOMMU_PLATFORM) |
33 		(1ULL << VIRTIO_F_RING_PACKED) |
34 		(1ULL << VIRTIO_F_ORDER_PLATFORM) |
35 		(1ULL << VIRTIO_RING_F_INDIRECT_DESC) |
36 		(1ULL << VIRTIO_RING_F_EVENT_IDX),
37 
38 	VHOST_VDPA_NET_FEATURES = VHOST_VDPA_FEATURES |
39 		(1ULL << VIRTIO_NET_F_CSUM) |
40 		(1ULL << VIRTIO_NET_F_GUEST_CSUM) |
41 		(1ULL << VIRTIO_NET_F_MTU) |
42 		(1ULL << VIRTIO_NET_F_MAC) |
43 		(1ULL << VIRTIO_NET_F_GUEST_TSO4) |
44 		(1ULL << VIRTIO_NET_F_GUEST_TSO6) |
45 		(1ULL << VIRTIO_NET_F_GUEST_ECN) |
46 		(1ULL << VIRTIO_NET_F_GUEST_UFO) |
47 		(1ULL << VIRTIO_NET_F_HOST_TSO4) |
48 		(1ULL << VIRTIO_NET_F_HOST_TSO6) |
49 		(1ULL << VIRTIO_NET_F_HOST_ECN) |
50 		(1ULL << VIRTIO_NET_F_HOST_UFO) |
51 		(1ULL << VIRTIO_NET_F_MRG_RXBUF) |
52 		(1ULL << VIRTIO_NET_F_STATUS) |
53 		(1ULL << VIRTIO_NET_F_SPEED_DUPLEX),
54 };
55 
56 /* Currently, only network backend w/o multiqueue is supported. */
57 #define VHOST_VDPA_VQ_MAX	2
58 
59 #define VHOST_VDPA_DEV_MAX (1U << MINORBITS)
60 
61 struct vhost_vdpa {
62 	struct vhost_dev vdev;
63 	struct iommu_domain *domain;
64 	struct vhost_virtqueue *vqs;
65 	struct completion completion;
66 	struct vdpa_device *vdpa;
67 	struct device dev;
68 	struct cdev cdev;
69 	atomic_t opened;
70 	int nvqs;
71 	int virtio_id;
72 	int minor;
73 };
74 
75 static DEFINE_IDA(vhost_vdpa_ida);
76 
77 static dev_t vhost_vdpa_major;
78 
79 static const u64 vhost_vdpa_features[] = {
80 	[VIRTIO_ID_NET] = VHOST_VDPA_NET_FEATURES,
81 };
82 
83 static void handle_vq_kick(struct vhost_work *work)
84 {
85 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
86 						  poll.work);
87 	struct vhost_vdpa *v = container_of(vq->dev, struct vhost_vdpa, vdev);
88 	const struct vdpa_config_ops *ops = v->vdpa->config;
89 
90 	ops->kick_vq(v->vdpa, vq - v->vqs);
91 }
92 
93 static irqreturn_t vhost_vdpa_virtqueue_cb(void *private)
94 {
95 	struct vhost_virtqueue *vq = private;
96 	struct eventfd_ctx *call_ctx = vq->call_ctx;
97 
98 	if (call_ctx)
99 		eventfd_signal(call_ctx, 1);
100 
101 	return IRQ_HANDLED;
102 }
103 
104 static void vhost_vdpa_reset(struct vhost_vdpa *v)
105 {
106 	struct vdpa_device *vdpa = v->vdpa;
107 	const struct vdpa_config_ops *ops = vdpa->config;
108 
109 	ops->set_status(vdpa, 0);
110 }
111 
112 static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp)
113 {
114 	struct vdpa_device *vdpa = v->vdpa;
115 	const struct vdpa_config_ops *ops = vdpa->config;
116 	u32 device_id;
117 
118 	device_id = ops->get_device_id(vdpa);
119 
120 	if (copy_to_user(argp, &device_id, sizeof(device_id)))
121 		return -EFAULT;
122 
123 	return 0;
124 }
125 
126 static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp)
127 {
128 	struct vdpa_device *vdpa = v->vdpa;
129 	const struct vdpa_config_ops *ops = vdpa->config;
130 	u8 status;
131 
132 	status = ops->get_status(vdpa);
133 
134 	if (copy_to_user(statusp, &status, sizeof(status)))
135 		return -EFAULT;
136 
137 	return 0;
138 }
139 
140 static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp)
141 {
142 	struct vdpa_device *vdpa = v->vdpa;
143 	const struct vdpa_config_ops *ops = vdpa->config;
144 	u8 status;
145 
146 	if (copy_from_user(&status, statusp, sizeof(status)))
147 		return -EFAULT;
148 
149 	/*
150 	 * Userspace shouldn't remove status bits unless reset the
151 	 * status to 0.
152 	 */
153 	if (status != 0 && (ops->get_status(vdpa) & ~status) != 0)
154 		return -EINVAL;
155 
156 	ops->set_status(vdpa, status);
157 
158 	return 0;
159 }
160 
161 static int vhost_vdpa_config_validate(struct vhost_vdpa *v,
162 				      struct vhost_vdpa_config *c)
163 {
164 	long size = 0;
165 
166 	switch (v->virtio_id) {
167 	case VIRTIO_ID_NET:
168 		size = sizeof(struct virtio_net_config);
169 		break;
170 	}
171 
172 	if (c->len == 0)
173 		return -EINVAL;
174 
175 	if (c->len > size - c->off)
176 		return -E2BIG;
177 
178 	return 0;
179 }
180 
181 static long vhost_vdpa_get_config(struct vhost_vdpa *v,
182 				  struct vhost_vdpa_config __user *c)
183 {
184 	struct vdpa_device *vdpa = v->vdpa;
185 	const struct vdpa_config_ops *ops = vdpa->config;
186 	struct vhost_vdpa_config config;
187 	unsigned long size = offsetof(struct vhost_vdpa_config, buf);
188 	u8 *buf;
189 
190 	if (copy_from_user(&config, c, size))
191 		return -EFAULT;
192 	if (vhost_vdpa_config_validate(v, &config))
193 		return -EINVAL;
194 	buf = kvzalloc(config.len, GFP_KERNEL);
195 	if (!buf)
196 		return -ENOMEM;
197 
198 	ops->get_config(vdpa, config.off, buf, config.len);
199 
200 	if (copy_to_user(c->buf, buf, config.len)) {
201 		kvfree(buf);
202 		return -EFAULT;
203 	}
204 
205 	kvfree(buf);
206 	return 0;
207 }
208 
209 static long vhost_vdpa_set_config(struct vhost_vdpa *v,
210 				  struct vhost_vdpa_config __user *c)
211 {
212 	struct vdpa_device *vdpa = v->vdpa;
213 	const struct vdpa_config_ops *ops = vdpa->config;
214 	struct vhost_vdpa_config config;
215 	unsigned long size = offsetof(struct vhost_vdpa_config, buf);
216 	u8 *buf;
217 
218 	if (copy_from_user(&config, c, size))
219 		return -EFAULT;
220 	if (vhost_vdpa_config_validate(v, &config))
221 		return -EINVAL;
222 	buf = kvzalloc(config.len, GFP_KERNEL);
223 	if (!buf)
224 		return -ENOMEM;
225 
226 	if (copy_from_user(buf, c->buf, config.len)) {
227 		kvfree(buf);
228 		return -EFAULT;
229 	}
230 
231 	ops->set_config(vdpa, config.off, buf, config.len);
232 
233 	kvfree(buf);
234 	return 0;
235 }
236 
237 static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep)
238 {
239 	struct vdpa_device *vdpa = v->vdpa;
240 	const struct vdpa_config_ops *ops = vdpa->config;
241 	u64 features;
242 
243 	features = ops->get_features(vdpa);
244 	features &= vhost_vdpa_features[v->virtio_id];
245 
246 	if (copy_to_user(featurep, &features, sizeof(features)))
247 		return -EFAULT;
248 
249 	return 0;
250 }
251 
252 static long vhost_vdpa_set_features(struct vhost_vdpa *v, u64 __user *featurep)
253 {
254 	struct vdpa_device *vdpa = v->vdpa;
255 	const struct vdpa_config_ops *ops = vdpa->config;
256 	u64 features;
257 
258 	/*
259 	 * It's not allowed to change the features after they have
260 	 * been negotiated.
261 	 */
262 	if (ops->get_status(vdpa) & VIRTIO_CONFIG_S_FEATURES_OK)
263 		return -EBUSY;
264 
265 	if (copy_from_user(&features, featurep, sizeof(features)))
266 		return -EFAULT;
267 
268 	if (features & ~vhost_vdpa_features[v->virtio_id])
269 		return -EINVAL;
270 
271 	if (ops->set_features(vdpa, features))
272 		return -EINVAL;
273 
274 	return 0;
275 }
276 
277 static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp)
278 {
279 	struct vdpa_device *vdpa = v->vdpa;
280 	const struct vdpa_config_ops *ops = vdpa->config;
281 	u16 num;
282 
283 	num = ops->get_vq_num_max(vdpa);
284 
285 	if (copy_to_user(argp, &num, sizeof(num)))
286 		return -EFAULT;
287 
288 	return 0;
289 }
290 
291 static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
292 				   void __user *argp)
293 {
294 	struct vdpa_device *vdpa = v->vdpa;
295 	const struct vdpa_config_ops *ops = vdpa->config;
296 	struct vdpa_callback cb;
297 	struct vhost_virtqueue *vq;
298 	struct vhost_vring_state s;
299 	u32 idx;
300 	long r;
301 
302 	r = get_user(idx, (u32 __user *)argp);
303 	if (r < 0)
304 		return r;
305 
306 	if (idx >= v->nvqs)
307 		return -ENOBUFS;
308 
309 	idx = array_index_nospec(idx, v->nvqs);
310 	vq = &v->vqs[idx];
311 
312 	if (cmd == VHOST_VDPA_SET_VRING_ENABLE) {
313 		if (copy_from_user(&s, argp, sizeof(s)))
314 			return -EFAULT;
315 		ops->set_vq_ready(vdpa, idx, s.num);
316 		return 0;
317 	}
318 
319 	if (cmd == VHOST_GET_VRING_BASE)
320 		vq->last_avail_idx = ops->get_vq_state(v->vdpa, idx);
321 
322 	r = vhost_vring_ioctl(&v->vdev, cmd, argp);
323 	if (r)
324 		return r;
325 
326 	switch (cmd) {
327 	case VHOST_SET_VRING_ADDR:
328 		if (ops->set_vq_address(vdpa, idx,
329 					(u64)(uintptr_t)vq->desc,
330 					(u64)(uintptr_t)vq->avail,
331 					(u64)(uintptr_t)vq->used))
332 			r = -EINVAL;
333 		break;
334 
335 	case VHOST_SET_VRING_BASE:
336 		if (ops->set_vq_state(vdpa, idx, vq->last_avail_idx))
337 			r = -EINVAL;
338 		break;
339 
340 	case VHOST_SET_VRING_CALL:
341 		if (vq->call_ctx) {
342 			cb.callback = vhost_vdpa_virtqueue_cb;
343 			cb.private = vq;
344 		} else {
345 			cb.callback = NULL;
346 			cb.private = NULL;
347 		}
348 		ops->set_vq_cb(vdpa, idx, &cb);
349 		break;
350 
351 	case VHOST_SET_VRING_NUM:
352 		ops->set_vq_num(vdpa, idx, vq->num);
353 		break;
354 	}
355 
356 	return r;
357 }
358 
359 static long vhost_vdpa_unlocked_ioctl(struct file *filep,
360 				      unsigned int cmd, unsigned long arg)
361 {
362 	struct vhost_vdpa *v = filep->private_data;
363 	struct vhost_dev *d = &v->vdev;
364 	void __user *argp = (void __user *)arg;
365 	long r;
366 
367 	mutex_lock(&d->mutex);
368 
369 	switch (cmd) {
370 	case VHOST_VDPA_GET_DEVICE_ID:
371 		r = vhost_vdpa_get_device_id(v, argp);
372 		break;
373 	case VHOST_VDPA_GET_STATUS:
374 		r = vhost_vdpa_get_status(v, argp);
375 		break;
376 	case VHOST_VDPA_SET_STATUS:
377 		r = vhost_vdpa_set_status(v, argp);
378 		break;
379 	case VHOST_VDPA_GET_CONFIG:
380 		r = vhost_vdpa_get_config(v, argp);
381 		break;
382 	case VHOST_VDPA_SET_CONFIG:
383 		r = vhost_vdpa_set_config(v, argp);
384 		break;
385 	case VHOST_GET_FEATURES:
386 		r = vhost_vdpa_get_features(v, argp);
387 		break;
388 	case VHOST_SET_FEATURES:
389 		r = vhost_vdpa_set_features(v, argp);
390 		break;
391 	case VHOST_VDPA_GET_VRING_NUM:
392 		r = vhost_vdpa_get_vring_num(v, argp);
393 		break;
394 	case VHOST_SET_LOG_BASE:
395 	case VHOST_SET_LOG_FD:
396 		r = -ENOIOCTLCMD;
397 		break;
398 	default:
399 		r = vhost_dev_ioctl(&v->vdev, cmd, argp);
400 		if (r == -ENOIOCTLCMD)
401 			r = vhost_vdpa_vring_ioctl(v, cmd, argp);
402 		break;
403 	}
404 
405 	mutex_unlock(&d->mutex);
406 	return r;
407 }
408 
409 static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v, u64 start, u64 last)
410 {
411 	struct vhost_dev *dev = &v->vdev;
412 	struct vhost_iotlb *iotlb = dev->iotlb;
413 	struct vhost_iotlb_map *map;
414 	struct page *page;
415 	unsigned long pfn, pinned;
416 
417 	while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) {
418 		pinned = map->size >> PAGE_SHIFT;
419 		for (pfn = map->addr >> PAGE_SHIFT;
420 		     pinned > 0; pfn++, pinned--) {
421 			page = pfn_to_page(pfn);
422 			if (map->perm & VHOST_ACCESS_WO)
423 				set_page_dirty_lock(page);
424 			unpin_user_page(page);
425 		}
426 		atomic64_sub(map->size >> PAGE_SHIFT, &dev->mm->pinned_vm);
427 		vhost_iotlb_map_free(iotlb, map);
428 	}
429 }
430 
431 static void vhost_vdpa_iotlb_free(struct vhost_vdpa *v)
432 {
433 	struct vhost_dev *dev = &v->vdev;
434 
435 	vhost_vdpa_iotlb_unmap(v, 0ULL, 0ULL - 1);
436 	kfree(dev->iotlb);
437 	dev->iotlb = NULL;
438 }
439 
440 static int perm_to_iommu_flags(u32 perm)
441 {
442 	int flags = 0;
443 
444 	switch (perm) {
445 	case VHOST_ACCESS_WO:
446 		flags |= IOMMU_WRITE;
447 		break;
448 	case VHOST_ACCESS_RO:
449 		flags |= IOMMU_READ;
450 		break;
451 	case VHOST_ACCESS_RW:
452 		flags |= (IOMMU_WRITE | IOMMU_READ);
453 		break;
454 	default:
455 		WARN(1, "invalidate vhost IOTLB permission\n");
456 		break;
457 	}
458 
459 	return flags | IOMMU_CACHE;
460 }
461 
462 static int vhost_vdpa_map(struct vhost_vdpa *v,
463 			  u64 iova, u64 size, u64 pa, u32 perm)
464 {
465 	struct vhost_dev *dev = &v->vdev;
466 	struct vdpa_device *vdpa = v->vdpa;
467 	const struct vdpa_config_ops *ops = vdpa->config;
468 	int r = 0;
469 
470 	r = vhost_iotlb_add_range(dev->iotlb, iova, iova + size - 1,
471 				  pa, perm);
472 	if (r)
473 		return r;
474 
475 	if (ops->dma_map)
476 		r = ops->dma_map(vdpa, iova, size, pa, perm);
477 	else if (ops->set_map)
478 		r = ops->set_map(vdpa, dev->iotlb);
479 	else
480 		r = iommu_map(v->domain, iova, pa, size,
481 			      perm_to_iommu_flags(perm));
482 
483 	return r;
484 }
485 
486 static void vhost_vdpa_unmap(struct vhost_vdpa *v, u64 iova, u64 size)
487 {
488 	struct vhost_dev *dev = &v->vdev;
489 	struct vdpa_device *vdpa = v->vdpa;
490 	const struct vdpa_config_ops *ops = vdpa->config;
491 
492 	vhost_vdpa_iotlb_unmap(v, iova, iova + size - 1);
493 
494 	if (ops->dma_map)
495 		ops->dma_unmap(vdpa, iova, size);
496 	else if (ops->set_map)
497 		ops->set_map(vdpa, dev->iotlb);
498 	else
499 		iommu_unmap(v->domain, iova, size);
500 }
501 
502 static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v,
503 					   struct vhost_iotlb_msg *msg)
504 {
505 	struct vhost_dev *dev = &v->vdev;
506 	struct vhost_iotlb *iotlb = dev->iotlb;
507 	struct page **page_list;
508 	unsigned long list_size = PAGE_SIZE / sizeof(struct page *);
509 	unsigned int gup_flags = FOLL_LONGTERM;
510 	unsigned long npages, cur_base, map_pfn, last_pfn = 0;
511 	unsigned long locked, lock_limit, pinned, i;
512 	u64 iova = msg->iova;
513 	int ret = 0;
514 
515 	if (vhost_iotlb_itree_first(iotlb, msg->iova,
516 				    msg->iova + msg->size - 1))
517 		return -EEXIST;
518 
519 	page_list = (struct page **) __get_free_page(GFP_KERNEL);
520 	if (!page_list)
521 		return -ENOMEM;
522 
523 	if (msg->perm & VHOST_ACCESS_WO)
524 		gup_flags |= FOLL_WRITE;
525 
526 	npages = PAGE_ALIGN(msg->size + (iova & ~PAGE_MASK)) >> PAGE_SHIFT;
527 	if (!npages)
528 		return -EINVAL;
529 
530 	down_read(&dev->mm->mmap_sem);
531 
532 	locked = atomic64_add_return(npages, &dev->mm->pinned_vm);
533 	lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
534 
535 	if (locked > lock_limit) {
536 		ret = -ENOMEM;
537 		goto out;
538 	}
539 
540 	cur_base = msg->uaddr & PAGE_MASK;
541 	iova &= PAGE_MASK;
542 
543 	while (npages) {
544 		pinned = min_t(unsigned long, npages, list_size);
545 		ret = pin_user_pages(cur_base, pinned,
546 				     gup_flags, page_list, NULL);
547 		if (ret != pinned)
548 			goto out;
549 
550 		if (!last_pfn)
551 			map_pfn = page_to_pfn(page_list[0]);
552 
553 		for (i = 0; i < ret; i++) {
554 			unsigned long this_pfn = page_to_pfn(page_list[i]);
555 			u64 csize;
556 
557 			if (last_pfn && (this_pfn != last_pfn + 1)) {
558 				/* Pin a contiguous chunk of memory */
559 				csize = (last_pfn - map_pfn + 1) << PAGE_SHIFT;
560 				if (vhost_vdpa_map(v, iova, csize,
561 						   map_pfn << PAGE_SHIFT,
562 						   msg->perm))
563 					goto out;
564 				map_pfn = this_pfn;
565 				iova += csize;
566 			}
567 
568 			last_pfn = this_pfn;
569 		}
570 
571 		cur_base += ret << PAGE_SHIFT;
572 		npages -= ret;
573 	}
574 
575 	/* Pin the rest chunk */
576 	ret = vhost_vdpa_map(v, iova, (last_pfn - map_pfn + 1) << PAGE_SHIFT,
577 			     map_pfn << PAGE_SHIFT, msg->perm);
578 out:
579 	if (ret) {
580 		vhost_vdpa_unmap(v, msg->iova, msg->size);
581 		atomic64_sub(npages, &dev->mm->pinned_vm);
582 	}
583 	up_read(&dev->mm->mmap_sem);
584 	free_page((unsigned long)page_list);
585 	return ret;
586 }
587 
588 static int vhost_vdpa_process_iotlb_msg(struct vhost_dev *dev,
589 					struct vhost_iotlb_msg *msg)
590 {
591 	struct vhost_vdpa *v = container_of(dev, struct vhost_vdpa, vdev);
592 	int r = 0;
593 
594 	r = vhost_dev_check_owner(dev);
595 	if (r)
596 		return r;
597 
598 	switch (msg->type) {
599 	case VHOST_IOTLB_UPDATE:
600 		r = vhost_vdpa_process_iotlb_update(v, msg);
601 		break;
602 	case VHOST_IOTLB_INVALIDATE:
603 		vhost_vdpa_unmap(v, msg->iova, msg->size);
604 		break;
605 	default:
606 		r = -EINVAL;
607 		break;
608 	}
609 
610 	return r;
611 }
612 
613 static ssize_t vhost_vdpa_chr_write_iter(struct kiocb *iocb,
614 					 struct iov_iter *from)
615 {
616 	struct file *file = iocb->ki_filp;
617 	struct vhost_vdpa *v = file->private_data;
618 	struct vhost_dev *dev = &v->vdev;
619 
620 	return vhost_chr_write_iter(dev, from);
621 }
622 
623 static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v)
624 {
625 	struct vdpa_device *vdpa = v->vdpa;
626 	const struct vdpa_config_ops *ops = vdpa->config;
627 	struct device *dma_dev = vdpa_get_dma_dev(vdpa);
628 	struct bus_type *bus;
629 	int ret;
630 
631 	/* Device want to do DMA by itself */
632 	if (ops->set_map || ops->dma_map)
633 		return 0;
634 
635 	bus = dma_dev->bus;
636 	if (!bus)
637 		return -EFAULT;
638 
639 	if (!iommu_capable(bus, IOMMU_CAP_CACHE_COHERENCY))
640 		return -ENOTSUPP;
641 
642 	v->domain = iommu_domain_alloc(bus);
643 	if (!v->domain)
644 		return -EIO;
645 
646 	ret = iommu_attach_device(v->domain, dma_dev);
647 	if (ret)
648 		goto err_attach;
649 
650 	return 0;
651 
652 err_attach:
653 	iommu_domain_free(v->domain);
654 	return ret;
655 }
656 
657 static void vhost_vdpa_free_domain(struct vhost_vdpa *v)
658 {
659 	struct vdpa_device *vdpa = v->vdpa;
660 	struct device *dma_dev = vdpa_get_dma_dev(vdpa);
661 
662 	if (v->domain) {
663 		iommu_detach_device(v->domain, dma_dev);
664 		iommu_domain_free(v->domain);
665 	}
666 
667 	v->domain = NULL;
668 }
669 
670 static int vhost_vdpa_open(struct inode *inode, struct file *filep)
671 {
672 	struct vhost_vdpa *v;
673 	struct vhost_dev *dev;
674 	struct vhost_virtqueue **vqs;
675 	int nvqs, i, r, opened;
676 
677 	v = container_of(inode->i_cdev, struct vhost_vdpa, cdev);
678 
679 	opened = atomic_cmpxchg(&v->opened, 0, 1);
680 	if (opened)
681 		return -EBUSY;
682 
683 	nvqs = v->nvqs;
684 	vhost_vdpa_reset(v);
685 
686 	vqs = kmalloc_array(nvqs, sizeof(*vqs), GFP_KERNEL);
687 	if (!vqs) {
688 		r = -ENOMEM;
689 		goto err;
690 	}
691 
692 	dev = &v->vdev;
693 	for (i = 0; i < nvqs; i++) {
694 		vqs[i] = &v->vqs[i];
695 		vqs[i]->handle_kick = handle_vq_kick;
696 	}
697 	vhost_dev_init(dev, vqs, nvqs, 0, 0, 0,
698 		       vhost_vdpa_process_iotlb_msg);
699 
700 	dev->iotlb = vhost_iotlb_alloc(0, 0);
701 	if (!dev->iotlb) {
702 		r = -ENOMEM;
703 		goto err_init_iotlb;
704 	}
705 
706 	r = vhost_vdpa_alloc_domain(v);
707 	if (r)
708 		goto err_init_iotlb;
709 
710 	filep->private_data = v;
711 
712 	return 0;
713 
714 err_init_iotlb:
715 	vhost_dev_cleanup(&v->vdev);
716 err:
717 	atomic_dec(&v->opened);
718 	return r;
719 }
720 
721 static int vhost_vdpa_release(struct inode *inode, struct file *filep)
722 {
723 	struct vhost_vdpa *v = filep->private_data;
724 	struct vhost_dev *d = &v->vdev;
725 
726 	mutex_lock(&d->mutex);
727 	filep->private_data = NULL;
728 	vhost_vdpa_reset(v);
729 	vhost_dev_stop(&v->vdev);
730 	vhost_vdpa_iotlb_free(v);
731 	vhost_vdpa_free_domain(v);
732 	vhost_dev_cleanup(&v->vdev);
733 	kfree(v->vdev.vqs);
734 	mutex_unlock(&d->mutex);
735 
736 	atomic_dec(&v->opened);
737 	complete(&v->completion);
738 
739 	return 0;
740 }
741 
742 static const struct file_operations vhost_vdpa_fops = {
743 	.owner		= THIS_MODULE,
744 	.open		= vhost_vdpa_open,
745 	.release	= vhost_vdpa_release,
746 	.write_iter	= vhost_vdpa_chr_write_iter,
747 	.unlocked_ioctl	= vhost_vdpa_unlocked_ioctl,
748 	.compat_ioctl	= compat_ptr_ioctl,
749 };
750 
751 static void vhost_vdpa_release_dev(struct device *device)
752 {
753 	struct vhost_vdpa *v =
754 	       container_of(device, struct vhost_vdpa, dev);
755 
756 	ida_simple_remove(&vhost_vdpa_ida, v->minor);
757 	kfree(v->vqs);
758 	kfree(v);
759 }
760 
761 static int vhost_vdpa_probe(struct vdpa_device *vdpa)
762 {
763 	const struct vdpa_config_ops *ops = vdpa->config;
764 	struct vhost_vdpa *v;
765 	int minor, nvqs = VHOST_VDPA_VQ_MAX;
766 	int r;
767 
768 	/* Currently, we only accept the network devices. */
769 	if (ops->get_device_id(vdpa) != VIRTIO_ID_NET)
770 		return -ENOTSUPP;
771 
772 	v = kzalloc(sizeof(*v), GFP_KERNEL | __GFP_RETRY_MAYFAIL);
773 	if (!v)
774 		return -ENOMEM;
775 
776 	minor = ida_simple_get(&vhost_vdpa_ida, 0,
777 			       VHOST_VDPA_DEV_MAX, GFP_KERNEL);
778 	if (minor < 0) {
779 		kfree(v);
780 		return minor;
781 	}
782 
783 	atomic_set(&v->opened, 0);
784 	v->minor = minor;
785 	v->vdpa = vdpa;
786 	v->nvqs = nvqs;
787 	v->virtio_id = ops->get_device_id(vdpa);
788 
789 	device_initialize(&v->dev);
790 	v->dev.release = vhost_vdpa_release_dev;
791 	v->dev.parent = &vdpa->dev;
792 	v->dev.devt = MKDEV(MAJOR(vhost_vdpa_major), minor);
793 	v->vqs = kmalloc_array(nvqs, sizeof(struct vhost_virtqueue),
794 			       GFP_KERNEL);
795 	if (!v->vqs) {
796 		r = -ENOMEM;
797 		goto err;
798 	}
799 
800 	r = dev_set_name(&v->dev, "vhost-vdpa-%u", minor);
801 	if (r)
802 		goto err;
803 
804 	cdev_init(&v->cdev, &vhost_vdpa_fops);
805 	v->cdev.owner = THIS_MODULE;
806 
807 	r = cdev_device_add(&v->cdev, &v->dev);
808 	if (r)
809 		goto err;
810 
811 	init_completion(&v->completion);
812 	vdpa_set_drvdata(vdpa, v);
813 
814 	return 0;
815 
816 err:
817 	put_device(&v->dev);
818 	return r;
819 }
820 
821 static void vhost_vdpa_remove(struct vdpa_device *vdpa)
822 {
823 	struct vhost_vdpa *v = vdpa_get_drvdata(vdpa);
824 	int opened;
825 
826 	cdev_device_del(&v->cdev, &v->dev);
827 
828 	do {
829 		opened = atomic_cmpxchg(&v->opened, 0, 1);
830 		if (!opened)
831 			break;
832 		wait_for_completion(&v->completion);
833 	} while (1);
834 
835 	put_device(&v->dev);
836 }
837 
838 static struct vdpa_driver vhost_vdpa_driver = {
839 	.driver = {
840 		.name	= "vhost_vdpa",
841 	},
842 	.probe	= vhost_vdpa_probe,
843 	.remove	= vhost_vdpa_remove,
844 };
845 
846 static int __init vhost_vdpa_init(void)
847 {
848 	int r;
849 
850 	r = alloc_chrdev_region(&vhost_vdpa_major, 0, VHOST_VDPA_DEV_MAX,
851 				"vhost-vdpa");
852 	if (r)
853 		goto err_alloc_chrdev;
854 
855 	r = vdpa_register_driver(&vhost_vdpa_driver);
856 	if (r)
857 		goto err_vdpa_register_driver;
858 
859 	return 0;
860 
861 err_vdpa_register_driver:
862 	unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX);
863 err_alloc_chrdev:
864 	return r;
865 }
866 module_init(vhost_vdpa_init);
867 
868 static void __exit vhost_vdpa_exit(void)
869 {
870 	vdpa_unregister_driver(&vhost_vdpa_driver);
871 	unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX);
872 }
873 module_exit(vhost_vdpa_exit);
874 
875 MODULE_VERSION("0.0.1");
876 MODULE_LICENSE("GPL v2");
877 MODULE_AUTHOR("Intel Corporation");
878 MODULE_DESCRIPTION("vDPA-based vhost backend for virtio");
879