xref: /openbmc/linux/drivers/vhost/vdpa.c (revision 82df5b73) 
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2018-2020 Intel Corporation.
4  * Copyright (C) 2020 Red Hat, Inc.
5  *
6  * Author: Tiwei Bie <tiwei.bie@intel.com>
7  *         Jason Wang <jasowang@redhat.com>
8  *
9  * Thanks Michael S. Tsirkin for the valuable comments and
10  * suggestions.  And thanks to Cunming Liang and Zhihong Wang for all
11  * their supports.
12  */
13 
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/cdev.h>
17 #include <linux/device.h>
18 #include <linux/mm.h>
19 #include <linux/iommu.h>
20 #include <linux/uuid.h>
21 #include <linux/vdpa.h>
22 #include <linux/nospec.h>
23 #include <linux/vhost.h>
24 #include <linux/virtio_net.h>
25 #include <linux/kernel.h>
26 
27 #include "vhost.h"
28 
29 enum {
30 	VHOST_VDPA_FEATURES =
31 		(1ULL << VIRTIO_F_NOTIFY_ON_EMPTY) |
32 		(1ULL << VIRTIO_F_ANY_LAYOUT) |
33 		(1ULL << VIRTIO_F_VERSION_1) |
34 		(1ULL << VIRTIO_F_IOMMU_PLATFORM) |
35 		(1ULL << VIRTIO_F_RING_PACKED) |
36 		(1ULL << VIRTIO_F_ORDER_PLATFORM) |
37 		(1ULL << VIRTIO_RING_F_INDIRECT_DESC) |
38 		(1ULL << VIRTIO_RING_F_EVENT_IDX),
39 
40 	VHOST_VDPA_NET_FEATURES = VHOST_VDPA_FEATURES |
41 		(1ULL << VIRTIO_NET_F_CSUM) |
42 		(1ULL << VIRTIO_NET_F_GUEST_CSUM) |
43 		(1ULL << VIRTIO_NET_F_MTU) |
44 		(1ULL << VIRTIO_NET_F_MAC) |
45 		(1ULL << VIRTIO_NET_F_GUEST_TSO4) |
46 		(1ULL << VIRTIO_NET_F_GUEST_TSO6) |
47 		(1ULL << VIRTIO_NET_F_GUEST_ECN) |
48 		(1ULL << VIRTIO_NET_F_GUEST_UFO) |
49 		(1ULL << VIRTIO_NET_F_HOST_TSO4) |
50 		(1ULL << VIRTIO_NET_F_HOST_TSO6) |
51 		(1ULL << VIRTIO_NET_F_HOST_ECN) |
52 		(1ULL << VIRTIO_NET_F_HOST_UFO) |
53 		(1ULL << VIRTIO_NET_F_MRG_RXBUF) |
54 		(1ULL << VIRTIO_NET_F_STATUS) |
55 		(1ULL << VIRTIO_NET_F_SPEED_DUPLEX),
56 };
57 
58 /* Currently, only network backend w/o multiqueue is supported. */
59 #define VHOST_VDPA_VQ_MAX	2
60 
61 #define VHOST_VDPA_DEV_MAX (1U << MINORBITS)
62 
63 struct vhost_vdpa {
64 	struct vhost_dev vdev;
65 	struct iommu_domain *domain;
66 	struct vhost_virtqueue *vqs;
67 	struct completion completion;
68 	struct vdpa_device *vdpa;
69 	struct device dev;
70 	struct cdev cdev;
71 	atomic_t opened;
72 	int nvqs;
73 	int virtio_id;
74 	int minor;
75 	struct eventfd_ctx *config_ctx;
76 };
77 
78 static DEFINE_IDA(vhost_vdpa_ida);
79 
80 static dev_t vhost_vdpa_major;
81 
82 static const u64 vhost_vdpa_features[] = {
83 	[VIRTIO_ID_NET] = VHOST_VDPA_NET_FEATURES,
84 };
85 
86 static void handle_vq_kick(struct vhost_work *work)
87 {
88 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
89 						  poll.work);
90 	struct vhost_vdpa *v = container_of(vq->dev, struct vhost_vdpa, vdev);
91 	const struct vdpa_config_ops *ops = v->vdpa->config;
92 
93 	ops->kick_vq(v->vdpa, vq - v->vqs);
94 }
95 
96 static irqreturn_t vhost_vdpa_virtqueue_cb(void *private)
97 {
98 	struct vhost_virtqueue *vq = private;
99 	struct eventfd_ctx *call_ctx = vq->call_ctx;
100 
101 	if (call_ctx)
102 		eventfd_signal(call_ctx, 1);
103 
104 	return IRQ_HANDLED;
105 }
106 
107 static irqreturn_t vhost_vdpa_config_cb(void *private)
108 {
109 	struct vhost_vdpa *v = private;
110 	struct eventfd_ctx *config_ctx = v->config_ctx;
111 
112 	if (config_ctx)
113 		eventfd_signal(config_ctx, 1);
114 
115 	return IRQ_HANDLED;
116 }
117 
118 static void vhost_vdpa_reset(struct vhost_vdpa *v)
119 {
120 	struct vdpa_device *vdpa = v->vdpa;
121 	const struct vdpa_config_ops *ops = vdpa->config;
122 
123 	ops->set_status(vdpa, 0);
124 }
125 
126 static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp)
127 {
128 	struct vdpa_device *vdpa = v->vdpa;
129 	const struct vdpa_config_ops *ops = vdpa->config;
130 	u32 device_id;
131 
132 	device_id = ops->get_device_id(vdpa);
133 
134 	if (copy_to_user(argp, &device_id, sizeof(device_id)))
135 		return -EFAULT;
136 
137 	return 0;
138 }
139 
140 static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp)
141 {
142 	struct vdpa_device *vdpa = v->vdpa;
143 	const struct vdpa_config_ops *ops = vdpa->config;
144 	u8 status;
145 
146 	status = ops->get_status(vdpa);
147 
148 	if (copy_to_user(statusp, &status, sizeof(status)))
149 		return -EFAULT;
150 
151 	return 0;
152 }
153 
154 static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp)
155 {
156 	struct vdpa_device *vdpa = v->vdpa;
157 	const struct vdpa_config_ops *ops = vdpa->config;
158 	u8 status;
159 
160 	if (copy_from_user(&status, statusp, sizeof(status)))
161 		return -EFAULT;
162 
163 	/*
164 	 * Userspace shouldn't remove status bits unless reset the
165 	 * status to 0.
166 	 */
167 	if (status != 0 && (ops->get_status(vdpa) & ~status) != 0)
168 		return -EINVAL;
169 
170 	ops->set_status(vdpa, status);
171 
172 	return 0;
173 }
174 
175 static int vhost_vdpa_config_validate(struct vhost_vdpa *v,
176 				      struct vhost_vdpa_config *c)
177 {
178 	long size = 0;
179 
180 	switch (v->virtio_id) {
181 	case VIRTIO_ID_NET:
182 		size = sizeof(struct virtio_net_config);
183 		break;
184 	}
185 
186 	if (c->len == 0)
187 		return -EINVAL;
188 
189 	if (c->len > size - c->off)
190 		return -E2BIG;
191 
192 	return 0;
193 }
194 
195 static long vhost_vdpa_get_config(struct vhost_vdpa *v,
196 				  struct vhost_vdpa_config __user *c)
197 {
198 	struct vdpa_device *vdpa = v->vdpa;
199 	const struct vdpa_config_ops *ops = vdpa->config;
200 	struct vhost_vdpa_config config;
201 	unsigned long size = offsetof(struct vhost_vdpa_config, buf);
202 	u8 *buf;
203 
204 	if (copy_from_user(&config, c, size))
205 		return -EFAULT;
206 	if (vhost_vdpa_config_validate(v, &config))
207 		return -EINVAL;
208 	buf = kvzalloc(config.len, GFP_KERNEL);
209 	if (!buf)
210 		return -ENOMEM;
211 
212 	ops->get_config(vdpa, config.off, buf, config.len);
213 
214 	if (copy_to_user(c->buf, buf, config.len)) {
215 		kvfree(buf);
216 		return -EFAULT;
217 	}
218 
219 	kvfree(buf);
220 	return 0;
221 }
222 
223 static long vhost_vdpa_set_config(struct vhost_vdpa *v,
224 				  struct vhost_vdpa_config __user *c)
225 {
226 	struct vdpa_device *vdpa = v->vdpa;
227 	const struct vdpa_config_ops *ops = vdpa->config;
228 	struct vhost_vdpa_config config;
229 	unsigned long size = offsetof(struct vhost_vdpa_config, buf);
230 	u8 *buf;
231 
232 	if (copy_from_user(&config, c, size))
233 		return -EFAULT;
234 	if (vhost_vdpa_config_validate(v, &config))
235 		return -EINVAL;
236 	buf = kvzalloc(config.len, GFP_KERNEL);
237 	if (!buf)
238 		return -ENOMEM;
239 
240 	if (copy_from_user(buf, c->buf, config.len)) {
241 		kvfree(buf);
242 		return -EFAULT;
243 	}
244 
245 	ops->set_config(vdpa, config.off, buf, config.len);
246 
247 	kvfree(buf);
248 	return 0;
249 }
250 
251 static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep)
252 {
253 	struct vdpa_device *vdpa = v->vdpa;
254 	const struct vdpa_config_ops *ops = vdpa->config;
255 	u64 features;
256 
257 	features = ops->get_features(vdpa);
258 	features &= vhost_vdpa_features[v->virtio_id];
259 
260 	if (copy_to_user(featurep, &features, sizeof(features)))
261 		return -EFAULT;
262 
263 	return 0;
264 }
265 
266 static long vhost_vdpa_set_features(struct vhost_vdpa *v, u64 __user *featurep)
267 {
268 	struct vdpa_device *vdpa = v->vdpa;
269 	const struct vdpa_config_ops *ops = vdpa->config;
270 	u64 features;
271 
272 	/*
273 	 * It's not allowed to change the features after they have
274 	 * been negotiated.
275 	 */
276 	if (ops->get_status(vdpa) & VIRTIO_CONFIG_S_FEATURES_OK)
277 		return -EBUSY;
278 
279 	if (copy_from_user(&features, featurep, sizeof(features)))
280 		return -EFAULT;
281 
282 	if (features & ~vhost_vdpa_features[v->virtio_id])
283 		return -EINVAL;
284 
285 	if (ops->set_features(vdpa, features))
286 		return -EINVAL;
287 
288 	return 0;
289 }
290 
291 static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp)
292 {
293 	struct vdpa_device *vdpa = v->vdpa;
294 	const struct vdpa_config_ops *ops = vdpa->config;
295 	u16 num;
296 
297 	num = ops->get_vq_num_max(vdpa);
298 
299 	if (copy_to_user(argp, &num, sizeof(num)))
300 		return -EFAULT;
301 
302 	return 0;
303 }
304 
305 static void vhost_vdpa_config_put(struct vhost_vdpa *v)
306 {
307 	if (v->config_ctx)
308 		eventfd_ctx_put(v->config_ctx);
309 }
310 
311 static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp)
312 {
313 	struct vdpa_callback cb;
314 	int fd;
315 	struct eventfd_ctx *ctx;
316 
317 	cb.callback = vhost_vdpa_config_cb;
318 	cb.private = v->vdpa;
319 	if (copy_from_user(&fd, argp, sizeof(fd)))
320 		return  -EFAULT;
321 
322 	ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd);
323 	swap(ctx, v->config_ctx);
324 
325 	if (!IS_ERR_OR_NULL(ctx))
326 		eventfd_ctx_put(ctx);
327 
328 	if (IS_ERR(v->config_ctx))
329 		return PTR_ERR(v->config_ctx);
330 
331 	v->vdpa->config->set_config_cb(v->vdpa, &cb);
332 
333 	return 0;
334 }
335 static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
336 				   void __user *argp)
337 {
338 	struct vdpa_device *vdpa = v->vdpa;
339 	const struct vdpa_config_ops *ops = vdpa->config;
340 	struct vdpa_callback cb;
341 	struct vhost_virtqueue *vq;
342 	struct vhost_vring_state s;
343 	u32 idx;
344 	long r;
345 
346 	r = get_user(idx, (u32 __user *)argp);
347 	if (r < 0)
348 		return r;
349 
350 	if (idx >= v->nvqs)
351 		return -ENOBUFS;
352 
353 	idx = array_index_nospec(idx, v->nvqs);
354 	vq = &v->vqs[idx];
355 
356 	if (cmd == VHOST_VDPA_SET_VRING_ENABLE) {
357 		if (copy_from_user(&s, argp, sizeof(s)))
358 			return -EFAULT;
359 		ops->set_vq_ready(vdpa, idx, s.num);
360 		return 0;
361 	}
362 
363 	if (cmd == VHOST_GET_VRING_BASE)
364 		vq->last_avail_idx = ops->get_vq_state(v->vdpa, idx);
365 
366 	r = vhost_vring_ioctl(&v->vdev, cmd, argp);
367 	if (r)
368 		return r;
369 
370 	switch (cmd) {
371 	case VHOST_SET_VRING_ADDR:
372 		if (ops->set_vq_address(vdpa, idx,
373 					(u64)(uintptr_t)vq->desc,
374 					(u64)(uintptr_t)vq->avail,
375 					(u64)(uintptr_t)vq->used))
376 			r = -EINVAL;
377 		break;
378 
379 	case VHOST_SET_VRING_BASE:
380 		if (ops->set_vq_state(vdpa, idx, vq->last_avail_idx))
381 			r = -EINVAL;
382 		break;
383 
384 	case VHOST_SET_VRING_CALL:
385 		if (vq->call_ctx) {
386 			cb.callback = vhost_vdpa_virtqueue_cb;
387 			cb.private = vq;
388 		} else {
389 			cb.callback = NULL;
390 			cb.private = NULL;
391 		}
392 		ops->set_vq_cb(vdpa, idx, &cb);
393 		break;
394 
395 	case VHOST_SET_VRING_NUM:
396 		ops->set_vq_num(vdpa, idx, vq->num);
397 		break;
398 	}
399 
400 	return r;
401 }
402 
403 static long vhost_vdpa_unlocked_ioctl(struct file *filep,
404 				      unsigned int cmd, unsigned long arg)
405 {
406 	struct vhost_vdpa *v = filep->private_data;
407 	struct vhost_dev *d = &v->vdev;
408 	void __user *argp = (void __user *)arg;
409 	long r;
410 
411 	mutex_lock(&d->mutex);
412 
413 	switch (cmd) {
414 	case VHOST_VDPA_GET_DEVICE_ID:
415 		r = vhost_vdpa_get_device_id(v, argp);
416 		break;
417 	case VHOST_VDPA_GET_STATUS:
418 		r = vhost_vdpa_get_status(v, argp);
419 		break;
420 	case VHOST_VDPA_SET_STATUS:
421 		r = vhost_vdpa_set_status(v, argp);
422 		break;
423 	case VHOST_VDPA_GET_CONFIG:
424 		r = vhost_vdpa_get_config(v, argp);
425 		break;
426 	case VHOST_VDPA_SET_CONFIG:
427 		r = vhost_vdpa_set_config(v, argp);
428 		break;
429 	case VHOST_GET_FEATURES:
430 		r = vhost_vdpa_get_features(v, argp);
431 		break;
432 	case VHOST_SET_FEATURES:
433 		r = vhost_vdpa_set_features(v, argp);
434 		break;
435 	case VHOST_VDPA_GET_VRING_NUM:
436 		r = vhost_vdpa_get_vring_num(v, argp);
437 		break;
438 	case VHOST_SET_LOG_BASE:
439 	case VHOST_SET_LOG_FD:
440 		r = -ENOIOCTLCMD;
441 		break;
442 	case VHOST_VDPA_SET_CONFIG_CALL:
443 		r = vhost_vdpa_set_config_call(v, argp);
444 		break;
445 	default:
446 		r = vhost_dev_ioctl(&v->vdev, cmd, argp);
447 		if (r == -ENOIOCTLCMD)
448 			r = vhost_vdpa_vring_ioctl(v, cmd, argp);
449 		break;
450 	}
451 
452 	mutex_unlock(&d->mutex);
453 	return r;
454 }
455 
456 static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v, u64 start, u64 last)
457 {
458 	struct vhost_dev *dev = &v->vdev;
459 	struct vhost_iotlb *iotlb = dev->iotlb;
460 	struct vhost_iotlb_map *map;
461 	struct page *page;
462 	unsigned long pfn, pinned;
463 
464 	while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) {
465 		pinned = map->size >> PAGE_SHIFT;
466 		for (pfn = map->addr >> PAGE_SHIFT;
467 		     pinned > 0; pfn++, pinned--) {
468 			page = pfn_to_page(pfn);
469 			if (map->perm & VHOST_ACCESS_WO)
470 				set_page_dirty_lock(page);
471 			unpin_user_page(page);
472 		}
473 		atomic64_sub(map->size >> PAGE_SHIFT, &dev->mm->pinned_vm);
474 		vhost_iotlb_map_free(iotlb, map);
475 	}
476 }
477 
478 static void vhost_vdpa_iotlb_free(struct vhost_vdpa *v)
479 {
480 	struct vhost_dev *dev = &v->vdev;
481 
482 	vhost_vdpa_iotlb_unmap(v, 0ULL, 0ULL - 1);
483 	kfree(dev->iotlb);
484 	dev->iotlb = NULL;
485 }
486 
487 static int perm_to_iommu_flags(u32 perm)
488 {
489 	int flags = 0;
490 
491 	switch (perm) {
492 	case VHOST_ACCESS_WO:
493 		flags |= IOMMU_WRITE;
494 		break;
495 	case VHOST_ACCESS_RO:
496 		flags |= IOMMU_READ;
497 		break;
498 	case VHOST_ACCESS_RW:
499 		flags |= (IOMMU_WRITE | IOMMU_READ);
500 		break;
501 	default:
502 		WARN(1, "invalidate vhost IOTLB permission\n");
503 		break;
504 	}
505 
506 	return flags | IOMMU_CACHE;
507 }
508 
509 static int vhost_vdpa_map(struct vhost_vdpa *v,
510 			  u64 iova, u64 size, u64 pa, u32 perm)
511 {
512 	struct vhost_dev *dev = &v->vdev;
513 	struct vdpa_device *vdpa = v->vdpa;
514 	const struct vdpa_config_ops *ops = vdpa->config;
515 	int r = 0;
516 
517 	r = vhost_iotlb_add_range(dev->iotlb, iova, iova + size - 1,
518 				  pa, perm);
519 	if (r)
520 		return r;
521 
522 	if (ops->dma_map)
523 		r = ops->dma_map(vdpa, iova, size, pa, perm);
524 	else if (ops->set_map)
525 		r = ops->set_map(vdpa, dev->iotlb);
526 	else
527 		r = iommu_map(v->domain, iova, pa, size,
528 			      perm_to_iommu_flags(perm));
529 
530 	return r;
531 }
532 
533 static void vhost_vdpa_unmap(struct vhost_vdpa *v, u64 iova, u64 size)
534 {
535 	struct vhost_dev *dev = &v->vdev;
536 	struct vdpa_device *vdpa = v->vdpa;
537 	const struct vdpa_config_ops *ops = vdpa->config;
538 
539 	vhost_vdpa_iotlb_unmap(v, iova, iova + size - 1);
540 
541 	if (ops->dma_map)
542 		ops->dma_unmap(vdpa, iova, size);
543 	else if (ops->set_map)
544 		ops->set_map(vdpa, dev->iotlb);
545 	else
546 		iommu_unmap(v->domain, iova, size);
547 }
548 
549 static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v,
550 					   struct vhost_iotlb_msg *msg)
551 {
552 	struct vhost_dev *dev = &v->vdev;
553 	struct vhost_iotlb *iotlb = dev->iotlb;
554 	struct page **page_list;
555 	unsigned long list_size = PAGE_SIZE / sizeof(struct page *);
556 	unsigned int gup_flags = FOLL_LONGTERM;
557 	unsigned long npages, cur_base, map_pfn, last_pfn = 0;
558 	unsigned long locked, lock_limit, pinned, i;
559 	u64 iova = msg->iova;
560 	int ret = 0;
561 
562 	if (vhost_iotlb_itree_first(iotlb, msg->iova,
563 				    msg->iova + msg->size - 1))
564 		return -EEXIST;
565 
566 	page_list = (struct page **) __get_free_page(GFP_KERNEL);
567 	if (!page_list)
568 		return -ENOMEM;
569 
570 	if (msg->perm & VHOST_ACCESS_WO)
571 		gup_flags |= FOLL_WRITE;
572 
573 	npages = PAGE_ALIGN(msg->size + (iova & ~PAGE_MASK)) >> PAGE_SHIFT;
574 	if (!npages)
575 		return -EINVAL;
576 
577 	mmap_read_lock(dev->mm);
578 
579 	locked = atomic64_add_return(npages, &dev->mm->pinned_vm);
580 	lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
581 
582 	if (locked > lock_limit) {
583 		ret = -ENOMEM;
584 		goto out;
585 	}
586 
587 	cur_base = msg->uaddr & PAGE_MASK;
588 	iova &= PAGE_MASK;
589 
590 	while (npages) {
591 		pinned = min_t(unsigned long, npages, list_size);
592 		ret = pin_user_pages(cur_base, pinned,
593 				     gup_flags, page_list, NULL);
594 		if (ret != pinned)
595 			goto out;
596 
597 		if (!last_pfn)
598 			map_pfn = page_to_pfn(page_list[0]);
599 
600 		for (i = 0; i < ret; i++) {
601 			unsigned long this_pfn = page_to_pfn(page_list[i]);
602 			u64 csize;
603 
604 			if (last_pfn && (this_pfn != last_pfn + 1)) {
605 				/* Pin a contiguous chunk of memory */
606 				csize = (last_pfn - map_pfn + 1) << PAGE_SHIFT;
607 				if (vhost_vdpa_map(v, iova, csize,
608 						   map_pfn << PAGE_SHIFT,
609 						   msg->perm))
610 					goto out;
611 				map_pfn = this_pfn;
612 				iova += csize;
613 			}
614 
615 			last_pfn = this_pfn;
616 		}
617 
618 		cur_base += ret << PAGE_SHIFT;
619 		npages -= ret;
620 	}
621 
622 	/* Pin the rest chunk */
623 	ret = vhost_vdpa_map(v, iova, (last_pfn - map_pfn + 1) << PAGE_SHIFT,
624 			     map_pfn << PAGE_SHIFT, msg->perm);
625 out:
626 	if (ret) {
627 		vhost_vdpa_unmap(v, msg->iova, msg->size);
628 		atomic64_sub(npages, &dev->mm->pinned_vm);
629 	}
630 	mmap_read_unlock(dev->mm);
631 	free_page((unsigned long)page_list);
632 	return ret;
633 }
634 
635 static int vhost_vdpa_process_iotlb_msg(struct vhost_dev *dev,
636 					struct vhost_iotlb_msg *msg)
637 {
638 	struct vhost_vdpa *v = container_of(dev, struct vhost_vdpa, vdev);
639 	int r = 0;
640 
641 	r = vhost_dev_check_owner(dev);
642 	if (r)
643 		return r;
644 
645 	switch (msg->type) {
646 	case VHOST_IOTLB_UPDATE:
647 		r = vhost_vdpa_process_iotlb_update(v, msg);
648 		break;
649 	case VHOST_IOTLB_INVALIDATE:
650 		vhost_vdpa_unmap(v, msg->iova, msg->size);
651 		break;
652 	default:
653 		r = -EINVAL;
654 		break;
655 	}
656 
657 	return r;
658 }
659 
660 static ssize_t vhost_vdpa_chr_write_iter(struct kiocb *iocb,
661 					 struct iov_iter *from)
662 {
663 	struct file *file = iocb->ki_filp;
664 	struct vhost_vdpa *v = file->private_data;
665 	struct vhost_dev *dev = &v->vdev;
666 
667 	return vhost_chr_write_iter(dev, from);
668 }
669 
670 static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v)
671 {
672 	struct vdpa_device *vdpa = v->vdpa;
673 	const struct vdpa_config_ops *ops = vdpa->config;
674 	struct device *dma_dev = vdpa_get_dma_dev(vdpa);
675 	struct bus_type *bus;
676 	int ret;
677 
678 	/* Device want to do DMA by itself */
679 	if (ops->set_map || ops->dma_map)
680 		return 0;
681 
682 	bus = dma_dev->bus;
683 	if (!bus)
684 		return -EFAULT;
685 
686 	if (!iommu_capable(bus, IOMMU_CAP_CACHE_COHERENCY))
687 		return -ENOTSUPP;
688 
689 	v->domain = iommu_domain_alloc(bus);
690 	if (!v->domain)
691 		return -EIO;
692 
693 	ret = iommu_attach_device(v->domain, dma_dev);
694 	if (ret)
695 		goto err_attach;
696 
697 	return 0;
698 
699 err_attach:
700 	iommu_domain_free(v->domain);
701 	return ret;
702 }
703 
704 static void vhost_vdpa_free_domain(struct vhost_vdpa *v)
705 {
706 	struct vdpa_device *vdpa = v->vdpa;
707 	struct device *dma_dev = vdpa_get_dma_dev(vdpa);
708 
709 	if (v->domain) {
710 		iommu_detach_device(v->domain, dma_dev);
711 		iommu_domain_free(v->domain);
712 	}
713 
714 	v->domain = NULL;
715 }
716 
717 static int vhost_vdpa_open(struct inode *inode, struct file *filep)
718 {
719 	struct vhost_vdpa *v;
720 	struct vhost_dev *dev;
721 	struct vhost_virtqueue **vqs;
722 	int nvqs, i, r, opened;
723 
724 	v = container_of(inode->i_cdev, struct vhost_vdpa, cdev);
725 
726 	opened = atomic_cmpxchg(&v->opened, 0, 1);
727 	if (opened)
728 		return -EBUSY;
729 
730 	nvqs = v->nvqs;
731 	vhost_vdpa_reset(v);
732 
733 	vqs = kmalloc_array(nvqs, sizeof(*vqs), GFP_KERNEL);
734 	if (!vqs) {
735 		r = -ENOMEM;
736 		goto err;
737 	}
738 
739 	dev = &v->vdev;
740 	for (i = 0; i < nvqs; i++) {
741 		vqs[i] = &v->vqs[i];
742 		vqs[i]->handle_kick = handle_vq_kick;
743 	}
744 	vhost_dev_init(dev, vqs, nvqs, 0, 0, 0, false,
745 		       vhost_vdpa_process_iotlb_msg);
746 
747 	dev->iotlb = vhost_iotlb_alloc(0, 0);
748 	if (!dev->iotlb) {
749 		r = -ENOMEM;
750 		goto err_init_iotlb;
751 	}
752 
753 	r = vhost_vdpa_alloc_domain(v);
754 	if (r)
755 		goto err_init_iotlb;
756 
757 	filep->private_data = v;
758 
759 	return 0;
760 
761 err_init_iotlb:
762 	vhost_dev_cleanup(&v->vdev);
763 err:
764 	atomic_dec(&v->opened);
765 	return r;
766 }
767 
768 static int vhost_vdpa_release(struct inode *inode, struct file *filep)
769 {
770 	struct vhost_vdpa *v = filep->private_data;
771 	struct vhost_dev *d = &v->vdev;
772 
773 	mutex_lock(&d->mutex);
774 	filep->private_data = NULL;
775 	vhost_vdpa_reset(v);
776 	vhost_dev_stop(&v->vdev);
777 	vhost_vdpa_iotlb_free(v);
778 	vhost_vdpa_free_domain(v);
779 	vhost_vdpa_config_put(v);
780 	vhost_dev_cleanup(&v->vdev);
781 	kfree(v->vdev.vqs);
782 	mutex_unlock(&d->mutex);
783 
784 	atomic_dec(&v->opened);
785 	complete(&v->completion);
786 
787 	return 0;
788 }
789 
790 #ifdef CONFIG_MMU
791 static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf)
792 {
793 	struct vhost_vdpa *v = vmf->vma->vm_file->private_data;
794 	struct vdpa_device *vdpa = v->vdpa;
795 	const struct vdpa_config_ops *ops = vdpa->config;
796 	struct vdpa_notification_area notify;
797 	struct vm_area_struct *vma = vmf->vma;
798 	u16 index = vma->vm_pgoff;
799 
800 	notify = ops->get_vq_notification(vdpa, index);
801 
802 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
803 	if (remap_pfn_range(vma, vmf->address & PAGE_MASK,
804 			    notify.addr >> PAGE_SHIFT, PAGE_SIZE,
805 			    vma->vm_page_prot))
806 		return VM_FAULT_SIGBUS;
807 
808 	return VM_FAULT_NOPAGE;
809 }
810 
811 static const struct vm_operations_struct vhost_vdpa_vm_ops = {
812 	.fault = vhost_vdpa_fault,
813 };
814 
815 static int vhost_vdpa_mmap(struct file *file, struct vm_area_struct *vma)
816 {
817 	struct vhost_vdpa *v = vma->vm_file->private_data;
818 	struct vdpa_device *vdpa = v->vdpa;
819 	const struct vdpa_config_ops *ops = vdpa->config;
820 	struct vdpa_notification_area notify;
821 	int index = vma->vm_pgoff;
822 
823 	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
824 		return -EINVAL;
825 	if ((vma->vm_flags & VM_SHARED) == 0)
826 		return -EINVAL;
827 	if (vma->vm_flags & VM_READ)
828 		return -EINVAL;
829 	if (index > 65535)
830 		return -EINVAL;
831 	if (!ops->get_vq_notification)
832 		return -ENOTSUPP;
833 
834 	/* To be safe and easily modelled by userspace, We only
835 	 * support the doorbell which sits on the page boundary and
836 	 * does not share the page with other registers.
837 	 */
838 	notify = ops->get_vq_notification(vdpa, index);
839 	if (notify.addr & (PAGE_SIZE - 1))
840 		return -EINVAL;
841 	if (vma->vm_end - vma->vm_start != notify.size)
842 		return -ENOTSUPP;
843 
844 	vma->vm_ops = &vhost_vdpa_vm_ops;
845 	return 0;
846 }
847 #endif /* CONFIG_MMU */
848 
849 static const struct file_operations vhost_vdpa_fops = {
850 	.owner		= THIS_MODULE,
851 	.open		= vhost_vdpa_open,
852 	.release	= vhost_vdpa_release,
853 	.write_iter	= vhost_vdpa_chr_write_iter,
854 	.unlocked_ioctl	= vhost_vdpa_unlocked_ioctl,
855 #ifdef CONFIG_MMU
856 	.mmap		= vhost_vdpa_mmap,
857 #endif /* CONFIG_MMU */
858 	.compat_ioctl	= compat_ptr_ioctl,
859 };
860 
861 static void vhost_vdpa_release_dev(struct device *device)
862 {
863 	struct vhost_vdpa *v =
864 	       container_of(device, struct vhost_vdpa, dev);
865 
866 	ida_simple_remove(&vhost_vdpa_ida, v->minor);
867 	kfree(v->vqs);
868 	kfree(v);
869 }
870 
871 static int vhost_vdpa_probe(struct vdpa_device *vdpa)
872 {
873 	const struct vdpa_config_ops *ops = vdpa->config;
874 	struct vhost_vdpa *v;
875 	int minor, nvqs = VHOST_VDPA_VQ_MAX;
876 	int r;
877 
878 	/* Currently, we only accept the network devices. */
879 	if (ops->get_device_id(vdpa) != VIRTIO_ID_NET)
880 		return -ENOTSUPP;
881 
882 	v = kzalloc(sizeof(*v), GFP_KERNEL | __GFP_RETRY_MAYFAIL);
883 	if (!v)
884 		return -ENOMEM;
885 
886 	minor = ida_simple_get(&vhost_vdpa_ida, 0,
887 			       VHOST_VDPA_DEV_MAX, GFP_KERNEL);
888 	if (minor < 0) {
889 		kfree(v);
890 		return minor;
891 	}
892 
893 	atomic_set(&v->opened, 0);
894 	v->minor = minor;
895 	v->vdpa = vdpa;
896 	v->nvqs = nvqs;
897 	v->virtio_id = ops->get_device_id(vdpa);
898 
899 	device_initialize(&v->dev);
900 	v->dev.release = vhost_vdpa_release_dev;
901 	v->dev.parent = &vdpa->dev;
902 	v->dev.devt = MKDEV(MAJOR(vhost_vdpa_major), minor);
903 	v->vqs = kmalloc_array(nvqs, sizeof(struct vhost_virtqueue),
904 			       GFP_KERNEL);
905 	if (!v->vqs) {
906 		r = -ENOMEM;
907 		goto err;
908 	}
909 
910 	r = dev_set_name(&v->dev, "vhost-vdpa-%u", minor);
911 	if (r)
912 		goto err;
913 
914 	cdev_init(&v->cdev, &vhost_vdpa_fops);
915 	v->cdev.owner = THIS_MODULE;
916 
917 	r = cdev_device_add(&v->cdev, &v->dev);
918 	if (r)
919 		goto err;
920 
921 	init_completion(&v->completion);
922 	vdpa_set_drvdata(vdpa, v);
923 
924 	return 0;
925 
926 err:
927 	put_device(&v->dev);
928 	return r;
929 }
930 
931 static void vhost_vdpa_remove(struct vdpa_device *vdpa)
932 {
933 	struct vhost_vdpa *v = vdpa_get_drvdata(vdpa);
934 	int opened;
935 
936 	cdev_device_del(&v->cdev, &v->dev);
937 
938 	do {
939 		opened = atomic_cmpxchg(&v->opened, 0, 1);
940 		if (!opened)
941 			break;
942 		wait_for_completion(&v->completion);
943 	} while (1);
944 
945 	put_device(&v->dev);
946 }
947 
948 static struct vdpa_driver vhost_vdpa_driver = {
949 	.driver = {
950 		.name	= "vhost_vdpa",
951 	},
952 	.probe	= vhost_vdpa_probe,
953 	.remove	= vhost_vdpa_remove,
954 };
955 
956 static int __init vhost_vdpa_init(void)
957 {
958 	int r;
959 
960 	r = alloc_chrdev_region(&vhost_vdpa_major, 0, VHOST_VDPA_DEV_MAX,
961 				"vhost-vdpa");
962 	if (r)
963 		goto err_alloc_chrdev;
964 
965 	r = vdpa_register_driver(&vhost_vdpa_driver);
966 	if (r)
967 		goto err_vdpa_register_driver;
968 
969 	return 0;
970 
971 err_vdpa_register_driver:
972 	unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX);
973 err_alloc_chrdev:
974 	return r;
975 }
976 module_init(vhost_vdpa_init);
977 
978 static void __exit vhost_vdpa_exit(void)
979 {
980 	vdpa_unregister_driver(&vhost_vdpa_driver);
981 	unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX);
982 }
983 module_exit(vhost_vdpa_exit);
984 
985 MODULE_VERSION("0.0.1");
986 MODULE_LICENSE("GPL v2");
987 MODULE_AUTHOR("Intel Corporation");
988 MODULE_DESCRIPTION("vDPA-based vhost backend for virtio");
989