1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * VDUSE: vDPA Device in Userspace
4  *
5  * Copyright (C) 2020-2021 Bytedance Inc. and/or its affiliates. All rights reserved.
6  *
7  * Author: Xie Yongji <xieyongji@bytedance.com>
8  *
9  */
10 
11 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/cdev.h>
14 #include <linux/device.h>
15 #include <linux/eventfd.h>
16 #include <linux/slab.h>
17 #include <linux/wait.h>
18 #include <linux/dma-map-ops.h>
19 #include <linux/poll.h>
20 #include <linux/file.h>
21 #include <linux/uio.h>
22 #include <linux/vdpa.h>
23 #include <linux/nospec.h>
24 #include <uapi/linux/vduse.h>
25 #include <uapi/linux/vdpa.h>
26 #include <uapi/linux/virtio_config.h>
27 #include <uapi/linux/virtio_ids.h>
28 #include <uapi/linux/virtio_blk.h>
29 #include <linux/mod_devicetable.h>
30 
31 #include "iova_domain.h"
32 
33 #define DRV_AUTHOR   "Yongji Xie <xieyongji@bytedance.com>"
34 #define DRV_DESC     "vDPA Device in Userspace"
35 #define DRV_LICENSE  "GPL v2"
36 
37 #define VDUSE_DEV_MAX (1U << MINORBITS)
38 #define VDUSE_BOUNCE_SIZE (64 * 1024 * 1024)
39 #define VDUSE_IOVA_SIZE (128 * 1024 * 1024)
40 #define VDUSE_MSG_DEFAULT_TIMEOUT 30
41 
42 struct vduse_virtqueue {
43 	u16 index;
44 	u16 num_max;
45 	u32 num;
46 	u64 desc_addr;
47 	u64 driver_addr;
48 	u64 device_addr;
49 	struct vdpa_vq_state state;
50 	bool ready;
51 	bool kicked;
52 	spinlock_t kick_lock;
53 	spinlock_t irq_lock;
54 	struct eventfd_ctx *kickfd;
55 	struct vdpa_callback cb;
56 	struct work_struct inject;
57 	struct work_struct kick;
58 };
59 
60 struct vduse_dev;
61 
62 struct vduse_vdpa {
63 	struct vdpa_device vdpa;
64 	struct vduse_dev *dev;
65 };
66 
67 struct vduse_dev {
68 	struct vduse_vdpa *vdev;
69 	struct device *dev;
70 	struct vduse_virtqueue *vqs;
71 	struct vduse_iova_domain *domain;
72 	char *name;
73 	struct mutex lock;
74 	spinlock_t msg_lock;
75 	u64 msg_unique;
76 	u32 msg_timeout;
77 	wait_queue_head_t waitq;
78 	struct list_head send_list;
79 	struct list_head recv_list;
80 	struct vdpa_callback config_cb;
81 	struct work_struct inject;
82 	spinlock_t irq_lock;
83 	struct rw_semaphore rwsem;
84 	int minor;
85 	bool broken;
86 	bool connected;
87 	u64 api_version;
88 	u64 device_features;
89 	u64 driver_features;
90 	u32 device_id;
91 	u32 vendor_id;
92 	u32 generation;
93 	u32 config_size;
94 	void *config;
95 	u8 status;
96 	u32 vq_num;
97 	u32 vq_align;
98 };
99 
100 struct vduse_dev_msg {
101 	struct vduse_dev_request req;
102 	struct vduse_dev_response resp;
103 	struct list_head list;
104 	wait_queue_head_t waitq;
105 	bool completed;
106 };
107 
108 struct vduse_control {
109 	u64 api_version;
110 };
111 
112 static DEFINE_MUTEX(vduse_lock);
113 static DEFINE_IDR(vduse_idr);
114 
115 static dev_t vduse_major;
116 static struct class *vduse_class;
117 static struct cdev vduse_ctrl_cdev;
118 static struct cdev vduse_cdev;
119 static struct workqueue_struct *vduse_irq_wq;
120 
121 static u32 allowed_device_id[] = {
122 	VIRTIO_ID_BLOCK,
123 };
124 
125 static inline struct vduse_dev *vdpa_to_vduse(struct vdpa_device *vdpa)
126 {
127 	struct vduse_vdpa *vdev = container_of(vdpa, struct vduse_vdpa, vdpa);
128 
129 	return vdev->dev;
130 }
131 
132 static inline struct vduse_dev *dev_to_vduse(struct device *dev)
133 {
134 	struct vdpa_device *vdpa = dev_to_vdpa(dev);
135 
136 	return vdpa_to_vduse(vdpa);
137 }
138 
139 static struct vduse_dev_msg *vduse_find_msg(struct list_head *head,
140 					    uint32_t request_id)
141 {
142 	struct vduse_dev_msg *msg;
143 
144 	list_for_each_entry(msg, head, list) {
145 		if (msg->req.request_id == request_id) {
146 			list_del(&msg->list);
147 			return msg;
148 		}
149 	}
150 
151 	return NULL;
152 }
153 
154 static struct vduse_dev_msg *vduse_dequeue_msg(struct list_head *head)
155 {
156 	struct vduse_dev_msg *msg = NULL;
157 
158 	if (!list_empty(head)) {
159 		msg = list_first_entry(head, struct vduse_dev_msg, list);
160 		list_del(&msg->list);
161 	}
162 
163 	return msg;
164 }
165 
166 static void vduse_enqueue_msg(struct list_head *head,
167 			      struct vduse_dev_msg *msg)
168 {
169 	list_add_tail(&msg->list, head);
170 }
171 
172 static void vduse_dev_broken(struct vduse_dev *dev)
173 {
174 	struct vduse_dev_msg *msg, *tmp;
175 
176 	if (unlikely(dev->broken))
177 		return;
178 
179 	list_splice_init(&dev->recv_list, &dev->send_list);
180 	list_for_each_entry_safe(msg, tmp, &dev->send_list, list) {
181 		list_del(&msg->list);
182 		msg->completed = 1;
183 		msg->resp.result = VDUSE_REQ_RESULT_FAILED;
184 		wake_up(&msg->waitq);
185 	}
186 	dev->broken = true;
187 	wake_up(&dev->waitq);
188 }
189 
190 static int vduse_dev_msg_sync(struct vduse_dev *dev,
191 			      struct vduse_dev_msg *msg)
192 {
193 	int ret;
194 
195 	if (unlikely(dev->broken))
196 		return -EIO;
197 
198 	init_waitqueue_head(&msg->waitq);
199 	spin_lock(&dev->msg_lock);
200 	if (unlikely(dev->broken)) {
201 		spin_unlock(&dev->msg_lock);
202 		return -EIO;
203 	}
204 	msg->req.request_id = dev->msg_unique++;
205 	vduse_enqueue_msg(&dev->send_list, msg);
206 	wake_up(&dev->waitq);
207 	spin_unlock(&dev->msg_lock);
208 	if (dev->msg_timeout)
209 		ret = wait_event_killable_timeout(msg->waitq, msg->completed,
210 						  (long)dev->msg_timeout * HZ);
211 	else
212 		ret = wait_event_killable(msg->waitq, msg->completed);
213 
214 	spin_lock(&dev->msg_lock);
215 	if (!msg->completed) {
216 		list_del(&msg->list);
217 		msg->resp.result = VDUSE_REQ_RESULT_FAILED;
218 		/* Mark the device as malfunction when there is a timeout */
219 		if (!ret)
220 			vduse_dev_broken(dev);
221 	}
222 	ret = (msg->resp.result == VDUSE_REQ_RESULT_OK) ? 0 : -EIO;
223 	spin_unlock(&dev->msg_lock);
224 
225 	return ret;
226 }
227 
228 static int vduse_dev_get_vq_state_packed(struct vduse_dev *dev,
229 					 struct vduse_virtqueue *vq,
230 					 struct vdpa_vq_state_packed *packed)
231 {
232 	struct vduse_dev_msg msg = { 0 };
233 	int ret;
234 
235 	msg.req.type = VDUSE_GET_VQ_STATE;
236 	msg.req.vq_state.index = vq->index;
237 
238 	ret = vduse_dev_msg_sync(dev, &msg);
239 	if (ret)
240 		return ret;
241 
242 	packed->last_avail_counter =
243 			msg.resp.vq_state.packed.last_avail_counter & 0x0001;
244 	packed->last_avail_idx =
245 			msg.resp.vq_state.packed.last_avail_idx & 0x7FFF;
246 	packed->last_used_counter =
247 			msg.resp.vq_state.packed.last_used_counter & 0x0001;
248 	packed->last_used_idx =
249 			msg.resp.vq_state.packed.last_used_idx & 0x7FFF;
250 
251 	return 0;
252 }
253 
254 static int vduse_dev_get_vq_state_split(struct vduse_dev *dev,
255 					struct vduse_virtqueue *vq,
256 					struct vdpa_vq_state_split *split)
257 {
258 	struct vduse_dev_msg msg = { 0 };
259 	int ret;
260 
261 	msg.req.type = VDUSE_GET_VQ_STATE;
262 	msg.req.vq_state.index = vq->index;
263 
264 	ret = vduse_dev_msg_sync(dev, &msg);
265 	if (ret)
266 		return ret;
267 
268 	split->avail_index = msg.resp.vq_state.split.avail_index;
269 
270 	return 0;
271 }
272 
273 static int vduse_dev_set_status(struct vduse_dev *dev, u8 status)
274 {
275 	struct vduse_dev_msg msg = { 0 };
276 
277 	msg.req.type = VDUSE_SET_STATUS;
278 	msg.req.s.status = status;
279 
280 	return vduse_dev_msg_sync(dev, &msg);
281 }
282 
283 static int vduse_dev_update_iotlb(struct vduse_dev *dev,
284 				  u64 start, u64 last)
285 {
286 	struct vduse_dev_msg msg = { 0 };
287 
288 	if (last < start)
289 		return -EINVAL;
290 
291 	msg.req.type = VDUSE_UPDATE_IOTLB;
292 	msg.req.iova.start = start;
293 	msg.req.iova.last = last;
294 
295 	return vduse_dev_msg_sync(dev, &msg);
296 }
297 
298 static ssize_t vduse_dev_read_iter(struct kiocb *iocb, struct iov_iter *to)
299 {
300 	struct file *file = iocb->ki_filp;
301 	struct vduse_dev *dev = file->private_data;
302 	struct vduse_dev_msg *msg;
303 	int size = sizeof(struct vduse_dev_request);
304 	ssize_t ret;
305 
306 	if (iov_iter_count(to) < size)
307 		return -EINVAL;
308 
309 	spin_lock(&dev->msg_lock);
310 	while (1) {
311 		msg = vduse_dequeue_msg(&dev->send_list);
312 		if (msg)
313 			break;
314 
315 		ret = -EAGAIN;
316 		if (file->f_flags & O_NONBLOCK)
317 			goto unlock;
318 
319 		spin_unlock(&dev->msg_lock);
320 		ret = wait_event_interruptible_exclusive(dev->waitq,
321 					!list_empty(&dev->send_list));
322 		if (ret)
323 			return ret;
324 
325 		spin_lock(&dev->msg_lock);
326 	}
327 	spin_unlock(&dev->msg_lock);
328 	ret = copy_to_iter(&msg->req, size, to);
329 	spin_lock(&dev->msg_lock);
330 	if (ret != size) {
331 		ret = -EFAULT;
332 		vduse_enqueue_msg(&dev->send_list, msg);
333 		goto unlock;
334 	}
335 	vduse_enqueue_msg(&dev->recv_list, msg);
336 unlock:
337 	spin_unlock(&dev->msg_lock);
338 
339 	return ret;
340 }
341 
342 static bool is_mem_zero(const char *ptr, int size)
343 {
344 	int i;
345 
346 	for (i = 0; i < size; i++) {
347 		if (ptr[i])
348 			return false;
349 	}
350 	return true;
351 }
352 
353 static ssize_t vduse_dev_write_iter(struct kiocb *iocb, struct iov_iter *from)
354 {
355 	struct file *file = iocb->ki_filp;
356 	struct vduse_dev *dev = file->private_data;
357 	struct vduse_dev_response resp;
358 	struct vduse_dev_msg *msg;
359 	size_t ret;
360 
361 	ret = copy_from_iter(&resp, sizeof(resp), from);
362 	if (ret != sizeof(resp))
363 		return -EINVAL;
364 
365 	if (!is_mem_zero((const char *)resp.reserved, sizeof(resp.reserved)))
366 		return -EINVAL;
367 
368 	spin_lock(&dev->msg_lock);
369 	msg = vduse_find_msg(&dev->recv_list, resp.request_id);
370 	if (!msg) {
371 		ret = -ENOENT;
372 		goto unlock;
373 	}
374 
375 	memcpy(&msg->resp, &resp, sizeof(resp));
376 	msg->completed = 1;
377 	wake_up(&msg->waitq);
378 unlock:
379 	spin_unlock(&dev->msg_lock);
380 
381 	return ret;
382 }
383 
384 static __poll_t vduse_dev_poll(struct file *file, poll_table *wait)
385 {
386 	struct vduse_dev *dev = file->private_data;
387 	__poll_t mask = 0;
388 
389 	poll_wait(file, &dev->waitq, wait);
390 
391 	spin_lock(&dev->msg_lock);
392 
393 	if (unlikely(dev->broken))
394 		mask |= EPOLLERR;
395 	if (!list_empty(&dev->send_list))
396 		mask |= EPOLLIN | EPOLLRDNORM;
397 	if (!list_empty(&dev->recv_list))
398 		mask |= EPOLLOUT | EPOLLWRNORM;
399 
400 	spin_unlock(&dev->msg_lock);
401 
402 	return mask;
403 }
404 
405 static void vduse_dev_reset(struct vduse_dev *dev)
406 {
407 	int i;
408 	struct vduse_iova_domain *domain = dev->domain;
409 
410 	/* The coherent mappings are handled in vduse_dev_free_coherent() */
411 	if (domain->bounce_map)
412 		vduse_domain_reset_bounce_map(domain);
413 
414 	down_write(&dev->rwsem);
415 
416 	dev->status = 0;
417 	dev->driver_features = 0;
418 	dev->generation++;
419 	spin_lock(&dev->irq_lock);
420 	dev->config_cb.callback = NULL;
421 	dev->config_cb.private = NULL;
422 	spin_unlock(&dev->irq_lock);
423 	flush_work(&dev->inject);
424 
425 	for (i = 0; i < dev->vq_num; i++) {
426 		struct vduse_virtqueue *vq = &dev->vqs[i];
427 
428 		vq->ready = false;
429 		vq->desc_addr = 0;
430 		vq->driver_addr = 0;
431 		vq->device_addr = 0;
432 		vq->num = 0;
433 		memset(&vq->state, 0, sizeof(vq->state));
434 
435 		spin_lock(&vq->kick_lock);
436 		vq->kicked = false;
437 		if (vq->kickfd)
438 			eventfd_ctx_put(vq->kickfd);
439 		vq->kickfd = NULL;
440 		spin_unlock(&vq->kick_lock);
441 
442 		spin_lock(&vq->irq_lock);
443 		vq->cb.callback = NULL;
444 		vq->cb.private = NULL;
445 		spin_unlock(&vq->irq_lock);
446 		flush_work(&vq->inject);
447 		flush_work(&vq->kick);
448 	}
449 
450 	up_write(&dev->rwsem);
451 }
452 
453 static int vduse_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 idx,
454 				u64 desc_area, u64 driver_area,
455 				u64 device_area)
456 {
457 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
458 	struct vduse_virtqueue *vq = &dev->vqs[idx];
459 
460 	vq->desc_addr = desc_area;
461 	vq->driver_addr = driver_area;
462 	vq->device_addr = device_area;
463 
464 	return 0;
465 }
466 
467 static void vduse_vq_kick(struct vduse_virtqueue *vq)
468 {
469 	spin_lock(&vq->kick_lock);
470 	if (!vq->ready)
471 		goto unlock;
472 
473 	if (vq->kickfd)
474 		eventfd_signal(vq->kickfd, 1);
475 	else
476 		vq->kicked = true;
477 unlock:
478 	spin_unlock(&vq->kick_lock);
479 }
480 
481 static void vduse_vq_kick_work(struct work_struct *work)
482 {
483 	struct vduse_virtqueue *vq = container_of(work,
484 					struct vduse_virtqueue, kick);
485 
486 	vduse_vq_kick(vq);
487 }
488 
489 static void vduse_vdpa_kick_vq(struct vdpa_device *vdpa, u16 idx)
490 {
491 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
492 	struct vduse_virtqueue *vq = &dev->vqs[idx];
493 
494 	if (!eventfd_signal_allowed()) {
495 		schedule_work(&vq->kick);
496 		return;
497 	}
498 	vduse_vq_kick(vq);
499 }
500 
501 static void vduse_vdpa_set_vq_cb(struct vdpa_device *vdpa, u16 idx,
502 			      struct vdpa_callback *cb)
503 {
504 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
505 	struct vduse_virtqueue *vq = &dev->vqs[idx];
506 
507 	spin_lock(&vq->irq_lock);
508 	vq->cb.callback = cb->callback;
509 	vq->cb.private = cb->private;
510 	spin_unlock(&vq->irq_lock);
511 }
512 
513 static void vduse_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num)
514 {
515 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
516 	struct vduse_virtqueue *vq = &dev->vqs[idx];
517 
518 	vq->num = num;
519 }
520 
521 static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa,
522 					u16 idx, bool ready)
523 {
524 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
525 	struct vduse_virtqueue *vq = &dev->vqs[idx];
526 
527 	vq->ready = ready;
528 }
529 
530 static bool vduse_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 idx)
531 {
532 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
533 	struct vduse_virtqueue *vq = &dev->vqs[idx];
534 
535 	return vq->ready;
536 }
537 
538 static int vduse_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 idx,
539 				const struct vdpa_vq_state *state)
540 {
541 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
542 	struct vduse_virtqueue *vq = &dev->vqs[idx];
543 
544 	if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
545 		vq->state.packed.last_avail_counter =
546 				state->packed.last_avail_counter;
547 		vq->state.packed.last_avail_idx = state->packed.last_avail_idx;
548 		vq->state.packed.last_used_counter =
549 				state->packed.last_used_counter;
550 		vq->state.packed.last_used_idx = state->packed.last_used_idx;
551 	} else
552 		vq->state.split.avail_index = state->split.avail_index;
553 
554 	return 0;
555 }
556 
557 static int vduse_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 idx,
558 				struct vdpa_vq_state *state)
559 {
560 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
561 	struct vduse_virtqueue *vq = &dev->vqs[idx];
562 
563 	if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED))
564 		return vduse_dev_get_vq_state_packed(dev, vq, &state->packed);
565 
566 	return vduse_dev_get_vq_state_split(dev, vq, &state->split);
567 }
568 
569 static u32 vduse_vdpa_get_vq_align(struct vdpa_device *vdpa)
570 {
571 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
572 
573 	return dev->vq_align;
574 }
575 
576 static u64 vduse_vdpa_get_device_features(struct vdpa_device *vdpa)
577 {
578 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
579 
580 	return dev->device_features;
581 }
582 
583 static int vduse_vdpa_set_driver_features(struct vdpa_device *vdpa, u64 features)
584 {
585 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
586 
587 	dev->driver_features = features;
588 	return 0;
589 }
590 
591 static u64 vduse_vdpa_get_driver_features(struct vdpa_device *vdpa)
592 {
593 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
594 
595 	return dev->driver_features;
596 }
597 
598 static void vduse_vdpa_set_config_cb(struct vdpa_device *vdpa,
599 				  struct vdpa_callback *cb)
600 {
601 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
602 
603 	spin_lock(&dev->irq_lock);
604 	dev->config_cb.callback = cb->callback;
605 	dev->config_cb.private = cb->private;
606 	spin_unlock(&dev->irq_lock);
607 }
608 
609 static u16 vduse_vdpa_get_vq_num_max(struct vdpa_device *vdpa)
610 {
611 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
612 	u16 num_max = 0;
613 	int i;
614 
615 	for (i = 0; i < dev->vq_num; i++)
616 		if (num_max < dev->vqs[i].num_max)
617 			num_max = dev->vqs[i].num_max;
618 
619 	return num_max;
620 }
621 
622 static u32 vduse_vdpa_get_device_id(struct vdpa_device *vdpa)
623 {
624 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
625 
626 	return dev->device_id;
627 }
628 
629 static u32 vduse_vdpa_get_vendor_id(struct vdpa_device *vdpa)
630 {
631 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
632 
633 	return dev->vendor_id;
634 }
635 
636 static u8 vduse_vdpa_get_status(struct vdpa_device *vdpa)
637 {
638 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
639 
640 	return dev->status;
641 }
642 
643 static void vduse_vdpa_set_status(struct vdpa_device *vdpa, u8 status)
644 {
645 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
646 
647 	if (vduse_dev_set_status(dev, status))
648 		return;
649 
650 	dev->status = status;
651 }
652 
653 static size_t vduse_vdpa_get_config_size(struct vdpa_device *vdpa)
654 {
655 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
656 
657 	return dev->config_size;
658 }
659 
660 static void vduse_vdpa_get_config(struct vdpa_device *vdpa, unsigned int offset,
661 				  void *buf, unsigned int len)
662 {
663 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
664 
665 	if (offset > dev->config_size ||
666 	    len > dev->config_size - offset)
667 		return;
668 
669 	memcpy(buf, dev->config + offset, len);
670 }
671 
672 static void vduse_vdpa_set_config(struct vdpa_device *vdpa, unsigned int offset,
673 			const void *buf, unsigned int len)
674 {
675 	/* Now we only support read-only configuration space */
676 }
677 
678 static int vduse_vdpa_reset(struct vdpa_device *vdpa)
679 {
680 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
681 	int ret = vduse_dev_set_status(dev, 0);
682 
683 	vduse_dev_reset(dev);
684 
685 	return ret;
686 }
687 
688 static u32 vduse_vdpa_get_generation(struct vdpa_device *vdpa)
689 {
690 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
691 
692 	return dev->generation;
693 }
694 
695 static int vduse_vdpa_set_map(struct vdpa_device *vdpa,
696 				unsigned int asid,
697 				struct vhost_iotlb *iotlb)
698 {
699 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
700 	int ret;
701 
702 	ret = vduse_domain_set_map(dev->domain, iotlb);
703 	if (ret)
704 		return ret;
705 
706 	ret = vduse_dev_update_iotlb(dev, 0ULL, ULLONG_MAX);
707 	if (ret) {
708 		vduse_domain_clear_map(dev->domain, iotlb);
709 		return ret;
710 	}
711 
712 	return 0;
713 }
714 
715 static void vduse_vdpa_free(struct vdpa_device *vdpa)
716 {
717 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
718 
719 	dev->vdev = NULL;
720 }
721 
722 static const struct vdpa_config_ops vduse_vdpa_config_ops = {
723 	.set_vq_address		= vduse_vdpa_set_vq_address,
724 	.kick_vq		= vduse_vdpa_kick_vq,
725 	.set_vq_cb		= vduse_vdpa_set_vq_cb,
726 	.set_vq_num             = vduse_vdpa_set_vq_num,
727 	.set_vq_ready		= vduse_vdpa_set_vq_ready,
728 	.get_vq_ready		= vduse_vdpa_get_vq_ready,
729 	.set_vq_state		= vduse_vdpa_set_vq_state,
730 	.get_vq_state		= vduse_vdpa_get_vq_state,
731 	.get_vq_align		= vduse_vdpa_get_vq_align,
732 	.get_device_features	= vduse_vdpa_get_device_features,
733 	.set_driver_features	= vduse_vdpa_set_driver_features,
734 	.get_driver_features	= vduse_vdpa_get_driver_features,
735 	.set_config_cb		= vduse_vdpa_set_config_cb,
736 	.get_vq_num_max		= vduse_vdpa_get_vq_num_max,
737 	.get_device_id		= vduse_vdpa_get_device_id,
738 	.get_vendor_id		= vduse_vdpa_get_vendor_id,
739 	.get_status		= vduse_vdpa_get_status,
740 	.set_status		= vduse_vdpa_set_status,
741 	.get_config_size	= vduse_vdpa_get_config_size,
742 	.get_config		= vduse_vdpa_get_config,
743 	.set_config		= vduse_vdpa_set_config,
744 	.get_generation		= vduse_vdpa_get_generation,
745 	.reset			= vduse_vdpa_reset,
746 	.set_map		= vduse_vdpa_set_map,
747 	.free			= vduse_vdpa_free,
748 };
749 
750 static dma_addr_t vduse_dev_map_page(struct device *dev, struct page *page,
751 				     unsigned long offset, size_t size,
752 				     enum dma_data_direction dir,
753 				     unsigned long attrs)
754 {
755 	struct vduse_dev *vdev = dev_to_vduse(dev);
756 	struct vduse_iova_domain *domain = vdev->domain;
757 
758 	return vduse_domain_map_page(domain, page, offset, size, dir, attrs);
759 }
760 
761 static void vduse_dev_unmap_page(struct device *dev, dma_addr_t dma_addr,
762 				size_t size, enum dma_data_direction dir,
763 				unsigned long attrs)
764 {
765 	struct vduse_dev *vdev = dev_to_vduse(dev);
766 	struct vduse_iova_domain *domain = vdev->domain;
767 
768 	return vduse_domain_unmap_page(domain, dma_addr, size, dir, attrs);
769 }
770 
771 static void *vduse_dev_alloc_coherent(struct device *dev, size_t size,
772 					dma_addr_t *dma_addr, gfp_t flag,
773 					unsigned long attrs)
774 {
775 	struct vduse_dev *vdev = dev_to_vduse(dev);
776 	struct vduse_iova_domain *domain = vdev->domain;
777 	unsigned long iova;
778 	void *addr;
779 
780 	*dma_addr = DMA_MAPPING_ERROR;
781 	addr = vduse_domain_alloc_coherent(domain, size,
782 				(dma_addr_t *)&iova, flag, attrs);
783 	if (!addr)
784 		return NULL;
785 
786 	*dma_addr = (dma_addr_t)iova;
787 
788 	return addr;
789 }
790 
791 static void vduse_dev_free_coherent(struct device *dev, size_t size,
792 					void *vaddr, dma_addr_t dma_addr,
793 					unsigned long attrs)
794 {
795 	struct vduse_dev *vdev = dev_to_vduse(dev);
796 	struct vduse_iova_domain *domain = vdev->domain;
797 
798 	vduse_domain_free_coherent(domain, size, vaddr, dma_addr, attrs);
799 }
800 
801 static size_t vduse_dev_max_mapping_size(struct device *dev)
802 {
803 	struct vduse_dev *vdev = dev_to_vduse(dev);
804 	struct vduse_iova_domain *domain = vdev->domain;
805 
806 	return domain->bounce_size;
807 }
808 
809 static const struct dma_map_ops vduse_dev_dma_ops = {
810 	.map_page = vduse_dev_map_page,
811 	.unmap_page = vduse_dev_unmap_page,
812 	.alloc = vduse_dev_alloc_coherent,
813 	.free = vduse_dev_free_coherent,
814 	.max_mapping_size = vduse_dev_max_mapping_size,
815 };
816 
817 static unsigned int perm_to_file_flags(u8 perm)
818 {
819 	unsigned int flags = 0;
820 
821 	switch (perm) {
822 	case VDUSE_ACCESS_WO:
823 		flags |= O_WRONLY;
824 		break;
825 	case VDUSE_ACCESS_RO:
826 		flags |= O_RDONLY;
827 		break;
828 	case VDUSE_ACCESS_RW:
829 		flags |= O_RDWR;
830 		break;
831 	default:
832 		WARN(1, "invalidate vhost IOTLB permission\n");
833 		break;
834 	}
835 
836 	return flags;
837 }
838 
839 static int vduse_kickfd_setup(struct vduse_dev *dev,
840 			struct vduse_vq_eventfd *eventfd)
841 {
842 	struct eventfd_ctx *ctx = NULL;
843 	struct vduse_virtqueue *vq;
844 	u32 index;
845 
846 	if (eventfd->index >= dev->vq_num)
847 		return -EINVAL;
848 
849 	index = array_index_nospec(eventfd->index, dev->vq_num);
850 	vq = &dev->vqs[index];
851 	if (eventfd->fd >= 0) {
852 		ctx = eventfd_ctx_fdget(eventfd->fd);
853 		if (IS_ERR(ctx))
854 			return PTR_ERR(ctx);
855 	} else if (eventfd->fd != VDUSE_EVENTFD_DEASSIGN)
856 		return 0;
857 
858 	spin_lock(&vq->kick_lock);
859 	if (vq->kickfd)
860 		eventfd_ctx_put(vq->kickfd);
861 	vq->kickfd = ctx;
862 	if (vq->ready && vq->kicked && vq->kickfd) {
863 		eventfd_signal(vq->kickfd, 1);
864 		vq->kicked = false;
865 	}
866 	spin_unlock(&vq->kick_lock);
867 
868 	return 0;
869 }
870 
871 static bool vduse_dev_is_ready(struct vduse_dev *dev)
872 {
873 	int i;
874 
875 	for (i = 0; i < dev->vq_num; i++)
876 		if (!dev->vqs[i].num_max)
877 			return false;
878 
879 	return true;
880 }
881 
882 static void vduse_dev_irq_inject(struct work_struct *work)
883 {
884 	struct vduse_dev *dev = container_of(work, struct vduse_dev, inject);
885 
886 	spin_lock_irq(&dev->irq_lock);
887 	if (dev->config_cb.callback)
888 		dev->config_cb.callback(dev->config_cb.private);
889 	spin_unlock_irq(&dev->irq_lock);
890 }
891 
892 static void vduse_vq_irq_inject(struct work_struct *work)
893 {
894 	struct vduse_virtqueue *vq = container_of(work,
895 					struct vduse_virtqueue, inject);
896 
897 	spin_lock_irq(&vq->irq_lock);
898 	if (vq->ready && vq->cb.callback)
899 		vq->cb.callback(vq->cb.private);
900 	spin_unlock_irq(&vq->irq_lock);
901 }
902 
903 static int vduse_dev_queue_irq_work(struct vduse_dev *dev,
904 				    struct work_struct *irq_work)
905 {
906 	int ret = -EINVAL;
907 
908 	down_read(&dev->rwsem);
909 	if (!(dev->status & VIRTIO_CONFIG_S_DRIVER_OK))
910 		goto unlock;
911 
912 	ret = 0;
913 	queue_work(vduse_irq_wq, irq_work);
914 unlock:
915 	up_read(&dev->rwsem);
916 
917 	return ret;
918 }
919 
920 static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
921 			    unsigned long arg)
922 {
923 	struct vduse_dev *dev = file->private_data;
924 	void __user *argp = (void __user *)arg;
925 	int ret;
926 
927 	if (unlikely(dev->broken))
928 		return -EPERM;
929 
930 	switch (cmd) {
931 	case VDUSE_IOTLB_GET_FD: {
932 		struct vduse_iotlb_entry entry;
933 		struct vhost_iotlb_map *map;
934 		struct vdpa_map_file *map_file;
935 		struct vduse_iova_domain *domain = dev->domain;
936 		struct file *f = NULL;
937 
938 		ret = -EFAULT;
939 		if (copy_from_user(&entry, argp, sizeof(entry)))
940 			break;
941 
942 		ret = -EINVAL;
943 		if (entry.start > entry.last)
944 			break;
945 
946 		spin_lock(&domain->iotlb_lock);
947 		map = vhost_iotlb_itree_first(domain->iotlb,
948 					      entry.start, entry.last);
949 		if (map) {
950 			map_file = (struct vdpa_map_file *)map->opaque;
951 			f = get_file(map_file->file);
952 			entry.offset = map_file->offset;
953 			entry.start = map->start;
954 			entry.last = map->last;
955 			entry.perm = map->perm;
956 		}
957 		spin_unlock(&domain->iotlb_lock);
958 		ret = -EINVAL;
959 		if (!f)
960 			break;
961 
962 		ret = -EFAULT;
963 		if (copy_to_user(argp, &entry, sizeof(entry))) {
964 			fput(f);
965 			break;
966 		}
967 		ret = receive_fd(f, perm_to_file_flags(entry.perm));
968 		fput(f);
969 		break;
970 	}
971 	case VDUSE_DEV_GET_FEATURES:
972 		/*
973 		 * Just mirror what driver wrote here.
974 		 * The driver is expected to check FEATURE_OK later.
975 		 */
976 		ret = put_user(dev->driver_features, (u64 __user *)argp);
977 		break;
978 	case VDUSE_DEV_SET_CONFIG: {
979 		struct vduse_config_data config;
980 		unsigned long size = offsetof(struct vduse_config_data,
981 					      buffer);
982 
983 		ret = -EFAULT;
984 		if (copy_from_user(&config, argp, size))
985 			break;
986 
987 		ret = -EINVAL;
988 		if (config.offset > dev->config_size ||
989 		    config.length == 0 ||
990 		    config.length > dev->config_size - config.offset)
991 			break;
992 
993 		ret = -EFAULT;
994 		if (copy_from_user(dev->config + config.offset, argp + size,
995 				   config.length))
996 			break;
997 
998 		ret = 0;
999 		break;
1000 	}
1001 	case VDUSE_DEV_INJECT_CONFIG_IRQ:
1002 		ret = vduse_dev_queue_irq_work(dev, &dev->inject);
1003 		break;
1004 	case VDUSE_VQ_SETUP: {
1005 		struct vduse_vq_config config;
1006 		u32 index;
1007 
1008 		ret = -EFAULT;
1009 		if (copy_from_user(&config, argp, sizeof(config)))
1010 			break;
1011 
1012 		ret = -EINVAL;
1013 		if (config.index >= dev->vq_num)
1014 			break;
1015 
1016 		if (!is_mem_zero((const char *)config.reserved,
1017 				 sizeof(config.reserved)))
1018 			break;
1019 
1020 		index = array_index_nospec(config.index, dev->vq_num);
1021 		dev->vqs[index].num_max = config.max_size;
1022 		ret = 0;
1023 		break;
1024 	}
1025 	case VDUSE_VQ_GET_INFO: {
1026 		struct vduse_vq_info vq_info;
1027 		struct vduse_virtqueue *vq;
1028 		u32 index;
1029 
1030 		ret = -EFAULT;
1031 		if (copy_from_user(&vq_info, argp, sizeof(vq_info)))
1032 			break;
1033 
1034 		ret = -EINVAL;
1035 		if (vq_info.index >= dev->vq_num)
1036 			break;
1037 
1038 		index = array_index_nospec(vq_info.index, dev->vq_num);
1039 		vq = &dev->vqs[index];
1040 		vq_info.desc_addr = vq->desc_addr;
1041 		vq_info.driver_addr = vq->driver_addr;
1042 		vq_info.device_addr = vq->device_addr;
1043 		vq_info.num = vq->num;
1044 
1045 		if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
1046 			vq_info.packed.last_avail_counter =
1047 				vq->state.packed.last_avail_counter;
1048 			vq_info.packed.last_avail_idx =
1049 				vq->state.packed.last_avail_idx;
1050 			vq_info.packed.last_used_counter =
1051 				vq->state.packed.last_used_counter;
1052 			vq_info.packed.last_used_idx =
1053 				vq->state.packed.last_used_idx;
1054 		} else
1055 			vq_info.split.avail_index =
1056 				vq->state.split.avail_index;
1057 
1058 		vq_info.ready = vq->ready;
1059 
1060 		ret = -EFAULT;
1061 		if (copy_to_user(argp, &vq_info, sizeof(vq_info)))
1062 			break;
1063 
1064 		ret = 0;
1065 		break;
1066 	}
1067 	case VDUSE_VQ_SETUP_KICKFD: {
1068 		struct vduse_vq_eventfd eventfd;
1069 
1070 		ret = -EFAULT;
1071 		if (copy_from_user(&eventfd, argp, sizeof(eventfd)))
1072 			break;
1073 
1074 		ret = vduse_kickfd_setup(dev, &eventfd);
1075 		break;
1076 	}
1077 	case VDUSE_VQ_INJECT_IRQ: {
1078 		u32 index;
1079 
1080 		ret = -EFAULT;
1081 		if (get_user(index, (u32 __user *)argp))
1082 			break;
1083 
1084 		ret = -EINVAL;
1085 		if (index >= dev->vq_num)
1086 			break;
1087 
1088 		index = array_index_nospec(index, dev->vq_num);
1089 		ret = vduse_dev_queue_irq_work(dev, &dev->vqs[index].inject);
1090 		break;
1091 	}
1092 	default:
1093 		ret = -ENOIOCTLCMD;
1094 		break;
1095 	}
1096 
1097 	return ret;
1098 }
1099 
1100 static int vduse_dev_release(struct inode *inode, struct file *file)
1101 {
1102 	struct vduse_dev *dev = file->private_data;
1103 
1104 	spin_lock(&dev->msg_lock);
1105 	/* Make sure the inflight messages can processed after reconncection */
1106 	list_splice_init(&dev->recv_list, &dev->send_list);
1107 	spin_unlock(&dev->msg_lock);
1108 	dev->connected = false;
1109 
1110 	return 0;
1111 }
1112 
1113 static struct vduse_dev *vduse_dev_get_from_minor(int minor)
1114 {
1115 	struct vduse_dev *dev;
1116 
1117 	mutex_lock(&vduse_lock);
1118 	dev = idr_find(&vduse_idr, minor);
1119 	mutex_unlock(&vduse_lock);
1120 
1121 	return dev;
1122 }
1123 
1124 static int vduse_dev_open(struct inode *inode, struct file *file)
1125 {
1126 	int ret;
1127 	struct vduse_dev *dev = vduse_dev_get_from_minor(iminor(inode));
1128 
1129 	if (!dev)
1130 		return -ENODEV;
1131 
1132 	ret = -EBUSY;
1133 	mutex_lock(&dev->lock);
1134 	if (dev->connected)
1135 		goto unlock;
1136 
1137 	ret = 0;
1138 	dev->connected = true;
1139 	file->private_data = dev;
1140 unlock:
1141 	mutex_unlock(&dev->lock);
1142 
1143 	return ret;
1144 }
1145 
1146 static const struct file_operations vduse_dev_fops = {
1147 	.owner		= THIS_MODULE,
1148 	.open		= vduse_dev_open,
1149 	.release	= vduse_dev_release,
1150 	.read_iter	= vduse_dev_read_iter,
1151 	.write_iter	= vduse_dev_write_iter,
1152 	.poll		= vduse_dev_poll,
1153 	.unlocked_ioctl	= vduse_dev_ioctl,
1154 	.compat_ioctl	= compat_ptr_ioctl,
1155 	.llseek		= noop_llseek,
1156 };
1157 
1158 static struct vduse_dev *vduse_dev_create(void)
1159 {
1160 	struct vduse_dev *dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1161 
1162 	if (!dev)
1163 		return NULL;
1164 
1165 	mutex_init(&dev->lock);
1166 	spin_lock_init(&dev->msg_lock);
1167 	INIT_LIST_HEAD(&dev->send_list);
1168 	INIT_LIST_HEAD(&dev->recv_list);
1169 	spin_lock_init(&dev->irq_lock);
1170 	init_rwsem(&dev->rwsem);
1171 
1172 	INIT_WORK(&dev->inject, vduse_dev_irq_inject);
1173 	init_waitqueue_head(&dev->waitq);
1174 
1175 	return dev;
1176 }
1177 
1178 static void vduse_dev_destroy(struct vduse_dev *dev)
1179 {
1180 	kfree(dev);
1181 }
1182 
1183 static struct vduse_dev *vduse_find_dev(const char *name)
1184 {
1185 	struct vduse_dev *dev;
1186 	int id;
1187 
1188 	idr_for_each_entry(&vduse_idr, dev, id)
1189 		if (!strcmp(dev->name, name))
1190 			return dev;
1191 
1192 	return NULL;
1193 }
1194 
1195 static int vduse_destroy_dev(char *name)
1196 {
1197 	struct vduse_dev *dev = vduse_find_dev(name);
1198 
1199 	if (!dev)
1200 		return -EINVAL;
1201 
1202 	mutex_lock(&dev->lock);
1203 	if (dev->vdev || dev->connected) {
1204 		mutex_unlock(&dev->lock);
1205 		return -EBUSY;
1206 	}
1207 	dev->connected = true;
1208 	mutex_unlock(&dev->lock);
1209 
1210 	vduse_dev_reset(dev);
1211 	device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));
1212 	idr_remove(&vduse_idr, dev->minor);
1213 	kvfree(dev->config);
1214 	kfree(dev->vqs);
1215 	vduse_domain_destroy(dev->domain);
1216 	kfree(dev->name);
1217 	vduse_dev_destroy(dev);
1218 	module_put(THIS_MODULE);
1219 
1220 	return 0;
1221 }
1222 
1223 static bool device_is_allowed(u32 device_id)
1224 {
1225 	int i;
1226 
1227 	for (i = 0; i < ARRAY_SIZE(allowed_device_id); i++)
1228 		if (allowed_device_id[i] == device_id)
1229 			return true;
1230 
1231 	return false;
1232 }
1233 
1234 static bool features_is_valid(u64 features)
1235 {
1236 	if (!(features & (1ULL << VIRTIO_F_ACCESS_PLATFORM)))
1237 		return false;
1238 
1239 	/* Now we only support read-only configuration space */
1240 	if (features & (1ULL << VIRTIO_BLK_F_CONFIG_WCE))
1241 		return false;
1242 
1243 	return true;
1244 }
1245 
1246 static bool vduse_validate_config(struct vduse_dev_config *config)
1247 {
1248 	if (!is_mem_zero((const char *)config->reserved,
1249 			 sizeof(config->reserved)))
1250 		return false;
1251 
1252 	if (config->vq_align > PAGE_SIZE)
1253 		return false;
1254 
1255 	if (config->config_size > PAGE_SIZE)
1256 		return false;
1257 
1258 	if (!device_is_allowed(config->device_id))
1259 		return false;
1260 
1261 	if (!features_is_valid(config->features))
1262 		return false;
1263 
1264 	return true;
1265 }
1266 
1267 static ssize_t msg_timeout_show(struct device *device,
1268 				struct device_attribute *attr, char *buf)
1269 {
1270 	struct vduse_dev *dev = dev_get_drvdata(device);
1271 
1272 	return sysfs_emit(buf, "%u\n", dev->msg_timeout);
1273 }
1274 
1275 static ssize_t msg_timeout_store(struct device *device,
1276 				 struct device_attribute *attr,
1277 				 const char *buf, size_t count)
1278 {
1279 	struct vduse_dev *dev = dev_get_drvdata(device);
1280 	int ret;
1281 
1282 	ret = kstrtouint(buf, 10, &dev->msg_timeout);
1283 	if (ret < 0)
1284 		return ret;
1285 
1286 	return count;
1287 }
1288 
1289 static DEVICE_ATTR_RW(msg_timeout);
1290 
1291 static struct attribute *vduse_dev_attrs[] = {
1292 	&dev_attr_msg_timeout.attr,
1293 	NULL
1294 };
1295 
1296 ATTRIBUTE_GROUPS(vduse_dev);
1297 
1298 static int vduse_create_dev(struct vduse_dev_config *config,
1299 			    void *config_buf, u64 api_version)
1300 {
1301 	int i, ret;
1302 	struct vduse_dev *dev;
1303 
1304 	ret = -EEXIST;
1305 	if (vduse_find_dev(config->name))
1306 		goto err;
1307 
1308 	ret = -ENOMEM;
1309 	dev = vduse_dev_create();
1310 	if (!dev)
1311 		goto err;
1312 
1313 	dev->api_version = api_version;
1314 	dev->device_features = config->features;
1315 	dev->device_id = config->device_id;
1316 	dev->vendor_id = config->vendor_id;
1317 	dev->name = kstrdup(config->name, GFP_KERNEL);
1318 	if (!dev->name)
1319 		goto err_str;
1320 
1321 	dev->domain = vduse_domain_create(VDUSE_IOVA_SIZE - 1,
1322 					  VDUSE_BOUNCE_SIZE);
1323 	if (!dev->domain)
1324 		goto err_domain;
1325 
1326 	dev->config = config_buf;
1327 	dev->config_size = config->config_size;
1328 	dev->vq_align = config->vq_align;
1329 	dev->vq_num = config->vq_num;
1330 	dev->vqs = kcalloc(dev->vq_num, sizeof(*dev->vqs), GFP_KERNEL);
1331 	if (!dev->vqs)
1332 		goto err_vqs;
1333 
1334 	for (i = 0; i < dev->vq_num; i++) {
1335 		dev->vqs[i].index = i;
1336 		INIT_WORK(&dev->vqs[i].inject, vduse_vq_irq_inject);
1337 		INIT_WORK(&dev->vqs[i].kick, vduse_vq_kick_work);
1338 		spin_lock_init(&dev->vqs[i].kick_lock);
1339 		spin_lock_init(&dev->vqs[i].irq_lock);
1340 	}
1341 
1342 	ret = idr_alloc(&vduse_idr, dev, 1, VDUSE_DEV_MAX, GFP_KERNEL);
1343 	if (ret < 0)
1344 		goto err_idr;
1345 
1346 	dev->minor = ret;
1347 	dev->msg_timeout = VDUSE_MSG_DEFAULT_TIMEOUT;
1348 	dev->dev = device_create_with_groups(vduse_class, NULL,
1349 				MKDEV(MAJOR(vduse_major), dev->minor),
1350 				dev, vduse_dev_groups, "%s", config->name);
1351 	if (IS_ERR(dev->dev)) {
1352 		ret = PTR_ERR(dev->dev);
1353 		goto err_dev;
1354 	}
1355 	__module_get(THIS_MODULE);
1356 
1357 	return 0;
1358 err_dev:
1359 	idr_remove(&vduse_idr, dev->minor);
1360 err_idr:
1361 	kfree(dev->vqs);
1362 err_vqs:
1363 	vduse_domain_destroy(dev->domain);
1364 err_domain:
1365 	kfree(dev->name);
1366 err_str:
1367 	vduse_dev_destroy(dev);
1368 err:
1369 	return ret;
1370 }
1371 
1372 static long vduse_ioctl(struct file *file, unsigned int cmd,
1373 			unsigned long arg)
1374 {
1375 	int ret;
1376 	void __user *argp = (void __user *)arg;
1377 	struct vduse_control *control = file->private_data;
1378 
1379 	mutex_lock(&vduse_lock);
1380 	switch (cmd) {
1381 	case VDUSE_GET_API_VERSION:
1382 		ret = put_user(control->api_version, (u64 __user *)argp);
1383 		break;
1384 	case VDUSE_SET_API_VERSION: {
1385 		u64 api_version;
1386 
1387 		ret = -EFAULT;
1388 		if (get_user(api_version, (u64 __user *)argp))
1389 			break;
1390 
1391 		ret = -EINVAL;
1392 		if (api_version > VDUSE_API_VERSION)
1393 			break;
1394 
1395 		ret = 0;
1396 		control->api_version = api_version;
1397 		break;
1398 	}
1399 	case VDUSE_CREATE_DEV: {
1400 		struct vduse_dev_config config;
1401 		unsigned long size = offsetof(struct vduse_dev_config, config);
1402 		void *buf;
1403 
1404 		ret = -EFAULT;
1405 		if (copy_from_user(&config, argp, size))
1406 			break;
1407 
1408 		ret = -EINVAL;
1409 		if (vduse_validate_config(&config) == false)
1410 			break;
1411 
1412 		buf = vmemdup_user(argp + size, config.config_size);
1413 		if (IS_ERR(buf)) {
1414 			ret = PTR_ERR(buf);
1415 			break;
1416 		}
1417 		config.name[VDUSE_NAME_MAX - 1] = '\0';
1418 		ret = vduse_create_dev(&config, buf, control->api_version);
1419 		if (ret)
1420 			kvfree(buf);
1421 		break;
1422 	}
1423 	case VDUSE_DESTROY_DEV: {
1424 		char name[VDUSE_NAME_MAX];
1425 
1426 		ret = -EFAULT;
1427 		if (copy_from_user(name, argp, VDUSE_NAME_MAX))
1428 			break;
1429 
1430 		name[VDUSE_NAME_MAX - 1] = '\0';
1431 		ret = vduse_destroy_dev(name);
1432 		break;
1433 	}
1434 	default:
1435 		ret = -EINVAL;
1436 		break;
1437 	}
1438 	mutex_unlock(&vduse_lock);
1439 
1440 	return ret;
1441 }
1442 
1443 static int vduse_release(struct inode *inode, struct file *file)
1444 {
1445 	struct vduse_control *control = file->private_data;
1446 
1447 	kfree(control);
1448 	return 0;
1449 }
1450 
1451 static int vduse_open(struct inode *inode, struct file *file)
1452 {
1453 	struct vduse_control *control;
1454 
1455 	control = kmalloc(sizeof(struct vduse_control), GFP_KERNEL);
1456 	if (!control)
1457 		return -ENOMEM;
1458 
1459 	control->api_version = VDUSE_API_VERSION;
1460 	file->private_data = control;
1461 
1462 	return 0;
1463 }
1464 
1465 static const struct file_operations vduse_ctrl_fops = {
1466 	.owner		= THIS_MODULE,
1467 	.open		= vduse_open,
1468 	.release	= vduse_release,
1469 	.unlocked_ioctl	= vduse_ioctl,
1470 	.compat_ioctl	= compat_ptr_ioctl,
1471 	.llseek		= noop_llseek,
1472 };
1473 
1474 static char *vduse_devnode(struct device *dev, umode_t *mode)
1475 {
1476 	return kasprintf(GFP_KERNEL, "vduse/%s", dev_name(dev));
1477 }
1478 
1479 static void vduse_mgmtdev_release(struct device *dev)
1480 {
1481 }
1482 
1483 static struct device vduse_mgmtdev = {
1484 	.init_name = "vduse",
1485 	.release = vduse_mgmtdev_release,
1486 };
1487 
1488 static struct vdpa_mgmt_dev mgmt_dev;
1489 
1490 static int vduse_dev_init_vdpa(struct vduse_dev *dev, const char *name)
1491 {
1492 	struct vduse_vdpa *vdev;
1493 	int ret;
1494 
1495 	if (dev->vdev)
1496 		return -EEXIST;
1497 
1498 	vdev = vdpa_alloc_device(struct vduse_vdpa, vdpa, dev->dev,
1499 				 &vduse_vdpa_config_ops, 1, 1, name, true);
1500 	if (IS_ERR(vdev))
1501 		return PTR_ERR(vdev);
1502 
1503 	dev->vdev = vdev;
1504 	vdev->dev = dev;
1505 	vdev->vdpa.dev.dma_mask = &vdev->vdpa.dev.coherent_dma_mask;
1506 	ret = dma_set_mask_and_coherent(&vdev->vdpa.dev, DMA_BIT_MASK(64));
1507 	if (ret) {
1508 		put_device(&vdev->vdpa.dev);
1509 		return ret;
1510 	}
1511 	set_dma_ops(&vdev->vdpa.dev, &vduse_dev_dma_ops);
1512 	vdev->vdpa.dma_dev = &vdev->vdpa.dev;
1513 	vdev->vdpa.mdev = &mgmt_dev;
1514 
1515 	return 0;
1516 }
1517 
1518 static int vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
1519 			const struct vdpa_dev_set_config *config)
1520 {
1521 	struct vduse_dev *dev;
1522 	int ret;
1523 
1524 	mutex_lock(&vduse_lock);
1525 	dev = vduse_find_dev(name);
1526 	if (!dev || !vduse_dev_is_ready(dev)) {
1527 		mutex_unlock(&vduse_lock);
1528 		return -EINVAL;
1529 	}
1530 	ret = vduse_dev_init_vdpa(dev, name);
1531 	mutex_unlock(&vduse_lock);
1532 	if (ret)
1533 		return ret;
1534 
1535 	ret = _vdpa_register_device(&dev->vdev->vdpa, dev->vq_num);
1536 	if (ret) {
1537 		put_device(&dev->vdev->vdpa.dev);
1538 		return ret;
1539 	}
1540 
1541 	return 0;
1542 }
1543 
1544 static void vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev)
1545 {
1546 	_vdpa_unregister_device(dev);
1547 }
1548 
1549 static const struct vdpa_mgmtdev_ops vdpa_dev_mgmtdev_ops = {
1550 	.dev_add = vdpa_dev_add,
1551 	.dev_del = vdpa_dev_del,
1552 };
1553 
1554 static struct virtio_device_id id_table[] = {
1555 	{ VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID },
1556 	{ 0 },
1557 };
1558 
1559 static struct vdpa_mgmt_dev mgmt_dev = {
1560 	.device = &vduse_mgmtdev,
1561 	.id_table = id_table,
1562 	.ops = &vdpa_dev_mgmtdev_ops,
1563 };
1564 
1565 static int vduse_mgmtdev_init(void)
1566 {
1567 	int ret;
1568 
1569 	ret = device_register(&vduse_mgmtdev);
1570 	if (ret)
1571 		return ret;
1572 
1573 	ret = vdpa_mgmtdev_register(&mgmt_dev);
1574 	if (ret)
1575 		goto err;
1576 
1577 	return 0;
1578 err:
1579 	device_unregister(&vduse_mgmtdev);
1580 	return ret;
1581 }
1582 
1583 static void vduse_mgmtdev_exit(void)
1584 {
1585 	vdpa_mgmtdev_unregister(&mgmt_dev);
1586 	device_unregister(&vduse_mgmtdev);
1587 }
1588 
1589 static int vduse_init(void)
1590 {
1591 	int ret;
1592 	struct device *dev;
1593 
1594 	vduse_class = class_create(THIS_MODULE, "vduse");
1595 	if (IS_ERR(vduse_class))
1596 		return PTR_ERR(vduse_class);
1597 
1598 	vduse_class->devnode = vduse_devnode;
1599 
1600 	ret = alloc_chrdev_region(&vduse_major, 0, VDUSE_DEV_MAX, "vduse");
1601 	if (ret)
1602 		goto err_chardev_region;
1603 
1604 	/* /dev/vduse/control */
1605 	cdev_init(&vduse_ctrl_cdev, &vduse_ctrl_fops);
1606 	vduse_ctrl_cdev.owner = THIS_MODULE;
1607 	ret = cdev_add(&vduse_ctrl_cdev, vduse_major, 1);
1608 	if (ret)
1609 		goto err_ctrl_cdev;
1610 
1611 	dev = device_create(vduse_class, NULL, vduse_major, NULL, "control");
1612 	if (IS_ERR(dev)) {
1613 		ret = PTR_ERR(dev);
1614 		goto err_device;
1615 	}
1616 
1617 	/* /dev/vduse/$DEVICE */
1618 	cdev_init(&vduse_cdev, &vduse_dev_fops);
1619 	vduse_cdev.owner = THIS_MODULE;
1620 	ret = cdev_add(&vduse_cdev, MKDEV(MAJOR(vduse_major), 1),
1621 		       VDUSE_DEV_MAX - 1);
1622 	if (ret)
1623 		goto err_cdev;
1624 
1625 	vduse_irq_wq = alloc_workqueue("vduse-irq",
1626 				WQ_HIGHPRI | WQ_SYSFS | WQ_UNBOUND, 0);
1627 	if (!vduse_irq_wq) {
1628 		ret = -ENOMEM;
1629 		goto err_wq;
1630 	}
1631 
1632 	ret = vduse_domain_init();
1633 	if (ret)
1634 		goto err_domain;
1635 
1636 	ret = vduse_mgmtdev_init();
1637 	if (ret)
1638 		goto err_mgmtdev;
1639 
1640 	return 0;
1641 err_mgmtdev:
1642 	vduse_domain_exit();
1643 err_domain:
1644 	destroy_workqueue(vduse_irq_wq);
1645 err_wq:
1646 	cdev_del(&vduse_cdev);
1647 err_cdev:
1648 	device_destroy(vduse_class, vduse_major);
1649 err_device:
1650 	cdev_del(&vduse_ctrl_cdev);
1651 err_ctrl_cdev:
1652 	unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);
1653 err_chardev_region:
1654 	class_destroy(vduse_class);
1655 	return ret;
1656 }
1657 module_init(vduse_init);
1658 
1659 static void vduse_exit(void)
1660 {
1661 	vduse_mgmtdev_exit();
1662 	vduse_domain_exit();
1663 	destroy_workqueue(vduse_irq_wq);
1664 	cdev_del(&vduse_cdev);
1665 	device_destroy(vduse_class, vduse_major);
1666 	cdev_del(&vduse_ctrl_cdev);
1667 	unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);
1668 	class_destroy(vduse_class);
1669 }
1670 module_exit(vduse_exit);
1671 
1672 MODULE_LICENSE(DRV_LICENSE);
1673 MODULE_AUTHOR(DRV_AUTHOR);
1674 MODULE_DESCRIPTION(DRV_DESC);
1675