1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * VDUSE: vDPA Device in Userspace
4  *
5  * Copyright (C) 2020-2021 Bytedance Inc. and/or its affiliates. All rights reserved.
6  *
7  * Author: Xie Yongji <xieyongji@bytedance.com>
8  *
9  */
10 
11 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/cdev.h>
14 #include <linux/device.h>
15 #include <linux/eventfd.h>
16 #include <linux/slab.h>
17 #include <linux/wait.h>
18 #include <linux/dma-map-ops.h>
19 #include <linux/poll.h>
20 #include <linux/file.h>
21 #include <linux/uio.h>
22 #include <linux/vdpa.h>
23 #include <linux/nospec.h>
24 #include <uapi/linux/vduse.h>
25 #include <uapi/linux/vdpa.h>
26 #include <uapi/linux/virtio_config.h>
27 #include <uapi/linux/virtio_ids.h>
28 #include <uapi/linux/virtio_blk.h>
29 #include <linux/mod_devicetable.h>
30 
31 #include "iova_domain.h"
32 
33 #define DRV_AUTHOR   "Yongji Xie <xieyongji@bytedance.com>"
34 #define DRV_DESC     "vDPA Device in Userspace"
35 #define DRV_LICENSE  "GPL v2"
36 
37 #define VDUSE_DEV_MAX (1U << MINORBITS)
38 #define VDUSE_BOUNCE_SIZE (64 * 1024 * 1024)
39 #define VDUSE_IOVA_SIZE (128 * 1024 * 1024)
40 #define VDUSE_MSG_DEFAULT_TIMEOUT 30
41 
42 struct vduse_virtqueue {
43 	u16 index;
44 	u16 num_max;
45 	u32 num;
46 	u64 desc_addr;
47 	u64 driver_addr;
48 	u64 device_addr;
49 	struct vdpa_vq_state state;
50 	bool ready;
51 	bool kicked;
52 	spinlock_t kick_lock;
53 	spinlock_t irq_lock;
54 	struct eventfd_ctx *kickfd;
55 	struct vdpa_callback cb;
56 	struct work_struct inject;
57 	struct work_struct kick;
58 };
59 
60 struct vduse_dev;
61 
62 struct vduse_vdpa {
63 	struct vdpa_device vdpa;
64 	struct vduse_dev *dev;
65 };
66 
67 struct vduse_dev {
68 	struct vduse_vdpa *vdev;
69 	struct device *dev;
70 	struct vduse_virtqueue *vqs;
71 	struct vduse_iova_domain *domain;
72 	char *name;
73 	struct mutex lock;
74 	spinlock_t msg_lock;
75 	u64 msg_unique;
76 	u32 msg_timeout;
77 	wait_queue_head_t waitq;
78 	struct list_head send_list;
79 	struct list_head recv_list;
80 	struct vdpa_callback config_cb;
81 	struct work_struct inject;
82 	spinlock_t irq_lock;
83 	struct rw_semaphore rwsem;
84 	int minor;
85 	bool broken;
86 	bool connected;
87 	u64 api_version;
88 	u64 device_features;
89 	u64 driver_features;
90 	u32 device_id;
91 	u32 vendor_id;
92 	u32 generation;
93 	u32 config_size;
94 	void *config;
95 	u8 status;
96 	u32 vq_num;
97 	u32 vq_align;
98 };
99 
100 struct vduse_dev_msg {
101 	struct vduse_dev_request req;
102 	struct vduse_dev_response resp;
103 	struct list_head list;
104 	wait_queue_head_t waitq;
105 	bool completed;
106 };
107 
108 struct vduse_control {
109 	u64 api_version;
110 };
111 
112 static DEFINE_MUTEX(vduse_lock);
113 static DEFINE_IDR(vduse_idr);
114 
115 static dev_t vduse_major;
116 static struct class *vduse_class;
117 static struct cdev vduse_ctrl_cdev;
118 static struct cdev vduse_cdev;
119 static struct workqueue_struct *vduse_irq_wq;
120 
121 static u32 allowed_device_id[] = {
122 	VIRTIO_ID_BLOCK,
123 };
124 
125 static inline struct vduse_dev *vdpa_to_vduse(struct vdpa_device *vdpa)
126 {
127 	struct vduse_vdpa *vdev = container_of(vdpa, struct vduse_vdpa, vdpa);
128 
129 	return vdev->dev;
130 }
131 
132 static inline struct vduse_dev *dev_to_vduse(struct device *dev)
133 {
134 	struct vdpa_device *vdpa = dev_to_vdpa(dev);
135 
136 	return vdpa_to_vduse(vdpa);
137 }
138 
139 static struct vduse_dev_msg *vduse_find_msg(struct list_head *head,
140 					    uint32_t request_id)
141 {
142 	struct vduse_dev_msg *msg;
143 
144 	list_for_each_entry(msg, head, list) {
145 		if (msg->req.request_id == request_id) {
146 			list_del(&msg->list);
147 			return msg;
148 		}
149 	}
150 
151 	return NULL;
152 }
153 
154 static struct vduse_dev_msg *vduse_dequeue_msg(struct list_head *head)
155 {
156 	struct vduse_dev_msg *msg = NULL;
157 
158 	if (!list_empty(head)) {
159 		msg = list_first_entry(head, struct vduse_dev_msg, list);
160 		list_del(&msg->list);
161 	}
162 
163 	return msg;
164 }
165 
166 static void vduse_enqueue_msg(struct list_head *head,
167 			      struct vduse_dev_msg *msg)
168 {
169 	list_add_tail(&msg->list, head);
170 }
171 
172 static void vduse_dev_broken(struct vduse_dev *dev)
173 {
174 	struct vduse_dev_msg *msg, *tmp;
175 
176 	if (unlikely(dev->broken))
177 		return;
178 
179 	list_splice_init(&dev->recv_list, &dev->send_list);
180 	list_for_each_entry_safe(msg, tmp, &dev->send_list, list) {
181 		list_del(&msg->list);
182 		msg->completed = 1;
183 		msg->resp.result = VDUSE_REQ_RESULT_FAILED;
184 		wake_up(&msg->waitq);
185 	}
186 	dev->broken = true;
187 	wake_up(&dev->waitq);
188 }
189 
190 static int vduse_dev_msg_sync(struct vduse_dev *dev,
191 			      struct vduse_dev_msg *msg)
192 {
193 	int ret;
194 
195 	if (unlikely(dev->broken))
196 		return -EIO;
197 
198 	init_waitqueue_head(&msg->waitq);
199 	spin_lock(&dev->msg_lock);
200 	if (unlikely(dev->broken)) {
201 		spin_unlock(&dev->msg_lock);
202 		return -EIO;
203 	}
204 	msg->req.request_id = dev->msg_unique++;
205 	vduse_enqueue_msg(&dev->send_list, msg);
206 	wake_up(&dev->waitq);
207 	spin_unlock(&dev->msg_lock);
208 	if (dev->msg_timeout)
209 		ret = wait_event_killable_timeout(msg->waitq, msg->completed,
210 						  (long)dev->msg_timeout * HZ);
211 	else
212 		ret = wait_event_killable(msg->waitq, msg->completed);
213 
214 	spin_lock(&dev->msg_lock);
215 	if (!msg->completed) {
216 		list_del(&msg->list);
217 		msg->resp.result = VDUSE_REQ_RESULT_FAILED;
218 		/* Mark the device as malfunction when there is a timeout */
219 		if (!ret)
220 			vduse_dev_broken(dev);
221 	}
222 	ret = (msg->resp.result == VDUSE_REQ_RESULT_OK) ? 0 : -EIO;
223 	spin_unlock(&dev->msg_lock);
224 
225 	return ret;
226 }
227 
228 static int vduse_dev_get_vq_state_packed(struct vduse_dev *dev,
229 					 struct vduse_virtqueue *vq,
230 					 struct vdpa_vq_state_packed *packed)
231 {
232 	struct vduse_dev_msg msg = { 0 };
233 	int ret;
234 
235 	msg.req.type = VDUSE_GET_VQ_STATE;
236 	msg.req.vq_state.index = vq->index;
237 
238 	ret = vduse_dev_msg_sync(dev, &msg);
239 	if (ret)
240 		return ret;
241 
242 	packed->last_avail_counter =
243 			msg.resp.vq_state.packed.last_avail_counter & 0x0001;
244 	packed->last_avail_idx =
245 			msg.resp.vq_state.packed.last_avail_idx & 0x7FFF;
246 	packed->last_used_counter =
247 			msg.resp.vq_state.packed.last_used_counter & 0x0001;
248 	packed->last_used_idx =
249 			msg.resp.vq_state.packed.last_used_idx & 0x7FFF;
250 
251 	return 0;
252 }
253 
254 static int vduse_dev_get_vq_state_split(struct vduse_dev *dev,
255 					struct vduse_virtqueue *vq,
256 					struct vdpa_vq_state_split *split)
257 {
258 	struct vduse_dev_msg msg = { 0 };
259 	int ret;
260 
261 	msg.req.type = VDUSE_GET_VQ_STATE;
262 	msg.req.vq_state.index = vq->index;
263 
264 	ret = vduse_dev_msg_sync(dev, &msg);
265 	if (ret)
266 		return ret;
267 
268 	split->avail_index = msg.resp.vq_state.split.avail_index;
269 
270 	return 0;
271 }
272 
273 static int vduse_dev_set_status(struct vduse_dev *dev, u8 status)
274 {
275 	struct vduse_dev_msg msg = { 0 };
276 
277 	msg.req.type = VDUSE_SET_STATUS;
278 	msg.req.s.status = status;
279 
280 	return vduse_dev_msg_sync(dev, &msg);
281 }
282 
283 static int vduse_dev_update_iotlb(struct vduse_dev *dev,
284 				  u64 start, u64 last)
285 {
286 	struct vduse_dev_msg msg = { 0 };
287 
288 	if (last < start)
289 		return -EINVAL;
290 
291 	msg.req.type = VDUSE_UPDATE_IOTLB;
292 	msg.req.iova.start = start;
293 	msg.req.iova.last = last;
294 
295 	return vduse_dev_msg_sync(dev, &msg);
296 }
297 
298 static ssize_t vduse_dev_read_iter(struct kiocb *iocb, struct iov_iter *to)
299 {
300 	struct file *file = iocb->ki_filp;
301 	struct vduse_dev *dev = file->private_data;
302 	struct vduse_dev_msg *msg;
303 	int size = sizeof(struct vduse_dev_request);
304 	ssize_t ret;
305 
306 	if (iov_iter_count(to) < size)
307 		return -EINVAL;
308 
309 	spin_lock(&dev->msg_lock);
310 	while (1) {
311 		msg = vduse_dequeue_msg(&dev->send_list);
312 		if (msg)
313 			break;
314 
315 		ret = -EAGAIN;
316 		if (file->f_flags & O_NONBLOCK)
317 			goto unlock;
318 
319 		spin_unlock(&dev->msg_lock);
320 		ret = wait_event_interruptible_exclusive(dev->waitq,
321 					!list_empty(&dev->send_list));
322 		if (ret)
323 			return ret;
324 
325 		spin_lock(&dev->msg_lock);
326 	}
327 	spin_unlock(&dev->msg_lock);
328 	ret = copy_to_iter(&msg->req, size, to);
329 	spin_lock(&dev->msg_lock);
330 	if (ret != size) {
331 		ret = -EFAULT;
332 		vduse_enqueue_msg(&dev->send_list, msg);
333 		goto unlock;
334 	}
335 	vduse_enqueue_msg(&dev->recv_list, msg);
336 unlock:
337 	spin_unlock(&dev->msg_lock);
338 
339 	return ret;
340 }
341 
342 static bool is_mem_zero(const char *ptr, int size)
343 {
344 	int i;
345 
346 	for (i = 0; i < size; i++) {
347 		if (ptr[i])
348 			return false;
349 	}
350 	return true;
351 }
352 
353 static ssize_t vduse_dev_write_iter(struct kiocb *iocb, struct iov_iter *from)
354 {
355 	struct file *file = iocb->ki_filp;
356 	struct vduse_dev *dev = file->private_data;
357 	struct vduse_dev_response resp;
358 	struct vduse_dev_msg *msg;
359 	size_t ret;
360 
361 	ret = copy_from_iter(&resp, sizeof(resp), from);
362 	if (ret != sizeof(resp))
363 		return -EINVAL;
364 
365 	if (!is_mem_zero((const char *)resp.reserved, sizeof(resp.reserved)))
366 		return -EINVAL;
367 
368 	spin_lock(&dev->msg_lock);
369 	msg = vduse_find_msg(&dev->recv_list, resp.request_id);
370 	if (!msg) {
371 		ret = -ENOENT;
372 		goto unlock;
373 	}
374 
375 	memcpy(&msg->resp, &resp, sizeof(resp));
376 	msg->completed = 1;
377 	wake_up(&msg->waitq);
378 unlock:
379 	spin_unlock(&dev->msg_lock);
380 
381 	return ret;
382 }
383 
384 static __poll_t vduse_dev_poll(struct file *file, poll_table *wait)
385 {
386 	struct vduse_dev *dev = file->private_data;
387 	__poll_t mask = 0;
388 
389 	poll_wait(file, &dev->waitq, wait);
390 
391 	spin_lock(&dev->msg_lock);
392 
393 	if (unlikely(dev->broken))
394 		mask |= EPOLLERR;
395 	if (!list_empty(&dev->send_list))
396 		mask |= EPOLLIN | EPOLLRDNORM;
397 	if (!list_empty(&dev->recv_list))
398 		mask |= EPOLLOUT | EPOLLWRNORM;
399 
400 	spin_unlock(&dev->msg_lock);
401 
402 	return mask;
403 }
404 
405 static void vduse_dev_reset(struct vduse_dev *dev)
406 {
407 	int i;
408 	struct vduse_iova_domain *domain = dev->domain;
409 
410 	/* The coherent mappings are handled in vduse_dev_free_coherent() */
411 	if (domain->bounce_map)
412 		vduse_domain_reset_bounce_map(domain);
413 
414 	down_write(&dev->rwsem);
415 
416 	dev->status = 0;
417 	dev->driver_features = 0;
418 	dev->generation++;
419 	spin_lock(&dev->irq_lock);
420 	dev->config_cb.callback = NULL;
421 	dev->config_cb.private = NULL;
422 	spin_unlock(&dev->irq_lock);
423 	flush_work(&dev->inject);
424 
425 	for (i = 0; i < dev->vq_num; i++) {
426 		struct vduse_virtqueue *vq = &dev->vqs[i];
427 
428 		vq->ready = false;
429 		vq->desc_addr = 0;
430 		vq->driver_addr = 0;
431 		vq->device_addr = 0;
432 		vq->num = 0;
433 		memset(&vq->state, 0, sizeof(vq->state));
434 
435 		spin_lock(&vq->kick_lock);
436 		vq->kicked = false;
437 		if (vq->kickfd)
438 			eventfd_ctx_put(vq->kickfd);
439 		vq->kickfd = NULL;
440 		spin_unlock(&vq->kick_lock);
441 
442 		spin_lock(&vq->irq_lock);
443 		vq->cb.callback = NULL;
444 		vq->cb.private = NULL;
445 		spin_unlock(&vq->irq_lock);
446 		flush_work(&vq->inject);
447 		flush_work(&vq->kick);
448 	}
449 
450 	up_write(&dev->rwsem);
451 }
452 
453 static int vduse_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 idx,
454 				u64 desc_area, u64 driver_area,
455 				u64 device_area)
456 {
457 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
458 	struct vduse_virtqueue *vq = &dev->vqs[idx];
459 
460 	vq->desc_addr = desc_area;
461 	vq->driver_addr = driver_area;
462 	vq->device_addr = device_area;
463 
464 	return 0;
465 }
466 
467 static void vduse_vq_kick(struct vduse_virtqueue *vq)
468 {
469 	spin_lock(&vq->kick_lock);
470 	if (!vq->ready)
471 		goto unlock;
472 
473 	if (vq->kickfd)
474 		eventfd_signal(vq->kickfd, 1);
475 	else
476 		vq->kicked = true;
477 unlock:
478 	spin_unlock(&vq->kick_lock);
479 }
480 
481 static void vduse_vq_kick_work(struct work_struct *work)
482 {
483 	struct vduse_virtqueue *vq = container_of(work,
484 					struct vduse_virtqueue, kick);
485 
486 	vduse_vq_kick(vq);
487 }
488 
489 static void vduse_vdpa_kick_vq(struct vdpa_device *vdpa, u16 idx)
490 {
491 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
492 	struct vduse_virtqueue *vq = &dev->vqs[idx];
493 
494 	if (!eventfd_signal_allowed()) {
495 		schedule_work(&vq->kick);
496 		return;
497 	}
498 	vduse_vq_kick(vq);
499 }
500 
501 static void vduse_vdpa_set_vq_cb(struct vdpa_device *vdpa, u16 idx,
502 			      struct vdpa_callback *cb)
503 {
504 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
505 	struct vduse_virtqueue *vq = &dev->vqs[idx];
506 
507 	spin_lock(&vq->irq_lock);
508 	vq->cb.callback = cb->callback;
509 	vq->cb.private = cb->private;
510 	spin_unlock(&vq->irq_lock);
511 }
512 
513 static void vduse_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num)
514 {
515 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
516 	struct vduse_virtqueue *vq = &dev->vqs[idx];
517 
518 	vq->num = num;
519 }
520 
521 static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa,
522 					u16 idx, bool ready)
523 {
524 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
525 	struct vduse_virtqueue *vq = &dev->vqs[idx];
526 
527 	vq->ready = ready;
528 }
529 
530 static bool vduse_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 idx)
531 {
532 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
533 	struct vduse_virtqueue *vq = &dev->vqs[idx];
534 
535 	return vq->ready;
536 }
537 
538 static int vduse_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 idx,
539 				const struct vdpa_vq_state *state)
540 {
541 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
542 	struct vduse_virtqueue *vq = &dev->vqs[idx];
543 
544 	if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
545 		vq->state.packed.last_avail_counter =
546 				state->packed.last_avail_counter;
547 		vq->state.packed.last_avail_idx = state->packed.last_avail_idx;
548 		vq->state.packed.last_used_counter =
549 				state->packed.last_used_counter;
550 		vq->state.packed.last_used_idx = state->packed.last_used_idx;
551 	} else
552 		vq->state.split.avail_index = state->split.avail_index;
553 
554 	return 0;
555 }
556 
557 static int vduse_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 idx,
558 				struct vdpa_vq_state *state)
559 {
560 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
561 	struct vduse_virtqueue *vq = &dev->vqs[idx];
562 
563 	if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED))
564 		return vduse_dev_get_vq_state_packed(dev, vq, &state->packed);
565 
566 	return vduse_dev_get_vq_state_split(dev, vq, &state->split);
567 }
568 
569 static u32 vduse_vdpa_get_vq_align(struct vdpa_device *vdpa)
570 {
571 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
572 
573 	return dev->vq_align;
574 }
575 
576 static u64 vduse_vdpa_get_device_features(struct vdpa_device *vdpa)
577 {
578 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
579 
580 	return dev->device_features;
581 }
582 
583 static int vduse_vdpa_set_driver_features(struct vdpa_device *vdpa, u64 features)
584 {
585 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
586 
587 	dev->driver_features = features;
588 	return 0;
589 }
590 
591 static u64 vduse_vdpa_get_driver_features(struct vdpa_device *vdpa)
592 {
593 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
594 
595 	return dev->driver_features;
596 }
597 
598 static void vduse_vdpa_set_config_cb(struct vdpa_device *vdpa,
599 				  struct vdpa_callback *cb)
600 {
601 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
602 
603 	spin_lock(&dev->irq_lock);
604 	dev->config_cb.callback = cb->callback;
605 	dev->config_cb.private = cb->private;
606 	spin_unlock(&dev->irq_lock);
607 }
608 
609 static u16 vduse_vdpa_get_vq_num_max(struct vdpa_device *vdpa)
610 {
611 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
612 	u16 num_max = 0;
613 	int i;
614 
615 	for (i = 0; i < dev->vq_num; i++)
616 		if (num_max < dev->vqs[i].num_max)
617 			num_max = dev->vqs[i].num_max;
618 
619 	return num_max;
620 }
621 
622 static u32 vduse_vdpa_get_device_id(struct vdpa_device *vdpa)
623 {
624 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
625 
626 	return dev->device_id;
627 }
628 
629 static u32 vduse_vdpa_get_vendor_id(struct vdpa_device *vdpa)
630 {
631 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
632 
633 	return dev->vendor_id;
634 }
635 
636 static u8 vduse_vdpa_get_status(struct vdpa_device *vdpa)
637 {
638 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
639 
640 	return dev->status;
641 }
642 
643 static void vduse_vdpa_set_status(struct vdpa_device *vdpa, u8 status)
644 {
645 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
646 
647 	if (vduse_dev_set_status(dev, status))
648 		return;
649 
650 	dev->status = status;
651 }
652 
653 static size_t vduse_vdpa_get_config_size(struct vdpa_device *vdpa)
654 {
655 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
656 
657 	return dev->config_size;
658 }
659 
660 static void vduse_vdpa_get_config(struct vdpa_device *vdpa, unsigned int offset,
661 				  void *buf, unsigned int len)
662 {
663 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
664 
665 	if (offset > dev->config_size ||
666 	    len > dev->config_size - offset)
667 		return;
668 
669 	memcpy(buf, dev->config + offset, len);
670 }
671 
672 static void vduse_vdpa_set_config(struct vdpa_device *vdpa, unsigned int offset,
673 			const void *buf, unsigned int len)
674 {
675 	/* Now we only support read-only configuration space */
676 }
677 
678 static int vduse_vdpa_reset(struct vdpa_device *vdpa)
679 {
680 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
681 	int ret = vduse_dev_set_status(dev, 0);
682 
683 	vduse_dev_reset(dev);
684 
685 	return ret;
686 }
687 
688 static u32 vduse_vdpa_get_generation(struct vdpa_device *vdpa)
689 {
690 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
691 
692 	return dev->generation;
693 }
694 
695 static int vduse_vdpa_set_map(struct vdpa_device *vdpa,
696 				struct vhost_iotlb *iotlb)
697 {
698 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
699 	int ret;
700 
701 	ret = vduse_domain_set_map(dev->domain, iotlb);
702 	if (ret)
703 		return ret;
704 
705 	ret = vduse_dev_update_iotlb(dev, 0ULL, ULLONG_MAX);
706 	if (ret) {
707 		vduse_domain_clear_map(dev->domain, iotlb);
708 		return ret;
709 	}
710 
711 	return 0;
712 }
713 
714 static void vduse_vdpa_free(struct vdpa_device *vdpa)
715 {
716 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
717 
718 	dev->vdev = NULL;
719 }
720 
721 static const struct vdpa_config_ops vduse_vdpa_config_ops = {
722 	.set_vq_address		= vduse_vdpa_set_vq_address,
723 	.kick_vq		= vduse_vdpa_kick_vq,
724 	.set_vq_cb		= vduse_vdpa_set_vq_cb,
725 	.set_vq_num             = vduse_vdpa_set_vq_num,
726 	.set_vq_ready		= vduse_vdpa_set_vq_ready,
727 	.get_vq_ready		= vduse_vdpa_get_vq_ready,
728 	.set_vq_state		= vduse_vdpa_set_vq_state,
729 	.get_vq_state		= vduse_vdpa_get_vq_state,
730 	.get_vq_align		= vduse_vdpa_get_vq_align,
731 	.get_device_features	= vduse_vdpa_get_device_features,
732 	.set_driver_features	= vduse_vdpa_set_driver_features,
733 	.get_driver_features	= vduse_vdpa_get_driver_features,
734 	.set_config_cb		= vduse_vdpa_set_config_cb,
735 	.get_vq_num_max		= vduse_vdpa_get_vq_num_max,
736 	.get_device_id		= vduse_vdpa_get_device_id,
737 	.get_vendor_id		= vduse_vdpa_get_vendor_id,
738 	.get_status		= vduse_vdpa_get_status,
739 	.set_status		= vduse_vdpa_set_status,
740 	.get_config_size	= vduse_vdpa_get_config_size,
741 	.get_config		= vduse_vdpa_get_config,
742 	.set_config		= vduse_vdpa_set_config,
743 	.get_generation		= vduse_vdpa_get_generation,
744 	.reset			= vduse_vdpa_reset,
745 	.set_map		= vduse_vdpa_set_map,
746 	.free			= vduse_vdpa_free,
747 };
748 
749 static dma_addr_t vduse_dev_map_page(struct device *dev, struct page *page,
750 				     unsigned long offset, size_t size,
751 				     enum dma_data_direction dir,
752 				     unsigned long attrs)
753 {
754 	struct vduse_dev *vdev = dev_to_vduse(dev);
755 	struct vduse_iova_domain *domain = vdev->domain;
756 
757 	return vduse_domain_map_page(domain, page, offset, size, dir, attrs);
758 }
759 
760 static void vduse_dev_unmap_page(struct device *dev, dma_addr_t dma_addr,
761 				size_t size, enum dma_data_direction dir,
762 				unsigned long attrs)
763 {
764 	struct vduse_dev *vdev = dev_to_vduse(dev);
765 	struct vduse_iova_domain *domain = vdev->domain;
766 
767 	return vduse_domain_unmap_page(domain, dma_addr, size, dir, attrs);
768 }
769 
770 static void *vduse_dev_alloc_coherent(struct device *dev, size_t size,
771 					dma_addr_t *dma_addr, gfp_t flag,
772 					unsigned long attrs)
773 {
774 	struct vduse_dev *vdev = dev_to_vduse(dev);
775 	struct vduse_iova_domain *domain = vdev->domain;
776 	unsigned long iova;
777 	void *addr;
778 
779 	*dma_addr = DMA_MAPPING_ERROR;
780 	addr = vduse_domain_alloc_coherent(domain, size,
781 				(dma_addr_t *)&iova, flag, attrs);
782 	if (!addr)
783 		return NULL;
784 
785 	*dma_addr = (dma_addr_t)iova;
786 
787 	return addr;
788 }
789 
790 static void vduse_dev_free_coherent(struct device *dev, size_t size,
791 					void *vaddr, dma_addr_t dma_addr,
792 					unsigned long attrs)
793 {
794 	struct vduse_dev *vdev = dev_to_vduse(dev);
795 	struct vduse_iova_domain *domain = vdev->domain;
796 
797 	vduse_domain_free_coherent(domain, size, vaddr, dma_addr, attrs);
798 }
799 
800 static size_t vduse_dev_max_mapping_size(struct device *dev)
801 {
802 	struct vduse_dev *vdev = dev_to_vduse(dev);
803 	struct vduse_iova_domain *domain = vdev->domain;
804 
805 	return domain->bounce_size;
806 }
807 
808 static const struct dma_map_ops vduse_dev_dma_ops = {
809 	.map_page = vduse_dev_map_page,
810 	.unmap_page = vduse_dev_unmap_page,
811 	.alloc = vduse_dev_alloc_coherent,
812 	.free = vduse_dev_free_coherent,
813 	.max_mapping_size = vduse_dev_max_mapping_size,
814 };
815 
816 static unsigned int perm_to_file_flags(u8 perm)
817 {
818 	unsigned int flags = 0;
819 
820 	switch (perm) {
821 	case VDUSE_ACCESS_WO:
822 		flags |= O_WRONLY;
823 		break;
824 	case VDUSE_ACCESS_RO:
825 		flags |= O_RDONLY;
826 		break;
827 	case VDUSE_ACCESS_RW:
828 		flags |= O_RDWR;
829 		break;
830 	default:
831 		WARN(1, "invalidate vhost IOTLB permission\n");
832 		break;
833 	}
834 
835 	return flags;
836 }
837 
838 static int vduse_kickfd_setup(struct vduse_dev *dev,
839 			struct vduse_vq_eventfd *eventfd)
840 {
841 	struct eventfd_ctx *ctx = NULL;
842 	struct vduse_virtqueue *vq;
843 	u32 index;
844 
845 	if (eventfd->index >= dev->vq_num)
846 		return -EINVAL;
847 
848 	index = array_index_nospec(eventfd->index, dev->vq_num);
849 	vq = &dev->vqs[index];
850 	if (eventfd->fd >= 0) {
851 		ctx = eventfd_ctx_fdget(eventfd->fd);
852 		if (IS_ERR(ctx))
853 			return PTR_ERR(ctx);
854 	} else if (eventfd->fd != VDUSE_EVENTFD_DEASSIGN)
855 		return 0;
856 
857 	spin_lock(&vq->kick_lock);
858 	if (vq->kickfd)
859 		eventfd_ctx_put(vq->kickfd);
860 	vq->kickfd = ctx;
861 	if (vq->ready && vq->kicked && vq->kickfd) {
862 		eventfd_signal(vq->kickfd, 1);
863 		vq->kicked = false;
864 	}
865 	spin_unlock(&vq->kick_lock);
866 
867 	return 0;
868 }
869 
870 static bool vduse_dev_is_ready(struct vduse_dev *dev)
871 {
872 	int i;
873 
874 	for (i = 0; i < dev->vq_num; i++)
875 		if (!dev->vqs[i].num_max)
876 			return false;
877 
878 	return true;
879 }
880 
881 static void vduse_dev_irq_inject(struct work_struct *work)
882 {
883 	struct vduse_dev *dev = container_of(work, struct vduse_dev, inject);
884 
885 	spin_lock_irq(&dev->irq_lock);
886 	if (dev->config_cb.callback)
887 		dev->config_cb.callback(dev->config_cb.private);
888 	spin_unlock_irq(&dev->irq_lock);
889 }
890 
891 static void vduse_vq_irq_inject(struct work_struct *work)
892 {
893 	struct vduse_virtqueue *vq = container_of(work,
894 					struct vduse_virtqueue, inject);
895 
896 	spin_lock_irq(&vq->irq_lock);
897 	if (vq->ready && vq->cb.callback)
898 		vq->cb.callback(vq->cb.private);
899 	spin_unlock_irq(&vq->irq_lock);
900 }
901 
902 static int vduse_dev_queue_irq_work(struct vduse_dev *dev,
903 				    struct work_struct *irq_work)
904 {
905 	int ret = -EINVAL;
906 
907 	down_read(&dev->rwsem);
908 	if (!(dev->status & VIRTIO_CONFIG_S_DRIVER_OK))
909 		goto unlock;
910 
911 	ret = 0;
912 	queue_work(vduse_irq_wq, irq_work);
913 unlock:
914 	up_read(&dev->rwsem);
915 
916 	return ret;
917 }
918 
919 static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
920 			    unsigned long arg)
921 {
922 	struct vduse_dev *dev = file->private_data;
923 	void __user *argp = (void __user *)arg;
924 	int ret;
925 
926 	if (unlikely(dev->broken))
927 		return -EPERM;
928 
929 	switch (cmd) {
930 	case VDUSE_IOTLB_GET_FD: {
931 		struct vduse_iotlb_entry entry;
932 		struct vhost_iotlb_map *map;
933 		struct vdpa_map_file *map_file;
934 		struct vduse_iova_domain *domain = dev->domain;
935 		struct file *f = NULL;
936 
937 		ret = -EFAULT;
938 		if (copy_from_user(&entry, argp, sizeof(entry)))
939 			break;
940 
941 		ret = -EINVAL;
942 		if (entry.start > entry.last)
943 			break;
944 
945 		spin_lock(&domain->iotlb_lock);
946 		map = vhost_iotlb_itree_first(domain->iotlb,
947 					      entry.start, entry.last);
948 		if (map) {
949 			map_file = (struct vdpa_map_file *)map->opaque;
950 			f = get_file(map_file->file);
951 			entry.offset = map_file->offset;
952 			entry.start = map->start;
953 			entry.last = map->last;
954 			entry.perm = map->perm;
955 		}
956 		spin_unlock(&domain->iotlb_lock);
957 		ret = -EINVAL;
958 		if (!f)
959 			break;
960 
961 		ret = -EFAULT;
962 		if (copy_to_user(argp, &entry, sizeof(entry))) {
963 			fput(f);
964 			break;
965 		}
966 		ret = receive_fd(f, perm_to_file_flags(entry.perm));
967 		fput(f);
968 		break;
969 	}
970 	case VDUSE_DEV_GET_FEATURES:
971 		/*
972 		 * Just mirror what driver wrote here.
973 		 * The driver is expected to check FEATURE_OK later.
974 		 */
975 		ret = put_user(dev->driver_features, (u64 __user *)argp);
976 		break;
977 	case VDUSE_DEV_SET_CONFIG: {
978 		struct vduse_config_data config;
979 		unsigned long size = offsetof(struct vduse_config_data,
980 					      buffer);
981 
982 		ret = -EFAULT;
983 		if (copy_from_user(&config, argp, size))
984 			break;
985 
986 		ret = -EINVAL;
987 		if (config.offset > dev->config_size ||
988 		    config.length == 0 ||
989 		    config.length > dev->config_size - config.offset)
990 			break;
991 
992 		ret = -EFAULT;
993 		if (copy_from_user(dev->config + config.offset, argp + size,
994 				   config.length))
995 			break;
996 
997 		ret = 0;
998 		break;
999 	}
1000 	case VDUSE_DEV_INJECT_CONFIG_IRQ:
1001 		ret = vduse_dev_queue_irq_work(dev, &dev->inject);
1002 		break;
1003 	case VDUSE_VQ_SETUP: {
1004 		struct vduse_vq_config config;
1005 		u32 index;
1006 
1007 		ret = -EFAULT;
1008 		if (copy_from_user(&config, argp, sizeof(config)))
1009 			break;
1010 
1011 		ret = -EINVAL;
1012 		if (config.index >= dev->vq_num)
1013 			break;
1014 
1015 		if (!is_mem_zero((const char *)config.reserved,
1016 				 sizeof(config.reserved)))
1017 			break;
1018 
1019 		index = array_index_nospec(config.index, dev->vq_num);
1020 		dev->vqs[index].num_max = config.max_size;
1021 		ret = 0;
1022 		break;
1023 	}
1024 	case VDUSE_VQ_GET_INFO: {
1025 		struct vduse_vq_info vq_info;
1026 		struct vduse_virtqueue *vq;
1027 		u32 index;
1028 
1029 		ret = -EFAULT;
1030 		if (copy_from_user(&vq_info, argp, sizeof(vq_info)))
1031 			break;
1032 
1033 		ret = -EINVAL;
1034 		if (vq_info.index >= dev->vq_num)
1035 			break;
1036 
1037 		index = array_index_nospec(vq_info.index, dev->vq_num);
1038 		vq = &dev->vqs[index];
1039 		vq_info.desc_addr = vq->desc_addr;
1040 		vq_info.driver_addr = vq->driver_addr;
1041 		vq_info.device_addr = vq->device_addr;
1042 		vq_info.num = vq->num;
1043 
1044 		if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
1045 			vq_info.packed.last_avail_counter =
1046 				vq->state.packed.last_avail_counter;
1047 			vq_info.packed.last_avail_idx =
1048 				vq->state.packed.last_avail_idx;
1049 			vq_info.packed.last_used_counter =
1050 				vq->state.packed.last_used_counter;
1051 			vq_info.packed.last_used_idx =
1052 				vq->state.packed.last_used_idx;
1053 		} else
1054 			vq_info.split.avail_index =
1055 				vq->state.split.avail_index;
1056 
1057 		vq_info.ready = vq->ready;
1058 
1059 		ret = -EFAULT;
1060 		if (copy_to_user(argp, &vq_info, sizeof(vq_info)))
1061 			break;
1062 
1063 		ret = 0;
1064 		break;
1065 	}
1066 	case VDUSE_VQ_SETUP_KICKFD: {
1067 		struct vduse_vq_eventfd eventfd;
1068 
1069 		ret = -EFAULT;
1070 		if (copy_from_user(&eventfd, argp, sizeof(eventfd)))
1071 			break;
1072 
1073 		ret = vduse_kickfd_setup(dev, &eventfd);
1074 		break;
1075 	}
1076 	case VDUSE_VQ_INJECT_IRQ: {
1077 		u32 index;
1078 
1079 		ret = -EFAULT;
1080 		if (get_user(index, (u32 __user *)argp))
1081 			break;
1082 
1083 		ret = -EINVAL;
1084 		if (index >= dev->vq_num)
1085 			break;
1086 
1087 		index = array_index_nospec(index, dev->vq_num);
1088 		ret = vduse_dev_queue_irq_work(dev, &dev->vqs[index].inject);
1089 		break;
1090 	}
1091 	default:
1092 		ret = -ENOIOCTLCMD;
1093 		break;
1094 	}
1095 
1096 	return ret;
1097 }
1098 
1099 static int vduse_dev_release(struct inode *inode, struct file *file)
1100 {
1101 	struct vduse_dev *dev = file->private_data;
1102 
1103 	spin_lock(&dev->msg_lock);
1104 	/* Make sure the inflight messages can processed after reconncection */
1105 	list_splice_init(&dev->recv_list, &dev->send_list);
1106 	spin_unlock(&dev->msg_lock);
1107 	dev->connected = false;
1108 
1109 	return 0;
1110 }
1111 
1112 static struct vduse_dev *vduse_dev_get_from_minor(int minor)
1113 {
1114 	struct vduse_dev *dev;
1115 
1116 	mutex_lock(&vduse_lock);
1117 	dev = idr_find(&vduse_idr, minor);
1118 	mutex_unlock(&vduse_lock);
1119 
1120 	return dev;
1121 }
1122 
1123 static int vduse_dev_open(struct inode *inode, struct file *file)
1124 {
1125 	int ret;
1126 	struct vduse_dev *dev = vduse_dev_get_from_minor(iminor(inode));
1127 
1128 	if (!dev)
1129 		return -ENODEV;
1130 
1131 	ret = -EBUSY;
1132 	mutex_lock(&dev->lock);
1133 	if (dev->connected)
1134 		goto unlock;
1135 
1136 	ret = 0;
1137 	dev->connected = true;
1138 	file->private_data = dev;
1139 unlock:
1140 	mutex_unlock(&dev->lock);
1141 
1142 	return ret;
1143 }
1144 
1145 static const struct file_operations vduse_dev_fops = {
1146 	.owner		= THIS_MODULE,
1147 	.open		= vduse_dev_open,
1148 	.release	= vduse_dev_release,
1149 	.read_iter	= vduse_dev_read_iter,
1150 	.write_iter	= vduse_dev_write_iter,
1151 	.poll		= vduse_dev_poll,
1152 	.unlocked_ioctl	= vduse_dev_ioctl,
1153 	.compat_ioctl	= compat_ptr_ioctl,
1154 	.llseek		= noop_llseek,
1155 };
1156 
1157 static struct vduse_dev *vduse_dev_create(void)
1158 {
1159 	struct vduse_dev *dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1160 
1161 	if (!dev)
1162 		return NULL;
1163 
1164 	mutex_init(&dev->lock);
1165 	spin_lock_init(&dev->msg_lock);
1166 	INIT_LIST_HEAD(&dev->send_list);
1167 	INIT_LIST_HEAD(&dev->recv_list);
1168 	spin_lock_init(&dev->irq_lock);
1169 	init_rwsem(&dev->rwsem);
1170 
1171 	INIT_WORK(&dev->inject, vduse_dev_irq_inject);
1172 	init_waitqueue_head(&dev->waitq);
1173 
1174 	return dev;
1175 }
1176 
1177 static void vduse_dev_destroy(struct vduse_dev *dev)
1178 {
1179 	kfree(dev);
1180 }
1181 
1182 static struct vduse_dev *vduse_find_dev(const char *name)
1183 {
1184 	struct vduse_dev *dev;
1185 	int id;
1186 
1187 	idr_for_each_entry(&vduse_idr, dev, id)
1188 		if (!strcmp(dev->name, name))
1189 			return dev;
1190 
1191 	return NULL;
1192 }
1193 
1194 static int vduse_destroy_dev(char *name)
1195 {
1196 	struct vduse_dev *dev = vduse_find_dev(name);
1197 
1198 	if (!dev)
1199 		return -EINVAL;
1200 
1201 	mutex_lock(&dev->lock);
1202 	if (dev->vdev || dev->connected) {
1203 		mutex_unlock(&dev->lock);
1204 		return -EBUSY;
1205 	}
1206 	dev->connected = true;
1207 	mutex_unlock(&dev->lock);
1208 
1209 	vduse_dev_reset(dev);
1210 	device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));
1211 	idr_remove(&vduse_idr, dev->minor);
1212 	kvfree(dev->config);
1213 	kfree(dev->vqs);
1214 	vduse_domain_destroy(dev->domain);
1215 	kfree(dev->name);
1216 	vduse_dev_destroy(dev);
1217 	module_put(THIS_MODULE);
1218 
1219 	return 0;
1220 }
1221 
1222 static bool device_is_allowed(u32 device_id)
1223 {
1224 	int i;
1225 
1226 	for (i = 0; i < ARRAY_SIZE(allowed_device_id); i++)
1227 		if (allowed_device_id[i] == device_id)
1228 			return true;
1229 
1230 	return false;
1231 }
1232 
1233 static bool features_is_valid(u64 features)
1234 {
1235 	if (!(features & (1ULL << VIRTIO_F_ACCESS_PLATFORM)))
1236 		return false;
1237 
1238 	/* Now we only support read-only configuration space */
1239 	if (features & (1ULL << VIRTIO_BLK_F_CONFIG_WCE))
1240 		return false;
1241 
1242 	return true;
1243 }
1244 
1245 static bool vduse_validate_config(struct vduse_dev_config *config)
1246 {
1247 	if (!is_mem_zero((const char *)config->reserved,
1248 			 sizeof(config->reserved)))
1249 		return false;
1250 
1251 	if (config->vq_align > PAGE_SIZE)
1252 		return false;
1253 
1254 	if (config->config_size > PAGE_SIZE)
1255 		return false;
1256 
1257 	if (!device_is_allowed(config->device_id))
1258 		return false;
1259 
1260 	if (!features_is_valid(config->features))
1261 		return false;
1262 
1263 	return true;
1264 }
1265 
1266 static ssize_t msg_timeout_show(struct device *device,
1267 				struct device_attribute *attr, char *buf)
1268 {
1269 	struct vduse_dev *dev = dev_get_drvdata(device);
1270 
1271 	return sysfs_emit(buf, "%u\n", dev->msg_timeout);
1272 }
1273 
1274 static ssize_t msg_timeout_store(struct device *device,
1275 				 struct device_attribute *attr,
1276 				 const char *buf, size_t count)
1277 {
1278 	struct vduse_dev *dev = dev_get_drvdata(device);
1279 	int ret;
1280 
1281 	ret = kstrtouint(buf, 10, &dev->msg_timeout);
1282 	if (ret < 0)
1283 		return ret;
1284 
1285 	return count;
1286 }
1287 
1288 static DEVICE_ATTR_RW(msg_timeout);
1289 
1290 static struct attribute *vduse_dev_attrs[] = {
1291 	&dev_attr_msg_timeout.attr,
1292 	NULL
1293 };
1294 
1295 ATTRIBUTE_GROUPS(vduse_dev);
1296 
1297 static int vduse_create_dev(struct vduse_dev_config *config,
1298 			    void *config_buf, u64 api_version)
1299 {
1300 	int i, ret;
1301 	struct vduse_dev *dev;
1302 
1303 	ret = -EEXIST;
1304 	if (vduse_find_dev(config->name))
1305 		goto err;
1306 
1307 	ret = -ENOMEM;
1308 	dev = vduse_dev_create();
1309 	if (!dev)
1310 		goto err;
1311 
1312 	dev->api_version = api_version;
1313 	dev->device_features = config->features;
1314 	dev->device_id = config->device_id;
1315 	dev->vendor_id = config->vendor_id;
1316 	dev->name = kstrdup(config->name, GFP_KERNEL);
1317 	if (!dev->name)
1318 		goto err_str;
1319 
1320 	dev->domain = vduse_domain_create(VDUSE_IOVA_SIZE - 1,
1321 					  VDUSE_BOUNCE_SIZE);
1322 	if (!dev->domain)
1323 		goto err_domain;
1324 
1325 	dev->config = config_buf;
1326 	dev->config_size = config->config_size;
1327 	dev->vq_align = config->vq_align;
1328 	dev->vq_num = config->vq_num;
1329 	dev->vqs = kcalloc(dev->vq_num, sizeof(*dev->vqs), GFP_KERNEL);
1330 	if (!dev->vqs)
1331 		goto err_vqs;
1332 
1333 	for (i = 0; i < dev->vq_num; i++) {
1334 		dev->vqs[i].index = i;
1335 		INIT_WORK(&dev->vqs[i].inject, vduse_vq_irq_inject);
1336 		INIT_WORK(&dev->vqs[i].kick, vduse_vq_kick_work);
1337 		spin_lock_init(&dev->vqs[i].kick_lock);
1338 		spin_lock_init(&dev->vqs[i].irq_lock);
1339 	}
1340 
1341 	ret = idr_alloc(&vduse_idr, dev, 1, VDUSE_DEV_MAX, GFP_KERNEL);
1342 	if (ret < 0)
1343 		goto err_idr;
1344 
1345 	dev->minor = ret;
1346 	dev->msg_timeout = VDUSE_MSG_DEFAULT_TIMEOUT;
1347 	dev->dev = device_create(vduse_class, NULL,
1348 				 MKDEV(MAJOR(vduse_major), dev->minor),
1349 				 dev, "%s", config->name);
1350 	if (IS_ERR(dev->dev)) {
1351 		ret = PTR_ERR(dev->dev);
1352 		goto err_dev;
1353 	}
1354 	__module_get(THIS_MODULE);
1355 
1356 	return 0;
1357 err_dev:
1358 	idr_remove(&vduse_idr, dev->minor);
1359 err_idr:
1360 	kfree(dev->vqs);
1361 err_vqs:
1362 	vduse_domain_destroy(dev->domain);
1363 err_domain:
1364 	kfree(dev->name);
1365 err_str:
1366 	vduse_dev_destroy(dev);
1367 err:
1368 	return ret;
1369 }
1370 
1371 static long vduse_ioctl(struct file *file, unsigned int cmd,
1372 			unsigned long arg)
1373 {
1374 	int ret;
1375 	void __user *argp = (void __user *)arg;
1376 	struct vduse_control *control = file->private_data;
1377 
1378 	mutex_lock(&vduse_lock);
1379 	switch (cmd) {
1380 	case VDUSE_GET_API_VERSION:
1381 		ret = put_user(control->api_version, (u64 __user *)argp);
1382 		break;
1383 	case VDUSE_SET_API_VERSION: {
1384 		u64 api_version;
1385 
1386 		ret = -EFAULT;
1387 		if (get_user(api_version, (u64 __user *)argp))
1388 			break;
1389 
1390 		ret = -EINVAL;
1391 		if (api_version > VDUSE_API_VERSION)
1392 			break;
1393 
1394 		ret = 0;
1395 		control->api_version = api_version;
1396 		break;
1397 	}
1398 	case VDUSE_CREATE_DEV: {
1399 		struct vduse_dev_config config;
1400 		unsigned long size = offsetof(struct vduse_dev_config, config);
1401 		void *buf;
1402 
1403 		ret = -EFAULT;
1404 		if (copy_from_user(&config, argp, size))
1405 			break;
1406 
1407 		ret = -EINVAL;
1408 		if (vduse_validate_config(&config) == false)
1409 			break;
1410 
1411 		buf = vmemdup_user(argp + size, config.config_size);
1412 		if (IS_ERR(buf)) {
1413 			ret = PTR_ERR(buf);
1414 			break;
1415 		}
1416 		config.name[VDUSE_NAME_MAX - 1] = '\0';
1417 		ret = vduse_create_dev(&config, buf, control->api_version);
1418 		if (ret)
1419 			kvfree(buf);
1420 		break;
1421 	}
1422 	case VDUSE_DESTROY_DEV: {
1423 		char name[VDUSE_NAME_MAX];
1424 
1425 		ret = -EFAULT;
1426 		if (copy_from_user(name, argp, VDUSE_NAME_MAX))
1427 			break;
1428 
1429 		name[VDUSE_NAME_MAX - 1] = '\0';
1430 		ret = vduse_destroy_dev(name);
1431 		break;
1432 	}
1433 	default:
1434 		ret = -EINVAL;
1435 		break;
1436 	}
1437 	mutex_unlock(&vduse_lock);
1438 
1439 	return ret;
1440 }
1441 
1442 static int vduse_release(struct inode *inode, struct file *file)
1443 {
1444 	struct vduse_control *control = file->private_data;
1445 
1446 	kfree(control);
1447 	return 0;
1448 }
1449 
1450 static int vduse_open(struct inode *inode, struct file *file)
1451 {
1452 	struct vduse_control *control;
1453 
1454 	control = kmalloc(sizeof(struct vduse_control), GFP_KERNEL);
1455 	if (!control)
1456 		return -ENOMEM;
1457 
1458 	control->api_version = VDUSE_API_VERSION;
1459 	file->private_data = control;
1460 
1461 	return 0;
1462 }
1463 
1464 static const struct file_operations vduse_ctrl_fops = {
1465 	.owner		= THIS_MODULE,
1466 	.open		= vduse_open,
1467 	.release	= vduse_release,
1468 	.unlocked_ioctl	= vduse_ioctl,
1469 	.compat_ioctl	= compat_ptr_ioctl,
1470 	.llseek		= noop_llseek,
1471 };
1472 
1473 static char *vduse_devnode(struct device *dev, umode_t *mode)
1474 {
1475 	return kasprintf(GFP_KERNEL, "vduse/%s", dev_name(dev));
1476 }
1477 
1478 static void vduse_mgmtdev_release(struct device *dev)
1479 {
1480 }
1481 
1482 static struct device vduse_mgmtdev = {
1483 	.init_name = "vduse",
1484 	.release = vduse_mgmtdev_release,
1485 };
1486 
1487 static struct vdpa_mgmt_dev mgmt_dev;
1488 
1489 static int vduse_dev_init_vdpa(struct vduse_dev *dev, const char *name)
1490 {
1491 	struct vduse_vdpa *vdev;
1492 	int ret;
1493 
1494 	if (dev->vdev)
1495 		return -EEXIST;
1496 
1497 	vdev = vdpa_alloc_device(struct vduse_vdpa, vdpa, dev->dev,
1498 				 &vduse_vdpa_config_ops, name, true);
1499 	if (IS_ERR(vdev))
1500 		return PTR_ERR(vdev);
1501 
1502 	dev->vdev = vdev;
1503 	vdev->dev = dev;
1504 	vdev->vdpa.dev.dma_mask = &vdev->vdpa.dev.coherent_dma_mask;
1505 	ret = dma_set_mask_and_coherent(&vdev->vdpa.dev, DMA_BIT_MASK(64));
1506 	if (ret) {
1507 		put_device(&vdev->vdpa.dev);
1508 		return ret;
1509 	}
1510 	set_dma_ops(&vdev->vdpa.dev, &vduse_dev_dma_ops);
1511 	vdev->vdpa.dma_dev = &vdev->vdpa.dev;
1512 	vdev->vdpa.mdev = &mgmt_dev;
1513 
1514 	return 0;
1515 }
1516 
1517 static int vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
1518 			const struct vdpa_dev_set_config *config)
1519 {
1520 	struct vduse_dev *dev;
1521 	int ret;
1522 
1523 	mutex_lock(&vduse_lock);
1524 	dev = vduse_find_dev(name);
1525 	if (!dev || !vduse_dev_is_ready(dev)) {
1526 		mutex_unlock(&vduse_lock);
1527 		return -EINVAL;
1528 	}
1529 	ret = vduse_dev_init_vdpa(dev, name);
1530 	mutex_unlock(&vduse_lock);
1531 	if (ret)
1532 		return ret;
1533 
1534 	ret = _vdpa_register_device(&dev->vdev->vdpa, dev->vq_num);
1535 	if (ret) {
1536 		put_device(&dev->vdev->vdpa.dev);
1537 		return ret;
1538 	}
1539 
1540 	return 0;
1541 }
1542 
1543 static void vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev)
1544 {
1545 	_vdpa_unregister_device(dev);
1546 }
1547 
1548 static const struct vdpa_mgmtdev_ops vdpa_dev_mgmtdev_ops = {
1549 	.dev_add = vdpa_dev_add,
1550 	.dev_del = vdpa_dev_del,
1551 };
1552 
1553 static struct virtio_device_id id_table[] = {
1554 	{ VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID },
1555 	{ 0 },
1556 };
1557 
1558 static struct vdpa_mgmt_dev mgmt_dev = {
1559 	.device = &vduse_mgmtdev,
1560 	.id_table = id_table,
1561 	.ops = &vdpa_dev_mgmtdev_ops,
1562 };
1563 
1564 static int vduse_mgmtdev_init(void)
1565 {
1566 	int ret;
1567 
1568 	ret = device_register(&vduse_mgmtdev);
1569 	if (ret)
1570 		return ret;
1571 
1572 	ret = vdpa_mgmtdev_register(&mgmt_dev);
1573 	if (ret)
1574 		goto err;
1575 
1576 	return 0;
1577 err:
1578 	device_unregister(&vduse_mgmtdev);
1579 	return ret;
1580 }
1581 
1582 static void vduse_mgmtdev_exit(void)
1583 {
1584 	vdpa_mgmtdev_unregister(&mgmt_dev);
1585 	device_unregister(&vduse_mgmtdev);
1586 }
1587 
1588 static int vduse_init(void)
1589 {
1590 	int ret;
1591 	struct device *dev;
1592 
1593 	vduse_class = class_create(THIS_MODULE, "vduse");
1594 	if (IS_ERR(vduse_class))
1595 		return PTR_ERR(vduse_class);
1596 
1597 	vduse_class->devnode = vduse_devnode;
1598 	vduse_class->dev_groups = vduse_dev_groups;
1599 
1600 	ret = alloc_chrdev_region(&vduse_major, 0, VDUSE_DEV_MAX, "vduse");
1601 	if (ret)
1602 		goto err_chardev_region;
1603 
1604 	/* /dev/vduse/control */
1605 	cdev_init(&vduse_ctrl_cdev, &vduse_ctrl_fops);
1606 	vduse_ctrl_cdev.owner = THIS_MODULE;
1607 	ret = cdev_add(&vduse_ctrl_cdev, vduse_major, 1);
1608 	if (ret)
1609 		goto err_ctrl_cdev;
1610 
1611 	dev = device_create(vduse_class, NULL, vduse_major, NULL, "control");
1612 	if (IS_ERR(dev)) {
1613 		ret = PTR_ERR(dev);
1614 		goto err_device;
1615 	}
1616 
1617 	/* /dev/vduse/$DEVICE */
1618 	cdev_init(&vduse_cdev, &vduse_dev_fops);
1619 	vduse_cdev.owner = THIS_MODULE;
1620 	ret = cdev_add(&vduse_cdev, MKDEV(MAJOR(vduse_major), 1),
1621 		       VDUSE_DEV_MAX - 1);
1622 	if (ret)
1623 		goto err_cdev;
1624 
1625 	vduse_irq_wq = alloc_workqueue("vduse-irq",
1626 				WQ_HIGHPRI | WQ_SYSFS | WQ_UNBOUND, 0);
1627 	if (!vduse_irq_wq) {
1628 		ret = -ENOMEM;
1629 		goto err_wq;
1630 	}
1631 
1632 	ret = vduse_domain_init();
1633 	if (ret)
1634 		goto err_domain;
1635 
1636 	ret = vduse_mgmtdev_init();
1637 	if (ret)
1638 		goto err_mgmtdev;
1639 
1640 	return 0;
1641 err_mgmtdev:
1642 	vduse_domain_exit();
1643 err_domain:
1644 	destroy_workqueue(vduse_irq_wq);
1645 err_wq:
1646 	cdev_del(&vduse_cdev);
1647 err_cdev:
1648 	device_destroy(vduse_class, vduse_major);
1649 err_device:
1650 	cdev_del(&vduse_ctrl_cdev);
1651 err_ctrl_cdev:
1652 	unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);
1653 err_chardev_region:
1654 	class_destroy(vduse_class);
1655 	return ret;
1656 }
1657 module_init(vduse_init);
1658 
1659 static void vduse_exit(void)
1660 {
1661 	vduse_mgmtdev_exit();
1662 	vduse_domain_exit();
1663 	destroy_workqueue(vduse_irq_wq);
1664 	cdev_del(&vduse_cdev);
1665 	device_destroy(vduse_class, vduse_major);
1666 	cdev_del(&vduse_ctrl_cdev);
1667 	unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);
1668 	class_destroy(vduse_class);
1669 }
1670 module_exit(vduse_exit);
1671 
1672 MODULE_LICENSE(DRV_LICENSE);
1673 MODULE_AUTHOR(DRV_AUTHOR);
1674 MODULE_DESCRIPTION(DRV_DESC);
1675