1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * VDUSE: vDPA Device in Userspace
4  *
5  * Copyright (C) 2020-2021 Bytedance Inc. and/or its affiliates. All rights reserved.
6  *
7  * Author: Xie Yongji <xieyongji@bytedance.com>
8  *
9  */
10 
11 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/cdev.h>
14 #include <linux/device.h>
15 #include <linux/eventfd.h>
16 #include <linux/slab.h>
17 #include <linux/wait.h>
18 #include <linux/dma-map-ops.h>
19 #include <linux/poll.h>
20 #include <linux/file.h>
21 #include <linux/uio.h>
22 #include <linux/vdpa.h>
23 #include <linux/nospec.h>
24 #include <linux/vmalloc.h>
25 #include <linux/sched/mm.h>
26 #include <uapi/linux/vduse.h>
27 #include <uapi/linux/vdpa.h>
28 #include <uapi/linux/virtio_config.h>
29 #include <uapi/linux/virtio_ids.h>
30 #include <uapi/linux/virtio_blk.h>
31 #include <linux/mod_devicetable.h>
32 
33 #include "iova_domain.h"
34 
35 #define DRV_AUTHOR   "Yongji Xie <xieyongji@bytedance.com>"
36 #define DRV_DESC     "vDPA Device in Userspace"
37 #define DRV_LICENSE  "GPL v2"
38 
39 #define VDUSE_DEV_MAX (1U << MINORBITS)
40 #define VDUSE_MAX_BOUNCE_SIZE (1024 * 1024 * 1024)
41 #define VDUSE_MIN_BOUNCE_SIZE (1024 * 1024)
42 #define VDUSE_BOUNCE_SIZE (64 * 1024 * 1024)
43 /* 128 MB reserved for virtqueue creation */
44 #define VDUSE_IOVA_SIZE (VDUSE_MAX_BOUNCE_SIZE + 128 * 1024 * 1024)
45 #define VDUSE_MSG_DEFAULT_TIMEOUT 30
46 
47 #define IRQ_UNBOUND -1
48 
49 struct vduse_virtqueue {
50 	u16 index;
51 	u16 num_max;
52 	u32 num;
53 	u64 desc_addr;
54 	u64 driver_addr;
55 	u64 device_addr;
56 	struct vdpa_vq_state state;
57 	bool ready;
58 	bool kicked;
59 	spinlock_t kick_lock;
60 	spinlock_t irq_lock;
61 	struct eventfd_ctx *kickfd;
62 	struct vdpa_callback cb;
63 	struct work_struct inject;
64 	struct work_struct kick;
65 	int irq_effective_cpu;
66 	struct cpumask irq_affinity;
67 	struct kobject kobj;
68 };
69 
70 struct vduse_dev;
71 
72 struct vduse_vdpa {
73 	struct vdpa_device vdpa;
74 	struct vduse_dev *dev;
75 };
76 
77 struct vduse_umem {
78 	unsigned long iova;
79 	unsigned long npages;
80 	struct page **pages;
81 	struct mm_struct *mm;
82 };
83 
84 struct vduse_dev {
85 	struct vduse_vdpa *vdev;
86 	struct device *dev;
87 	struct vduse_virtqueue **vqs;
88 	struct vduse_iova_domain *domain;
89 	char *name;
90 	struct mutex lock;
91 	spinlock_t msg_lock;
92 	u64 msg_unique;
93 	u32 msg_timeout;
94 	wait_queue_head_t waitq;
95 	struct list_head send_list;
96 	struct list_head recv_list;
97 	struct vdpa_callback config_cb;
98 	struct work_struct inject;
99 	spinlock_t irq_lock;
100 	struct rw_semaphore rwsem;
101 	int minor;
102 	bool broken;
103 	bool connected;
104 	u64 api_version;
105 	u64 device_features;
106 	u64 driver_features;
107 	u32 device_id;
108 	u32 vendor_id;
109 	u32 generation;
110 	u32 config_size;
111 	void *config;
112 	u8 status;
113 	u32 vq_num;
114 	u32 vq_align;
115 	struct vduse_umem *umem;
116 	struct mutex mem_lock;
117 	unsigned int bounce_size;
118 	struct mutex domain_lock;
119 };
120 
121 struct vduse_dev_msg {
122 	struct vduse_dev_request req;
123 	struct vduse_dev_response resp;
124 	struct list_head list;
125 	wait_queue_head_t waitq;
126 	bool completed;
127 };
128 
129 struct vduse_control {
130 	u64 api_version;
131 };
132 
133 static DEFINE_MUTEX(vduse_lock);
134 static DEFINE_IDR(vduse_idr);
135 
136 static dev_t vduse_major;
137 static struct class *vduse_class;
138 static struct cdev vduse_ctrl_cdev;
139 static struct cdev vduse_cdev;
140 static struct workqueue_struct *vduse_irq_wq;
141 static struct workqueue_struct *vduse_irq_bound_wq;
142 
143 static u32 allowed_device_id[] = {
144 	VIRTIO_ID_BLOCK,
145 };
146 
147 static inline struct vduse_dev *vdpa_to_vduse(struct vdpa_device *vdpa)
148 {
149 	struct vduse_vdpa *vdev = container_of(vdpa, struct vduse_vdpa, vdpa);
150 
151 	return vdev->dev;
152 }
153 
154 static inline struct vduse_dev *dev_to_vduse(struct device *dev)
155 {
156 	struct vdpa_device *vdpa = dev_to_vdpa(dev);
157 
158 	return vdpa_to_vduse(vdpa);
159 }
160 
161 static struct vduse_dev_msg *vduse_find_msg(struct list_head *head,
162 					    uint32_t request_id)
163 {
164 	struct vduse_dev_msg *msg;
165 
166 	list_for_each_entry(msg, head, list) {
167 		if (msg->req.request_id == request_id) {
168 			list_del(&msg->list);
169 			return msg;
170 		}
171 	}
172 
173 	return NULL;
174 }
175 
176 static struct vduse_dev_msg *vduse_dequeue_msg(struct list_head *head)
177 {
178 	struct vduse_dev_msg *msg = NULL;
179 
180 	if (!list_empty(head)) {
181 		msg = list_first_entry(head, struct vduse_dev_msg, list);
182 		list_del(&msg->list);
183 	}
184 
185 	return msg;
186 }
187 
188 static void vduse_enqueue_msg(struct list_head *head,
189 			      struct vduse_dev_msg *msg)
190 {
191 	list_add_tail(&msg->list, head);
192 }
193 
194 static void vduse_dev_broken(struct vduse_dev *dev)
195 {
196 	struct vduse_dev_msg *msg, *tmp;
197 
198 	if (unlikely(dev->broken))
199 		return;
200 
201 	list_splice_init(&dev->recv_list, &dev->send_list);
202 	list_for_each_entry_safe(msg, tmp, &dev->send_list, list) {
203 		list_del(&msg->list);
204 		msg->completed = 1;
205 		msg->resp.result = VDUSE_REQ_RESULT_FAILED;
206 		wake_up(&msg->waitq);
207 	}
208 	dev->broken = true;
209 	wake_up(&dev->waitq);
210 }
211 
212 static int vduse_dev_msg_sync(struct vduse_dev *dev,
213 			      struct vduse_dev_msg *msg)
214 {
215 	int ret;
216 
217 	if (unlikely(dev->broken))
218 		return -EIO;
219 
220 	init_waitqueue_head(&msg->waitq);
221 	spin_lock(&dev->msg_lock);
222 	if (unlikely(dev->broken)) {
223 		spin_unlock(&dev->msg_lock);
224 		return -EIO;
225 	}
226 	msg->req.request_id = dev->msg_unique++;
227 	vduse_enqueue_msg(&dev->send_list, msg);
228 	wake_up(&dev->waitq);
229 	spin_unlock(&dev->msg_lock);
230 	if (dev->msg_timeout)
231 		ret = wait_event_killable_timeout(msg->waitq, msg->completed,
232 						  (long)dev->msg_timeout * HZ);
233 	else
234 		ret = wait_event_killable(msg->waitq, msg->completed);
235 
236 	spin_lock(&dev->msg_lock);
237 	if (!msg->completed) {
238 		list_del(&msg->list);
239 		msg->resp.result = VDUSE_REQ_RESULT_FAILED;
240 		/* Mark the device as malfunction when there is a timeout */
241 		if (!ret)
242 			vduse_dev_broken(dev);
243 	}
244 	ret = (msg->resp.result == VDUSE_REQ_RESULT_OK) ? 0 : -EIO;
245 	spin_unlock(&dev->msg_lock);
246 
247 	return ret;
248 }
249 
250 static int vduse_dev_get_vq_state_packed(struct vduse_dev *dev,
251 					 struct vduse_virtqueue *vq,
252 					 struct vdpa_vq_state_packed *packed)
253 {
254 	struct vduse_dev_msg msg = { 0 };
255 	int ret;
256 
257 	msg.req.type = VDUSE_GET_VQ_STATE;
258 	msg.req.vq_state.index = vq->index;
259 
260 	ret = vduse_dev_msg_sync(dev, &msg);
261 	if (ret)
262 		return ret;
263 
264 	packed->last_avail_counter =
265 			msg.resp.vq_state.packed.last_avail_counter & 0x0001;
266 	packed->last_avail_idx =
267 			msg.resp.vq_state.packed.last_avail_idx & 0x7FFF;
268 	packed->last_used_counter =
269 			msg.resp.vq_state.packed.last_used_counter & 0x0001;
270 	packed->last_used_idx =
271 			msg.resp.vq_state.packed.last_used_idx & 0x7FFF;
272 
273 	return 0;
274 }
275 
276 static int vduse_dev_get_vq_state_split(struct vduse_dev *dev,
277 					struct vduse_virtqueue *vq,
278 					struct vdpa_vq_state_split *split)
279 {
280 	struct vduse_dev_msg msg = { 0 };
281 	int ret;
282 
283 	msg.req.type = VDUSE_GET_VQ_STATE;
284 	msg.req.vq_state.index = vq->index;
285 
286 	ret = vduse_dev_msg_sync(dev, &msg);
287 	if (ret)
288 		return ret;
289 
290 	split->avail_index = msg.resp.vq_state.split.avail_index;
291 
292 	return 0;
293 }
294 
295 static int vduse_dev_set_status(struct vduse_dev *dev, u8 status)
296 {
297 	struct vduse_dev_msg msg = { 0 };
298 
299 	msg.req.type = VDUSE_SET_STATUS;
300 	msg.req.s.status = status;
301 
302 	return vduse_dev_msg_sync(dev, &msg);
303 }
304 
305 static int vduse_dev_update_iotlb(struct vduse_dev *dev,
306 				  u64 start, u64 last)
307 {
308 	struct vduse_dev_msg msg = { 0 };
309 
310 	if (last < start)
311 		return -EINVAL;
312 
313 	msg.req.type = VDUSE_UPDATE_IOTLB;
314 	msg.req.iova.start = start;
315 	msg.req.iova.last = last;
316 
317 	return vduse_dev_msg_sync(dev, &msg);
318 }
319 
320 static ssize_t vduse_dev_read_iter(struct kiocb *iocb, struct iov_iter *to)
321 {
322 	struct file *file = iocb->ki_filp;
323 	struct vduse_dev *dev = file->private_data;
324 	struct vduse_dev_msg *msg;
325 	int size = sizeof(struct vduse_dev_request);
326 	ssize_t ret;
327 
328 	if (iov_iter_count(to) < size)
329 		return -EINVAL;
330 
331 	spin_lock(&dev->msg_lock);
332 	while (1) {
333 		msg = vduse_dequeue_msg(&dev->send_list);
334 		if (msg)
335 			break;
336 
337 		ret = -EAGAIN;
338 		if (file->f_flags & O_NONBLOCK)
339 			goto unlock;
340 
341 		spin_unlock(&dev->msg_lock);
342 		ret = wait_event_interruptible_exclusive(dev->waitq,
343 					!list_empty(&dev->send_list));
344 		if (ret)
345 			return ret;
346 
347 		spin_lock(&dev->msg_lock);
348 	}
349 	spin_unlock(&dev->msg_lock);
350 	ret = copy_to_iter(&msg->req, size, to);
351 	spin_lock(&dev->msg_lock);
352 	if (ret != size) {
353 		ret = -EFAULT;
354 		vduse_enqueue_msg(&dev->send_list, msg);
355 		goto unlock;
356 	}
357 	vduse_enqueue_msg(&dev->recv_list, msg);
358 unlock:
359 	spin_unlock(&dev->msg_lock);
360 
361 	return ret;
362 }
363 
364 static bool is_mem_zero(const char *ptr, int size)
365 {
366 	int i;
367 
368 	for (i = 0; i < size; i++) {
369 		if (ptr[i])
370 			return false;
371 	}
372 	return true;
373 }
374 
375 static ssize_t vduse_dev_write_iter(struct kiocb *iocb, struct iov_iter *from)
376 {
377 	struct file *file = iocb->ki_filp;
378 	struct vduse_dev *dev = file->private_data;
379 	struct vduse_dev_response resp;
380 	struct vduse_dev_msg *msg;
381 	size_t ret;
382 
383 	ret = copy_from_iter(&resp, sizeof(resp), from);
384 	if (ret != sizeof(resp))
385 		return -EINVAL;
386 
387 	if (!is_mem_zero((const char *)resp.reserved, sizeof(resp.reserved)))
388 		return -EINVAL;
389 
390 	spin_lock(&dev->msg_lock);
391 	msg = vduse_find_msg(&dev->recv_list, resp.request_id);
392 	if (!msg) {
393 		ret = -ENOENT;
394 		goto unlock;
395 	}
396 
397 	memcpy(&msg->resp, &resp, sizeof(resp));
398 	msg->completed = 1;
399 	wake_up(&msg->waitq);
400 unlock:
401 	spin_unlock(&dev->msg_lock);
402 
403 	return ret;
404 }
405 
406 static __poll_t vduse_dev_poll(struct file *file, poll_table *wait)
407 {
408 	struct vduse_dev *dev = file->private_data;
409 	__poll_t mask = 0;
410 
411 	poll_wait(file, &dev->waitq, wait);
412 
413 	spin_lock(&dev->msg_lock);
414 
415 	if (unlikely(dev->broken))
416 		mask |= EPOLLERR;
417 	if (!list_empty(&dev->send_list))
418 		mask |= EPOLLIN | EPOLLRDNORM;
419 	if (!list_empty(&dev->recv_list))
420 		mask |= EPOLLOUT | EPOLLWRNORM;
421 
422 	spin_unlock(&dev->msg_lock);
423 
424 	return mask;
425 }
426 
427 static void vduse_dev_reset(struct vduse_dev *dev)
428 {
429 	int i;
430 	struct vduse_iova_domain *domain = dev->domain;
431 
432 	/* The coherent mappings are handled in vduse_dev_free_coherent() */
433 	if (domain && domain->bounce_map)
434 		vduse_domain_reset_bounce_map(domain);
435 
436 	down_write(&dev->rwsem);
437 
438 	dev->status = 0;
439 	dev->driver_features = 0;
440 	dev->generation++;
441 	spin_lock(&dev->irq_lock);
442 	dev->config_cb.callback = NULL;
443 	dev->config_cb.private = NULL;
444 	spin_unlock(&dev->irq_lock);
445 	flush_work(&dev->inject);
446 
447 	for (i = 0; i < dev->vq_num; i++) {
448 		struct vduse_virtqueue *vq = dev->vqs[i];
449 
450 		vq->ready = false;
451 		vq->desc_addr = 0;
452 		vq->driver_addr = 0;
453 		vq->device_addr = 0;
454 		vq->num = 0;
455 		memset(&vq->state, 0, sizeof(vq->state));
456 
457 		spin_lock(&vq->kick_lock);
458 		vq->kicked = false;
459 		if (vq->kickfd)
460 			eventfd_ctx_put(vq->kickfd);
461 		vq->kickfd = NULL;
462 		spin_unlock(&vq->kick_lock);
463 
464 		spin_lock(&vq->irq_lock);
465 		vq->cb.callback = NULL;
466 		vq->cb.private = NULL;
467 		vq->cb.trigger = NULL;
468 		spin_unlock(&vq->irq_lock);
469 		flush_work(&vq->inject);
470 		flush_work(&vq->kick);
471 	}
472 
473 	up_write(&dev->rwsem);
474 }
475 
476 static int vduse_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 idx,
477 				u64 desc_area, u64 driver_area,
478 				u64 device_area)
479 {
480 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
481 	struct vduse_virtqueue *vq = dev->vqs[idx];
482 
483 	vq->desc_addr = desc_area;
484 	vq->driver_addr = driver_area;
485 	vq->device_addr = device_area;
486 
487 	return 0;
488 }
489 
490 static void vduse_vq_kick(struct vduse_virtqueue *vq)
491 {
492 	spin_lock(&vq->kick_lock);
493 	if (!vq->ready)
494 		goto unlock;
495 
496 	if (vq->kickfd)
497 		eventfd_signal(vq->kickfd, 1);
498 	else
499 		vq->kicked = true;
500 unlock:
501 	spin_unlock(&vq->kick_lock);
502 }
503 
504 static void vduse_vq_kick_work(struct work_struct *work)
505 {
506 	struct vduse_virtqueue *vq = container_of(work,
507 					struct vduse_virtqueue, kick);
508 
509 	vduse_vq_kick(vq);
510 }
511 
512 static void vduse_vdpa_kick_vq(struct vdpa_device *vdpa, u16 idx)
513 {
514 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
515 	struct vduse_virtqueue *vq = dev->vqs[idx];
516 
517 	if (!eventfd_signal_allowed()) {
518 		schedule_work(&vq->kick);
519 		return;
520 	}
521 	vduse_vq_kick(vq);
522 }
523 
524 static void vduse_vdpa_set_vq_cb(struct vdpa_device *vdpa, u16 idx,
525 			      struct vdpa_callback *cb)
526 {
527 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
528 	struct vduse_virtqueue *vq = dev->vqs[idx];
529 
530 	spin_lock(&vq->irq_lock);
531 	vq->cb.callback = cb->callback;
532 	vq->cb.private = cb->private;
533 	vq->cb.trigger = cb->trigger;
534 	spin_unlock(&vq->irq_lock);
535 }
536 
537 static void vduse_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num)
538 {
539 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
540 	struct vduse_virtqueue *vq = dev->vqs[idx];
541 
542 	vq->num = num;
543 }
544 
545 static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa,
546 					u16 idx, bool ready)
547 {
548 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
549 	struct vduse_virtqueue *vq = dev->vqs[idx];
550 
551 	vq->ready = ready;
552 }
553 
554 static bool vduse_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 idx)
555 {
556 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
557 	struct vduse_virtqueue *vq = dev->vqs[idx];
558 
559 	return vq->ready;
560 }
561 
562 static int vduse_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 idx,
563 				const struct vdpa_vq_state *state)
564 {
565 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
566 	struct vduse_virtqueue *vq = dev->vqs[idx];
567 
568 	if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
569 		vq->state.packed.last_avail_counter =
570 				state->packed.last_avail_counter;
571 		vq->state.packed.last_avail_idx = state->packed.last_avail_idx;
572 		vq->state.packed.last_used_counter =
573 				state->packed.last_used_counter;
574 		vq->state.packed.last_used_idx = state->packed.last_used_idx;
575 	} else
576 		vq->state.split.avail_index = state->split.avail_index;
577 
578 	return 0;
579 }
580 
581 static int vduse_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 idx,
582 				struct vdpa_vq_state *state)
583 {
584 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
585 	struct vduse_virtqueue *vq = dev->vqs[idx];
586 
587 	if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED))
588 		return vduse_dev_get_vq_state_packed(dev, vq, &state->packed);
589 
590 	return vduse_dev_get_vq_state_split(dev, vq, &state->split);
591 }
592 
593 static u32 vduse_vdpa_get_vq_align(struct vdpa_device *vdpa)
594 {
595 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
596 
597 	return dev->vq_align;
598 }
599 
600 static u64 vduse_vdpa_get_device_features(struct vdpa_device *vdpa)
601 {
602 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
603 
604 	return dev->device_features;
605 }
606 
607 static int vduse_vdpa_set_driver_features(struct vdpa_device *vdpa, u64 features)
608 {
609 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
610 
611 	dev->driver_features = features;
612 	return 0;
613 }
614 
615 static u64 vduse_vdpa_get_driver_features(struct vdpa_device *vdpa)
616 {
617 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
618 
619 	return dev->driver_features;
620 }
621 
622 static void vduse_vdpa_set_config_cb(struct vdpa_device *vdpa,
623 				  struct vdpa_callback *cb)
624 {
625 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
626 
627 	spin_lock(&dev->irq_lock);
628 	dev->config_cb.callback = cb->callback;
629 	dev->config_cb.private = cb->private;
630 	spin_unlock(&dev->irq_lock);
631 }
632 
633 static u16 vduse_vdpa_get_vq_num_max(struct vdpa_device *vdpa)
634 {
635 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
636 	u16 num_max = 0;
637 	int i;
638 
639 	for (i = 0; i < dev->vq_num; i++)
640 		if (num_max < dev->vqs[i]->num_max)
641 			num_max = dev->vqs[i]->num_max;
642 
643 	return num_max;
644 }
645 
646 static u32 vduse_vdpa_get_device_id(struct vdpa_device *vdpa)
647 {
648 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
649 
650 	return dev->device_id;
651 }
652 
653 static u32 vduse_vdpa_get_vendor_id(struct vdpa_device *vdpa)
654 {
655 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
656 
657 	return dev->vendor_id;
658 }
659 
660 static u8 vduse_vdpa_get_status(struct vdpa_device *vdpa)
661 {
662 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
663 
664 	return dev->status;
665 }
666 
667 static void vduse_vdpa_set_status(struct vdpa_device *vdpa, u8 status)
668 {
669 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
670 
671 	if (vduse_dev_set_status(dev, status))
672 		return;
673 
674 	dev->status = status;
675 }
676 
677 static size_t vduse_vdpa_get_config_size(struct vdpa_device *vdpa)
678 {
679 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
680 
681 	return dev->config_size;
682 }
683 
684 static void vduse_vdpa_get_config(struct vdpa_device *vdpa, unsigned int offset,
685 				  void *buf, unsigned int len)
686 {
687 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
688 
689 	/* Initialize the buffer in case of partial copy. */
690 	memset(buf, 0, len);
691 
692 	if (offset > dev->config_size)
693 		return;
694 
695 	if (len > dev->config_size - offset)
696 		len = dev->config_size - offset;
697 
698 	memcpy(buf, dev->config + offset, len);
699 }
700 
701 static void vduse_vdpa_set_config(struct vdpa_device *vdpa, unsigned int offset,
702 			const void *buf, unsigned int len)
703 {
704 	/* Now we only support read-only configuration space */
705 }
706 
707 static int vduse_vdpa_reset(struct vdpa_device *vdpa)
708 {
709 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
710 	int ret = vduse_dev_set_status(dev, 0);
711 
712 	vduse_dev_reset(dev);
713 
714 	return ret;
715 }
716 
717 static u32 vduse_vdpa_get_generation(struct vdpa_device *vdpa)
718 {
719 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
720 
721 	return dev->generation;
722 }
723 
724 static int vduse_vdpa_set_vq_affinity(struct vdpa_device *vdpa, u16 idx,
725 				      const struct cpumask *cpu_mask)
726 {
727 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
728 
729 	cpumask_copy(&dev->vqs[idx]->irq_affinity, cpu_mask);
730 	return 0;
731 }
732 
733 static const struct cpumask *
734 vduse_vdpa_get_vq_affinity(struct vdpa_device *vdpa, u16 idx)
735 {
736 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
737 
738 	return &dev->vqs[idx]->irq_affinity;
739 }
740 
741 static int vduse_vdpa_set_map(struct vdpa_device *vdpa,
742 				unsigned int asid,
743 				struct vhost_iotlb *iotlb)
744 {
745 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
746 	int ret;
747 
748 	ret = vduse_domain_set_map(dev->domain, iotlb);
749 	if (ret)
750 		return ret;
751 
752 	ret = vduse_dev_update_iotlb(dev, 0ULL, ULLONG_MAX);
753 	if (ret) {
754 		vduse_domain_clear_map(dev->domain, iotlb);
755 		return ret;
756 	}
757 
758 	return 0;
759 }
760 
761 static void vduse_vdpa_free(struct vdpa_device *vdpa)
762 {
763 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
764 
765 	dev->vdev = NULL;
766 }
767 
768 static const struct vdpa_config_ops vduse_vdpa_config_ops = {
769 	.set_vq_address		= vduse_vdpa_set_vq_address,
770 	.kick_vq		= vduse_vdpa_kick_vq,
771 	.set_vq_cb		= vduse_vdpa_set_vq_cb,
772 	.set_vq_num             = vduse_vdpa_set_vq_num,
773 	.set_vq_ready		= vduse_vdpa_set_vq_ready,
774 	.get_vq_ready		= vduse_vdpa_get_vq_ready,
775 	.set_vq_state		= vduse_vdpa_set_vq_state,
776 	.get_vq_state		= vduse_vdpa_get_vq_state,
777 	.get_vq_align		= vduse_vdpa_get_vq_align,
778 	.get_device_features	= vduse_vdpa_get_device_features,
779 	.set_driver_features	= vduse_vdpa_set_driver_features,
780 	.get_driver_features	= vduse_vdpa_get_driver_features,
781 	.set_config_cb		= vduse_vdpa_set_config_cb,
782 	.get_vq_num_max		= vduse_vdpa_get_vq_num_max,
783 	.get_device_id		= vduse_vdpa_get_device_id,
784 	.get_vendor_id		= vduse_vdpa_get_vendor_id,
785 	.get_status		= vduse_vdpa_get_status,
786 	.set_status		= vduse_vdpa_set_status,
787 	.get_config_size	= vduse_vdpa_get_config_size,
788 	.get_config		= vduse_vdpa_get_config,
789 	.set_config		= vduse_vdpa_set_config,
790 	.get_generation		= vduse_vdpa_get_generation,
791 	.set_vq_affinity	= vduse_vdpa_set_vq_affinity,
792 	.get_vq_affinity	= vduse_vdpa_get_vq_affinity,
793 	.reset			= vduse_vdpa_reset,
794 	.set_map		= vduse_vdpa_set_map,
795 	.free			= vduse_vdpa_free,
796 };
797 
798 static dma_addr_t vduse_dev_map_page(struct device *dev, struct page *page,
799 				     unsigned long offset, size_t size,
800 				     enum dma_data_direction dir,
801 				     unsigned long attrs)
802 {
803 	struct vduse_dev *vdev = dev_to_vduse(dev);
804 	struct vduse_iova_domain *domain = vdev->domain;
805 
806 	return vduse_domain_map_page(domain, page, offset, size, dir, attrs);
807 }
808 
809 static void vduse_dev_unmap_page(struct device *dev, dma_addr_t dma_addr,
810 				size_t size, enum dma_data_direction dir,
811 				unsigned long attrs)
812 {
813 	struct vduse_dev *vdev = dev_to_vduse(dev);
814 	struct vduse_iova_domain *domain = vdev->domain;
815 
816 	return vduse_domain_unmap_page(domain, dma_addr, size, dir, attrs);
817 }
818 
819 static void *vduse_dev_alloc_coherent(struct device *dev, size_t size,
820 					dma_addr_t *dma_addr, gfp_t flag,
821 					unsigned long attrs)
822 {
823 	struct vduse_dev *vdev = dev_to_vduse(dev);
824 	struct vduse_iova_domain *domain = vdev->domain;
825 	unsigned long iova;
826 	void *addr;
827 
828 	*dma_addr = DMA_MAPPING_ERROR;
829 	addr = vduse_domain_alloc_coherent(domain, size,
830 				(dma_addr_t *)&iova, flag, attrs);
831 	if (!addr)
832 		return NULL;
833 
834 	*dma_addr = (dma_addr_t)iova;
835 
836 	return addr;
837 }
838 
839 static void vduse_dev_free_coherent(struct device *dev, size_t size,
840 					void *vaddr, dma_addr_t dma_addr,
841 					unsigned long attrs)
842 {
843 	struct vduse_dev *vdev = dev_to_vduse(dev);
844 	struct vduse_iova_domain *domain = vdev->domain;
845 
846 	vduse_domain_free_coherent(domain, size, vaddr, dma_addr, attrs);
847 }
848 
849 static size_t vduse_dev_max_mapping_size(struct device *dev)
850 {
851 	struct vduse_dev *vdev = dev_to_vduse(dev);
852 	struct vduse_iova_domain *domain = vdev->domain;
853 
854 	return domain->bounce_size;
855 }
856 
857 static const struct dma_map_ops vduse_dev_dma_ops = {
858 	.map_page = vduse_dev_map_page,
859 	.unmap_page = vduse_dev_unmap_page,
860 	.alloc = vduse_dev_alloc_coherent,
861 	.free = vduse_dev_free_coherent,
862 	.max_mapping_size = vduse_dev_max_mapping_size,
863 };
864 
865 static unsigned int perm_to_file_flags(u8 perm)
866 {
867 	unsigned int flags = 0;
868 
869 	switch (perm) {
870 	case VDUSE_ACCESS_WO:
871 		flags |= O_WRONLY;
872 		break;
873 	case VDUSE_ACCESS_RO:
874 		flags |= O_RDONLY;
875 		break;
876 	case VDUSE_ACCESS_RW:
877 		flags |= O_RDWR;
878 		break;
879 	default:
880 		WARN(1, "invalidate vhost IOTLB permission\n");
881 		break;
882 	}
883 
884 	return flags;
885 }
886 
887 static int vduse_kickfd_setup(struct vduse_dev *dev,
888 			struct vduse_vq_eventfd *eventfd)
889 {
890 	struct eventfd_ctx *ctx = NULL;
891 	struct vduse_virtqueue *vq;
892 	u32 index;
893 
894 	if (eventfd->index >= dev->vq_num)
895 		return -EINVAL;
896 
897 	index = array_index_nospec(eventfd->index, dev->vq_num);
898 	vq = dev->vqs[index];
899 	if (eventfd->fd >= 0) {
900 		ctx = eventfd_ctx_fdget(eventfd->fd);
901 		if (IS_ERR(ctx))
902 			return PTR_ERR(ctx);
903 	} else if (eventfd->fd != VDUSE_EVENTFD_DEASSIGN)
904 		return 0;
905 
906 	spin_lock(&vq->kick_lock);
907 	if (vq->kickfd)
908 		eventfd_ctx_put(vq->kickfd);
909 	vq->kickfd = ctx;
910 	if (vq->ready && vq->kicked && vq->kickfd) {
911 		eventfd_signal(vq->kickfd, 1);
912 		vq->kicked = false;
913 	}
914 	spin_unlock(&vq->kick_lock);
915 
916 	return 0;
917 }
918 
919 static bool vduse_dev_is_ready(struct vduse_dev *dev)
920 {
921 	int i;
922 
923 	for (i = 0; i < dev->vq_num; i++)
924 		if (!dev->vqs[i]->num_max)
925 			return false;
926 
927 	return true;
928 }
929 
930 static void vduse_dev_irq_inject(struct work_struct *work)
931 {
932 	struct vduse_dev *dev = container_of(work, struct vduse_dev, inject);
933 
934 	spin_lock_irq(&dev->irq_lock);
935 	if (dev->config_cb.callback)
936 		dev->config_cb.callback(dev->config_cb.private);
937 	spin_unlock_irq(&dev->irq_lock);
938 }
939 
940 static void vduse_vq_irq_inject(struct work_struct *work)
941 {
942 	struct vduse_virtqueue *vq = container_of(work,
943 					struct vduse_virtqueue, inject);
944 
945 	spin_lock_irq(&vq->irq_lock);
946 	if (vq->ready && vq->cb.callback)
947 		vq->cb.callback(vq->cb.private);
948 	spin_unlock_irq(&vq->irq_lock);
949 }
950 
951 static bool vduse_vq_signal_irqfd(struct vduse_virtqueue *vq)
952 {
953 	bool signal = false;
954 
955 	if (!vq->cb.trigger)
956 		return false;
957 
958 	spin_lock_irq(&vq->irq_lock);
959 	if (vq->ready && vq->cb.trigger) {
960 		eventfd_signal(vq->cb.trigger, 1);
961 		signal = true;
962 	}
963 	spin_unlock_irq(&vq->irq_lock);
964 
965 	return signal;
966 }
967 
968 static int vduse_dev_queue_irq_work(struct vduse_dev *dev,
969 				    struct work_struct *irq_work,
970 				    int irq_effective_cpu)
971 {
972 	int ret = -EINVAL;
973 
974 	down_read(&dev->rwsem);
975 	if (!(dev->status & VIRTIO_CONFIG_S_DRIVER_OK))
976 		goto unlock;
977 
978 	ret = 0;
979 	if (irq_effective_cpu == IRQ_UNBOUND)
980 		queue_work(vduse_irq_wq, irq_work);
981 	else
982 		queue_work_on(irq_effective_cpu,
983 			      vduse_irq_bound_wq, irq_work);
984 unlock:
985 	up_read(&dev->rwsem);
986 
987 	return ret;
988 }
989 
990 static int vduse_dev_dereg_umem(struct vduse_dev *dev,
991 				u64 iova, u64 size)
992 {
993 	int ret;
994 
995 	mutex_lock(&dev->mem_lock);
996 	ret = -ENOENT;
997 	if (!dev->umem)
998 		goto unlock;
999 
1000 	ret = -EINVAL;
1001 	if (!dev->domain)
1002 		goto unlock;
1003 
1004 	if (dev->umem->iova != iova || size != dev->domain->bounce_size)
1005 		goto unlock;
1006 
1007 	vduse_domain_remove_user_bounce_pages(dev->domain);
1008 	unpin_user_pages_dirty_lock(dev->umem->pages,
1009 				    dev->umem->npages, true);
1010 	atomic64_sub(dev->umem->npages, &dev->umem->mm->pinned_vm);
1011 	mmdrop(dev->umem->mm);
1012 	vfree(dev->umem->pages);
1013 	kfree(dev->umem);
1014 	dev->umem = NULL;
1015 	ret = 0;
1016 unlock:
1017 	mutex_unlock(&dev->mem_lock);
1018 	return ret;
1019 }
1020 
1021 static int vduse_dev_reg_umem(struct vduse_dev *dev,
1022 			      u64 iova, u64 uaddr, u64 size)
1023 {
1024 	struct page **page_list = NULL;
1025 	struct vduse_umem *umem = NULL;
1026 	long pinned = 0;
1027 	unsigned long npages, lock_limit;
1028 	int ret;
1029 
1030 	if (!dev->domain || !dev->domain->bounce_map ||
1031 	    size != dev->domain->bounce_size ||
1032 	    iova != 0 || uaddr & ~PAGE_MASK)
1033 		return -EINVAL;
1034 
1035 	mutex_lock(&dev->mem_lock);
1036 	ret = -EEXIST;
1037 	if (dev->umem)
1038 		goto unlock;
1039 
1040 	ret = -ENOMEM;
1041 	npages = size >> PAGE_SHIFT;
1042 	page_list = __vmalloc(array_size(npages, sizeof(struct page *)),
1043 			      GFP_KERNEL_ACCOUNT);
1044 	umem = kzalloc(sizeof(*umem), GFP_KERNEL);
1045 	if (!page_list || !umem)
1046 		goto unlock;
1047 
1048 	mmap_read_lock(current->mm);
1049 
1050 	lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK));
1051 	if (npages + atomic64_read(&current->mm->pinned_vm) > lock_limit)
1052 		goto out;
1053 
1054 	pinned = pin_user_pages(uaddr, npages, FOLL_LONGTERM | FOLL_WRITE,
1055 				page_list, NULL);
1056 	if (pinned != npages) {
1057 		ret = pinned < 0 ? pinned : -ENOMEM;
1058 		goto out;
1059 	}
1060 
1061 	ret = vduse_domain_add_user_bounce_pages(dev->domain,
1062 						 page_list, pinned);
1063 	if (ret)
1064 		goto out;
1065 
1066 	atomic64_add(npages, &current->mm->pinned_vm);
1067 
1068 	umem->pages = page_list;
1069 	umem->npages = pinned;
1070 	umem->iova = iova;
1071 	umem->mm = current->mm;
1072 	mmgrab(current->mm);
1073 
1074 	dev->umem = umem;
1075 out:
1076 	if (ret && pinned > 0)
1077 		unpin_user_pages(page_list, pinned);
1078 
1079 	mmap_read_unlock(current->mm);
1080 unlock:
1081 	if (ret) {
1082 		vfree(page_list);
1083 		kfree(umem);
1084 	}
1085 	mutex_unlock(&dev->mem_lock);
1086 	return ret;
1087 }
1088 
1089 static void vduse_vq_update_effective_cpu(struct vduse_virtqueue *vq)
1090 {
1091 	int curr_cpu = vq->irq_effective_cpu;
1092 
1093 	while (true) {
1094 		curr_cpu = cpumask_next(curr_cpu, &vq->irq_affinity);
1095 		if (cpu_online(curr_cpu))
1096 			break;
1097 
1098 		if (curr_cpu >= nr_cpu_ids)
1099 			curr_cpu = IRQ_UNBOUND;
1100 	}
1101 
1102 	vq->irq_effective_cpu = curr_cpu;
1103 }
1104 
1105 static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
1106 			    unsigned long arg)
1107 {
1108 	struct vduse_dev *dev = file->private_data;
1109 	void __user *argp = (void __user *)arg;
1110 	int ret;
1111 
1112 	if (unlikely(dev->broken))
1113 		return -EPERM;
1114 
1115 	switch (cmd) {
1116 	case VDUSE_IOTLB_GET_FD: {
1117 		struct vduse_iotlb_entry entry;
1118 		struct vhost_iotlb_map *map;
1119 		struct vdpa_map_file *map_file;
1120 		struct file *f = NULL;
1121 
1122 		ret = -EFAULT;
1123 		if (copy_from_user(&entry, argp, sizeof(entry)))
1124 			break;
1125 
1126 		ret = -EINVAL;
1127 		if (entry.start > entry.last)
1128 			break;
1129 
1130 		mutex_lock(&dev->domain_lock);
1131 		if (!dev->domain) {
1132 			mutex_unlock(&dev->domain_lock);
1133 			break;
1134 		}
1135 		spin_lock(&dev->domain->iotlb_lock);
1136 		map = vhost_iotlb_itree_first(dev->domain->iotlb,
1137 					      entry.start, entry.last);
1138 		if (map) {
1139 			map_file = (struct vdpa_map_file *)map->opaque;
1140 			f = get_file(map_file->file);
1141 			entry.offset = map_file->offset;
1142 			entry.start = map->start;
1143 			entry.last = map->last;
1144 			entry.perm = map->perm;
1145 		}
1146 		spin_unlock(&dev->domain->iotlb_lock);
1147 		mutex_unlock(&dev->domain_lock);
1148 		ret = -EINVAL;
1149 		if (!f)
1150 			break;
1151 
1152 		ret = -EFAULT;
1153 		if (copy_to_user(argp, &entry, sizeof(entry))) {
1154 			fput(f);
1155 			break;
1156 		}
1157 		ret = receive_fd(f, perm_to_file_flags(entry.perm));
1158 		fput(f);
1159 		break;
1160 	}
1161 	case VDUSE_DEV_GET_FEATURES:
1162 		/*
1163 		 * Just mirror what driver wrote here.
1164 		 * The driver is expected to check FEATURE_OK later.
1165 		 */
1166 		ret = put_user(dev->driver_features, (u64 __user *)argp);
1167 		break;
1168 	case VDUSE_DEV_SET_CONFIG: {
1169 		struct vduse_config_data config;
1170 		unsigned long size = offsetof(struct vduse_config_data,
1171 					      buffer);
1172 
1173 		ret = -EFAULT;
1174 		if (copy_from_user(&config, argp, size))
1175 			break;
1176 
1177 		ret = -EINVAL;
1178 		if (config.offset > dev->config_size ||
1179 		    config.length == 0 ||
1180 		    config.length > dev->config_size - config.offset)
1181 			break;
1182 
1183 		ret = -EFAULT;
1184 		if (copy_from_user(dev->config + config.offset, argp + size,
1185 				   config.length))
1186 			break;
1187 
1188 		ret = 0;
1189 		break;
1190 	}
1191 	case VDUSE_DEV_INJECT_CONFIG_IRQ:
1192 		ret = vduse_dev_queue_irq_work(dev, &dev->inject, IRQ_UNBOUND);
1193 		break;
1194 	case VDUSE_VQ_SETUP: {
1195 		struct vduse_vq_config config;
1196 		u32 index;
1197 
1198 		ret = -EFAULT;
1199 		if (copy_from_user(&config, argp, sizeof(config)))
1200 			break;
1201 
1202 		ret = -EINVAL;
1203 		if (config.index >= dev->vq_num)
1204 			break;
1205 
1206 		if (!is_mem_zero((const char *)config.reserved,
1207 				 sizeof(config.reserved)))
1208 			break;
1209 
1210 		index = array_index_nospec(config.index, dev->vq_num);
1211 		dev->vqs[index]->num_max = config.max_size;
1212 		ret = 0;
1213 		break;
1214 	}
1215 	case VDUSE_VQ_GET_INFO: {
1216 		struct vduse_vq_info vq_info;
1217 		struct vduse_virtqueue *vq;
1218 		u32 index;
1219 
1220 		ret = -EFAULT;
1221 		if (copy_from_user(&vq_info, argp, sizeof(vq_info)))
1222 			break;
1223 
1224 		ret = -EINVAL;
1225 		if (vq_info.index >= dev->vq_num)
1226 			break;
1227 
1228 		index = array_index_nospec(vq_info.index, dev->vq_num);
1229 		vq = dev->vqs[index];
1230 		vq_info.desc_addr = vq->desc_addr;
1231 		vq_info.driver_addr = vq->driver_addr;
1232 		vq_info.device_addr = vq->device_addr;
1233 		vq_info.num = vq->num;
1234 
1235 		if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
1236 			vq_info.packed.last_avail_counter =
1237 				vq->state.packed.last_avail_counter;
1238 			vq_info.packed.last_avail_idx =
1239 				vq->state.packed.last_avail_idx;
1240 			vq_info.packed.last_used_counter =
1241 				vq->state.packed.last_used_counter;
1242 			vq_info.packed.last_used_idx =
1243 				vq->state.packed.last_used_idx;
1244 		} else
1245 			vq_info.split.avail_index =
1246 				vq->state.split.avail_index;
1247 
1248 		vq_info.ready = vq->ready;
1249 
1250 		ret = -EFAULT;
1251 		if (copy_to_user(argp, &vq_info, sizeof(vq_info)))
1252 			break;
1253 
1254 		ret = 0;
1255 		break;
1256 	}
1257 	case VDUSE_VQ_SETUP_KICKFD: {
1258 		struct vduse_vq_eventfd eventfd;
1259 
1260 		ret = -EFAULT;
1261 		if (copy_from_user(&eventfd, argp, sizeof(eventfd)))
1262 			break;
1263 
1264 		ret = vduse_kickfd_setup(dev, &eventfd);
1265 		break;
1266 	}
1267 	case VDUSE_VQ_INJECT_IRQ: {
1268 		u32 index;
1269 
1270 		ret = -EFAULT;
1271 		if (get_user(index, (u32 __user *)argp))
1272 			break;
1273 
1274 		ret = -EINVAL;
1275 		if (index >= dev->vq_num)
1276 			break;
1277 
1278 		ret = 0;
1279 		index = array_index_nospec(index, dev->vq_num);
1280 		if (!vduse_vq_signal_irqfd(dev->vqs[index])) {
1281 			vduse_vq_update_effective_cpu(dev->vqs[index]);
1282 			ret = vduse_dev_queue_irq_work(dev,
1283 						&dev->vqs[index]->inject,
1284 						dev->vqs[index]->irq_effective_cpu);
1285 		}
1286 		break;
1287 	}
1288 	case VDUSE_IOTLB_REG_UMEM: {
1289 		struct vduse_iova_umem umem;
1290 
1291 		ret = -EFAULT;
1292 		if (copy_from_user(&umem, argp, sizeof(umem)))
1293 			break;
1294 
1295 		ret = -EINVAL;
1296 		if (!is_mem_zero((const char *)umem.reserved,
1297 				 sizeof(umem.reserved)))
1298 			break;
1299 
1300 		mutex_lock(&dev->domain_lock);
1301 		ret = vduse_dev_reg_umem(dev, umem.iova,
1302 					 umem.uaddr, umem.size);
1303 		mutex_unlock(&dev->domain_lock);
1304 		break;
1305 	}
1306 	case VDUSE_IOTLB_DEREG_UMEM: {
1307 		struct vduse_iova_umem umem;
1308 
1309 		ret = -EFAULT;
1310 		if (copy_from_user(&umem, argp, sizeof(umem)))
1311 			break;
1312 
1313 		ret = -EINVAL;
1314 		if (!is_mem_zero((const char *)umem.reserved,
1315 				 sizeof(umem.reserved)))
1316 			break;
1317 		mutex_lock(&dev->domain_lock);
1318 		ret = vduse_dev_dereg_umem(dev, umem.iova,
1319 					   umem.size);
1320 		mutex_unlock(&dev->domain_lock);
1321 		break;
1322 	}
1323 	case VDUSE_IOTLB_GET_INFO: {
1324 		struct vduse_iova_info info;
1325 		struct vhost_iotlb_map *map;
1326 
1327 		ret = -EFAULT;
1328 		if (copy_from_user(&info, argp, sizeof(info)))
1329 			break;
1330 
1331 		ret = -EINVAL;
1332 		if (info.start > info.last)
1333 			break;
1334 
1335 		if (!is_mem_zero((const char *)info.reserved,
1336 				 sizeof(info.reserved)))
1337 			break;
1338 
1339 		mutex_lock(&dev->domain_lock);
1340 		if (!dev->domain) {
1341 			mutex_unlock(&dev->domain_lock);
1342 			break;
1343 		}
1344 		spin_lock(&dev->domain->iotlb_lock);
1345 		map = vhost_iotlb_itree_first(dev->domain->iotlb,
1346 					      info.start, info.last);
1347 		if (map) {
1348 			info.start = map->start;
1349 			info.last = map->last;
1350 			info.capability = 0;
1351 			if (dev->domain->bounce_map && map->start == 0 &&
1352 			    map->last == dev->domain->bounce_size - 1)
1353 				info.capability |= VDUSE_IOVA_CAP_UMEM;
1354 		}
1355 		spin_unlock(&dev->domain->iotlb_lock);
1356 		mutex_unlock(&dev->domain_lock);
1357 		if (!map)
1358 			break;
1359 
1360 		ret = -EFAULT;
1361 		if (copy_to_user(argp, &info, sizeof(info)))
1362 			break;
1363 
1364 		ret = 0;
1365 		break;
1366 	}
1367 	default:
1368 		ret = -ENOIOCTLCMD;
1369 		break;
1370 	}
1371 
1372 	return ret;
1373 }
1374 
1375 static int vduse_dev_release(struct inode *inode, struct file *file)
1376 {
1377 	struct vduse_dev *dev = file->private_data;
1378 
1379 	mutex_lock(&dev->domain_lock);
1380 	if (dev->domain)
1381 		vduse_dev_dereg_umem(dev, 0, dev->domain->bounce_size);
1382 	mutex_unlock(&dev->domain_lock);
1383 	spin_lock(&dev->msg_lock);
1384 	/* Make sure the inflight messages can processed after reconncection */
1385 	list_splice_init(&dev->recv_list, &dev->send_list);
1386 	spin_unlock(&dev->msg_lock);
1387 	dev->connected = false;
1388 
1389 	return 0;
1390 }
1391 
1392 static struct vduse_dev *vduse_dev_get_from_minor(int minor)
1393 {
1394 	struct vduse_dev *dev;
1395 
1396 	mutex_lock(&vduse_lock);
1397 	dev = idr_find(&vduse_idr, minor);
1398 	mutex_unlock(&vduse_lock);
1399 
1400 	return dev;
1401 }
1402 
1403 static int vduse_dev_open(struct inode *inode, struct file *file)
1404 {
1405 	int ret;
1406 	struct vduse_dev *dev = vduse_dev_get_from_minor(iminor(inode));
1407 
1408 	if (!dev)
1409 		return -ENODEV;
1410 
1411 	ret = -EBUSY;
1412 	mutex_lock(&dev->lock);
1413 	if (dev->connected)
1414 		goto unlock;
1415 
1416 	ret = 0;
1417 	dev->connected = true;
1418 	file->private_data = dev;
1419 unlock:
1420 	mutex_unlock(&dev->lock);
1421 
1422 	return ret;
1423 }
1424 
1425 static const struct file_operations vduse_dev_fops = {
1426 	.owner		= THIS_MODULE,
1427 	.open		= vduse_dev_open,
1428 	.release	= vduse_dev_release,
1429 	.read_iter	= vduse_dev_read_iter,
1430 	.write_iter	= vduse_dev_write_iter,
1431 	.poll		= vduse_dev_poll,
1432 	.unlocked_ioctl	= vduse_dev_ioctl,
1433 	.compat_ioctl	= compat_ptr_ioctl,
1434 	.llseek		= noop_llseek,
1435 };
1436 
1437 static ssize_t irq_cb_affinity_show(struct vduse_virtqueue *vq, char *buf)
1438 {
1439 	return sprintf(buf, "%*pb\n", cpumask_pr_args(&vq->irq_affinity));
1440 }
1441 
1442 static ssize_t irq_cb_affinity_store(struct vduse_virtqueue *vq,
1443 				     const char *buf, size_t count)
1444 {
1445 	cpumask_var_t new_value;
1446 	int ret;
1447 
1448 	if (!zalloc_cpumask_var(&new_value, GFP_KERNEL))
1449 		return -ENOMEM;
1450 
1451 	ret = cpumask_parse(buf, new_value);
1452 	if (ret)
1453 		goto free_mask;
1454 
1455 	ret = -EINVAL;
1456 	if (!cpumask_intersects(new_value, cpu_online_mask))
1457 		goto free_mask;
1458 
1459 	cpumask_copy(&vq->irq_affinity, new_value);
1460 	ret = count;
1461 free_mask:
1462 	free_cpumask_var(new_value);
1463 	return ret;
1464 }
1465 
1466 struct vq_sysfs_entry {
1467 	struct attribute attr;
1468 	ssize_t (*show)(struct vduse_virtqueue *vq, char *buf);
1469 	ssize_t (*store)(struct vduse_virtqueue *vq, const char *buf,
1470 			 size_t count);
1471 };
1472 
1473 static struct vq_sysfs_entry irq_cb_affinity_attr = __ATTR_RW(irq_cb_affinity);
1474 
1475 static struct attribute *vq_attrs[] = {
1476 	&irq_cb_affinity_attr.attr,
1477 	NULL,
1478 };
1479 ATTRIBUTE_GROUPS(vq);
1480 
1481 static ssize_t vq_attr_show(struct kobject *kobj, struct attribute *attr,
1482 			    char *buf)
1483 {
1484 	struct vduse_virtqueue *vq = container_of(kobj,
1485 					struct vduse_virtqueue, kobj);
1486 	struct vq_sysfs_entry *entry = container_of(attr,
1487 					struct vq_sysfs_entry, attr);
1488 
1489 	if (!entry->show)
1490 		return -EIO;
1491 
1492 	return entry->show(vq, buf);
1493 }
1494 
1495 static ssize_t vq_attr_store(struct kobject *kobj, struct attribute *attr,
1496 			     const char *buf, size_t count)
1497 {
1498 	struct vduse_virtqueue *vq = container_of(kobj,
1499 					struct vduse_virtqueue, kobj);
1500 	struct vq_sysfs_entry *entry = container_of(attr,
1501 					struct vq_sysfs_entry, attr);
1502 
1503 	if (!entry->store)
1504 		return -EIO;
1505 
1506 	return entry->store(vq, buf, count);
1507 }
1508 
1509 static const struct sysfs_ops vq_sysfs_ops = {
1510 	.show = vq_attr_show,
1511 	.store = vq_attr_store,
1512 };
1513 
1514 static void vq_release(struct kobject *kobj)
1515 {
1516 	struct vduse_virtqueue *vq = container_of(kobj,
1517 					struct vduse_virtqueue, kobj);
1518 	kfree(vq);
1519 }
1520 
1521 static const struct kobj_type vq_type = {
1522 	.release	= vq_release,
1523 	.sysfs_ops	= &vq_sysfs_ops,
1524 	.default_groups	= vq_groups,
1525 };
1526 
1527 static void vduse_dev_deinit_vqs(struct vduse_dev *dev)
1528 {
1529 	int i;
1530 
1531 	if (!dev->vqs)
1532 		return;
1533 
1534 	for (i = 0; i < dev->vq_num; i++)
1535 		kobject_put(&dev->vqs[i]->kobj);
1536 	kfree(dev->vqs);
1537 }
1538 
1539 static int vduse_dev_init_vqs(struct vduse_dev *dev, u32 vq_align, u32 vq_num)
1540 {
1541 	int ret, i;
1542 
1543 	dev->vq_align = vq_align;
1544 	dev->vq_num = vq_num;
1545 	dev->vqs = kcalloc(dev->vq_num, sizeof(*dev->vqs), GFP_KERNEL);
1546 	if (!dev->vqs)
1547 		return -ENOMEM;
1548 
1549 	for (i = 0; i < vq_num; i++) {
1550 		dev->vqs[i] = kzalloc(sizeof(*dev->vqs[i]), GFP_KERNEL);
1551 		if (!dev->vqs[i]) {
1552 			ret = -ENOMEM;
1553 			goto err;
1554 		}
1555 
1556 		dev->vqs[i]->index = i;
1557 		dev->vqs[i]->irq_effective_cpu = IRQ_UNBOUND;
1558 		INIT_WORK(&dev->vqs[i]->inject, vduse_vq_irq_inject);
1559 		INIT_WORK(&dev->vqs[i]->kick, vduse_vq_kick_work);
1560 		spin_lock_init(&dev->vqs[i]->kick_lock);
1561 		spin_lock_init(&dev->vqs[i]->irq_lock);
1562 		cpumask_setall(&dev->vqs[i]->irq_affinity);
1563 
1564 		kobject_init(&dev->vqs[i]->kobj, &vq_type);
1565 		ret = kobject_add(&dev->vqs[i]->kobj,
1566 				  &dev->dev->kobj, "vq%d", i);
1567 		if (ret) {
1568 			kfree(dev->vqs[i]);
1569 			goto err;
1570 		}
1571 	}
1572 
1573 	return 0;
1574 err:
1575 	while (i--)
1576 		kobject_put(&dev->vqs[i]->kobj);
1577 	kfree(dev->vqs);
1578 	dev->vqs = NULL;
1579 	return ret;
1580 }
1581 
1582 static struct vduse_dev *vduse_dev_create(void)
1583 {
1584 	struct vduse_dev *dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1585 
1586 	if (!dev)
1587 		return NULL;
1588 
1589 	mutex_init(&dev->lock);
1590 	mutex_init(&dev->mem_lock);
1591 	mutex_init(&dev->domain_lock);
1592 	spin_lock_init(&dev->msg_lock);
1593 	INIT_LIST_HEAD(&dev->send_list);
1594 	INIT_LIST_HEAD(&dev->recv_list);
1595 	spin_lock_init(&dev->irq_lock);
1596 	init_rwsem(&dev->rwsem);
1597 
1598 	INIT_WORK(&dev->inject, vduse_dev_irq_inject);
1599 	init_waitqueue_head(&dev->waitq);
1600 
1601 	return dev;
1602 }
1603 
1604 static void vduse_dev_destroy(struct vduse_dev *dev)
1605 {
1606 	kfree(dev);
1607 }
1608 
1609 static struct vduse_dev *vduse_find_dev(const char *name)
1610 {
1611 	struct vduse_dev *dev;
1612 	int id;
1613 
1614 	idr_for_each_entry(&vduse_idr, dev, id)
1615 		if (!strcmp(dev->name, name))
1616 			return dev;
1617 
1618 	return NULL;
1619 }
1620 
1621 static int vduse_destroy_dev(char *name)
1622 {
1623 	struct vduse_dev *dev = vduse_find_dev(name);
1624 
1625 	if (!dev)
1626 		return -EINVAL;
1627 
1628 	mutex_lock(&dev->lock);
1629 	if (dev->vdev || dev->connected) {
1630 		mutex_unlock(&dev->lock);
1631 		return -EBUSY;
1632 	}
1633 	dev->connected = true;
1634 	mutex_unlock(&dev->lock);
1635 
1636 	vduse_dev_reset(dev);
1637 	device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));
1638 	idr_remove(&vduse_idr, dev->minor);
1639 	kvfree(dev->config);
1640 	vduse_dev_deinit_vqs(dev);
1641 	if (dev->domain)
1642 		vduse_domain_destroy(dev->domain);
1643 	kfree(dev->name);
1644 	vduse_dev_destroy(dev);
1645 	module_put(THIS_MODULE);
1646 
1647 	return 0;
1648 }
1649 
1650 static bool device_is_allowed(u32 device_id)
1651 {
1652 	int i;
1653 
1654 	for (i = 0; i < ARRAY_SIZE(allowed_device_id); i++)
1655 		if (allowed_device_id[i] == device_id)
1656 			return true;
1657 
1658 	return false;
1659 }
1660 
1661 static bool features_is_valid(u64 features)
1662 {
1663 	if (!(features & (1ULL << VIRTIO_F_ACCESS_PLATFORM)))
1664 		return false;
1665 
1666 	/* Now we only support read-only configuration space */
1667 	if (features & (1ULL << VIRTIO_BLK_F_CONFIG_WCE))
1668 		return false;
1669 
1670 	return true;
1671 }
1672 
1673 static bool vduse_validate_config(struct vduse_dev_config *config)
1674 {
1675 	if (!is_mem_zero((const char *)config->reserved,
1676 			 sizeof(config->reserved)))
1677 		return false;
1678 
1679 	if (config->vq_align > PAGE_SIZE)
1680 		return false;
1681 
1682 	if (config->config_size > PAGE_SIZE)
1683 		return false;
1684 
1685 	if (config->vq_num > 0xffff)
1686 		return false;
1687 
1688 	if (!device_is_allowed(config->device_id))
1689 		return false;
1690 
1691 	if (!features_is_valid(config->features))
1692 		return false;
1693 
1694 	return true;
1695 }
1696 
1697 static ssize_t msg_timeout_show(struct device *device,
1698 				struct device_attribute *attr, char *buf)
1699 {
1700 	struct vduse_dev *dev = dev_get_drvdata(device);
1701 
1702 	return sysfs_emit(buf, "%u\n", dev->msg_timeout);
1703 }
1704 
1705 static ssize_t msg_timeout_store(struct device *device,
1706 				 struct device_attribute *attr,
1707 				 const char *buf, size_t count)
1708 {
1709 	struct vduse_dev *dev = dev_get_drvdata(device);
1710 	int ret;
1711 
1712 	ret = kstrtouint(buf, 10, &dev->msg_timeout);
1713 	if (ret < 0)
1714 		return ret;
1715 
1716 	return count;
1717 }
1718 
1719 static DEVICE_ATTR_RW(msg_timeout);
1720 
1721 static ssize_t bounce_size_show(struct device *device,
1722 				struct device_attribute *attr, char *buf)
1723 {
1724 	struct vduse_dev *dev = dev_get_drvdata(device);
1725 
1726 	return sysfs_emit(buf, "%u\n", dev->bounce_size);
1727 }
1728 
1729 static ssize_t bounce_size_store(struct device *device,
1730 				 struct device_attribute *attr,
1731 				 const char *buf, size_t count)
1732 {
1733 	struct vduse_dev *dev = dev_get_drvdata(device);
1734 	unsigned int bounce_size;
1735 	int ret;
1736 
1737 	ret = -EPERM;
1738 	mutex_lock(&dev->domain_lock);
1739 	if (dev->domain)
1740 		goto unlock;
1741 
1742 	ret = kstrtouint(buf, 10, &bounce_size);
1743 	if (ret < 0)
1744 		goto unlock;
1745 
1746 	ret = -EINVAL;
1747 	if (bounce_size > VDUSE_MAX_BOUNCE_SIZE ||
1748 	    bounce_size < VDUSE_MIN_BOUNCE_SIZE)
1749 		goto unlock;
1750 
1751 	dev->bounce_size = bounce_size & PAGE_MASK;
1752 	ret = count;
1753 unlock:
1754 	mutex_unlock(&dev->domain_lock);
1755 	return ret;
1756 }
1757 
1758 static DEVICE_ATTR_RW(bounce_size);
1759 
1760 static struct attribute *vduse_dev_attrs[] = {
1761 	&dev_attr_msg_timeout.attr,
1762 	&dev_attr_bounce_size.attr,
1763 	NULL
1764 };
1765 
1766 ATTRIBUTE_GROUPS(vduse_dev);
1767 
1768 static int vduse_create_dev(struct vduse_dev_config *config,
1769 			    void *config_buf, u64 api_version)
1770 {
1771 	int ret;
1772 	struct vduse_dev *dev;
1773 
1774 	ret = -EEXIST;
1775 	if (vduse_find_dev(config->name))
1776 		goto err;
1777 
1778 	ret = -ENOMEM;
1779 	dev = vduse_dev_create();
1780 	if (!dev)
1781 		goto err;
1782 
1783 	dev->api_version = api_version;
1784 	dev->device_features = config->features;
1785 	dev->device_id = config->device_id;
1786 	dev->vendor_id = config->vendor_id;
1787 	dev->name = kstrdup(config->name, GFP_KERNEL);
1788 	if (!dev->name)
1789 		goto err_str;
1790 
1791 	dev->bounce_size = VDUSE_BOUNCE_SIZE;
1792 	dev->config = config_buf;
1793 	dev->config_size = config->config_size;
1794 
1795 	ret = idr_alloc(&vduse_idr, dev, 1, VDUSE_DEV_MAX, GFP_KERNEL);
1796 	if (ret < 0)
1797 		goto err_idr;
1798 
1799 	dev->minor = ret;
1800 	dev->msg_timeout = VDUSE_MSG_DEFAULT_TIMEOUT;
1801 	dev->dev = device_create_with_groups(vduse_class, NULL,
1802 				MKDEV(MAJOR(vduse_major), dev->minor),
1803 				dev, vduse_dev_groups, "%s", config->name);
1804 	if (IS_ERR(dev->dev)) {
1805 		ret = PTR_ERR(dev->dev);
1806 		goto err_dev;
1807 	}
1808 
1809 	ret = vduse_dev_init_vqs(dev, config->vq_align, config->vq_num);
1810 	if (ret)
1811 		goto err_vqs;
1812 
1813 	__module_get(THIS_MODULE);
1814 
1815 	return 0;
1816 err_vqs:
1817 	device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));
1818 err_dev:
1819 	idr_remove(&vduse_idr, dev->minor);
1820 err_idr:
1821 	kfree(dev->name);
1822 err_str:
1823 	vduse_dev_destroy(dev);
1824 err:
1825 	return ret;
1826 }
1827 
1828 static long vduse_ioctl(struct file *file, unsigned int cmd,
1829 			unsigned long arg)
1830 {
1831 	int ret;
1832 	void __user *argp = (void __user *)arg;
1833 	struct vduse_control *control = file->private_data;
1834 
1835 	mutex_lock(&vduse_lock);
1836 	switch (cmd) {
1837 	case VDUSE_GET_API_VERSION:
1838 		ret = put_user(control->api_version, (u64 __user *)argp);
1839 		break;
1840 	case VDUSE_SET_API_VERSION: {
1841 		u64 api_version;
1842 
1843 		ret = -EFAULT;
1844 		if (get_user(api_version, (u64 __user *)argp))
1845 			break;
1846 
1847 		ret = -EINVAL;
1848 		if (api_version > VDUSE_API_VERSION)
1849 			break;
1850 
1851 		ret = 0;
1852 		control->api_version = api_version;
1853 		break;
1854 	}
1855 	case VDUSE_CREATE_DEV: {
1856 		struct vduse_dev_config config;
1857 		unsigned long size = offsetof(struct vduse_dev_config, config);
1858 		void *buf;
1859 
1860 		ret = -EFAULT;
1861 		if (copy_from_user(&config, argp, size))
1862 			break;
1863 
1864 		ret = -EINVAL;
1865 		if (vduse_validate_config(&config) == false)
1866 			break;
1867 
1868 		buf = vmemdup_user(argp + size, config.config_size);
1869 		if (IS_ERR(buf)) {
1870 			ret = PTR_ERR(buf);
1871 			break;
1872 		}
1873 		config.name[VDUSE_NAME_MAX - 1] = '\0';
1874 		ret = vduse_create_dev(&config, buf, control->api_version);
1875 		if (ret)
1876 			kvfree(buf);
1877 		break;
1878 	}
1879 	case VDUSE_DESTROY_DEV: {
1880 		char name[VDUSE_NAME_MAX];
1881 
1882 		ret = -EFAULT;
1883 		if (copy_from_user(name, argp, VDUSE_NAME_MAX))
1884 			break;
1885 
1886 		name[VDUSE_NAME_MAX - 1] = '\0';
1887 		ret = vduse_destroy_dev(name);
1888 		break;
1889 	}
1890 	default:
1891 		ret = -EINVAL;
1892 		break;
1893 	}
1894 	mutex_unlock(&vduse_lock);
1895 
1896 	return ret;
1897 }
1898 
1899 static int vduse_release(struct inode *inode, struct file *file)
1900 {
1901 	struct vduse_control *control = file->private_data;
1902 
1903 	kfree(control);
1904 	return 0;
1905 }
1906 
1907 static int vduse_open(struct inode *inode, struct file *file)
1908 {
1909 	struct vduse_control *control;
1910 
1911 	control = kmalloc(sizeof(struct vduse_control), GFP_KERNEL);
1912 	if (!control)
1913 		return -ENOMEM;
1914 
1915 	control->api_version = VDUSE_API_VERSION;
1916 	file->private_data = control;
1917 
1918 	return 0;
1919 }
1920 
1921 static const struct file_operations vduse_ctrl_fops = {
1922 	.owner		= THIS_MODULE,
1923 	.open		= vduse_open,
1924 	.release	= vduse_release,
1925 	.unlocked_ioctl	= vduse_ioctl,
1926 	.compat_ioctl	= compat_ptr_ioctl,
1927 	.llseek		= noop_llseek,
1928 };
1929 
1930 static char *vduse_devnode(const struct device *dev, umode_t *mode)
1931 {
1932 	return kasprintf(GFP_KERNEL, "vduse/%s", dev_name(dev));
1933 }
1934 
1935 struct vduse_mgmt_dev {
1936 	struct vdpa_mgmt_dev mgmt_dev;
1937 	struct device dev;
1938 };
1939 
1940 static struct vduse_mgmt_dev *vduse_mgmt;
1941 
1942 static int vduse_dev_init_vdpa(struct vduse_dev *dev, const char *name)
1943 {
1944 	struct vduse_vdpa *vdev;
1945 	int ret;
1946 
1947 	if (dev->vdev)
1948 		return -EEXIST;
1949 
1950 	vdev = vdpa_alloc_device(struct vduse_vdpa, vdpa, dev->dev,
1951 				 &vduse_vdpa_config_ops, 1, 1, name, true);
1952 	if (IS_ERR(vdev))
1953 		return PTR_ERR(vdev);
1954 
1955 	dev->vdev = vdev;
1956 	vdev->dev = dev;
1957 	vdev->vdpa.dev.dma_mask = &vdev->vdpa.dev.coherent_dma_mask;
1958 	ret = dma_set_mask_and_coherent(&vdev->vdpa.dev, DMA_BIT_MASK(64));
1959 	if (ret) {
1960 		put_device(&vdev->vdpa.dev);
1961 		return ret;
1962 	}
1963 	set_dma_ops(&vdev->vdpa.dev, &vduse_dev_dma_ops);
1964 	vdev->vdpa.dma_dev = &vdev->vdpa.dev;
1965 	vdev->vdpa.mdev = &vduse_mgmt->mgmt_dev;
1966 
1967 	return 0;
1968 }
1969 
1970 static int vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
1971 			const struct vdpa_dev_set_config *config)
1972 {
1973 	struct vduse_dev *dev;
1974 	int ret;
1975 
1976 	mutex_lock(&vduse_lock);
1977 	dev = vduse_find_dev(name);
1978 	if (!dev || !vduse_dev_is_ready(dev)) {
1979 		mutex_unlock(&vduse_lock);
1980 		return -EINVAL;
1981 	}
1982 	ret = vduse_dev_init_vdpa(dev, name);
1983 	mutex_unlock(&vduse_lock);
1984 	if (ret)
1985 		return ret;
1986 
1987 	mutex_lock(&dev->domain_lock);
1988 	if (!dev->domain)
1989 		dev->domain = vduse_domain_create(VDUSE_IOVA_SIZE - 1,
1990 						  dev->bounce_size);
1991 	mutex_unlock(&dev->domain_lock);
1992 	if (!dev->domain) {
1993 		put_device(&dev->vdev->vdpa.dev);
1994 		return -ENOMEM;
1995 	}
1996 
1997 	ret = _vdpa_register_device(&dev->vdev->vdpa, dev->vq_num);
1998 	if (ret) {
1999 		put_device(&dev->vdev->vdpa.dev);
2000 		mutex_lock(&dev->domain_lock);
2001 		vduse_domain_destroy(dev->domain);
2002 		dev->domain = NULL;
2003 		mutex_unlock(&dev->domain_lock);
2004 		return ret;
2005 	}
2006 
2007 	return 0;
2008 }
2009 
2010 static void vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev)
2011 {
2012 	_vdpa_unregister_device(dev);
2013 }
2014 
2015 static const struct vdpa_mgmtdev_ops vdpa_dev_mgmtdev_ops = {
2016 	.dev_add = vdpa_dev_add,
2017 	.dev_del = vdpa_dev_del,
2018 };
2019 
2020 static struct virtio_device_id id_table[] = {
2021 	{ VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID },
2022 	{ 0 },
2023 };
2024 
2025 static void vduse_mgmtdev_release(struct device *dev)
2026 {
2027 	struct vduse_mgmt_dev *mgmt_dev;
2028 
2029 	mgmt_dev = container_of(dev, struct vduse_mgmt_dev, dev);
2030 	kfree(mgmt_dev);
2031 }
2032 
2033 static int vduse_mgmtdev_init(void)
2034 {
2035 	int ret;
2036 
2037 	vduse_mgmt = kzalloc(sizeof(*vduse_mgmt), GFP_KERNEL);
2038 	if (!vduse_mgmt)
2039 		return -ENOMEM;
2040 
2041 	ret = dev_set_name(&vduse_mgmt->dev, "vduse");
2042 	if (ret) {
2043 		kfree(vduse_mgmt);
2044 		return ret;
2045 	}
2046 
2047 	vduse_mgmt->dev.release = vduse_mgmtdev_release;
2048 
2049 	ret = device_register(&vduse_mgmt->dev);
2050 	if (ret)
2051 		goto dev_reg_err;
2052 
2053 	vduse_mgmt->mgmt_dev.id_table = id_table;
2054 	vduse_mgmt->mgmt_dev.ops = &vdpa_dev_mgmtdev_ops;
2055 	vduse_mgmt->mgmt_dev.device = &vduse_mgmt->dev;
2056 	ret = vdpa_mgmtdev_register(&vduse_mgmt->mgmt_dev);
2057 	if (ret)
2058 		device_unregister(&vduse_mgmt->dev);
2059 
2060 	return ret;
2061 
2062 dev_reg_err:
2063 	put_device(&vduse_mgmt->dev);
2064 	return ret;
2065 }
2066 
2067 static void vduse_mgmtdev_exit(void)
2068 {
2069 	vdpa_mgmtdev_unregister(&vduse_mgmt->mgmt_dev);
2070 	device_unregister(&vduse_mgmt->dev);
2071 }
2072 
2073 static int vduse_init(void)
2074 {
2075 	int ret;
2076 	struct device *dev;
2077 
2078 	vduse_class = class_create("vduse");
2079 	if (IS_ERR(vduse_class))
2080 		return PTR_ERR(vduse_class);
2081 
2082 	vduse_class->devnode = vduse_devnode;
2083 
2084 	ret = alloc_chrdev_region(&vduse_major, 0, VDUSE_DEV_MAX, "vduse");
2085 	if (ret)
2086 		goto err_chardev_region;
2087 
2088 	/* /dev/vduse/control */
2089 	cdev_init(&vduse_ctrl_cdev, &vduse_ctrl_fops);
2090 	vduse_ctrl_cdev.owner = THIS_MODULE;
2091 	ret = cdev_add(&vduse_ctrl_cdev, vduse_major, 1);
2092 	if (ret)
2093 		goto err_ctrl_cdev;
2094 
2095 	dev = device_create(vduse_class, NULL, vduse_major, NULL, "control");
2096 	if (IS_ERR(dev)) {
2097 		ret = PTR_ERR(dev);
2098 		goto err_device;
2099 	}
2100 
2101 	/* /dev/vduse/$DEVICE */
2102 	cdev_init(&vduse_cdev, &vduse_dev_fops);
2103 	vduse_cdev.owner = THIS_MODULE;
2104 	ret = cdev_add(&vduse_cdev, MKDEV(MAJOR(vduse_major), 1),
2105 		       VDUSE_DEV_MAX - 1);
2106 	if (ret)
2107 		goto err_cdev;
2108 
2109 	ret = -ENOMEM;
2110 	vduse_irq_wq = alloc_workqueue("vduse-irq",
2111 				WQ_HIGHPRI | WQ_SYSFS | WQ_UNBOUND, 0);
2112 	if (!vduse_irq_wq)
2113 		goto err_wq;
2114 
2115 	vduse_irq_bound_wq = alloc_workqueue("vduse-irq-bound", WQ_HIGHPRI, 0);
2116 	if (!vduse_irq_bound_wq)
2117 		goto err_bound_wq;
2118 
2119 	ret = vduse_domain_init();
2120 	if (ret)
2121 		goto err_domain;
2122 
2123 	ret = vduse_mgmtdev_init();
2124 	if (ret)
2125 		goto err_mgmtdev;
2126 
2127 	return 0;
2128 err_mgmtdev:
2129 	vduse_domain_exit();
2130 err_domain:
2131 	destroy_workqueue(vduse_irq_bound_wq);
2132 err_bound_wq:
2133 	destroy_workqueue(vduse_irq_wq);
2134 err_wq:
2135 	cdev_del(&vduse_cdev);
2136 err_cdev:
2137 	device_destroy(vduse_class, vduse_major);
2138 err_device:
2139 	cdev_del(&vduse_ctrl_cdev);
2140 err_ctrl_cdev:
2141 	unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);
2142 err_chardev_region:
2143 	class_destroy(vduse_class);
2144 	return ret;
2145 }
2146 module_init(vduse_init);
2147 
2148 static void vduse_exit(void)
2149 {
2150 	vduse_mgmtdev_exit();
2151 	vduse_domain_exit();
2152 	destroy_workqueue(vduse_irq_bound_wq);
2153 	destroy_workqueue(vduse_irq_wq);
2154 	cdev_del(&vduse_cdev);
2155 	device_destroy(vduse_class, vduse_major);
2156 	cdev_del(&vduse_ctrl_cdev);
2157 	unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);
2158 	class_destroy(vduse_class);
2159 }
2160 module_exit(vduse_exit);
2161 
2162 MODULE_LICENSE(DRV_LICENSE);
2163 MODULE_AUTHOR(DRV_AUTHOR);
2164 MODULE_DESCRIPTION(DRV_DESC);
2165