1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * VDUSE: vDPA Device in Userspace
4 *
5 * Copyright (C) 2020-2021 Bytedance Inc. and/or its affiliates. All rights reserved.
6 *
7 * Author: Xie Yongji <xieyongji@bytedance.com>
8 *
9 */
10
11 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/cdev.h>
14 #include <linux/device.h>
15 #include <linux/eventfd.h>
16 #include <linux/slab.h>
17 #include <linux/wait.h>
18 #include <linux/dma-map-ops.h>
19 #include <linux/poll.h>
20 #include <linux/file.h>
21 #include <linux/uio.h>
22 #include <linux/vdpa.h>
23 #include <linux/nospec.h>
24 #include <linux/vmalloc.h>
25 #include <linux/sched/mm.h>
26 #include <uapi/linux/vduse.h>
27 #include <uapi/linux/vdpa.h>
28 #include <uapi/linux/virtio_config.h>
29 #include <uapi/linux/virtio_ids.h>
30 #include <uapi/linux/virtio_blk.h>
31 #include <linux/mod_devicetable.h>
32
33 #include "iova_domain.h"
34
35 #define DRV_AUTHOR "Yongji Xie <xieyongji@bytedance.com>"
36 #define DRV_DESC "vDPA Device in Userspace"
37 #define DRV_LICENSE "GPL v2"
38
39 #define VDUSE_DEV_MAX (1U << MINORBITS)
40 #define VDUSE_MAX_BOUNCE_SIZE (1024 * 1024 * 1024)
41 #define VDUSE_MIN_BOUNCE_SIZE (1024 * 1024)
42 #define VDUSE_BOUNCE_SIZE (64 * 1024 * 1024)
43 /* 128 MB reserved for virtqueue creation */
44 #define VDUSE_IOVA_SIZE (VDUSE_MAX_BOUNCE_SIZE + 128 * 1024 * 1024)
45 #define VDUSE_MSG_DEFAULT_TIMEOUT 30
46
47 #define IRQ_UNBOUND -1
48
49 struct vduse_virtqueue {
50 u16 index;
51 u16 num_max;
52 u32 num;
53 u64 desc_addr;
54 u64 driver_addr;
55 u64 device_addr;
56 struct vdpa_vq_state state;
57 bool ready;
58 bool kicked;
59 spinlock_t kick_lock;
60 spinlock_t irq_lock;
61 struct eventfd_ctx *kickfd;
62 struct vdpa_callback cb;
63 struct work_struct inject;
64 struct work_struct kick;
65 int irq_effective_cpu;
66 struct cpumask irq_affinity;
67 struct kobject kobj;
68 };
69
70 struct vduse_dev;
71
72 struct vduse_vdpa {
73 struct vdpa_device vdpa;
74 struct vduse_dev *dev;
75 };
76
77 struct vduse_umem {
78 unsigned long iova;
79 unsigned long npages;
80 struct page **pages;
81 struct mm_struct *mm;
82 };
83
84 struct vduse_dev {
85 struct vduse_vdpa *vdev;
86 struct device *dev;
87 struct vduse_virtqueue **vqs;
88 struct vduse_iova_domain *domain;
89 char *name;
90 struct mutex lock;
91 spinlock_t msg_lock;
92 u64 msg_unique;
93 u32 msg_timeout;
94 wait_queue_head_t waitq;
95 struct list_head send_list;
96 struct list_head recv_list;
97 struct vdpa_callback config_cb;
98 struct work_struct inject;
99 spinlock_t irq_lock;
100 struct rw_semaphore rwsem;
101 int minor;
102 bool broken;
103 bool connected;
104 u64 api_version;
105 u64 device_features;
106 u64 driver_features;
107 u32 device_id;
108 u32 vendor_id;
109 u32 generation;
110 u32 config_size;
111 void *config;
112 u8 status;
113 u32 vq_num;
114 u32 vq_align;
115 struct vduse_umem *umem;
116 struct mutex mem_lock;
117 unsigned int bounce_size;
118 struct mutex domain_lock;
119 };
120
121 struct vduse_dev_msg {
122 struct vduse_dev_request req;
123 struct vduse_dev_response resp;
124 struct list_head list;
125 wait_queue_head_t waitq;
126 bool completed;
127 };
128
129 struct vduse_control {
130 u64 api_version;
131 };
132
133 static DEFINE_MUTEX(vduse_lock);
134 static DEFINE_IDR(vduse_idr);
135
136 static dev_t vduse_major;
137 static struct class *vduse_class;
138 static struct cdev vduse_ctrl_cdev;
139 static struct cdev vduse_cdev;
140 static struct workqueue_struct *vduse_irq_wq;
141 static struct workqueue_struct *vduse_irq_bound_wq;
142
143 static u32 allowed_device_id[] = {
144 VIRTIO_ID_BLOCK,
145 };
146
vdpa_to_vduse(struct vdpa_device * vdpa)147 static inline struct vduse_dev *vdpa_to_vduse(struct vdpa_device *vdpa)
148 {
149 struct vduse_vdpa *vdev = container_of(vdpa, struct vduse_vdpa, vdpa);
150
151 return vdev->dev;
152 }
153
dev_to_vduse(struct device * dev)154 static inline struct vduse_dev *dev_to_vduse(struct device *dev)
155 {
156 struct vdpa_device *vdpa = dev_to_vdpa(dev);
157
158 return vdpa_to_vduse(vdpa);
159 }
160
vduse_find_msg(struct list_head * head,uint32_t request_id)161 static struct vduse_dev_msg *vduse_find_msg(struct list_head *head,
162 uint32_t request_id)
163 {
164 struct vduse_dev_msg *msg;
165
166 list_for_each_entry(msg, head, list) {
167 if (msg->req.request_id == request_id) {
168 list_del(&msg->list);
169 return msg;
170 }
171 }
172
173 return NULL;
174 }
175
vduse_dequeue_msg(struct list_head * head)176 static struct vduse_dev_msg *vduse_dequeue_msg(struct list_head *head)
177 {
178 struct vduse_dev_msg *msg = NULL;
179
180 if (!list_empty(head)) {
181 msg = list_first_entry(head, struct vduse_dev_msg, list);
182 list_del(&msg->list);
183 }
184
185 return msg;
186 }
187
vduse_enqueue_msg(struct list_head * head,struct vduse_dev_msg * msg)188 static void vduse_enqueue_msg(struct list_head *head,
189 struct vduse_dev_msg *msg)
190 {
191 list_add_tail(&msg->list, head);
192 }
193
vduse_dev_broken(struct vduse_dev * dev)194 static void vduse_dev_broken(struct vduse_dev *dev)
195 {
196 struct vduse_dev_msg *msg, *tmp;
197
198 if (unlikely(dev->broken))
199 return;
200
201 list_splice_init(&dev->recv_list, &dev->send_list);
202 list_for_each_entry_safe(msg, tmp, &dev->send_list, list) {
203 list_del(&msg->list);
204 msg->completed = 1;
205 msg->resp.result = VDUSE_REQ_RESULT_FAILED;
206 wake_up(&msg->waitq);
207 }
208 dev->broken = true;
209 wake_up(&dev->waitq);
210 }
211
vduse_dev_msg_sync(struct vduse_dev * dev,struct vduse_dev_msg * msg)212 static int vduse_dev_msg_sync(struct vduse_dev *dev,
213 struct vduse_dev_msg *msg)
214 {
215 int ret;
216
217 if (unlikely(dev->broken))
218 return -EIO;
219
220 init_waitqueue_head(&msg->waitq);
221 spin_lock(&dev->msg_lock);
222 if (unlikely(dev->broken)) {
223 spin_unlock(&dev->msg_lock);
224 return -EIO;
225 }
226 msg->req.request_id = dev->msg_unique++;
227 vduse_enqueue_msg(&dev->send_list, msg);
228 wake_up(&dev->waitq);
229 spin_unlock(&dev->msg_lock);
230 if (dev->msg_timeout)
231 ret = wait_event_killable_timeout(msg->waitq, msg->completed,
232 (long)dev->msg_timeout * HZ);
233 else
234 ret = wait_event_killable(msg->waitq, msg->completed);
235
236 spin_lock(&dev->msg_lock);
237 if (!msg->completed) {
238 list_del(&msg->list);
239 msg->resp.result = VDUSE_REQ_RESULT_FAILED;
240 /* Mark the device as malfunction when there is a timeout */
241 if (!ret)
242 vduse_dev_broken(dev);
243 }
244 ret = (msg->resp.result == VDUSE_REQ_RESULT_OK) ? 0 : -EIO;
245 spin_unlock(&dev->msg_lock);
246
247 return ret;
248 }
249
vduse_dev_get_vq_state_packed(struct vduse_dev * dev,struct vduse_virtqueue * vq,struct vdpa_vq_state_packed * packed)250 static int vduse_dev_get_vq_state_packed(struct vduse_dev *dev,
251 struct vduse_virtqueue *vq,
252 struct vdpa_vq_state_packed *packed)
253 {
254 struct vduse_dev_msg msg = { 0 };
255 int ret;
256
257 msg.req.type = VDUSE_GET_VQ_STATE;
258 msg.req.vq_state.index = vq->index;
259
260 ret = vduse_dev_msg_sync(dev, &msg);
261 if (ret)
262 return ret;
263
264 packed->last_avail_counter =
265 msg.resp.vq_state.packed.last_avail_counter & 0x0001;
266 packed->last_avail_idx =
267 msg.resp.vq_state.packed.last_avail_idx & 0x7FFF;
268 packed->last_used_counter =
269 msg.resp.vq_state.packed.last_used_counter & 0x0001;
270 packed->last_used_idx =
271 msg.resp.vq_state.packed.last_used_idx & 0x7FFF;
272
273 return 0;
274 }
275
vduse_dev_get_vq_state_split(struct vduse_dev * dev,struct vduse_virtqueue * vq,struct vdpa_vq_state_split * split)276 static int vduse_dev_get_vq_state_split(struct vduse_dev *dev,
277 struct vduse_virtqueue *vq,
278 struct vdpa_vq_state_split *split)
279 {
280 struct vduse_dev_msg msg = { 0 };
281 int ret;
282
283 msg.req.type = VDUSE_GET_VQ_STATE;
284 msg.req.vq_state.index = vq->index;
285
286 ret = vduse_dev_msg_sync(dev, &msg);
287 if (ret)
288 return ret;
289
290 split->avail_index = msg.resp.vq_state.split.avail_index;
291
292 return 0;
293 }
294
vduse_dev_set_status(struct vduse_dev * dev,u8 status)295 static int vduse_dev_set_status(struct vduse_dev *dev, u8 status)
296 {
297 struct vduse_dev_msg msg = { 0 };
298
299 msg.req.type = VDUSE_SET_STATUS;
300 msg.req.s.status = status;
301
302 return vduse_dev_msg_sync(dev, &msg);
303 }
304
vduse_dev_update_iotlb(struct vduse_dev * dev,u64 start,u64 last)305 static int vduse_dev_update_iotlb(struct vduse_dev *dev,
306 u64 start, u64 last)
307 {
308 struct vduse_dev_msg msg = { 0 };
309
310 if (last < start)
311 return -EINVAL;
312
313 msg.req.type = VDUSE_UPDATE_IOTLB;
314 msg.req.iova.start = start;
315 msg.req.iova.last = last;
316
317 return vduse_dev_msg_sync(dev, &msg);
318 }
319
vduse_dev_read_iter(struct kiocb * iocb,struct iov_iter * to)320 static ssize_t vduse_dev_read_iter(struct kiocb *iocb, struct iov_iter *to)
321 {
322 struct file *file = iocb->ki_filp;
323 struct vduse_dev *dev = file->private_data;
324 struct vduse_dev_msg *msg;
325 int size = sizeof(struct vduse_dev_request);
326 ssize_t ret;
327
328 if (iov_iter_count(to) < size)
329 return -EINVAL;
330
331 spin_lock(&dev->msg_lock);
332 while (1) {
333 msg = vduse_dequeue_msg(&dev->send_list);
334 if (msg)
335 break;
336
337 ret = -EAGAIN;
338 if (file->f_flags & O_NONBLOCK)
339 goto unlock;
340
341 spin_unlock(&dev->msg_lock);
342 ret = wait_event_interruptible_exclusive(dev->waitq,
343 !list_empty(&dev->send_list));
344 if (ret)
345 return ret;
346
347 spin_lock(&dev->msg_lock);
348 }
349 spin_unlock(&dev->msg_lock);
350 ret = copy_to_iter(&msg->req, size, to);
351 spin_lock(&dev->msg_lock);
352 if (ret != size) {
353 ret = -EFAULT;
354 vduse_enqueue_msg(&dev->send_list, msg);
355 goto unlock;
356 }
357 vduse_enqueue_msg(&dev->recv_list, msg);
358 unlock:
359 spin_unlock(&dev->msg_lock);
360
361 return ret;
362 }
363
is_mem_zero(const char * ptr,int size)364 static bool is_mem_zero(const char *ptr, int size)
365 {
366 int i;
367
368 for (i = 0; i < size; i++) {
369 if (ptr[i])
370 return false;
371 }
372 return true;
373 }
374
vduse_dev_write_iter(struct kiocb * iocb,struct iov_iter * from)375 static ssize_t vduse_dev_write_iter(struct kiocb *iocb, struct iov_iter *from)
376 {
377 struct file *file = iocb->ki_filp;
378 struct vduse_dev *dev = file->private_data;
379 struct vduse_dev_response resp;
380 struct vduse_dev_msg *msg;
381 size_t ret;
382
383 ret = copy_from_iter(&resp, sizeof(resp), from);
384 if (ret != sizeof(resp))
385 return -EINVAL;
386
387 if (!is_mem_zero((const char *)resp.reserved, sizeof(resp.reserved)))
388 return -EINVAL;
389
390 spin_lock(&dev->msg_lock);
391 msg = vduse_find_msg(&dev->recv_list, resp.request_id);
392 if (!msg) {
393 ret = -ENOENT;
394 goto unlock;
395 }
396
397 memcpy(&msg->resp, &resp, sizeof(resp));
398 msg->completed = 1;
399 wake_up(&msg->waitq);
400 unlock:
401 spin_unlock(&dev->msg_lock);
402
403 return ret;
404 }
405
vduse_dev_poll(struct file * file,poll_table * wait)406 static __poll_t vduse_dev_poll(struct file *file, poll_table *wait)
407 {
408 struct vduse_dev *dev = file->private_data;
409 __poll_t mask = 0;
410
411 poll_wait(file, &dev->waitq, wait);
412
413 spin_lock(&dev->msg_lock);
414
415 if (unlikely(dev->broken))
416 mask |= EPOLLERR;
417 if (!list_empty(&dev->send_list))
418 mask |= EPOLLIN | EPOLLRDNORM;
419 if (!list_empty(&dev->recv_list))
420 mask |= EPOLLOUT | EPOLLWRNORM;
421
422 spin_unlock(&dev->msg_lock);
423
424 return mask;
425 }
426
vduse_dev_reset(struct vduse_dev * dev)427 static void vduse_dev_reset(struct vduse_dev *dev)
428 {
429 int i;
430 struct vduse_iova_domain *domain = dev->domain;
431
432 /* The coherent mappings are handled in vduse_dev_free_coherent() */
433 if (domain && domain->bounce_map)
434 vduse_domain_reset_bounce_map(domain);
435
436 down_write(&dev->rwsem);
437
438 dev->status = 0;
439 dev->driver_features = 0;
440 dev->generation++;
441 spin_lock(&dev->irq_lock);
442 dev->config_cb.callback = NULL;
443 dev->config_cb.private = NULL;
444 spin_unlock(&dev->irq_lock);
445 flush_work(&dev->inject);
446
447 for (i = 0; i < dev->vq_num; i++) {
448 struct vduse_virtqueue *vq = dev->vqs[i];
449
450 vq->ready = false;
451 vq->desc_addr = 0;
452 vq->driver_addr = 0;
453 vq->device_addr = 0;
454 vq->num = 0;
455 memset(&vq->state, 0, sizeof(vq->state));
456
457 spin_lock(&vq->kick_lock);
458 vq->kicked = false;
459 if (vq->kickfd)
460 eventfd_ctx_put(vq->kickfd);
461 vq->kickfd = NULL;
462 spin_unlock(&vq->kick_lock);
463
464 spin_lock(&vq->irq_lock);
465 vq->cb.callback = NULL;
466 vq->cb.private = NULL;
467 vq->cb.trigger = NULL;
468 spin_unlock(&vq->irq_lock);
469 flush_work(&vq->inject);
470 flush_work(&vq->kick);
471 }
472
473 up_write(&dev->rwsem);
474 }
475
vduse_vdpa_set_vq_address(struct vdpa_device * vdpa,u16 idx,u64 desc_area,u64 driver_area,u64 device_area)476 static int vduse_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 idx,
477 u64 desc_area, u64 driver_area,
478 u64 device_area)
479 {
480 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
481 struct vduse_virtqueue *vq = dev->vqs[idx];
482
483 vq->desc_addr = desc_area;
484 vq->driver_addr = driver_area;
485 vq->device_addr = device_area;
486
487 return 0;
488 }
489
vduse_vq_kick(struct vduse_virtqueue * vq)490 static void vduse_vq_kick(struct vduse_virtqueue *vq)
491 {
492 spin_lock(&vq->kick_lock);
493 if (!vq->ready)
494 goto unlock;
495
496 if (vq->kickfd)
497 eventfd_signal(vq->kickfd, 1);
498 else
499 vq->kicked = true;
500 unlock:
501 spin_unlock(&vq->kick_lock);
502 }
503
vduse_vq_kick_work(struct work_struct * work)504 static void vduse_vq_kick_work(struct work_struct *work)
505 {
506 struct vduse_virtqueue *vq = container_of(work,
507 struct vduse_virtqueue, kick);
508
509 vduse_vq_kick(vq);
510 }
511
vduse_vdpa_kick_vq(struct vdpa_device * vdpa,u16 idx)512 static void vduse_vdpa_kick_vq(struct vdpa_device *vdpa, u16 idx)
513 {
514 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
515 struct vduse_virtqueue *vq = dev->vqs[idx];
516
517 if (!eventfd_signal_allowed()) {
518 schedule_work(&vq->kick);
519 return;
520 }
521 vduse_vq_kick(vq);
522 }
523
vduse_vdpa_set_vq_cb(struct vdpa_device * vdpa,u16 idx,struct vdpa_callback * cb)524 static void vduse_vdpa_set_vq_cb(struct vdpa_device *vdpa, u16 idx,
525 struct vdpa_callback *cb)
526 {
527 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
528 struct vduse_virtqueue *vq = dev->vqs[idx];
529
530 spin_lock(&vq->irq_lock);
531 vq->cb.callback = cb->callback;
532 vq->cb.private = cb->private;
533 vq->cb.trigger = cb->trigger;
534 spin_unlock(&vq->irq_lock);
535 }
536
vduse_vdpa_set_vq_num(struct vdpa_device * vdpa,u16 idx,u32 num)537 static void vduse_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num)
538 {
539 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
540 struct vduse_virtqueue *vq = dev->vqs[idx];
541
542 vq->num = num;
543 }
544
vduse_vdpa_set_vq_ready(struct vdpa_device * vdpa,u16 idx,bool ready)545 static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa,
546 u16 idx, bool ready)
547 {
548 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
549 struct vduse_virtqueue *vq = dev->vqs[idx];
550
551 vq->ready = ready;
552 }
553
vduse_vdpa_get_vq_ready(struct vdpa_device * vdpa,u16 idx)554 static bool vduse_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 idx)
555 {
556 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
557 struct vduse_virtqueue *vq = dev->vqs[idx];
558
559 return vq->ready;
560 }
561
vduse_vdpa_set_vq_state(struct vdpa_device * vdpa,u16 idx,const struct vdpa_vq_state * state)562 static int vduse_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 idx,
563 const struct vdpa_vq_state *state)
564 {
565 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
566 struct vduse_virtqueue *vq = dev->vqs[idx];
567
568 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
569 vq->state.packed.last_avail_counter =
570 state->packed.last_avail_counter;
571 vq->state.packed.last_avail_idx = state->packed.last_avail_idx;
572 vq->state.packed.last_used_counter =
573 state->packed.last_used_counter;
574 vq->state.packed.last_used_idx = state->packed.last_used_idx;
575 } else
576 vq->state.split.avail_index = state->split.avail_index;
577
578 return 0;
579 }
580
vduse_vdpa_get_vq_state(struct vdpa_device * vdpa,u16 idx,struct vdpa_vq_state * state)581 static int vduse_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 idx,
582 struct vdpa_vq_state *state)
583 {
584 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
585 struct vduse_virtqueue *vq = dev->vqs[idx];
586
587 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED))
588 return vduse_dev_get_vq_state_packed(dev, vq, &state->packed);
589
590 return vduse_dev_get_vq_state_split(dev, vq, &state->split);
591 }
592
vduse_vdpa_get_vq_align(struct vdpa_device * vdpa)593 static u32 vduse_vdpa_get_vq_align(struct vdpa_device *vdpa)
594 {
595 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
596
597 return dev->vq_align;
598 }
599
vduse_vdpa_get_device_features(struct vdpa_device * vdpa)600 static u64 vduse_vdpa_get_device_features(struct vdpa_device *vdpa)
601 {
602 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
603
604 return dev->device_features;
605 }
606
vduse_vdpa_set_driver_features(struct vdpa_device * vdpa,u64 features)607 static int vduse_vdpa_set_driver_features(struct vdpa_device *vdpa, u64 features)
608 {
609 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
610
611 dev->driver_features = features;
612 return 0;
613 }
614
vduse_vdpa_get_driver_features(struct vdpa_device * vdpa)615 static u64 vduse_vdpa_get_driver_features(struct vdpa_device *vdpa)
616 {
617 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
618
619 return dev->driver_features;
620 }
621
vduse_vdpa_set_config_cb(struct vdpa_device * vdpa,struct vdpa_callback * cb)622 static void vduse_vdpa_set_config_cb(struct vdpa_device *vdpa,
623 struct vdpa_callback *cb)
624 {
625 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
626
627 spin_lock(&dev->irq_lock);
628 dev->config_cb.callback = cb->callback;
629 dev->config_cb.private = cb->private;
630 spin_unlock(&dev->irq_lock);
631 }
632
vduse_vdpa_get_vq_num_max(struct vdpa_device * vdpa)633 static u16 vduse_vdpa_get_vq_num_max(struct vdpa_device *vdpa)
634 {
635 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
636 u16 num_max = 0;
637 int i;
638
639 for (i = 0; i < dev->vq_num; i++)
640 if (num_max < dev->vqs[i]->num_max)
641 num_max = dev->vqs[i]->num_max;
642
643 return num_max;
644 }
645
vduse_vdpa_get_device_id(struct vdpa_device * vdpa)646 static u32 vduse_vdpa_get_device_id(struct vdpa_device *vdpa)
647 {
648 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
649
650 return dev->device_id;
651 }
652
vduse_vdpa_get_vendor_id(struct vdpa_device * vdpa)653 static u32 vduse_vdpa_get_vendor_id(struct vdpa_device *vdpa)
654 {
655 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
656
657 return dev->vendor_id;
658 }
659
vduse_vdpa_get_status(struct vdpa_device * vdpa)660 static u8 vduse_vdpa_get_status(struct vdpa_device *vdpa)
661 {
662 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
663
664 return dev->status;
665 }
666
vduse_vdpa_set_status(struct vdpa_device * vdpa,u8 status)667 static void vduse_vdpa_set_status(struct vdpa_device *vdpa, u8 status)
668 {
669 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
670
671 if (vduse_dev_set_status(dev, status))
672 return;
673
674 dev->status = status;
675 }
676
vduse_vdpa_get_config_size(struct vdpa_device * vdpa)677 static size_t vduse_vdpa_get_config_size(struct vdpa_device *vdpa)
678 {
679 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
680
681 return dev->config_size;
682 }
683
vduse_vdpa_get_config(struct vdpa_device * vdpa,unsigned int offset,void * buf,unsigned int len)684 static void vduse_vdpa_get_config(struct vdpa_device *vdpa, unsigned int offset,
685 void *buf, unsigned int len)
686 {
687 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
688
689 /* Initialize the buffer in case of partial copy. */
690 memset(buf, 0, len);
691
692 if (offset > dev->config_size)
693 return;
694
695 if (len > dev->config_size - offset)
696 len = dev->config_size - offset;
697
698 memcpy(buf, dev->config + offset, len);
699 }
700
vduse_vdpa_set_config(struct vdpa_device * vdpa,unsigned int offset,const void * buf,unsigned int len)701 static void vduse_vdpa_set_config(struct vdpa_device *vdpa, unsigned int offset,
702 const void *buf, unsigned int len)
703 {
704 /* Now we only support read-only configuration space */
705 }
706
vduse_vdpa_reset(struct vdpa_device * vdpa)707 static int vduse_vdpa_reset(struct vdpa_device *vdpa)
708 {
709 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
710 int ret = vduse_dev_set_status(dev, 0);
711
712 vduse_dev_reset(dev);
713
714 return ret;
715 }
716
vduse_vdpa_get_generation(struct vdpa_device * vdpa)717 static u32 vduse_vdpa_get_generation(struct vdpa_device *vdpa)
718 {
719 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
720
721 return dev->generation;
722 }
723
vduse_vdpa_set_vq_affinity(struct vdpa_device * vdpa,u16 idx,const struct cpumask * cpu_mask)724 static int vduse_vdpa_set_vq_affinity(struct vdpa_device *vdpa, u16 idx,
725 const struct cpumask *cpu_mask)
726 {
727 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
728
729 if (cpu_mask)
730 cpumask_copy(&dev->vqs[idx]->irq_affinity, cpu_mask);
731 else
732 cpumask_setall(&dev->vqs[idx]->irq_affinity);
733
734 return 0;
735 }
736
737 static const struct cpumask *
vduse_vdpa_get_vq_affinity(struct vdpa_device * vdpa,u16 idx)738 vduse_vdpa_get_vq_affinity(struct vdpa_device *vdpa, u16 idx)
739 {
740 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
741
742 return &dev->vqs[idx]->irq_affinity;
743 }
744
vduse_vdpa_set_map(struct vdpa_device * vdpa,unsigned int asid,struct vhost_iotlb * iotlb)745 static int vduse_vdpa_set_map(struct vdpa_device *vdpa,
746 unsigned int asid,
747 struct vhost_iotlb *iotlb)
748 {
749 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
750 int ret;
751
752 ret = vduse_domain_set_map(dev->domain, iotlb);
753 if (ret)
754 return ret;
755
756 ret = vduse_dev_update_iotlb(dev, 0ULL, ULLONG_MAX);
757 if (ret) {
758 vduse_domain_clear_map(dev->domain, iotlb);
759 return ret;
760 }
761
762 return 0;
763 }
764
vduse_vdpa_free(struct vdpa_device * vdpa)765 static void vduse_vdpa_free(struct vdpa_device *vdpa)
766 {
767 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
768
769 dev->vdev = NULL;
770 }
771
772 static const struct vdpa_config_ops vduse_vdpa_config_ops = {
773 .set_vq_address = vduse_vdpa_set_vq_address,
774 .kick_vq = vduse_vdpa_kick_vq,
775 .set_vq_cb = vduse_vdpa_set_vq_cb,
776 .set_vq_num = vduse_vdpa_set_vq_num,
777 .set_vq_ready = vduse_vdpa_set_vq_ready,
778 .get_vq_ready = vduse_vdpa_get_vq_ready,
779 .set_vq_state = vduse_vdpa_set_vq_state,
780 .get_vq_state = vduse_vdpa_get_vq_state,
781 .get_vq_align = vduse_vdpa_get_vq_align,
782 .get_device_features = vduse_vdpa_get_device_features,
783 .set_driver_features = vduse_vdpa_set_driver_features,
784 .get_driver_features = vduse_vdpa_get_driver_features,
785 .set_config_cb = vduse_vdpa_set_config_cb,
786 .get_vq_num_max = vduse_vdpa_get_vq_num_max,
787 .get_device_id = vduse_vdpa_get_device_id,
788 .get_vendor_id = vduse_vdpa_get_vendor_id,
789 .get_status = vduse_vdpa_get_status,
790 .set_status = vduse_vdpa_set_status,
791 .get_config_size = vduse_vdpa_get_config_size,
792 .get_config = vduse_vdpa_get_config,
793 .set_config = vduse_vdpa_set_config,
794 .get_generation = vduse_vdpa_get_generation,
795 .set_vq_affinity = vduse_vdpa_set_vq_affinity,
796 .get_vq_affinity = vduse_vdpa_get_vq_affinity,
797 .reset = vduse_vdpa_reset,
798 .set_map = vduse_vdpa_set_map,
799 .free = vduse_vdpa_free,
800 };
801
vduse_dev_map_page(struct device * dev,struct page * page,unsigned long offset,size_t size,enum dma_data_direction dir,unsigned long attrs)802 static dma_addr_t vduse_dev_map_page(struct device *dev, struct page *page,
803 unsigned long offset, size_t size,
804 enum dma_data_direction dir,
805 unsigned long attrs)
806 {
807 struct vduse_dev *vdev = dev_to_vduse(dev);
808 struct vduse_iova_domain *domain = vdev->domain;
809
810 return vduse_domain_map_page(domain, page, offset, size, dir, attrs);
811 }
812
vduse_dev_unmap_page(struct device * dev,dma_addr_t dma_addr,size_t size,enum dma_data_direction dir,unsigned long attrs)813 static void vduse_dev_unmap_page(struct device *dev, dma_addr_t dma_addr,
814 size_t size, enum dma_data_direction dir,
815 unsigned long attrs)
816 {
817 struct vduse_dev *vdev = dev_to_vduse(dev);
818 struct vduse_iova_domain *domain = vdev->domain;
819
820 return vduse_domain_unmap_page(domain, dma_addr, size, dir, attrs);
821 }
822
vduse_dev_alloc_coherent(struct device * dev,size_t size,dma_addr_t * dma_addr,gfp_t flag,unsigned long attrs)823 static void *vduse_dev_alloc_coherent(struct device *dev, size_t size,
824 dma_addr_t *dma_addr, gfp_t flag,
825 unsigned long attrs)
826 {
827 struct vduse_dev *vdev = dev_to_vduse(dev);
828 struct vduse_iova_domain *domain = vdev->domain;
829 unsigned long iova;
830 void *addr;
831
832 *dma_addr = DMA_MAPPING_ERROR;
833 addr = vduse_domain_alloc_coherent(domain, size,
834 (dma_addr_t *)&iova, flag, attrs);
835 if (!addr)
836 return NULL;
837
838 *dma_addr = (dma_addr_t)iova;
839
840 return addr;
841 }
842
vduse_dev_free_coherent(struct device * dev,size_t size,void * vaddr,dma_addr_t dma_addr,unsigned long attrs)843 static void vduse_dev_free_coherent(struct device *dev, size_t size,
844 void *vaddr, dma_addr_t dma_addr,
845 unsigned long attrs)
846 {
847 struct vduse_dev *vdev = dev_to_vduse(dev);
848 struct vduse_iova_domain *domain = vdev->domain;
849
850 vduse_domain_free_coherent(domain, size, vaddr, dma_addr, attrs);
851 }
852
vduse_dev_max_mapping_size(struct device * dev)853 static size_t vduse_dev_max_mapping_size(struct device *dev)
854 {
855 struct vduse_dev *vdev = dev_to_vduse(dev);
856 struct vduse_iova_domain *domain = vdev->domain;
857
858 return domain->bounce_size;
859 }
860
861 static const struct dma_map_ops vduse_dev_dma_ops = {
862 .map_page = vduse_dev_map_page,
863 .unmap_page = vduse_dev_unmap_page,
864 .alloc = vduse_dev_alloc_coherent,
865 .free = vduse_dev_free_coherent,
866 .max_mapping_size = vduse_dev_max_mapping_size,
867 };
868
perm_to_file_flags(u8 perm)869 static unsigned int perm_to_file_flags(u8 perm)
870 {
871 unsigned int flags = 0;
872
873 switch (perm) {
874 case VDUSE_ACCESS_WO:
875 flags |= O_WRONLY;
876 break;
877 case VDUSE_ACCESS_RO:
878 flags |= O_RDONLY;
879 break;
880 case VDUSE_ACCESS_RW:
881 flags |= O_RDWR;
882 break;
883 default:
884 WARN(1, "invalidate vhost IOTLB permission\n");
885 break;
886 }
887
888 return flags;
889 }
890
vduse_kickfd_setup(struct vduse_dev * dev,struct vduse_vq_eventfd * eventfd)891 static int vduse_kickfd_setup(struct vduse_dev *dev,
892 struct vduse_vq_eventfd *eventfd)
893 {
894 struct eventfd_ctx *ctx = NULL;
895 struct vduse_virtqueue *vq;
896 u32 index;
897
898 if (eventfd->index >= dev->vq_num)
899 return -EINVAL;
900
901 index = array_index_nospec(eventfd->index, dev->vq_num);
902 vq = dev->vqs[index];
903 if (eventfd->fd >= 0) {
904 ctx = eventfd_ctx_fdget(eventfd->fd);
905 if (IS_ERR(ctx))
906 return PTR_ERR(ctx);
907 } else if (eventfd->fd != VDUSE_EVENTFD_DEASSIGN)
908 return 0;
909
910 spin_lock(&vq->kick_lock);
911 if (vq->kickfd)
912 eventfd_ctx_put(vq->kickfd);
913 vq->kickfd = ctx;
914 if (vq->ready && vq->kicked && vq->kickfd) {
915 eventfd_signal(vq->kickfd, 1);
916 vq->kicked = false;
917 }
918 spin_unlock(&vq->kick_lock);
919
920 return 0;
921 }
922
vduse_dev_is_ready(struct vduse_dev * dev)923 static bool vduse_dev_is_ready(struct vduse_dev *dev)
924 {
925 int i;
926
927 for (i = 0; i < dev->vq_num; i++)
928 if (!dev->vqs[i]->num_max)
929 return false;
930
931 return true;
932 }
933
vduse_dev_irq_inject(struct work_struct * work)934 static void vduse_dev_irq_inject(struct work_struct *work)
935 {
936 struct vduse_dev *dev = container_of(work, struct vduse_dev, inject);
937
938 spin_lock_bh(&dev->irq_lock);
939 if (dev->config_cb.callback)
940 dev->config_cb.callback(dev->config_cb.private);
941 spin_unlock_bh(&dev->irq_lock);
942 }
943
vduse_vq_irq_inject(struct work_struct * work)944 static void vduse_vq_irq_inject(struct work_struct *work)
945 {
946 struct vduse_virtqueue *vq = container_of(work,
947 struct vduse_virtqueue, inject);
948
949 spin_lock_bh(&vq->irq_lock);
950 if (vq->ready && vq->cb.callback)
951 vq->cb.callback(vq->cb.private);
952 spin_unlock_bh(&vq->irq_lock);
953 }
954
vduse_vq_signal_irqfd(struct vduse_virtqueue * vq)955 static bool vduse_vq_signal_irqfd(struct vduse_virtqueue *vq)
956 {
957 bool signal = false;
958
959 if (!vq->cb.trigger)
960 return false;
961
962 spin_lock_irq(&vq->irq_lock);
963 if (vq->ready && vq->cb.trigger) {
964 eventfd_signal(vq->cb.trigger, 1);
965 signal = true;
966 }
967 spin_unlock_irq(&vq->irq_lock);
968
969 return signal;
970 }
971
vduse_dev_queue_irq_work(struct vduse_dev * dev,struct work_struct * irq_work,int irq_effective_cpu)972 static int vduse_dev_queue_irq_work(struct vduse_dev *dev,
973 struct work_struct *irq_work,
974 int irq_effective_cpu)
975 {
976 int ret = -EINVAL;
977
978 down_read(&dev->rwsem);
979 if (!(dev->status & VIRTIO_CONFIG_S_DRIVER_OK))
980 goto unlock;
981
982 ret = 0;
983 if (irq_effective_cpu == IRQ_UNBOUND)
984 queue_work(vduse_irq_wq, irq_work);
985 else
986 queue_work_on(irq_effective_cpu,
987 vduse_irq_bound_wq, irq_work);
988 unlock:
989 up_read(&dev->rwsem);
990
991 return ret;
992 }
993
vduse_dev_dereg_umem(struct vduse_dev * dev,u64 iova,u64 size)994 static int vduse_dev_dereg_umem(struct vduse_dev *dev,
995 u64 iova, u64 size)
996 {
997 int ret;
998
999 mutex_lock(&dev->mem_lock);
1000 ret = -ENOENT;
1001 if (!dev->umem)
1002 goto unlock;
1003
1004 ret = -EINVAL;
1005 if (!dev->domain)
1006 goto unlock;
1007
1008 if (dev->umem->iova != iova || size != dev->domain->bounce_size)
1009 goto unlock;
1010
1011 vduse_domain_remove_user_bounce_pages(dev->domain);
1012 unpin_user_pages_dirty_lock(dev->umem->pages,
1013 dev->umem->npages, true);
1014 atomic64_sub(dev->umem->npages, &dev->umem->mm->pinned_vm);
1015 mmdrop(dev->umem->mm);
1016 vfree(dev->umem->pages);
1017 kfree(dev->umem);
1018 dev->umem = NULL;
1019 ret = 0;
1020 unlock:
1021 mutex_unlock(&dev->mem_lock);
1022 return ret;
1023 }
1024
vduse_dev_reg_umem(struct vduse_dev * dev,u64 iova,u64 uaddr,u64 size)1025 static int vduse_dev_reg_umem(struct vduse_dev *dev,
1026 u64 iova, u64 uaddr, u64 size)
1027 {
1028 struct page **page_list = NULL;
1029 struct vduse_umem *umem = NULL;
1030 long pinned = 0;
1031 unsigned long npages, lock_limit;
1032 int ret;
1033
1034 if (!dev->domain || !dev->domain->bounce_map ||
1035 size != dev->domain->bounce_size ||
1036 iova != 0 || uaddr & ~PAGE_MASK)
1037 return -EINVAL;
1038
1039 mutex_lock(&dev->mem_lock);
1040 ret = -EEXIST;
1041 if (dev->umem)
1042 goto unlock;
1043
1044 ret = -ENOMEM;
1045 npages = size >> PAGE_SHIFT;
1046 page_list = __vmalloc(array_size(npages, sizeof(struct page *)),
1047 GFP_KERNEL_ACCOUNT);
1048 umem = kzalloc(sizeof(*umem), GFP_KERNEL);
1049 if (!page_list || !umem)
1050 goto unlock;
1051
1052 mmap_read_lock(current->mm);
1053
1054 lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK));
1055 if (npages + atomic64_read(¤t->mm->pinned_vm) > lock_limit)
1056 goto out;
1057
1058 pinned = pin_user_pages(uaddr, npages, FOLL_LONGTERM | FOLL_WRITE,
1059 page_list);
1060 if (pinned != npages) {
1061 ret = pinned < 0 ? pinned : -ENOMEM;
1062 goto out;
1063 }
1064
1065 ret = vduse_domain_add_user_bounce_pages(dev->domain,
1066 page_list, pinned);
1067 if (ret)
1068 goto out;
1069
1070 atomic64_add(npages, ¤t->mm->pinned_vm);
1071
1072 umem->pages = page_list;
1073 umem->npages = pinned;
1074 umem->iova = iova;
1075 umem->mm = current->mm;
1076 mmgrab(current->mm);
1077
1078 dev->umem = umem;
1079 out:
1080 if (ret && pinned > 0)
1081 unpin_user_pages(page_list, pinned);
1082
1083 mmap_read_unlock(current->mm);
1084 unlock:
1085 if (ret) {
1086 vfree(page_list);
1087 kfree(umem);
1088 }
1089 mutex_unlock(&dev->mem_lock);
1090 return ret;
1091 }
1092
vduse_vq_update_effective_cpu(struct vduse_virtqueue * vq)1093 static void vduse_vq_update_effective_cpu(struct vduse_virtqueue *vq)
1094 {
1095 int curr_cpu = vq->irq_effective_cpu;
1096
1097 while (true) {
1098 curr_cpu = cpumask_next(curr_cpu, &vq->irq_affinity);
1099 if (cpu_online(curr_cpu))
1100 break;
1101
1102 if (curr_cpu >= nr_cpu_ids)
1103 curr_cpu = IRQ_UNBOUND;
1104 }
1105
1106 vq->irq_effective_cpu = curr_cpu;
1107 }
1108
vduse_dev_ioctl(struct file * file,unsigned int cmd,unsigned long arg)1109 static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
1110 unsigned long arg)
1111 {
1112 struct vduse_dev *dev = file->private_data;
1113 void __user *argp = (void __user *)arg;
1114 int ret;
1115
1116 if (unlikely(dev->broken))
1117 return -EPERM;
1118
1119 switch (cmd) {
1120 case VDUSE_IOTLB_GET_FD: {
1121 struct vduse_iotlb_entry entry;
1122 struct vhost_iotlb_map *map;
1123 struct vdpa_map_file *map_file;
1124 struct file *f = NULL;
1125
1126 ret = -EFAULT;
1127 if (copy_from_user(&entry, argp, sizeof(entry)))
1128 break;
1129
1130 ret = -EINVAL;
1131 if (entry.start > entry.last)
1132 break;
1133
1134 mutex_lock(&dev->domain_lock);
1135 if (!dev->domain) {
1136 mutex_unlock(&dev->domain_lock);
1137 break;
1138 }
1139 spin_lock(&dev->domain->iotlb_lock);
1140 map = vhost_iotlb_itree_first(dev->domain->iotlb,
1141 entry.start, entry.last);
1142 if (map) {
1143 map_file = (struct vdpa_map_file *)map->opaque;
1144 f = get_file(map_file->file);
1145 entry.offset = map_file->offset;
1146 entry.start = map->start;
1147 entry.last = map->last;
1148 entry.perm = map->perm;
1149 }
1150 spin_unlock(&dev->domain->iotlb_lock);
1151 mutex_unlock(&dev->domain_lock);
1152 ret = -EINVAL;
1153 if (!f)
1154 break;
1155
1156 ret = -EFAULT;
1157 if (copy_to_user(argp, &entry, sizeof(entry))) {
1158 fput(f);
1159 break;
1160 }
1161 ret = receive_fd(f, perm_to_file_flags(entry.perm));
1162 fput(f);
1163 break;
1164 }
1165 case VDUSE_DEV_GET_FEATURES:
1166 /*
1167 * Just mirror what driver wrote here.
1168 * The driver is expected to check FEATURE_OK later.
1169 */
1170 ret = put_user(dev->driver_features, (u64 __user *)argp);
1171 break;
1172 case VDUSE_DEV_SET_CONFIG: {
1173 struct vduse_config_data config;
1174 unsigned long size = offsetof(struct vduse_config_data,
1175 buffer);
1176
1177 ret = -EFAULT;
1178 if (copy_from_user(&config, argp, size))
1179 break;
1180
1181 ret = -EINVAL;
1182 if (config.offset > dev->config_size ||
1183 config.length == 0 ||
1184 config.length > dev->config_size - config.offset)
1185 break;
1186
1187 ret = -EFAULT;
1188 if (copy_from_user(dev->config + config.offset, argp + size,
1189 config.length))
1190 break;
1191
1192 ret = 0;
1193 break;
1194 }
1195 case VDUSE_DEV_INJECT_CONFIG_IRQ:
1196 ret = vduse_dev_queue_irq_work(dev, &dev->inject, IRQ_UNBOUND);
1197 break;
1198 case VDUSE_VQ_SETUP: {
1199 struct vduse_vq_config config;
1200 u32 index;
1201
1202 ret = -EFAULT;
1203 if (copy_from_user(&config, argp, sizeof(config)))
1204 break;
1205
1206 ret = -EINVAL;
1207 if (config.index >= dev->vq_num)
1208 break;
1209
1210 if (!is_mem_zero((const char *)config.reserved,
1211 sizeof(config.reserved)))
1212 break;
1213
1214 index = array_index_nospec(config.index, dev->vq_num);
1215 dev->vqs[index]->num_max = config.max_size;
1216 ret = 0;
1217 break;
1218 }
1219 case VDUSE_VQ_GET_INFO: {
1220 struct vduse_vq_info vq_info;
1221 struct vduse_virtqueue *vq;
1222 u32 index;
1223
1224 ret = -EFAULT;
1225 if (copy_from_user(&vq_info, argp, sizeof(vq_info)))
1226 break;
1227
1228 ret = -EINVAL;
1229 if (vq_info.index >= dev->vq_num)
1230 break;
1231
1232 index = array_index_nospec(vq_info.index, dev->vq_num);
1233 vq = dev->vqs[index];
1234 vq_info.desc_addr = vq->desc_addr;
1235 vq_info.driver_addr = vq->driver_addr;
1236 vq_info.device_addr = vq->device_addr;
1237 vq_info.num = vq->num;
1238
1239 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
1240 vq_info.packed.last_avail_counter =
1241 vq->state.packed.last_avail_counter;
1242 vq_info.packed.last_avail_idx =
1243 vq->state.packed.last_avail_idx;
1244 vq_info.packed.last_used_counter =
1245 vq->state.packed.last_used_counter;
1246 vq_info.packed.last_used_idx =
1247 vq->state.packed.last_used_idx;
1248 } else
1249 vq_info.split.avail_index =
1250 vq->state.split.avail_index;
1251
1252 vq_info.ready = vq->ready;
1253
1254 ret = -EFAULT;
1255 if (copy_to_user(argp, &vq_info, sizeof(vq_info)))
1256 break;
1257
1258 ret = 0;
1259 break;
1260 }
1261 case VDUSE_VQ_SETUP_KICKFD: {
1262 struct vduse_vq_eventfd eventfd;
1263
1264 ret = -EFAULT;
1265 if (copy_from_user(&eventfd, argp, sizeof(eventfd)))
1266 break;
1267
1268 ret = vduse_kickfd_setup(dev, &eventfd);
1269 break;
1270 }
1271 case VDUSE_VQ_INJECT_IRQ: {
1272 u32 index;
1273
1274 ret = -EFAULT;
1275 if (get_user(index, (u32 __user *)argp))
1276 break;
1277
1278 ret = -EINVAL;
1279 if (index >= dev->vq_num)
1280 break;
1281
1282 ret = 0;
1283 index = array_index_nospec(index, dev->vq_num);
1284 if (!vduse_vq_signal_irqfd(dev->vqs[index])) {
1285 vduse_vq_update_effective_cpu(dev->vqs[index]);
1286 ret = vduse_dev_queue_irq_work(dev,
1287 &dev->vqs[index]->inject,
1288 dev->vqs[index]->irq_effective_cpu);
1289 }
1290 break;
1291 }
1292 case VDUSE_IOTLB_REG_UMEM: {
1293 struct vduse_iova_umem umem;
1294
1295 ret = -EFAULT;
1296 if (copy_from_user(&umem, argp, sizeof(umem)))
1297 break;
1298
1299 ret = -EINVAL;
1300 if (!is_mem_zero((const char *)umem.reserved,
1301 sizeof(umem.reserved)))
1302 break;
1303
1304 mutex_lock(&dev->domain_lock);
1305 ret = vduse_dev_reg_umem(dev, umem.iova,
1306 umem.uaddr, umem.size);
1307 mutex_unlock(&dev->domain_lock);
1308 break;
1309 }
1310 case VDUSE_IOTLB_DEREG_UMEM: {
1311 struct vduse_iova_umem umem;
1312
1313 ret = -EFAULT;
1314 if (copy_from_user(&umem, argp, sizeof(umem)))
1315 break;
1316
1317 ret = -EINVAL;
1318 if (!is_mem_zero((const char *)umem.reserved,
1319 sizeof(umem.reserved)))
1320 break;
1321 mutex_lock(&dev->domain_lock);
1322 ret = vduse_dev_dereg_umem(dev, umem.iova,
1323 umem.size);
1324 mutex_unlock(&dev->domain_lock);
1325 break;
1326 }
1327 case VDUSE_IOTLB_GET_INFO: {
1328 struct vduse_iova_info info;
1329 struct vhost_iotlb_map *map;
1330
1331 ret = -EFAULT;
1332 if (copy_from_user(&info, argp, sizeof(info)))
1333 break;
1334
1335 ret = -EINVAL;
1336 if (info.start > info.last)
1337 break;
1338
1339 if (!is_mem_zero((const char *)info.reserved,
1340 sizeof(info.reserved)))
1341 break;
1342
1343 mutex_lock(&dev->domain_lock);
1344 if (!dev->domain) {
1345 mutex_unlock(&dev->domain_lock);
1346 break;
1347 }
1348 spin_lock(&dev->domain->iotlb_lock);
1349 map = vhost_iotlb_itree_first(dev->domain->iotlb,
1350 info.start, info.last);
1351 if (map) {
1352 info.start = map->start;
1353 info.last = map->last;
1354 info.capability = 0;
1355 if (dev->domain->bounce_map && map->start == 0 &&
1356 map->last == dev->domain->bounce_size - 1)
1357 info.capability |= VDUSE_IOVA_CAP_UMEM;
1358 }
1359 spin_unlock(&dev->domain->iotlb_lock);
1360 mutex_unlock(&dev->domain_lock);
1361 if (!map)
1362 break;
1363
1364 ret = -EFAULT;
1365 if (copy_to_user(argp, &info, sizeof(info)))
1366 break;
1367
1368 ret = 0;
1369 break;
1370 }
1371 default:
1372 ret = -ENOIOCTLCMD;
1373 break;
1374 }
1375
1376 return ret;
1377 }
1378
vduse_dev_release(struct inode * inode,struct file * file)1379 static int vduse_dev_release(struct inode *inode, struct file *file)
1380 {
1381 struct vduse_dev *dev = file->private_data;
1382
1383 mutex_lock(&dev->domain_lock);
1384 if (dev->domain)
1385 vduse_dev_dereg_umem(dev, 0, dev->domain->bounce_size);
1386 mutex_unlock(&dev->domain_lock);
1387 spin_lock(&dev->msg_lock);
1388 /* Make sure the inflight messages can processed after reconncection */
1389 list_splice_init(&dev->recv_list, &dev->send_list);
1390 spin_unlock(&dev->msg_lock);
1391 dev->connected = false;
1392
1393 return 0;
1394 }
1395
vduse_dev_get_from_minor(int minor)1396 static struct vduse_dev *vduse_dev_get_from_minor(int minor)
1397 {
1398 struct vduse_dev *dev;
1399
1400 mutex_lock(&vduse_lock);
1401 dev = idr_find(&vduse_idr, minor);
1402 mutex_unlock(&vduse_lock);
1403
1404 return dev;
1405 }
1406
vduse_dev_open(struct inode * inode,struct file * file)1407 static int vduse_dev_open(struct inode *inode, struct file *file)
1408 {
1409 int ret;
1410 struct vduse_dev *dev = vduse_dev_get_from_minor(iminor(inode));
1411
1412 if (!dev)
1413 return -ENODEV;
1414
1415 ret = -EBUSY;
1416 mutex_lock(&dev->lock);
1417 if (dev->connected)
1418 goto unlock;
1419
1420 ret = 0;
1421 dev->connected = true;
1422 file->private_data = dev;
1423 unlock:
1424 mutex_unlock(&dev->lock);
1425
1426 return ret;
1427 }
1428
1429 static const struct file_operations vduse_dev_fops = {
1430 .owner = THIS_MODULE,
1431 .open = vduse_dev_open,
1432 .release = vduse_dev_release,
1433 .read_iter = vduse_dev_read_iter,
1434 .write_iter = vduse_dev_write_iter,
1435 .poll = vduse_dev_poll,
1436 .unlocked_ioctl = vduse_dev_ioctl,
1437 .compat_ioctl = compat_ptr_ioctl,
1438 .llseek = noop_llseek,
1439 };
1440
irq_cb_affinity_show(struct vduse_virtqueue * vq,char * buf)1441 static ssize_t irq_cb_affinity_show(struct vduse_virtqueue *vq, char *buf)
1442 {
1443 return sprintf(buf, "%*pb\n", cpumask_pr_args(&vq->irq_affinity));
1444 }
1445
irq_cb_affinity_store(struct vduse_virtqueue * vq,const char * buf,size_t count)1446 static ssize_t irq_cb_affinity_store(struct vduse_virtqueue *vq,
1447 const char *buf, size_t count)
1448 {
1449 cpumask_var_t new_value;
1450 int ret;
1451
1452 if (!zalloc_cpumask_var(&new_value, GFP_KERNEL))
1453 return -ENOMEM;
1454
1455 ret = cpumask_parse(buf, new_value);
1456 if (ret)
1457 goto free_mask;
1458
1459 ret = -EINVAL;
1460 if (!cpumask_intersects(new_value, cpu_online_mask))
1461 goto free_mask;
1462
1463 cpumask_copy(&vq->irq_affinity, new_value);
1464 ret = count;
1465 free_mask:
1466 free_cpumask_var(new_value);
1467 return ret;
1468 }
1469
1470 struct vq_sysfs_entry {
1471 struct attribute attr;
1472 ssize_t (*show)(struct vduse_virtqueue *vq, char *buf);
1473 ssize_t (*store)(struct vduse_virtqueue *vq, const char *buf,
1474 size_t count);
1475 };
1476
1477 static struct vq_sysfs_entry irq_cb_affinity_attr = __ATTR_RW(irq_cb_affinity);
1478
1479 static struct attribute *vq_attrs[] = {
1480 &irq_cb_affinity_attr.attr,
1481 NULL,
1482 };
1483 ATTRIBUTE_GROUPS(vq);
1484
vq_attr_show(struct kobject * kobj,struct attribute * attr,char * buf)1485 static ssize_t vq_attr_show(struct kobject *kobj, struct attribute *attr,
1486 char *buf)
1487 {
1488 struct vduse_virtqueue *vq = container_of(kobj,
1489 struct vduse_virtqueue, kobj);
1490 struct vq_sysfs_entry *entry = container_of(attr,
1491 struct vq_sysfs_entry, attr);
1492
1493 if (!entry->show)
1494 return -EIO;
1495
1496 return entry->show(vq, buf);
1497 }
1498
vq_attr_store(struct kobject * kobj,struct attribute * attr,const char * buf,size_t count)1499 static ssize_t vq_attr_store(struct kobject *kobj, struct attribute *attr,
1500 const char *buf, size_t count)
1501 {
1502 struct vduse_virtqueue *vq = container_of(kobj,
1503 struct vduse_virtqueue, kobj);
1504 struct vq_sysfs_entry *entry = container_of(attr,
1505 struct vq_sysfs_entry, attr);
1506
1507 if (!entry->store)
1508 return -EIO;
1509
1510 return entry->store(vq, buf, count);
1511 }
1512
1513 static const struct sysfs_ops vq_sysfs_ops = {
1514 .show = vq_attr_show,
1515 .store = vq_attr_store,
1516 };
1517
vq_release(struct kobject * kobj)1518 static void vq_release(struct kobject *kobj)
1519 {
1520 struct vduse_virtqueue *vq = container_of(kobj,
1521 struct vduse_virtqueue, kobj);
1522 kfree(vq);
1523 }
1524
1525 static const struct kobj_type vq_type = {
1526 .release = vq_release,
1527 .sysfs_ops = &vq_sysfs_ops,
1528 .default_groups = vq_groups,
1529 };
1530
vduse_dev_deinit_vqs(struct vduse_dev * dev)1531 static void vduse_dev_deinit_vqs(struct vduse_dev *dev)
1532 {
1533 int i;
1534
1535 if (!dev->vqs)
1536 return;
1537
1538 for (i = 0; i < dev->vq_num; i++)
1539 kobject_put(&dev->vqs[i]->kobj);
1540 kfree(dev->vqs);
1541 }
1542
vduse_dev_init_vqs(struct vduse_dev * dev,u32 vq_align,u32 vq_num)1543 static int vduse_dev_init_vqs(struct vduse_dev *dev, u32 vq_align, u32 vq_num)
1544 {
1545 int ret, i;
1546
1547 dev->vq_align = vq_align;
1548 dev->vq_num = vq_num;
1549 dev->vqs = kcalloc(dev->vq_num, sizeof(*dev->vqs), GFP_KERNEL);
1550 if (!dev->vqs)
1551 return -ENOMEM;
1552
1553 for (i = 0; i < vq_num; i++) {
1554 dev->vqs[i] = kzalloc(sizeof(*dev->vqs[i]), GFP_KERNEL);
1555 if (!dev->vqs[i]) {
1556 ret = -ENOMEM;
1557 goto err;
1558 }
1559
1560 dev->vqs[i]->index = i;
1561 dev->vqs[i]->irq_effective_cpu = IRQ_UNBOUND;
1562 INIT_WORK(&dev->vqs[i]->inject, vduse_vq_irq_inject);
1563 INIT_WORK(&dev->vqs[i]->kick, vduse_vq_kick_work);
1564 spin_lock_init(&dev->vqs[i]->kick_lock);
1565 spin_lock_init(&dev->vqs[i]->irq_lock);
1566 cpumask_setall(&dev->vqs[i]->irq_affinity);
1567
1568 kobject_init(&dev->vqs[i]->kobj, &vq_type);
1569 ret = kobject_add(&dev->vqs[i]->kobj,
1570 &dev->dev->kobj, "vq%d", i);
1571 if (ret) {
1572 kfree(dev->vqs[i]);
1573 goto err;
1574 }
1575 }
1576
1577 return 0;
1578 err:
1579 while (i--)
1580 kobject_put(&dev->vqs[i]->kobj);
1581 kfree(dev->vqs);
1582 dev->vqs = NULL;
1583 return ret;
1584 }
1585
vduse_dev_create(void)1586 static struct vduse_dev *vduse_dev_create(void)
1587 {
1588 struct vduse_dev *dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1589
1590 if (!dev)
1591 return NULL;
1592
1593 mutex_init(&dev->lock);
1594 mutex_init(&dev->mem_lock);
1595 mutex_init(&dev->domain_lock);
1596 spin_lock_init(&dev->msg_lock);
1597 INIT_LIST_HEAD(&dev->send_list);
1598 INIT_LIST_HEAD(&dev->recv_list);
1599 spin_lock_init(&dev->irq_lock);
1600 init_rwsem(&dev->rwsem);
1601
1602 INIT_WORK(&dev->inject, vduse_dev_irq_inject);
1603 init_waitqueue_head(&dev->waitq);
1604
1605 return dev;
1606 }
1607
vduse_dev_destroy(struct vduse_dev * dev)1608 static void vduse_dev_destroy(struct vduse_dev *dev)
1609 {
1610 kfree(dev);
1611 }
1612
vduse_find_dev(const char * name)1613 static struct vduse_dev *vduse_find_dev(const char *name)
1614 {
1615 struct vduse_dev *dev;
1616 int id;
1617
1618 idr_for_each_entry(&vduse_idr, dev, id)
1619 if (!strcmp(dev->name, name))
1620 return dev;
1621
1622 return NULL;
1623 }
1624
vduse_destroy_dev(char * name)1625 static int vduse_destroy_dev(char *name)
1626 {
1627 struct vduse_dev *dev = vduse_find_dev(name);
1628
1629 if (!dev)
1630 return -EINVAL;
1631
1632 mutex_lock(&dev->lock);
1633 if (dev->vdev || dev->connected) {
1634 mutex_unlock(&dev->lock);
1635 return -EBUSY;
1636 }
1637 dev->connected = true;
1638 mutex_unlock(&dev->lock);
1639
1640 vduse_dev_reset(dev);
1641 device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));
1642 idr_remove(&vduse_idr, dev->minor);
1643 kvfree(dev->config);
1644 vduse_dev_deinit_vqs(dev);
1645 if (dev->domain)
1646 vduse_domain_destroy(dev->domain);
1647 kfree(dev->name);
1648 vduse_dev_destroy(dev);
1649 module_put(THIS_MODULE);
1650
1651 return 0;
1652 }
1653
device_is_allowed(u32 device_id)1654 static bool device_is_allowed(u32 device_id)
1655 {
1656 int i;
1657
1658 for (i = 0; i < ARRAY_SIZE(allowed_device_id); i++)
1659 if (allowed_device_id[i] == device_id)
1660 return true;
1661
1662 return false;
1663 }
1664
features_is_valid(u64 features)1665 static bool features_is_valid(u64 features)
1666 {
1667 if (!(features & (1ULL << VIRTIO_F_ACCESS_PLATFORM)))
1668 return false;
1669
1670 /* Now we only support read-only configuration space */
1671 if (features & (1ULL << VIRTIO_BLK_F_CONFIG_WCE))
1672 return false;
1673
1674 return true;
1675 }
1676
vduse_validate_config(struct vduse_dev_config * config)1677 static bool vduse_validate_config(struct vduse_dev_config *config)
1678 {
1679 if (!is_mem_zero((const char *)config->reserved,
1680 sizeof(config->reserved)))
1681 return false;
1682
1683 if (config->vq_align > PAGE_SIZE)
1684 return false;
1685
1686 if (config->config_size > PAGE_SIZE)
1687 return false;
1688
1689 if (config->vq_num > 0xffff)
1690 return false;
1691
1692 if (!config->name[0])
1693 return false;
1694
1695 if (!device_is_allowed(config->device_id))
1696 return false;
1697
1698 if (!features_is_valid(config->features))
1699 return false;
1700
1701 return true;
1702 }
1703
msg_timeout_show(struct device * device,struct device_attribute * attr,char * buf)1704 static ssize_t msg_timeout_show(struct device *device,
1705 struct device_attribute *attr, char *buf)
1706 {
1707 struct vduse_dev *dev = dev_get_drvdata(device);
1708
1709 return sysfs_emit(buf, "%u\n", dev->msg_timeout);
1710 }
1711
msg_timeout_store(struct device * device,struct device_attribute * attr,const char * buf,size_t count)1712 static ssize_t msg_timeout_store(struct device *device,
1713 struct device_attribute *attr,
1714 const char *buf, size_t count)
1715 {
1716 struct vduse_dev *dev = dev_get_drvdata(device);
1717 int ret;
1718
1719 ret = kstrtouint(buf, 10, &dev->msg_timeout);
1720 if (ret < 0)
1721 return ret;
1722
1723 return count;
1724 }
1725
1726 static DEVICE_ATTR_RW(msg_timeout);
1727
bounce_size_show(struct device * device,struct device_attribute * attr,char * buf)1728 static ssize_t bounce_size_show(struct device *device,
1729 struct device_attribute *attr, char *buf)
1730 {
1731 struct vduse_dev *dev = dev_get_drvdata(device);
1732
1733 return sysfs_emit(buf, "%u\n", dev->bounce_size);
1734 }
1735
bounce_size_store(struct device * device,struct device_attribute * attr,const char * buf,size_t count)1736 static ssize_t bounce_size_store(struct device *device,
1737 struct device_attribute *attr,
1738 const char *buf, size_t count)
1739 {
1740 struct vduse_dev *dev = dev_get_drvdata(device);
1741 unsigned int bounce_size;
1742 int ret;
1743
1744 ret = -EPERM;
1745 mutex_lock(&dev->domain_lock);
1746 if (dev->domain)
1747 goto unlock;
1748
1749 ret = kstrtouint(buf, 10, &bounce_size);
1750 if (ret < 0)
1751 goto unlock;
1752
1753 ret = -EINVAL;
1754 if (bounce_size > VDUSE_MAX_BOUNCE_SIZE ||
1755 bounce_size < VDUSE_MIN_BOUNCE_SIZE)
1756 goto unlock;
1757
1758 dev->bounce_size = bounce_size & PAGE_MASK;
1759 ret = count;
1760 unlock:
1761 mutex_unlock(&dev->domain_lock);
1762 return ret;
1763 }
1764
1765 static DEVICE_ATTR_RW(bounce_size);
1766
1767 static struct attribute *vduse_dev_attrs[] = {
1768 &dev_attr_msg_timeout.attr,
1769 &dev_attr_bounce_size.attr,
1770 NULL
1771 };
1772
1773 ATTRIBUTE_GROUPS(vduse_dev);
1774
vduse_create_dev(struct vduse_dev_config * config,void * config_buf,u64 api_version)1775 static int vduse_create_dev(struct vduse_dev_config *config,
1776 void *config_buf, u64 api_version)
1777 {
1778 int ret;
1779 struct vduse_dev *dev;
1780
1781 ret = -EEXIST;
1782 if (vduse_find_dev(config->name))
1783 goto err;
1784
1785 ret = -ENOMEM;
1786 dev = vduse_dev_create();
1787 if (!dev)
1788 goto err;
1789
1790 dev->api_version = api_version;
1791 dev->device_features = config->features;
1792 dev->device_id = config->device_id;
1793 dev->vendor_id = config->vendor_id;
1794 dev->name = kstrdup(config->name, GFP_KERNEL);
1795 if (!dev->name)
1796 goto err_str;
1797
1798 dev->bounce_size = VDUSE_BOUNCE_SIZE;
1799 dev->config = config_buf;
1800 dev->config_size = config->config_size;
1801
1802 ret = idr_alloc(&vduse_idr, dev, 1, VDUSE_DEV_MAX, GFP_KERNEL);
1803 if (ret < 0)
1804 goto err_idr;
1805
1806 dev->minor = ret;
1807 dev->msg_timeout = VDUSE_MSG_DEFAULT_TIMEOUT;
1808 dev->dev = device_create_with_groups(vduse_class, NULL,
1809 MKDEV(MAJOR(vduse_major), dev->minor),
1810 dev, vduse_dev_groups, "%s", config->name);
1811 if (IS_ERR(dev->dev)) {
1812 ret = PTR_ERR(dev->dev);
1813 goto err_dev;
1814 }
1815
1816 ret = vduse_dev_init_vqs(dev, config->vq_align, config->vq_num);
1817 if (ret)
1818 goto err_vqs;
1819
1820 __module_get(THIS_MODULE);
1821
1822 return 0;
1823 err_vqs:
1824 device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));
1825 err_dev:
1826 idr_remove(&vduse_idr, dev->minor);
1827 err_idr:
1828 kfree(dev->name);
1829 err_str:
1830 vduse_dev_destroy(dev);
1831 err:
1832 return ret;
1833 }
1834
vduse_ioctl(struct file * file,unsigned int cmd,unsigned long arg)1835 static long vduse_ioctl(struct file *file, unsigned int cmd,
1836 unsigned long arg)
1837 {
1838 int ret;
1839 void __user *argp = (void __user *)arg;
1840 struct vduse_control *control = file->private_data;
1841
1842 mutex_lock(&vduse_lock);
1843 switch (cmd) {
1844 case VDUSE_GET_API_VERSION:
1845 ret = put_user(control->api_version, (u64 __user *)argp);
1846 break;
1847 case VDUSE_SET_API_VERSION: {
1848 u64 api_version;
1849
1850 ret = -EFAULT;
1851 if (get_user(api_version, (u64 __user *)argp))
1852 break;
1853
1854 ret = -EINVAL;
1855 if (api_version > VDUSE_API_VERSION)
1856 break;
1857
1858 ret = 0;
1859 control->api_version = api_version;
1860 break;
1861 }
1862 case VDUSE_CREATE_DEV: {
1863 struct vduse_dev_config config;
1864 unsigned long size = offsetof(struct vduse_dev_config, config);
1865 void *buf;
1866
1867 ret = -EFAULT;
1868 if (copy_from_user(&config, argp, size))
1869 break;
1870
1871 ret = -EINVAL;
1872 if (vduse_validate_config(&config) == false)
1873 break;
1874
1875 buf = vmemdup_user(argp + size, config.config_size);
1876 if (IS_ERR(buf)) {
1877 ret = PTR_ERR(buf);
1878 break;
1879 }
1880 config.name[VDUSE_NAME_MAX - 1] = '\0';
1881 ret = vduse_create_dev(&config, buf, control->api_version);
1882 if (ret)
1883 kvfree(buf);
1884 break;
1885 }
1886 case VDUSE_DESTROY_DEV: {
1887 char name[VDUSE_NAME_MAX];
1888
1889 ret = -EFAULT;
1890 if (copy_from_user(name, argp, VDUSE_NAME_MAX))
1891 break;
1892
1893 name[VDUSE_NAME_MAX - 1] = '\0';
1894 ret = vduse_destroy_dev(name);
1895 break;
1896 }
1897 default:
1898 ret = -EINVAL;
1899 break;
1900 }
1901 mutex_unlock(&vduse_lock);
1902
1903 return ret;
1904 }
1905
vduse_release(struct inode * inode,struct file * file)1906 static int vduse_release(struct inode *inode, struct file *file)
1907 {
1908 struct vduse_control *control = file->private_data;
1909
1910 kfree(control);
1911 return 0;
1912 }
1913
vduse_open(struct inode * inode,struct file * file)1914 static int vduse_open(struct inode *inode, struct file *file)
1915 {
1916 struct vduse_control *control;
1917
1918 control = kmalloc(sizeof(struct vduse_control), GFP_KERNEL);
1919 if (!control)
1920 return -ENOMEM;
1921
1922 control->api_version = VDUSE_API_VERSION;
1923 file->private_data = control;
1924
1925 return 0;
1926 }
1927
1928 static const struct file_operations vduse_ctrl_fops = {
1929 .owner = THIS_MODULE,
1930 .open = vduse_open,
1931 .release = vduse_release,
1932 .unlocked_ioctl = vduse_ioctl,
1933 .compat_ioctl = compat_ptr_ioctl,
1934 .llseek = noop_llseek,
1935 };
1936
vduse_devnode(const struct device * dev,umode_t * mode)1937 static char *vduse_devnode(const struct device *dev, umode_t *mode)
1938 {
1939 return kasprintf(GFP_KERNEL, "vduse/%s", dev_name(dev));
1940 }
1941
1942 struct vduse_mgmt_dev {
1943 struct vdpa_mgmt_dev mgmt_dev;
1944 struct device dev;
1945 };
1946
1947 static struct vduse_mgmt_dev *vduse_mgmt;
1948
vduse_dev_init_vdpa(struct vduse_dev * dev,const char * name)1949 static int vduse_dev_init_vdpa(struct vduse_dev *dev, const char *name)
1950 {
1951 struct vduse_vdpa *vdev;
1952 int ret;
1953
1954 if (dev->vdev)
1955 return -EEXIST;
1956
1957 vdev = vdpa_alloc_device(struct vduse_vdpa, vdpa, dev->dev,
1958 &vduse_vdpa_config_ops, 1, 1, name, true);
1959 if (IS_ERR(vdev))
1960 return PTR_ERR(vdev);
1961
1962 dev->vdev = vdev;
1963 vdev->dev = dev;
1964 vdev->vdpa.dev.dma_mask = &vdev->vdpa.dev.coherent_dma_mask;
1965 ret = dma_set_mask_and_coherent(&vdev->vdpa.dev, DMA_BIT_MASK(64));
1966 if (ret) {
1967 put_device(&vdev->vdpa.dev);
1968 return ret;
1969 }
1970 set_dma_ops(&vdev->vdpa.dev, &vduse_dev_dma_ops);
1971 vdev->vdpa.dma_dev = &vdev->vdpa.dev;
1972 vdev->vdpa.mdev = &vduse_mgmt->mgmt_dev;
1973
1974 return 0;
1975 }
1976
vdpa_dev_add(struct vdpa_mgmt_dev * mdev,const char * name,const struct vdpa_dev_set_config * config)1977 static int vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
1978 const struct vdpa_dev_set_config *config)
1979 {
1980 struct vduse_dev *dev;
1981 int ret;
1982
1983 mutex_lock(&vduse_lock);
1984 dev = vduse_find_dev(name);
1985 if (!dev || !vduse_dev_is_ready(dev)) {
1986 mutex_unlock(&vduse_lock);
1987 return -EINVAL;
1988 }
1989 ret = vduse_dev_init_vdpa(dev, name);
1990 mutex_unlock(&vduse_lock);
1991 if (ret)
1992 return ret;
1993
1994 mutex_lock(&dev->domain_lock);
1995 if (!dev->domain)
1996 dev->domain = vduse_domain_create(VDUSE_IOVA_SIZE - 1,
1997 dev->bounce_size);
1998 mutex_unlock(&dev->domain_lock);
1999 if (!dev->domain) {
2000 put_device(&dev->vdev->vdpa.dev);
2001 return -ENOMEM;
2002 }
2003
2004 ret = _vdpa_register_device(&dev->vdev->vdpa, dev->vq_num);
2005 if (ret) {
2006 put_device(&dev->vdev->vdpa.dev);
2007 mutex_lock(&dev->domain_lock);
2008 vduse_domain_destroy(dev->domain);
2009 dev->domain = NULL;
2010 mutex_unlock(&dev->domain_lock);
2011 return ret;
2012 }
2013
2014 return 0;
2015 }
2016
vdpa_dev_del(struct vdpa_mgmt_dev * mdev,struct vdpa_device * dev)2017 static void vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev)
2018 {
2019 _vdpa_unregister_device(dev);
2020 }
2021
2022 static const struct vdpa_mgmtdev_ops vdpa_dev_mgmtdev_ops = {
2023 .dev_add = vdpa_dev_add,
2024 .dev_del = vdpa_dev_del,
2025 };
2026
2027 static struct virtio_device_id id_table[] = {
2028 { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID },
2029 { 0 },
2030 };
2031
vduse_mgmtdev_release(struct device * dev)2032 static void vduse_mgmtdev_release(struct device *dev)
2033 {
2034 struct vduse_mgmt_dev *mgmt_dev;
2035
2036 mgmt_dev = container_of(dev, struct vduse_mgmt_dev, dev);
2037 kfree(mgmt_dev);
2038 }
2039
vduse_mgmtdev_init(void)2040 static int vduse_mgmtdev_init(void)
2041 {
2042 int ret;
2043
2044 vduse_mgmt = kzalloc(sizeof(*vduse_mgmt), GFP_KERNEL);
2045 if (!vduse_mgmt)
2046 return -ENOMEM;
2047
2048 ret = dev_set_name(&vduse_mgmt->dev, "vduse");
2049 if (ret) {
2050 kfree(vduse_mgmt);
2051 return ret;
2052 }
2053
2054 vduse_mgmt->dev.release = vduse_mgmtdev_release;
2055
2056 ret = device_register(&vduse_mgmt->dev);
2057 if (ret)
2058 goto dev_reg_err;
2059
2060 vduse_mgmt->mgmt_dev.id_table = id_table;
2061 vduse_mgmt->mgmt_dev.ops = &vdpa_dev_mgmtdev_ops;
2062 vduse_mgmt->mgmt_dev.device = &vduse_mgmt->dev;
2063 ret = vdpa_mgmtdev_register(&vduse_mgmt->mgmt_dev);
2064 if (ret)
2065 device_unregister(&vduse_mgmt->dev);
2066
2067 return ret;
2068
2069 dev_reg_err:
2070 put_device(&vduse_mgmt->dev);
2071 return ret;
2072 }
2073
vduse_mgmtdev_exit(void)2074 static void vduse_mgmtdev_exit(void)
2075 {
2076 vdpa_mgmtdev_unregister(&vduse_mgmt->mgmt_dev);
2077 device_unregister(&vduse_mgmt->dev);
2078 }
2079
vduse_init(void)2080 static int vduse_init(void)
2081 {
2082 int ret;
2083 struct device *dev;
2084
2085 vduse_class = class_create("vduse");
2086 if (IS_ERR(vduse_class))
2087 return PTR_ERR(vduse_class);
2088
2089 vduse_class->devnode = vduse_devnode;
2090
2091 ret = alloc_chrdev_region(&vduse_major, 0, VDUSE_DEV_MAX, "vduse");
2092 if (ret)
2093 goto err_chardev_region;
2094
2095 /* /dev/vduse/control */
2096 cdev_init(&vduse_ctrl_cdev, &vduse_ctrl_fops);
2097 vduse_ctrl_cdev.owner = THIS_MODULE;
2098 ret = cdev_add(&vduse_ctrl_cdev, vduse_major, 1);
2099 if (ret)
2100 goto err_ctrl_cdev;
2101
2102 dev = device_create(vduse_class, NULL, vduse_major, NULL, "control");
2103 if (IS_ERR(dev)) {
2104 ret = PTR_ERR(dev);
2105 goto err_device;
2106 }
2107
2108 /* /dev/vduse/$DEVICE */
2109 cdev_init(&vduse_cdev, &vduse_dev_fops);
2110 vduse_cdev.owner = THIS_MODULE;
2111 ret = cdev_add(&vduse_cdev, MKDEV(MAJOR(vduse_major), 1),
2112 VDUSE_DEV_MAX - 1);
2113 if (ret)
2114 goto err_cdev;
2115
2116 ret = -ENOMEM;
2117 vduse_irq_wq = alloc_workqueue("vduse-irq",
2118 WQ_HIGHPRI | WQ_SYSFS | WQ_UNBOUND, 0);
2119 if (!vduse_irq_wq)
2120 goto err_wq;
2121
2122 vduse_irq_bound_wq = alloc_workqueue("vduse-irq-bound", WQ_HIGHPRI, 0);
2123 if (!vduse_irq_bound_wq)
2124 goto err_bound_wq;
2125
2126 ret = vduse_domain_init();
2127 if (ret)
2128 goto err_domain;
2129
2130 ret = vduse_mgmtdev_init();
2131 if (ret)
2132 goto err_mgmtdev;
2133
2134 return 0;
2135 err_mgmtdev:
2136 vduse_domain_exit();
2137 err_domain:
2138 destroy_workqueue(vduse_irq_bound_wq);
2139 err_bound_wq:
2140 destroy_workqueue(vduse_irq_wq);
2141 err_wq:
2142 cdev_del(&vduse_cdev);
2143 err_cdev:
2144 device_destroy(vduse_class, vduse_major);
2145 err_device:
2146 cdev_del(&vduse_ctrl_cdev);
2147 err_ctrl_cdev:
2148 unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);
2149 err_chardev_region:
2150 class_destroy(vduse_class);
2151 return ret;
2152 }
2153 module_init(vduse_init);
2154
vduse_exit(void)2155 static void vduse_exit(void)
2156 {
2157 vduse_mgmtdev_exit();
2158 vduse_domain_exit();
2159 destroy_workqueue(vduse_irq_bound_wq);
2160 destroy_workqueue(vduse_irq_wq);
2161 cdev_del(&vduse_cdev);
2162 device_destroy(vduse_class, vduse_major);
2163 cdev_del(&vduse_ctrl_cdev);
2164 unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);
2165 class_destroy(vduse_class);
2166 }
2167 module_exit(vduse_exit);
2168
2169 MODULE_LICENSE(DRV_LICENSE);
2170 MODULE_AUTHOR(DRV_AUTHOR);
2171 MODULE_DESCRIPTION(DRV_DESC);
2172