Lines Matching +full:- +full:v
1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2018-2020 Intel Corporation.
68 static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
76 return as->id; in iotlb_to_asid()
79 static struct vhost_vdpa_as *asid_to_as(struct vhost_vdpa *v, u32 asid) in asid_to_as() argument
81 struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS]; in asid_to_as()
85 if (as->id == asid) in asid_to_as()
91 static struct vhost_iotlb *asid_to_iotlb(struct vhost_vdpa *v, u32 asid) in asid_to_iotlb() argument
93 struct vhost_vdpa_as *as = asid_to_as(v, asid); in asid_to_iotlb()
98 return &as->iotlb; in asid_to_iotlb()
101 static struct vhost_vdpa_as *vhost_vdpa_alloc_as(struct vhost_vdpa *v, u32 asid) in vhost_vdpa_alloc_as() argument
103 struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS]; in vhost_vdpa_alloc_as()
106 if (asid_to_as(v, asid)) in vhost_vdpa_alloc_as()
109 if (asid >= v->vdpa->nas) in vhost_vdpa_alloc_as()
116 vhost_iotlb_init(&as->iotlb, 0, 0); in vhost_vdpa_alloc_as()
117 as->id = asid; in vhost_vdpa_alloc_as()
118 hlist_add_head(&as->hash_link, head); in vhost_vdpa_alloc_as()
123 static struct vhost_vdpa_as *vhost_vdpa_find_alloc_as(struct vhost_vdpa *v, in vhost_vdpa_find_alloc_as() argument
126 struct vhost_vdpa_as *as = asid_to_as(v, asid); in vhost_vdpa_find_alloc_as()
131 return vhost_vdpa_alloc_as(v, asid); in vhost_vdpa_find_alloc_as()
134 static int vhost_vdpa_remove_as(struct vhost_vdpa *v, u32 asid) in vhost_vdpa_remove_as() argument
136 struct vhost_vdpa_as *as = asid_to_as(v, asid); in vhost_vdpa_remove_as()
139 return -EINVAL; in vhost_vdpa_remove_as()
141 hlist_del(&as->hash_link); in vhost_vdpa_remove_as()
142 vhost_vdpa_iotlb_unmap(v, &as->iotlb, 0ULL, 0ULL - 1, asid); in vhost_vdpa_remove_as()
152 struct vhost_vdpa *v = container_of(vq->dev, struct vhost_vdpa, vdev); in handle_vq_kick() local
153 const struct vdpa_config_ops *ops = v->vdpa->config; in handle_vq_kick()
155 ops->kick_vq(v->vdpa, vq - v->vqs); in handle_vq_kick()
161 struct eventfd_ctx *call_ctx = vq->call_ctx.ctx; in vhost_vdpa_virtqueue_cb()
171 struct vhost_vdpa *v = private; in vhost_vdpa_config_cb() local
172 struct eventfd_ctx *config_ctx = v->config_ctx; in vhost_vdpa_config_cb()
180 static void vhost_vdpa_setup_vq_irq(struct vhost_vdpa *v, u16 qid) in vhost_vdpa_setup_vq_irq() argument
182 struct vhost_virtqueue *vq = &v->vqs[qid]; in vhost_vdpa_setup_vq_irq()
183 const struct vdpa_config_ops *ops = v->vdpa->config; in vhost_vdpa_setup_vq_irq()
184 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_setup_vq_irq()
187 if (!ops->get_vq_irq) in vhost_vdpa_setup_vq_irq()
190 irq = ops->get_vq_irq(vdpa, qid); in vhost_vdpa_setup_vq_irq()
194 if (!vq->call_ctx.ctx) in vhost_vdpa_setup_vq_irq()
197 vq->call_ctx.producer.irq = irq; in vhost_vdpa_setup_vq_irq()
198 ret = irq_bypass_register_producer(&vq->call_ctx.producer); in vhost_vdpa_setup_vq_irq()
200 dev_info(&v->dev, "vq %u, irq bypass producer (token %p) registration fails, ret = %d\n", in vhost_vdpa_setup_vq_irq()
201 qid, vq->call_ctx.producer.token, ret); in vhost_vdpa_setup_vq_irq()
204 static void vhost_vdpa_unsetup_vq_irq(struct vhost_vdpa *v, u16 qid) in vhost_vdpa_unsetup_vq_irq() argument
206 struct vhost_virtqueue *vq = &v->vqs[qid]; in vhost_vdpa_unsetup_vq_irq()
208 irq_bypass_unregister_producer(&vq->call_ctx.producer); in vhost_vdpa_unsetup_vq_irq()
211 static int vhost_vdpa_reset(struct vhost_vdpa *v) in vhost_vdpa_reset() argument
213 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_reset()
215 v->in_batch = 0; in vhost_vdpa_reset()
220 static long vhost_vdpa_bind_mm(struct vhost_vdpa *v) in vhost_vdpa_bind_mm() argument
222 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_bind_mm()
223 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_bind_mm()
225 if (!vdpa->use_va || !ops->bind_mm) in vhost_vdpa_bind_mm()
228 return ops->bind_mm(vdpa, v->vdev.mm); in vhost_vdpa_bind_mm()
231 static void vhost_vdpa_unbind_mm(struct vhost_vdpa *v) in vhost_vdpa_unbind_mm() argument
233 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_unbind_mm()
234 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_unbind_mm()
236 if (!vdpa->use_va || !ops->unbind_mm) in vhost_vdpa_unbind_mm()
239 ops->unbind_mm(vdpa); in vhost_vdpa_unbind_mm()
242 static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp) in vhost_vdpa_get_device_id() argument
244 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_device_id()
245 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_get_device_id()
248 device_id = ops->get_device_id(vdpa); in vhost_vdpa_get_device_id()
251 return -EFAULT; in vhost_vdpa_get_device_id()
256 static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp) in vhost_vdpa_get_status() argument
258 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_status()
259 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_get_status()
262 status = ops->get_status(vdpa); in vhost_vdpa_get_status()
265 return -EFAULT; in vhost_vdpa_get_status()
270 static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp) in vhost_vdpa_set_status() argument
272 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_set_status()
273 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_set_status()
275 u32 nvqs = v->nvqs; in vhost_vdpa_set_status()
280 return -EFAULT; in vhost_vdpa_set_status()
282 status_old = ops->get_status(vdpa); in vhost_vdpa_set_status()
289 return -EINVAL; in vhost_vdpa_set_status()
293 vhost_vdpa_unsetup_vq_irq(v, i); in vhost_vdpa_set_status()
304 vhost_vdpa_setup_vq_irq(v, i); in vhost_vdpa_set_status()
309 static int vhost_vdpa_config_validate(struct vhost_vdpa *v, in vhost_vdpa_config_validate() argument
312 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_config_validate()
313 size_t size = vdpa->config->get_config_size(vdpa); in vhost_vdpa_config_validate()
315 if (c->len == 0 || c->off > size) in vhost_vdpa_config_validate()
316 return -EINVAL; in vhost_vdpa_config_validate()
318 if (c->len > size - c->off) in vhost_vdpa_config_validate()
319 return -E2BIG; in vhost_vdpa_config_validate()
324 static long vhost_vdpa_get_config(struct vhost_vdpa *v, in vhost_vdpa_get_config() argument
327 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_config()
333 return -EFAULT; in vhost_vdpa_get_config()
334 if (vhost_vdpa_config_validate(v, &config)) in vhost_vdpa_get_config()
335 return -EINVAL; in vhost_vdpa_get_config()
338 return -ENOMEM; in vhost_vdpa_get_config()
342 if (copy_to_user(c->buf, buf, config.len)) { in vhost_vdpa_get_config()
344 return -EFAULT; in vhost_vdpa_get_config()
351 static long vhost_vdpa_set_config(struct vhost_vdpa *v, in vhost_vdpa_set_config() argument
354 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_set_config()
360 return -EFAULT; in vhost_vdpa_set_config()
361 if (vhost_vdpa_config_validate(v, &config)) in vhost_vdpa_set_config()
362 return -EINVAL; in vhost_vdpa_set_config()
364 buf = vmemdup_user(c->buf, config.len); in vhost_vdpa_set_config()
374 static bool vhost_vdpa_can_suspend(const struct vhost_vdpa *v) in vhost_vdpa_can_suspend() argument
376 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_can_suspend()
377 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_can_suspend()
379 return ops->suspend; in vhost_vdpa_can_suspend()
382 static bool vhost_vdpa_can_resume(const struct vhost_vdpa *v) in vhost_vdpa_can_resume() argument
384 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_can_resume()
385 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_can_resume()
387 return ops->resume; in vhost_vdpa_can_resume()
390 static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep) in vhost_vdpa_get_features() argument
392 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_features()
393 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_get_features()
396 features = ops->get_device_features(vdpa); in vhost_vdpa_get_features()
399 return -EFAULT; in vhost_vdpa_get_features()
404 static u64 vhost_vdpa_get_backend_features(const struct vhost_vdpa *v) in vhost_vdpa_get_backend_features() argument
406 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_backend_features()
407 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_get_backend_features()
409 if (!ops->get_backend_features) in vhost_vdpa_get_backend_features()
412 return ops->get_backend_features(vdpa); in vhost_vdpa_get_backend_features()
415 static long vhost_vdpa_set_features(struct vhost_vdpa *v, u64 __user *featurep) in vhost_vdpa_set_features() argument
417 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_set_features()
418 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_set_features()
419 struct vhost_dev *d = &v->vdev; in vhost_vdpa_set_features()
428 if (ops->get_status(vdpa) & VIRTIO_CONFIG_S_FEATURES_OK) in vhost_vdpa_set_features()
429 return -EBUSY; in vhost_vdpa_set_features()
432 return -EFAULT; in vhost_vdpa_set_features()
435 return -EINVAL; in vhost_vdpa_set_features()
438 actual_features = ops->get_driver_features(vdpa); in vhost_vdpa_set_features()
439 for (i = 0; i < d->nvqs; ++i) { in vhost_vdpa_set_features()
440 struct vhost_virtqueue *vq = d->vqs[i]; in vhost_vdpa_set_features()
442 mutex_lock(&vq->mutex); in vhost_vdpa_set_features()
443 vq->acked_features = actual_features; in vhost_vdpa_set_features()
444 mutex_unlock(&vq->mutex); in vhost_vdpa_set_features()
450 static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp) in vhost_vdpa_get_vring_num() argument
452 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_vring_num()
453 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_get_vring_num()
456 num = ops->get_vq_num_max(vdpa); in vhost_vdpa_get_vring_num()
459 return -EFAULT; in vhost_vdpa_get_vring_num()
464 static void vhost_vdpa_config_put(struct vhost_vdpa *v) in vhost_vdpa_config_put() argument
466 if (v->config_ctx) { in vhost_vdpa_config_put()
467 eventfd_ctx_put(v->config_ctx); in vhost_vdpa_config_put()
468 v->config_ctx = NULL; in vhost_vdpa_config_put()
472 static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp) in vhost_vdpa_set_config_call() argument
479 cb.private = v; in vhost_vdpa_set_config_call()
481 return -EFAULT; in vhost_vdpa_set_config_call()
484 swap(ctx, v->config_ctx); in vhost_vdpa_set_config_call()
489 if (IS_ERR(v->config_ctx)) { in vhost_vdpa_set_config_call()
490 long ret = PTR_ERR(v->config_ctx); in vhost_vdpa_set_config_call()
492 v->config_ctx = NULL; in vhost_vdpa_set_config_call()
496 v->vdpa->config->set_config_cb(v->vdpa, &cb); in vhost_vdpa_set_config_call()
501 static long vhost_vdpa_get_iova_range(struct vhost_vdpa *v, u32 __user *argp) in vhost_vdpa_get_iova_range() argument
504 .first = v->range.first, in vhost_vdpa_get_iova_range()
505 .last = v->range.last, in vhost_vdpa_get_iova_range()
509 return -EFAULT; in vhost_vdpa_get_iova_range()
513 static long vhost_vdpa_get_config_size(struct vhost_vdpa *v, u32 __user *argp) in vhost_vdpa_get_config_size() argument
515 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_config_size()
516 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_get_config_size()
519 size = ops->get_config_size(vdpa); in vhost_vdpa_get_config_size()
522 return -EFAULT; in vhost_vdpa_get_config_size()
527 static long vhost_vdpa_get_vqs_count(struct vhost_vdpa *v, u32 __user *argp) in vhost_vdpa_get_vqs_count() argument
529 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_get_vqs_count()
531 if (copy_to_user(argp, &vdpa->nvqs, sizeof(vdpa->nvqs))) in vhost_vdpa_get_vqs_count()
532 return -EFAULT; in vhost_vdpa_get_vqs_count()
542 static long vhost_vdpa_suspend(struct vhost_vdpa *v) in vhost_vdpa_suspend() argument
544 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_suspend()
545 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_suspend()
547 if (!ops->suspend) in vhost_vdpa_suspend()
548 return -EOPNOTSUPP; in vhost_vdpa_suspend()
550 return ops->suspend(vdpa); in vhost_vdpa_suspend()
557 static long vhost_vdpa_resume(struct vhost_vdpa *v) in vhost_vdpa_resume() argument
559 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_resume()
560 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_resume()
562 if (!ops->resume) in vhost_vdpa_resume()
563 return -EOPNOTSUPP; in vhost_vdpa_resume()
565 return ops->resume(vdpa); in vhost_vdpa_resume()
568 static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, in vhost_vdpa_vring_ioctl() argument
571 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_vring_ioctl()
572 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_vring_ioctl()
584 if (idx >= v->nvqs) in vhost_vdpa_vring_ioctl()
585 return -ENOBUFS; in vhost_vdpa_vring_ioctl()
587 idx = array_index_nospec(idx, v->nvqs); in vhost_vdpa_vring_ioctl()
588 vq = &v->vqs[idx]; in vhost_vdpa_vring_ioctl()
593 return -EFAULT; in vhost_vdpa_vring_ioctl()
594 ops->set_vq_ready(vdpa, idx, s.num); in vhost_vdpa_vring_ioctl()
597 if (!ops->get_vq_group) in vhost_vdpa_vring_ioctl()
598 return -EOPNOTSUPP; in vhost_vdpa_vring_ioctl()
600 s.num = ops->get_vq_group(vdpa, idx); in vhost_vdpa_vring_ioctl()
601 if (s.num >= vdpa->ngroups) in vhost_vdpa_vring_ioctl()
602 return -EIO; in vhost_vdpa_vring_ioctl()
604 return -EFAULT; in vhost_vdpa_vring_ioctl()
608 return -EFAULT; in vhost_vdpa_vring_ioctl()
609 if (s.num >= vdpa->nas) in vhost_vdpa_vring_ioctl()
610 return -EINVAL; in vhost_vdpa_vring_ioctl()
611 if (!ops->set_group_asid) in vhost_vdpa_vring_ioctl()
612 return -EOPNOTSUPP; in vhost_vdpa_vring_ioctl()
613 return ops->set_group_asid(vdpa, idx, s.num); in vhost_vdpa_vring_ioctl()
615 r = ops->get_vq_state(v->vdpa, idx, &vq_state); in vhost_vdpa_vring_ioctl()
620 vq->last_avail_idx = vq_state.packed.last_avail_idx | in vhost_vdpa_vring_ioctl()
622 vq->last_used_idx = vq_state.packed.last_used_idx | in vhost_vdpa_vring_ioctl()
625 vq->last_avail_idx = vq_state.split.avail_index; in vhost_vdpa_vring_ioctl()
629 if (vq->call_ctx.ctx) { in vhost_vdpa_vring_ioctl()
630 if (ops->get_status(vdpa) & in vhost_vdpa_vring_ioctl()
632 vhost_vdpa_unsetup_vq_irq(v, idx); in vhost_vdpa_vring_ioctl()
633 vq->call_ctx.producer.token = NULL; in vhost_vdpa_vring_ioctl()
638 r = vhost_vring_ioctl(&v->vdev, cmd, argp); in vhost_vdpa_vring_ioctl()
644 if (ops->set_vq_address(vdpa, idx, in vhost_vdpa_vring_ioctl()
645 (u64)(uintptr_t)vq->desc, in vhost_vdpa_vring_ioctl()
646 (u64)(uintptr_t)vq->avail, in vhost_vdpa_vring_ioctl()
647 (u64)(uintptr_t)vq->used)) in vhost_vdpa_vring_ioctl()
648 r = -EINVAL; in vhost_vdpa_vring_ioctl()
653 vq_state.packed.last_avail_idx = vq->last_avail_idx & 0x7fff; in vhost_vdpa_vring_ioctl()
654 vq_state.packed.last_avail_counter = !!(vq->last_avail_idx & 0x8000); in vhost_vdpa_vring_ioctl()
655 vq_state.packed.last_used_idx = vq->last_used_idx & 0x7fff; in vhost_vdpa_vring_ioctl()
656 vq_state.packed.last_used_counter = !!(vq->last_used_idx & 0x8000); in vhost_vdpa_vring_ioctl()
658 vq_state.split.avail_index = vq->last_avail_idx; in vhost_vdpa_vring_ioctl()
660 r = ops->set_vq_state(vdpa, idx, &vq_state); in vhost_vdpa_vring_ioctl()
664 if (vq->call_ctx.ctx) { in vhost_vdpa_vring_ioctl()
667 cb.trigger = vq->call_ctx.ctx; in vhost_vdpa_vring_ioctl()
668 vq->call_ctx.producer.token = vq->call_ctx.ctx; in vhost_vdpa_vring_ioctl()
669 if (ops->get_status(vdpa) & in vhost_vdpa_vring_ioctl()
671 vhost_vdpa_setup_vq_irq(v, idx); in vhost_vdpa_vring_ioctl()
677 ops->set_vq_cb(vdpa, idx, &cb); in vhost_vdpa_vring_ioctl()
681 ops->set_vq_num(vdpa, idx, vq->num); in vhost_vdpa_vring_ioctl()
691 struct vhost_vdpa *v = filep->private_data; in vhost_vdpa_unlocked_ioctl() local
692 struct vhost_dev *d = &v->vdev; in vhost_vdpa_unlocked_ioctl()
700 return -EFAULT; in vhost_vdpa_unlocked_ioctl()
705 return -EOPNOTSUPP; in vhost_vdpa_unlocked_ioctl()
707 !vhost_vdpa_can_suspend(v)) in vhost_vdpa_unlocked_ioctl()
708 return -EOPNOTSUPP; in vhost_vdpa_unlocked_ioctl()
710 !vhost_vdpa_can_resume(v)) in vhost_vdpa_unlocked_ioctl()
711 return -EOPNOTSUPP; in vhost_vdpa_unlocked_ioctl()
712 vhost_set_backend_features(&v->vdev, features); in vhost_vdpa_unlocked_ioctl()
716 mutex_lock(&d->mutex); in vhost_vdpa_unlocked_ioctl()
720 r = vhost_vdpa_get_device_id(v, argp); in vhost_vdpa_unlocked_ioctl()
723 r = vhost_vdpa_get_status(v, argp); in vhost_vdpa_unlocked_ioctl()
726 r = vhost_vdpa_set_status(v, argp); in vhost_vdpa_unlocked_ioctl()
729 r = vhost_vdpa_get_config(v, argp); in vhost_vdpa_unlocked_ioctl()
732 r = vhost_vdpa_set_config(v, argp); in vhost_vdpa_unlocked_ioctl()
735 r = vhost_vdpa_get_features(v, argp); in vhost_vdpa_unlocked_ioctl()
738 r = vhost_vdpa_set_features(v, argp); in vhost_vdpa_unlocked_ioctl()
741 r = vhost_vdpa_get_vring_num(v, argp); in vhost_vdpa_unlocked_ioctl()
744 if (copy_to_user(argp, &v->vdpa->ngroups, in vhost_vdpa_unlocked_ioctl()
745 sizeof(v->vdpa->ngroups))) in vhost_vdpa_unlocked_ioctl()
746 r = -EFAULT; in vhost_vdpa_unlocked_ioctl()
749 if (copy_to_user(argp, &v->vdpa->nas, sizeof(v->vdpa->nas))) in vhost_vdpa_unlocked_ioctl()
750 r = -EFAULT; in vhost_vdpa_unlocked_ioctl()
754 r = -ENOIOCTLCMD; in vhost_vdpa_unlocked_ioctl()
757 r = vhost_vdpa_set_config_call(v, argp); in vhost_vdpa_unlocked_ioctl()
761 if (vhost_vdpa_can_suspend(v)) in vhost_vdpa_unlocked_ioctl()
763 if (vhost_vdpa_can_resume(v)) in vhost_vdpa_unlocked_ioctl()
765 features |= vhost_vdpa_get_backend_features(v); in vhost_vdpa_unlocked_ioctl()
767 r = -EFAULT; in vhost_vdpa_unlocked_ioctl()
770 r = vhost_vdpa_get_iova_range(v, argp); in vhost_vdpa_unlocked_ioctl()
773 r = vhost_vdpa_get_config_size(v, argp); in vhost_vdpa_unlocked_ioctl()
776 r = vhost_vdpa_get_vqs_count(v, argp); in vhost_vdpa_unlocked_ioctl()
779 r = vhost_vdpa_suspend(v); in vhost_vdpa_unlocked_ioctl()
782 r = vhost_vdpa_resume(v); in vhost_vdpa_unlocked_ioctl()
785 r = vhost_dev_ioctl(&v->vdev, cmd, argp); in vhost_vdpa_unlocked_ioctl()
786 if (r == -ENOIOCTLCMD) in vhost_vdpa_unlocked_ioctl()
787 r = vhost_vdpa_vring_ioctl(v, cmd, argp); in vhost_vdpa_unlocked_ioctl()
796 r = vhost_vdpa_bind_mm(v); in vhost_vdpa_unlocked_ioctl()
802 mutex_unlock(&d->mutex); in vhost_vdpa_unlocked_ioctl()
805 static void vhost_vdpa_general_unmap(struct vhost_vdpa *v, in vhost_vdpa_general_unmap() argument
808 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_general_unmap()
809 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_general_unmap()
810 if (ops->dma_map) { in vhost_vdpa_general_unmap()
811 ops->dma_unmap(vdpa, asid, map->start, map->size); in vhost_vdpa_general_unmap()
812 } else if (ops->set_map == NULL) { in vhost_vdpa_general_unmap()
813 iommu_unmap(v->domain, map->start, map->size); in vhost_vdpa_general_unmap()
817 static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, struct vhost_iotlb *iotlb, in vhost_vdpa_pa_unmap() argument
820 struct vhost_dev *dev = &v->vdev; in vhost_vdpa_pa_unmap()
826 pinned = PFN_DOWN(map->size); in vhost_vdpa_pa_unmap()
827 for (pfn = PFN_DOWN(map->addr); in vhost_vdpa_pa_unmap()
828 pinned > 0; pfn++, pinned--) { in vhost_vdpa_pa_unmap()
830 if (map->perm & VHOST_ACCESS_WO) in vhost_vdpa_pa_unmap()
834 atomic64_sub(PFN_DOWN(map->size), &dev->mm->pinned_vm); in vhost_vdpa_pa_unmap()
835 vhost_vdpa_general_unmap(v, map, asid); in vhost_vdpa_pa_unmap()
840 static void vhost_vdpa_va_unmap(struct vhost_vdpa *v, struct vhost_iotlb *iotlb, in vhost_vdpa_va_unmap() argument
847 map_file = (struct vdpa_map_file *)map->opaque; in vhost_vdpa_va_unmap()
848 fput(map_file->file); in vhost_vdpa_va_unmap()
850 vhost_vdpa_general_unmap(v, map, asid); in vhost_vdpa_va_unmap()
855 static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v, in vhost_vdpa_iotlb_unmap() argument
859 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_iotlb_unmap()
861 if (vdpa->use_va) in vhost_vdpa_iotlb_unmap()
862 return vhost_vdpa_va_unmap(v, iotlb, start, last, asid); in vhost_vdpa_iotlb_unmap()
864 return vhost_vdpa_pa_unmap(v, iotlb, start, last, asid); in vhost_vdpa_iotlb_unmap()
889 static int vhost_vdpa_map(struct vhost_vdpa *v, struct vhost_iotlb *iotlb, in vhost_vdpa_map() argument
892 struct vhost_dev *dev = &v->vdev; in vhost_vdpa_map()
893 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_map()
894 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_map()
898 r = vhost_iotlb_add_range_ctx(iotlb, iova, iova + size - 1, in vhost_vdpa_map()
903 if (ops->dma_map) { in vhost_vdpa_map()
904 r = ops->dma_map(vdpa, asid, iova, size, pa, perm, opaque); in vhost_vdpa_map()
905 } else if (ops->set_map) { in vhost_vdpa_map()
906 if (!v->in_batch) in vhost_vdpa_map()
907 r = ops->set_map(vdpa, asid, iotlb); in vhost_vdpa_map()
909 r = iommu_map(v->domain, iova, pa, size, in vhost_vdpa_map()
913 vhost_iotlb_del_range(iotlb, iova, iova + size - 1); in vhost_vdpa_map()
917 if (!vdpa->use_va) in vhost_vdpa_map()
918 atomic64_add(PFN_DOWN(size), &dev->mm->pinned_vm); in vhost_vdpa_map()
923 static void vhost_vdpa_unmap(struct vhost_vdpa *v, in vhost_vdpa_unmap() argument
927 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_unmap()
928 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_unmap()
931 vhost_vdpa_iotlb_unmap(v, iotlb, iova, iova + size - 1, asid); in vhost_vdpa_unmap()
933 if (ops->set_map) { in vhost_vdpa_unmap()
934 if (!v->in_batch) in vhost_vdpa_unmap()
935 ops->set_map(vdpa, asid, iotlb); in vhost_vdpa_unmap()
940 static int vhost_vdpa_va_map(struct vhost_vdpa *v, in vhost_vdpa_va_map() argument
944 struct vhost_dev *dev = &v->vdev; in vhost_vdpa_va_map()
950 mmap_read_lock(dev->mm); in vhost_vdpa_va_map()
953 vma = find_vma(dev->mm, uaddr); in vhost_vdpa_va_map()
955 ret = -EINVAL; in vhost_vdpa_va_map()
958 map_size = min(size, vma->vm_end - uaddr); in vhost_vdpa_va_map()
959 if (!(vma->vm_file && (vma->vm_flags & VM_SHARED) && in vhost_vdpa_va_map()
960 !(vma->vm_flags & (VM_IO | VM_PFNMAP)))) in vhost_vdpa_va_map()
965 ret = -ENOMEM; in vhost_vdpa_va_map()
968 offset = (vma->vm_pgoff << PAGE_SHIFT) + uaddr - vma->vm_start; in vhost_vdpa_va_map()
969 map_file->offset = offset; in vhost_vdpa_va_map()
970 map_file->file = get_file(vma->vm_file); in vhost_vdpa_va_map()
971 ret = vhost_vdpa_map(v, iotlb, map_iova, map_size, uaddr, in vhost_vdpa_va_map()
974 fput(map_file->file); in vhost_vdpa_va_map()
979 size -= map_size; in vhost_vdpa_va_map()
984 vhost_vdpa_unmap(v, iotlb, iova, map_iova - iova); in vhost_vdpa_va_map()
986 mmap_read_unlock(dev->mm); in vhost_vdpa_va_map()
991 static int vhost_vdpa_pa_map(struct vhost_vdpa *v, in vhost_vdpa_pa_map() argument
995 struct vhost_dev *dev = &v->vdev; in vhost_vdpa_pa_map()
1008 return -ENOMEM; in vhost_vdpa_pa_map()
1015 ret = -EINVAL; in vhost_vdpa_pa_map()
1019 mmap_read_lock(dev->mm); in vhost_vdpa_pa_map()
1022 if (npages + atomic64_read(&dev->mm->pinned_vm) > lock_limit) { in vhost_vdpa_pa_map()
1023 ret = -ENOMEM; in vhost_vdpa_pa_map()
1040 ret = -ENOMEM; in vhost_vdpa_pa_map()
1055 csize = PFN_PHYS(last_pfn - map_pfn + 1); in vhost_vdpa_pa_map()
1056 ret = vhost_vdpa_map(v, iotlb, iova, csize, in vhost_vdpa_pa_map()
1069 pinned - i); in vhost_vdpa_pa_map()
1082 npages -= pinned; in vhost_vdpa_pa_map()
1086 ret = vhost_vdpa_map(v, iotlb, iova, PFN_PHYS(last_pfn - map_pfn + 1), in vhost_vdpa_pa_map()
1106 vhost_vdpa_unmap(v, iotlb, start, size); in vhost_vdpa_pa_map()
1109 mmap_read_unlock(dev->mm); in vhost_vdpa_pa_map()
1116 static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v, in vhost_vdpa_process_iotlb_update() argument
1120 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_process_iotlb_update()
1122 if (msg->iova < v->range.first || !msg->size || in vhost_vdpa_process_iotlb_update()
1123 msg->iova > U64_MAX - msg->size + 1 || in vhost_vdpa_process_iotlb_update()
1124 msg->iova + msg->size - 1 > v->range.last) in vhost_vdpa_process_iotlb_update()
1125 return -EINVAL; in vhost_vdpa_process_iotlb_update()
1127 if (vhost_iotlb_itree_first(iotlb, msg->iova, in vhost_vdpa_process_iotlb_update()
1128 msg->iova + msg->size - 1)) in vhost_vdpa_process_iotlb_update()
1129 return -EEXIST; in vhost_vdpa_process_iotlb_update()
1131 if (vdpa->use_va) in vhost_vdpa_process_iotlb_update()
1132 return vhost_vdpa_va_map(v, iotlb, msg->iova, msg->size, in vhost_vdpa_process_iotlb_update()
1133 msg->uaddr, msg->perm); in vhost_vdpa_process_iotlb_update()
1135 return vhost_vdpa_pa_map(v, iotlb, msg->iova, msg->size, msg->uaddr, in vhost_vdpa_process_iotlb_update()
1136 msg->perm); in vhost_vdpa_process_iotlb_update()
1142 struct vhost_vdpa *v = container_of(dev, struct vhost_vdpa, vdev); in vhost_vdpa_process_iotlb_msg() local
1143 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_process_iotlb_msg()
1144 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_process_iotlb_msg()
1149 mutex_lock(&dev->mutex); in vhost_vdpa_process_iotlb_msg()
1155 if (msg->type == VHOST_IOTLB_UPDATE || in vhost_vdpa_process_iotlb_msg()
1156 msg->type == VHOST_IOTLB_BATCH_BEGIN) { in vhost_vdpa_process_iotlb_msg()
1157 as = vhost_vdpa_find_alloc_as(v, asid); in vhost_vdpa_process_iotlb_msg()
1159 dev_err(&v->dev, "can't find and alloc asid %d\n", in vhost_vdpa_process_iotlb_msg()
1161 r = -EINVAL; in vhost_vdpa_process_iotlb_msg()
1164 iotlb = &as->iotlb; in vhost_vdpa_process_iotlb_msg()
1166 iotlb = asid_to_iotlb(v, asid); in vhost_vdpa_process_iotlb_msg()
1168 if ((v->in_batch && v->batch_asid != asid) || !iotlb) { in vhost_vdpa_process_iotlb_msg()
1169 if (v->in_batch && v->batch_asid != asid) { in vhost_vdpa_process_iotlb_msg()
1170 dev_info(&v->dev, "batch id %d asid %d\n", in vhost_vdpa_process_iotlb_msg()
1171 v->batch_asid, asid); in vhost_vdpa_process_iotlb_msg()
1174 dev_err(&v->dev, "no iotlb for asid %d\n", asid); in vhost_vdpa_process_iotlb_msg()
1175 r = -EINVAL; in vhost_vdpa_process_iotlb_msg()
1179 switch (msg->type) { in vhost_vdpa_process_iotlb_msg()
1181 r = vhost_vdpa_process_iotlb_update(v, iotlb, msg); in vhost_vdpa_process_iotlb_msg()
1184 vhost_vdpa_unmap(v, iotlb, msg->iova, msg->size); in vhost_vdpa_process_iotlb_msg()
1187 v->batch_asid = asid; in vhost_vdpa_process_iotlb_msg()
1188 v->in_batch = true; in vhost_vdpa_process_iotlb_msg()
1191 if (v->in_batch && ops->set_map) in vhost_vdpa_process_iotlb_msg()
1192 ops->set_map(vdpa, asid, iotlb); in vhost_vdpa_process_iotlb_msg()
1193 v->in_batch = false; in vhost_vdpa_process_iotlb_msg()
1196 r = -EINVAL; in vhost_vdpa_process_iotlb_msg()
1200 mutex_unlock(&dev->mutex); in vhost_vdpa_process_iotlb_msg()
1208 struct file *file = iocb->ki_filp; in vhost_vdpa_chr_write_iter()
1209 struct vhost_vdpa *v = file->private_data; in vhost_vdpa_chr_write_iter() local
1210 struct vhost_dev *dev = &v->vdev; in vhost_vdpa_chr_write_iter()
1215 static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v) in vhost_vdpa_alloc_domain() argument
1217 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_alloc_domain()
1218 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_alloc_domain()
1224 if (ops->set_map || ops->dma_map) in vhost_vdpa_alloc_domain()
1227 bus = dma_dev->bus; in vhost_vdpa_alloc_domain()
1229 return -EFAULT; in vhost_vdpa_alloc_domain()
1232 dev_warn_once(&v->dev, in vhost_vdpa_alloc_domain()
1234 return -ENOTSUPP; in vhost_vdpa_alloc_domain()
1237 v->domain = iommu_domain_alloc(bus); in vhost_vdpa_alloc_domain()
1238 if (!v->domain) in vhost_vdpa_alloc_domain()
1239 return -EIO; in vhost_vdpa_alloc_domain()
1241 ret = iommu_attach_device(v->domain, dma_dev); in vhost_vdpa_alloc_domain()
1248 iommu_domain_free(v->domain); in vhost_vdpa_alloc_domain()
1249 v->domain = NULL; in vhost_vdpa_alloc_domain()
1253 static void vhost_vdpa_free_domain(struct vhost_vdpa *v) in vhost_vdpa_free_domain() argument
1255 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_free_domain()
1258 if (v->domain) { in vhost_vdpa_free_domain()
1259 iommu_detach_device(v->domain, dma_dev); in vhost_vdpa_free_domain()
1260 iommu_domain_free(v->domain); in vhost_vdpa_free_domain()
1263 v->domain = NULL; in vhost_vdpa_free_domain()
1266 static void vhost_vdpa_set_iova_range(struct vhost_vdpa *v) in vhost_vdpa_set_iova_range() argument
1268 struct vdpa_iova_range *range = &v->range; in vhost_vdpa_set_iova_range()
1269 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_set_iova_range()
1270 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_set_iova_range()
1272 if (ops->get_iova_range) { in vhost_vdpa_set_iova_range()
1273 *range = ops->get_iova_range(vdpa); in vhost_vdpa_set_iova_range()
1274 } else if (v->domain && v->domain->geometry.force_aperture) { in vhost_vdpa_set_iova_range()
1275 range->first = v->domain->geometry.aperture_start; in vhost_vdpa_set_iova_range()
1276 range->last = v->domain->geometry.aperture_end; in vhost_vdpa_set_iova_range()
1278 range->first = 0; in vhost_vdpa_set_iova_range()
1279 range->last = ULLONG_MAX; in vhost_vdpa_set_iova_range()
1283 static void vhost_vdpa_cleanup(struct vhost_vdpa *v) in vhost_vdpa_cleanup() argument
1288 for (asid = 0; asid < v->vdpa->nas; asid++) { in vhost_vdpa_cleanup()
1289 as = asid_to_as(v, asid); in vhost_vdpa_cleanup()
1291 vhost_vdpa_remove_as(v, asid); in vhost_vdpa_cleanup()
1294 vhost_vdpa_free_domain(v); in vhost_vdpa_cleanup()
1295 vhost_dev_cleanup(&v->vdev); in vhost_vdpa_cleanup()
1296 kfree(v->vdev.vqs); in vhost_vdpa_cleanup()
1301 struct vhost_vdpa *v; in vhost_vdpa_open() local
1307 v = container_of(inode->i_cdev, struct vhost_vdpa, cdev); in vhost_vdpa_open()
1309 opened = atomic_cmpxchg(&v->opened, 0, 1); in vhost_vdpa_open()
1311 return -EBUSY; in vhost_vdpa_open()
1313 nvqs = v->nvqs; in vhost_vdpa_open()
1314 r = vhost_vdpa_reset(v); in vhost_vdpa_open()
1320 r = -ENOMEM; in vhost_vdpa_open()
1324 dev = &v->vdev; in vhost_vdpa_open()
1326 vqs[i] = &v->vqs[i]; in vhost_vdpa_open()
1327 vqs[i]->handle_kick = handle_vq_kick; in vhost_vdpa_open()
1328 vqs[i]->call_ctx.ctx = NULL; in vhost_vdpa_open()
1333 r = vhost_vdpa_alloc_domain(v); in vhost_vdpa_open()
1337 vhost_vdpa_set_iova_range(v); in vhost_vdpa_open()
1339 filep->private_data = v; in vhost_vdpa_open()
1344 vhost_vdpa_cleanup(v); in vhost_vdpa_open()
1346 atomic_dec(&v->opened); in vhost_vdpa_open()
1350 static void vhost_vdpa_clean_irq(struct vhost_vdpa *v) in vhost_vdpa_clean_irq() argument
1354 for (i = 0; i < v->nvqs; i++) in vhost_vdpa_clean_irq()
1355 vhost_vdpa_unsetup_vq_irq(v, i); in vhost_vdpa_clean_irq()
1360 struct vhost_vdpa *v = filep->private_data; in vhost_vdpa_release() local
1361 struct vhost_dev *d = &v->vdev; in vhost_vdpa_release()
1363 mutex_lock(&d->mutex); in vhost_vdpa_release()
1364 filep->private_data = NULL; in vhost_vdpa_release()
1365 vhost_vdpa_clean_irq(v); in vhost_vdpa_release()
1366 vhost_vdpa_reset(v); in vhost_vdpa_release()
1367 vhost_dev_stop(&v->vdev); in vhost_vdpa_release()
1368 vhost_vdpa_unbind_mm(v); in vhost_vdpa_release()
1369 vhost_vdpa_config_put(v); in vhost_vdpa_release()
1370 vhost_vdpa_cleanup(v); in vhost_vdpa_release()
1371 mutex_unlock(&d->mutex); in vhost_vdpa_release()
1373 atomic_dec(&v->opened); in vhost_vdpa_release()
1374 complete(&v->completion); in vhost_vdpa_release()
1382 struct vhost_vdpa *v = vmf->vma->vm_file->private_data; in vhost_vdpa_fault() local
1383 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_fault()
1384 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_fault()
1386 struct vm_area_struct *vma = vmf->vma; in vhost_vdpa_fault()
1387 u16 index = vma->vm_pgoff; in vhost_vdpa_fault()
1389 notify = ops->get_vq_notification(vdpa, index); in vhost_vdpa_fault()
1391 return vmf_insert_pfn(vma, vmf->address & PAGE_MASK, PFN_DOWN(notify.addr)); in vhost_vdpa_fault()
1400 struct vhost_vdpa *v = vma->vm_file->private_data; in vhost_vdpa_mmap() local
1401 struct vdpa_device *vdpa = v->vdpa; in vhost_vdpa_mmap()
1402 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_mmap()
1404 unsigned long index = vma->vm_pgoff; in vhost_vdpa_mmap()
1406 if (vma->vm_end - vma->vm_start != PAGE_SIZE) in vhost_vdpa_mmap()
1407 return -EINVAL; in vhost_vdpa_mmap()
1408 if ((vma->vm_flags & VM_SHARED) == 0) in vhost_vdpa_mmap()
1409 return -EINVAL; in vhost_vdpa_mmap()
1410 if (vma->vm_flags & VM_READ) in vhost_vdpa_mmap()
1411 return -EINVAL; in vhost_vdpa_mmap()
1413 return -EINVAL; in vhost_vdpa_mmap()
1414 if (!ops->get_vq_notification) in vhost_vdpa_mmap()
1415 return -ENOTSUPP; in vhost_vdpa_mmap()
1421 notify = ops->get_vq_notification(vdpa, index); in vhost_vdpa_mmap()
1422 if (notify.addr & (PAGE_SIZE - 1)) in vhost_vdpa_mmap()
1423 return -EINVAL; in vhost_vdpa_mmap()
1424 if (vma->vm_end - vma->vm_start != notify.size) in vhost_vdpa_mmap()
1425 return -ENOTSUPP; in vhost_vdpa_mmap()
1428 vma->vm_ops = &vhost_vdpa_vm_ops; in vhost_vdpa_mmap()
1447 struct vhost_vdpa *v = in vhost_vdpa_release_dev() local
1450 ida_simple_remove(&vhost_vdpa_ida, v->minor); in vhost_vdpa_release_dev()
1451 kfree(v->vqs); in vhost_vdpa_release_dev()
1452 kfree(v); in vhost_vdpa_release_dev()
1457 const struct vdpa_config_ops *ops = vdpa->config; in vhost_vdpa_probe()
1458 struct vhost_vdpa *v; in vhost_vdpa_probe() local
1465 if (!ops->set_map && !ops->dma_map && in vhost_vdpa_probe()
1466 (vdpa->ngroups > 1 || vdpa->nas > 1)) in vhost_vdpa_probe()
1467 return -EOPNOTSUPP; in vhost_vdpa_probe()
1469 v = kzalloc(sizeof(*v), GFP_KERNEL | __GFP_RETRY_MAYFAIL); in vhost_vdpa_probe()
1470 if (!v) in vhost_vdpa_probe()
1471 return -ENOMEM; in vhost_vdpa_probe()
1476 kfree(v); in vhost_vdpa_probe()
1480 atomic_set(&v->opened, 0); in vhost_vdpa_probe()
1481 v->minor = minor; in vhost_vdpa_probe()
1482 v->vdpa = vdpa; in vhost_vdpa_probe()
1483 v->nvqs = vdpa->nvqs; in vhost_vdpa_probe()
1484 v->virtio_id = ops->get_device_id(vdpa); in vhost_vdpa_probe()
1486 device_initialize(&v->dev); in vhost_vdpa_probe()
1487 v->dev.release = vhost_vdpa_release_dev; in vhost_vdpa_probe()
1488 v->dev.parent = &vdpa->dev; in vhost_vdpa_probe()
1489 v->dev.devt = MKDEV(MAJOR(vhost_vdpa_major), minor); in vhost_vdpa_probe()
1490 v->vqs = kmalloc_array(v->nvqs, sizeof(struct vhost_virtqueue), in vhost_vdpa_probe()
1492 if (!v->vqs) { in vhost_vdpa_probe()
1493 r = -ENOMEM; in vhost_vdpa_probe()
1497 r = dev_set_name(&v->dev, "vhost-vdpa-%u", minor); in vhost_vdpa_probe()
1501 cdev_init(&v->cdev, &vhost_vdpa_fops); in vhost_vdpa_probe()
1502 v->cdev.owner = THIS_MODULE; in vhost_vdpa_probe()
1504 r = cdev_device_add(&v->cdev, &v->dev); in vhost_vdpa_probe()
1508 init_completion(&v->completion); in vhost_vdpa_probe()
1509 vdpa_set_drvdata(vdpa, v); in vhost_vdpa_probe()
1512 INIT_HLIST_HEAD(&v->as[i]); in vhost_vdpa_probe()
1517 put_device(&v->dev); in vhost_vdpa_probe()
1523 struct vhost_vdpa *v = vdpa_get_drvdata(vdpa); in vhost_vdpa_remove() local
1526 cdev_device_del(&v->cdev, &v->dev); in vhost_vdpa_remove()
1529 opened = atomic_cmpxchg(&v->opened, 0, 1); in vhost_vdpa_remove()
1532 wait_for_completion(&v->completion); in vhost_vdpa_remove()
1535 put_device(&v->dev); in vhost_vdpa_remove()
1551 "vhost-vdpa"); in vhost_vdpa_init()
1578 MODULE_DESCRIPTION("vDPA-based vhost backend for virtio");