Lines Matching refs:dev
76 struct vduse_dev *dev; member
88 struct device *dev; member
153 return vdev->dev; in vdpa_to_vduse()
156 static inline struct vduse_dev *dev_to_vduse(struct device *dev) in dev_to_vduse() argument
158 struct vdpa_device *vdpa = dev_to_vdpa(dev); in dev_to_vduse()
196 static void vduse_dev_broken(struct vduse_dev *dev) in vduse_dev_broken() argument
200 if (unlikely(dev->broken)) in vduse_dev_broken()
203 list_splice_init(&dev->recv_list, &dev->send_list); in vduse_dev_broken()
204 list_for_each_entry_safe(msg, tmp, &dev->send_list, list) { in vduse_dev_broken()
210 dev->broken = true; in vduse_dev_broken()
211 wake_up(&dev->waitq); in vduse_dev_broken()
214 static int vduse_dev_msg_sync(struct vduse_dev *dev, in vduse_dev_msg_sync() argument
219 if (unlikely(dev->broken)) in vduse_dev_msg_sync()
223 spin_lock(&dev->msg_lock); in vduse_dev_msg_sync()
224 if (unlikely(dev->broken)) { in vduse_dev_msg_sync()
225 spin_unlock(&dev->msg_lock); in vduse_dev_msg_sync()
228 msg->req.request_id = dev->msg_unique++; in vduse_dev_msg_sync()
229 vduse_enqueue_msg(&dev->send_list, msg); in vduse_dev_msg_sync()
230 wake_up(&dev->waitq); in vduse_dev_msg_sync()
231 spin_unlock(&dev->msg_lock); in vduse_dev_msg_sync()
232 if (dev->msg_timeout) in vduse_dev_msg_sync()
234 (long)dev->msg_timeout * HZ); in vduse_dev_msg_sync()
238 spin_lock(&dev->msg_lock); in vduse_dev_msg_sync()
244 vduse_dev_broken(dev); in vduse_dev_msg_sync()
247 spin_unlock(&dev->msg_lock); in vduse_dev_msg_sync()
252 static int vduse_dev_get_vq_state_packed(struct vduse_dev *dev, in vduse_dev_get_vq_state_packed() argument
262 ret = vduse_dev_msg_sync(dev, &msg); in vduse_dev_get_vq_state_packed()
278 static int vduse_dev_get_vq_state_split(struct vduse_dev *dev, in vduse_dev_get_vq_state_split() argument
288 ret = vduse_dev_msg_sync(dev, &msg); in vduse_dev_get_vq_state_split()
297 static int vduse_dev_set_status(struct vduse_dev *dev, u8 status) in vduse_dev_set_status() argument
304 return vduse_dev_msg_sync(dev, &msg); in vduse_dev_set_status()
307 static int vduse_dev_update_iotlb(struct vduse_dev *dev, in vduse_dev_update_iotlb() argument
319 return vduse_dev_msg_sync(dev, &msg); in vduse_dev_update_iotlb()
325 struct vduse_dev *dev = file->private_data; in vduse_dev_read_iter() local
333 spin_lock(&dev->msg_lock); in vduse_dev_read_iter()
335 msg = vduse_dequeue_msg(&dev->send_list); in vduse_dev_read_iter()
343 spin_unlock(&dev->msg_lock); in vduse_dev_read_iter()
344 ret = wait_event_interruptible_exclusive(dev->waitq, in vduse_dev_read_iter()
345 !list_empty(&dev->send_list)); in vduse_dev_read_iter()
349 spin_lock(&dev->msg_lock); in vduse_dev_read_iter()
351 spin_unlock(&dev->msg_lock); in vduse_dev_read_iter()
353 spin_lock(&dev->msg_lock); in vduse_dev_read_iter()
356 vduse_enqueue_msg(&dev->send_list, msg); in vduse_dev_read_iter()
359 vduse_enqueue_msg(&dev->recv_list, msg); in vduse_dev_read_iter()
361 spin_unlock(&dev->msg_lock); in vduse_dev_read_iter()
380 struct vduse_dev *dev = file->private_data; in vduse_dev_write_iter() local
392 spin_lock(&dev->msg_lock); in vduse_dev_write_iter()
393 msg = vduse_find_msg(&dev->recv_list, resp.request_id); in vduse_dev_write_iter()
403 spin_unlock(&dev->msg_lock); in vduse_dev_write_iter()
410 struct vduse_dev *dev = file->private_data; in vduse_dev_poll() local
413 poll_wait(file, &dev->waitq, wait); in vduse_dev_poll()
415 spin_lock(&dev->msg_lock); in vduse_dev_poll()
417 if (unlikely(dev->broken)) in vduse_dev_poll()
419 if (!list_empty(&dev->send_list)) in vduse_dev_poll()
421 if (!list_empty(&dev->recv_list)) in vduse_dev_poll()
424 spin_unlock(&dev->msg_lock); in vduse_dev_poll()
429 static void vduse_dev_reset(struct vduse_dev *dev) in vduse_dev_reset() argument
432 struct vduse_iova_domain *domain = dev->domain; in vduse_dev_reset()
438 down_write(&dev->rwsem); in vduse_dev_reset()
440 dev->status = 0; in vduse_dev_reset()
441 dev->driver_features = 0; in vduse_dev_reset()
442 dev->generation++; in vduse_dev_reset()
443 spin_lock(&dev->irq_lock); in vduse_dev_reset()
444 dev->config_cb.callback = NULL; in vduse_dev_reset()
445 dev->config_cb.private = NULL; in vduse_dev_reset()
446 spin_unlock(&dev->irq_lock); in vduse_dev_reset()
447 flush_work(&dev->inject); in vduse_dev_reset()
449 for (i = 0; i < dev->vq_num; i++) { in vduse_dev_reset()
450 struct vduse_virtqueue *vq = dev->vqs[i]; in vduse_dev_reset()
475 up_write(&dev->rwsem); in vduse_dev_reset()
482 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_set_vq_address() local
483 struct vduse_virtqueue *vq = dev->vqs[idx]; in vduse_vdpa_set_vq_address()
516 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_kick_vq() local
517 struct vduse_virtqueue *vq = dev->vqs[idx]; in vduse_vdpa_kick_vq()
529 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_set_vq_cb() local
530 struct vduse_virtqueue *vq = dev->vqs[idx]; in vduse_vdpa_set_vq_cb()
541 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_set_vq_num() local
542 struct vduse_virtqueue *vq = dev->vqs[idx]; in vduse_vdpa_set_vq_num()
550 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_set_vq_ready() local
551 struct vduse_virtqueue *vq = dev->vqs[idx]; in vduse_vdpa_set_vq_ready()
558 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_get_vq_ready() local
559 struct vduse_virtqueue *vq = dev->vqs[idx]; in vduse_vdpa_get_vq_ready()
567 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_set_vq_state() local
568 struct vduse_virtqueue *vq = dev->vqs[idx]; in vduse_vdpa_set_vq_state()
570 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) { in vduse_vdpa_set_vq_state()
586 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_get_vq_state() local
587 struct vduse_virtqueue *vq = dev->vqs[idx]; in vduse_vdpa_get_vq_state()
589 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) in vduse_vdpa_get_vq_state()
590 return vduse_dev_get_vq_state_packed(dev, vq, &state->packed); in vduse_vdpa_get_vq_state()
592 return vduse_dev_get_vq_state_split(dev, vq, &state->split); in vduse_vdpa_get_vq_state()
597 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_get_vq_align() local
599 return dev->vq_align; in vduse_vdpa_get_vq_align()
604 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_get_device_features() local
606 return dev->device_features; in vduse_vdpa_get_device_features()
611 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_set_driver_features() local
613 dev->driver_features = features; in vduse_vdpa_set_driver_features()
619 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_get_driver_features() local
621 return dev->driver_features; in vduse_vdpa_get_driver_features()
627 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_set_config_cb() local
629 spin_lock(&dev->irq_lock); in vduse_vdpa_set_config_cb()
630 dev->config_cb.callback = cb->callback; in vduse_vdpa_set_config_cb()
631 dev->config_cb.private = cb->private; in vduse_vdpa_set_config_cb()
632 spin_unlock(&dev->irq_lock); in vduse_vdpa_set_config_cb()
637 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_get_vq_num_max() local
641 for (i = 0; i < dev->vq_num; i++) in vduse_vdpa_get_vq_num_max()
642 if (num_max < dev->vqs[i]->num_max) in vduse_vdpa_get_vq_num_max()
643 num_max = dev->vqs[i]->num_max; in vduse_vdpa_get_vq_num_max()
650 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_get_device_id() local
652 return dev->device_id; in vduse_vdpa_get_device_id()
657 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_get_vendor_id() local
659 return dev->vendor_id; in vduse_vdpa_get_vendor_id()
664 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_get_status() local
666 return dev->status; in vduse_vdpa_get_status()
671 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_set_status() local
673 if (vduse_dev_set_status(dev, status)) in vduse_vdpa_set_status()
676 dev->status = status; in vduse_vdpa_set_status()
681 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_get_config_size() local
683 return dev->config_size; in vduse_vdpa_get_config_size()
689 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_get_config() local
694 if (offset > dev->config_size) in vduse_vdpa_get_config()
697 if (len > dev->config_size - offset) in vduse_vdpa_get_config()
698 len = dev->config_size - offset; in vduse_vdpa_get_config()
700 memcpy(buf, dev->config + offset, len); in vduse_vdpa_get_config()
711 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_reset() local
712 int ret = vduse_dev_set_status(dev, 0); in vduse_vdpa_reset()
714 vduse_dev_reset(dev); in vduse_vdpa_reset()
721 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_get_generation() local
723 return dev->generation; in vduse_vdpa_get_generation()
729 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_set_vq_affinity() local
732 cpumask_copy(&dev->vqs[idx]->irq_affinity, cpu_mask); in vduse_vdpa_set_vq_affinity()
734 cpumask_setall(&dev->vqs[idx]->irq_affinity); in vduse_vdpa_set_vq_affinity()
742 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_get_vq_affinity() local
744 return &dev->vqs[idx]->irq_affinity; in vduse_vdpa_get_vq_affinity()
751 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_set_map() local
754 ret = vduse_domain_set_map(dev->domain, iotlb); in vduse_vdpa_set_map()
758 ret = vduse_dev_update_iotlb(dev, 0ULL, ULLONG_MAX); in vduse_vdpa_set_map()
760 vduse_domain_clear_map(dev->domain, iotlb); in vduse_vdpa_set_map()
769 struct vduse_dev *dev = vdpa_to_vduse(vdpa); in vduse_vdpa_free() local
771 dev->vdev = NULL; in vduse_vdpa_free()
804 static dma_addr_t vduse_dev_map_page(struct device *dev, struct page *page, in vduse_dev_map_page() argument
809 struct vduse_dev *vdev = dev_to_vduse(dev); in vduse_dev_map_page()
815 static void vduse_dev_unmap_page(struct device *dev, dma_addr_t dma_addr, in vduse_dev_unmap_page() argument
819 struct vduse_dev *vdev = dev_to_vduse(dev); in vduse_dev_unmap_page()
825 static void *vduse_dev_alloc_coherent(struct device *dev, size_t size, in vduse_dev_alloc_coherent() argument
829 struct vduse_dev *vdev = dev_to_vduse(dev); in vduse_dev_alloc_coherent()
845 static void vduse_dev_free_coherent(struct device *dev, size_t size, in vduse_dev_free_coherent() argument
849 struct vduse_dev *vdev = dev_to_vduse(dev); in vduse_dev_free_coherent()
855 static size_t vduse_dev_max_mapping_size(struct device *dev) in vduse_dev_max_mapping_size() argument
857 struct vduse_dev *vdev = dev_to_vduse(dev); in vduse_dev_max_mapping_size()
893 static int vduse_kickfd_setup(struct vduse_dev *dev, in vduse_kickfd_setup() argument
900 if (eventfd->index >= dev->vq_num) in vduse_kickfd_setup()
903 index = array_index_nospec(eventfd->index, dev->vq_num); in vduse_kickfd_setup()
904 vq = dev->vqs[index]; in vduse_kickfd_setup()
925 static bool vduse_dev_is_ready(struct vduse_dev *dev) in vduse_dev_is_ready() argument
929 for (i = 0; i < dev->vq_num; i++) in vduse_dev_is_ready()
930 if (!dev->vqs[i]->num_max) in vduse_dev_is_ready()
938 struct vduse_dev *dev = container_of(work, struct vduse_dev, inject); in vduse_dev_irq_inject() local
940 spin_lock_bh(&dev->irq_lock); in vduse_dev_irq_inject()
941 if (dev->config_cb.callback) in vduse_dev_irq_inject()
942 dev->config_cb.callback(dev->config_cb.private); in vduse_dev_irq_inject()
943 spin_unlock_bh(&dev->irq_lock); in vduse_dev_irq_inject()
974 static int vduse_dev_queue_irq_work(struct vduse_dev *dev, in vduse_dev_queue_irq_work() argument
980 down_read(&dev->rwsem); in vduse_dev_queue_irq_work()
981 if (!(dev->status & VIRTIO_CONFIG_S_DRIVER_OK)) in vduse_dev_queue_irq_work()
991 up_read(&dev->rwsem); in vduse_dev_queue_irq_work()
996 static int vduse_dev_dereg_umem(struct vduse_dev *dev, in vduse_dev_dereg_umem() argument
1001 mutex_lock(&dev->mem_lock); in vduse_dev_dereg_umem()
1003 if (!dev->umem) in vduse_dev_dereg_umem()
1007 if (!dev->domain) in vduse_dev_dereg_umem()
1010 if (dev->umem->iova != iova || size != dev->domain->bounce_size) in vduse_dev_dereg_umem()
1013 vduse_domain_remove_user_bounce_pages(dev->domain); in vduse_dev_dereg_umem()
1014 unpin_user_pages_dirty_lock(dev->umem->pages, in vduse_dev_dereg_umem()
1015 dev->umem->npages, true); in vduse_dev_dereg_umem()
1016 atomic64_sub(dev->umem->npages, &dev->umem->mm->pinned_vm); in vduse_dev_dereg_umem()
1017 mmdrop(dev->umem->mm); in vduse_dev_dereg_umem()
1018 vfree(dev->umem->pages); in vduse_dev_dereg_umem()
1019 kfree(dev->umem); in vduse_dev_dereg_umem()
1020 dev->umem = NULL; in vduse_dev_dereg_umem()
1023 mutex_unlock(&dev->mem_lock); in vduse_dev_dereg_umem()
1027 static int vduse_dev_reg_umem(struct vduse_dev *dev, in vduse_dev_reg_umem() argument
1036 if (!dev->domain || !dev->domain->bounce_map || in vduse_dev_reg_umem()
1037 size != dev->domain->bounce_size || in vduse_dev_reg_umem()
1041 mutex_lock(&dev->mem_lock); in vduse_dev_reg_umem()
1043 if (dev->umem) in vduse_dev_reg_umem()
1067 ret = vduse_domain_add_user_bounce_pages(dev->domain, in vduse_dev_reg_umem()
1080 dev->umem = umem; in vduse_dev_reg_umem()
1091 mutex_unlock(&dev->mem_lock); in vduse_dev_reg_umem()
1114 struct vduse_dev *dev = file->private_data; in vduse_dev_ioctl() local
1118 if (unlikely(dev->broken)) in vduse_dev_ioctl()
1136 mutex_lock(&dev->domain_lock); in vduse_dev_ioctl()
1137 if (!dev->domain) { in vduse_dev_ioctl()
1138 mutex_unlock(&dev->domain_lock); in vduse_dev_ioctl()
1141 spin_lock(&dev->domain->iotlb_lock); in vduse_dev_ioctl()
1142 map = vhost_iotlb_itree_first(dev->domain->iotlb, in vduse_dev_ioctl()
1152 spin_unlock(&dev->domain->iotlb_lock); in vduse_dev_ioctl()
1153 mutex_unlock(&dev->domain_lock); in vduse_dev_ioctl()
1172 ret = put_user(dev->driver_features, (u64 __user *)argp); in vduse_dev_ioctl()
1184 if (config.offset > dev->config_size || in vduse_dev_ioctl()
1186 config.length > dev->config_size - config.offset) in vduse_dev_ioctl()
1190 if (copy_from_user(dev->config + config.offset, argp + size, in vduse_dev_ioctl()
1198 ret = vduse_dev_queue_irq_work(dev, &dev->inject, IRQ_UNBOUND); in vduse_dev_ioctl()
1209 if (config.index >= dev->vq_num) in vduse_dev_ioctl()
1216 index = array_index_nospec(config.index, dev->vq_num); in vduse_dev_ioctl()
1217 dev->vqs[index]->num_max = config.max_size; in vduse_dev_ioctl()
1231 if (vq_info.index >= dev->vq_num) in vduse_dev_ioctl()
1234 index = array_index_nospec(vq_info.index, dev->vq_num); in vduse_dev_ioctl()
1235 vq = dev->vqs[index]; in vduse_dev_ioctl()
1241 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) { in vduse_dev_ioctl()
1270 ret = vduse_kickfd_setup(dev, &eventfd); in vduse_dev_ioctl()
1281 if (index >= dev->vq_num) in vduse_dev_ioctl()
1285 index = array_index_nospec(index, dev->vq_num); in vduse_dev_ioctl()
1286 if (!vduse_vq_signal_irqfd(dev->vqs[index])) { in vduse_dev_ioctl()
1287 vduse_vq_update_effective_cpu(dev->vqs[index]); in vduse_dev_ioctl()
1288 ret = vduse_dev_queue_irq_work(dev, in vduse_dev_ioctl()
1289 &dev->vqs[index]->inject, in vduse_dev_ioctl()
1290 dev->vqs[index]->irq_effective_cpu); in vduse_dev_ioctl()
1306 mutex_lock(&dev->domain_lock); in vduse_dev_ioctl()
1307 ret = vduse_dev_reg_umem(dev, umem.iova, in vduse_dev_ioctl()
1309 mutex_unlock(&dev->domain_lock); in vduse_dev_ioctl()
1323 mutex_lock(&dev->domain_lock); in vduse_dev_ioctl()
1324 ret = vduse_dev_dereg_umem(dev, umem.iova, in vduse_dev_ioctl()
1326 mutex_unlock(&dev->domain_lock); in vduse_dev_ioctl()
1345 mutex_lock(&dev->domain_lock); in vduse_dev_ioctl()
1346 if (!dev->domain) { in vduse_dev_ioctl()
1347 mutex_unlock(&dev->domain_lock); in vduse_dev_ioctl()
1350 spin_lock(&dev->domain->iotlb_lock); in vduse_dev_ioctl()
1351 map = vhost_iotlb_itree_first(dev->domain->iotlb, in vduse_dev_ioctl()
1357 if (dev->domain->bounce_map && map->start == 0 && in vduse_dev_ioctl()
1358 map->last == dev->domain->bounce_size - 1) in vduse_dev_ioctl()
1361 spin_unlock(&dev->domain->iotlb_lock); in vduse_dev_ioctl()
1362 mutex_unlock(&dev->domain_lock); in vduse_dev_ioctl()
1383 struct vduse_dev *dev = file->private_data; in vduse_dev_release() local
1385 mutex_lock(&dev->domain_lock); in vduse_dev_release()
1386 if (dev->domain) in vduse_dev_release()
1387 vduse_dev_dereg_umem(dev, 0, dev->domain->bounce_size); in vduse_dev_release()
1388 mutex_unlock(&dev->domain_lock); in vduse_dev_release()
1389 spin_lock(&dev->msg_lock); in vduse_dev_release()
1391 list_splice_init(&dev->recv_list, &dev->send_list); in vduse_dev_release()
1392 spin_unlock(&dev->msg_lock); in vduse_dev_release()
1393 dev->connected = false; in vduse_dev_release()
1400 struct vduse_dev *dev; in vduse_dev_get_from_minor() local
1403 dev = idr_find(&vduse_idr, minor); in vduse_dev_get_from_minor()
1406 return dev; in vduse_dev_get_from_minor()
1412 struct vduse_dev *dev = vduse_dev_get_from_minor(iminor(inode)); in vduse_dev_open() local
1414 if (!dev) in vduse_dev_open()
1418 mutex_lock(&dev->lock); in vduse_dev_open()
1419 if (dev->connected) in vduse_dev_open()
1423 dev->connected = true; in vduse_dev_open()
1424 file->private_data = dev; in vduse_dev_open()
1426 mutex_unlock(&dev->lock); in vduse_dev_open()
1533 static void vduse_dev_deinit_vqs(struct vduse_dev *dev) in vduse_dev_deinit_vqs() argument
1537 if (!dev->vqs) in vduse_dev_deinit_vqs()
1540 for (i = 0; i < dev->vq_num; i++) in vduse_dev_deinit_vqs()
1541 kobject_put(&dev->vqs[i]->kobj); in vduse_dev_deinit_vqs()
1542 kfree(dev->vqs); in vduse_dev_deinit_vqs()
1545 static int vduse_dev_init_vqs(struct vduse_dev *dev, u32 vq_align, u32 vq_num) in vduse_dev_init_vqs() argument
1549 dev->vq_align = vq_align; in vduse_dev_init_vqs()
1550 dev->vq_num = vq_num; in vduse_dev_init_vqs()
1551 dev->vqs = kcalloc(dev->vq_num, sizeof(*dev->vqs), GFP_KERNEL); in vduse_dev_init_vqs()
1552 if (!dev->vqs) in vduse_dev_init_vqs()
1556 dev->vqs[i] = kzalloc(sizeof(*dev->vqs[i]), GFP_KERNEL); in vduse_dev_init_vqs()
1557 if (!dev->vqs[i]) { in vduse_dev_init_vqs()
1562 dev->vqs[i]->index = i; in vduse_dev_init_vqs()
1563 dev->vqs[i]->irq_effective_cpu = IRQ_UNBOUND; in vduse_dev_init_vqs()
1564 INIT_WORK(&dev->vqs[i]->inject, vduse_vq_irq_inject); in vduse_dev_init_vqs()
1565 INIT_WORK(&dev->vqs[i]->kick, vduse_vq_kick_work); in vduse_dev_init_vqs()
1566 spin_lock_init(&dev->vqs[i]->kick_lock); in vduse_dev_init_vqs()
1567 spin_lock_init(&dev->vqs[i]->irq_lock); in vduse_dev_init_vqs()
1568 cpumask_setall(&dev->vqs[i]->irq_affinity); in vduse_dev_init_vqs()
1570 kobject_init(&dev->vqs[i]->kobj, &vq_type); in vduse_dev_init_vqs()
1571 ret = kobject_add(&dev->vqs[i]->kobj, in vduse_dev_init_vqs()
1572 &dev->dev->kobj, "vq%d", i); in vduse_dev_init_vqs()
1574 kfree(dev->vqs[i]); in vduse_dev_init_vqs()
1582 kobject_put(&dev->vqs[i]->kobj); in vduse_dev_init_vqs()
1583 kfree(dev->vqs); in vduse_dev_init_vqs()
1584 dev->vqs = NULL; in vduse_dev_init_vqs()
1590 struct vduse_dev *dev = kzalloc(sizeof(*dev), GFP_KERNEL); in vduse_dev_create() local
1592 if (!dev) in vduse_dev_create()
1595 mutex_init(&dev->lock); in vduse_dev_create()
1596 mutex_init(&dev->mem_lock); in vduse_dev_create()
1597 mutex_init(&dev->domain_lock); in vduse_dev_create()
1598 spin_lock_init(&dev->msg_lock); in vduse_dev_create()
1599 INIT_LIST_HEAD(&dev->send_list); in vduse_dev_create()
1600 INIT_LIST_HEAD(&dev->recv_list); in vduse_dev_create()
1601 spin_lock_init(&dev->irq_lock); in vduse_dev_create()
1602 init_rwsem(&dev->rwsem); in vduse_dev_create()
1604 INIT_WORK(&dev->inject, vduse_dev_irq_inject); in vduse_dev_create()
1605 init_waitqueue_head(&dev->waitq); in vduse_dev_create()
1607 return dev; in vduse_dev_create()
1610 static void vduse_dev_destroy(struct vduse_dev *dev) in vduse_dev_destroy() argument
1612 kfree(dev); in vduse_dev_destroy()
1617 struct vduse_dev *dev; in vduse_find_dev() local
1620 idr_for_each_entry(&vduse_idr, dev, id) in vduse_find_dev()
1621 if (!strcmp(dev->name, name)) in vduse_find_dev()
1622 return dev; in vduse_find_dev()
1629 struct vduse_dev *dev = vduse_find_dev(name); in vduse_destroy_dev() local
1631 if (!dev) in vduse_destroy_dev()
1634 mutex_lock(&dev->lock); in vduse_destroy_dev()
1635 if (dev->vdev || dev->connected) { in vduse_destroy_dev()
1636 mutex_unlock(&dev->lock); in vduse_destroy_dev()
1639 dev->connected = true; in vduse_destroy_dev()
1640 mutex_unlock(&dev->lock); in vduse_destroy_dev()
1642 vduse_dev_reset(dev); in vduse_destroy_dev()
1643 device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor)); in vduse_destroy_dev()
1644 idr_remove(&vduse_idr, dev->minor); in vduse_destroy_dev()
1645 kvfree(dev->config); in vduse_destroy_dev()
1646 vduse_dev_deinit_vqs(dev); in vduse_destroy_dev()
1647 if (dev->domain) in vduse_destroy_dev()
1648 vduse_domain_destroy(dev->domain); in vduse_destroy_dev()
1649 kfree(dev->name); in vduse_destroy_dev()
1650 vduse_dev_destroy(dev); in vduse_destroy_dev()
1713 struct vduse_dev *dev = dev_get_drvdata(device); in msg_timeout_show() local
1715 return sysfs_emit(buf, "%u\n", dev->msg_timeout); in msg_timeout_show()
1722 struct vduse_dev *dev = dev_get_drvdata(device); in msg_timeout_store() local
1725 ret = kstrtouint(buf, 10, &dev->msg_timeout); in msg_timeout_store()
1737 struct vduse_dev *dev = dev_get_drvdata(device); in bounce_size_show() local
1739 return sysfs_emit(buf, "%u\n", dev->bounce_size); in bounce_size_show()
1746 struct vduse_dev *dev = dev_get_drvdata(device); in bounce_size_store() local
1751 mutex_lock(&dev->domain_lock); in bounce_size_store()
1752 if (dev->domain) in bounce_size_store()
1764 dev->bounce_size = bounce_size & PAGE_MASK; in bounce_size_store()
1767 mutex_unlock(&dev->domain_lock); in bounce_size_store()
1785 struct vduse_dev *dev; in vduse_create_dev() local
1792 dev = vduse_dev_create(); in vduse_create_dev()
1793 if (!dev) in vduse_create_dev()
1796 dev->api_version = api_version; in vduse_create_dev()
1797 dev->device_features = config->features; in vduse_create_dev()
1798 dev->device_id = config->device_id; in vduse_create_dev()
1799 dev->vendor_id = config->vendor_id; in vduse_create_dev()
1800 dev->name = kstrdup(config->name, GFP_KERNEL); in vduse_create_dev()
1801 if (!dev->name) in vduse_create_dev()
1804 dev->bounce_size = VDUSE_BOUNCE_SIZE; in vduse_create_dev()
1805 dev->config = config_buf; in vduse_create_dev()
1806 dev->config_size = config->config_size; in vduse_create_dev()
1808 ret = idr_alloc(&vduse_idr, dev, 1, VDUSE_DEV_MAX, GFP_KERNEL); in vduse_create_dev()
1812 dev->minor = ret; in vduse_create_dev()
1813 dev->msg_timeout = VDUSE_MSG_DEFAULT_TIMEOUT; in vduse_create_dev()
1814 dev->dev = device_create_with_groups(vduse_class, NULL, in vduse_create_dev()
1815 MKDEV(MAJOR(vduse_major), dev->minor), in vduse_create_dev()
1816 dev, vduse_dev_groups, "%s", config->name); in vduse_create_dev()
1817 if (IS_ERR(dev->dev)) { in vduse_create_dev()
1818 ret = PTR_ERR(dev->dev); in vduse_create_dev()
1822 ret = vduse_dev_init_vqs(dev, config->vq_align, config->vq_num); in vduse_create_dev()
1830 device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor)); in vduse_create_dev()
1832 idr_remove(&vduse_idr, dev->minor); in vduse_create_dev()
1834 kfree(dev->name); in vduse_create_dev()
1836 vduse_dev_destroy(dev); in vduse_create_dev()
1943 static char *vduse_devnode(const struct device *dev, umode_t *mode) in vduse_devnode() argument
1945 return kasprintf(GFP_KERNEL, "vduse/%s", dev_name(dev)); in vduse_devnode()
1950 struct device dev; member
1955 static int vduse_dev_init_vdpa(struct vduse_dev *dev, const char *name) in vduse_dev_init_vdpa() argument
1960 if (dev->vdev) in vduse_dev_init_vdpa()
1963 vdev = vdpa_alloc_device(struct vduse_vdpa, vdpa, dev->dev, in vduse_dev_init_vdpa()
1968 dev->vdev = vdev; in vduse_dev_init_vdpa()
1969 vdev->dev = dev; in vduse_dev_init_vdpa()
1970 vdev->vdpa.dev.dma_mask = &vdev->vdpa.dev.coherent_dma_mask; in vduse_dev_init_vdpa()
1971 ret = dma_set_mask_and_coherent(&vdev->vdpa.dev, DMA_BIT_MASK(64)); in vduse_dev_init_vdpa()
1973 put_device(&vdev->vdpa.dev); in vduse_dev_init_vdpa()
1976 set_dma_ops(&vdev->vdpa.dev, &vduse_dev_dma_ops); in vduse_dev_init_vdpa()
1977 vdev->vdpa.dma_dev = &vdev->vdpa.dev; in vduse_dev_init_vdpa()
1986 struct vduse_dev *dev; in vdpa_dev_add() local
1990 dev = vduse_find_dev(name); in vdpa_dev_add()
1991 if (!dev || !vduse_dev_is_ready(dev)) { in vdpa_dev_add()
1995 ret = vduse_dev_init_vdpa(dev, name); in vdpa_dev_add()
2000 mutex_lock(&dev->domain_lock); in vdpa_dev_add()
2001 if (!dev->domain) in vdpa_dev_add()
2002 dev->domain = vduse_domain_create(VDUSE_IOVA_SIZE - 1, in vdpa_dev_add()
2003 dev->bounce_size); in vdpa_dev_add()
2004 mutex_unlock(&dev->domain_lock); in vdpa_dev_add()
2005 if (!dev->domain) { in vdpa_dev_add()
2006 put_device(&dev->vdev->vdpa.dev); in vdpa_dev_add()
2010 ret = _vdpa_register_device(&dev->vdev->vdpa, dev->vq_num); in vdpa_dev_add()
2012 put_device(&dev->vdev->vdpa.dev); in vdpa_dev_add()
2013 mutex_lock(&dev->domain_lock); in vdpa_dev_add()
2014 vduse_domain_destroy(dev->domain); in vdpa_dev_add()
2015 dev->domain = NULL; in vdpa_dev_add()
2016 mutex_unlock(&dev->domain_lock); in vdpa_dev_add()
2023 static void vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev) in vdpa_dev_del() argument
2025 _vdpa_unregister_device(dev); in vdpa_dev_del()
2038 static void vduse_mgmtdev_release(struct device *dev) in vduse_mgmtdev_release() argument
2042 mgmt_dev = container_of(dev, struct vduse_mgmt_dev, dev); in vduse_mgmtdev_release()
2054 ret = dev_set_name(&vduse_mgmt->dev, "vduse"); in vduse_mgmtdev_init()
2060 vduse_mgmt->dev.release = vduse_mgmtdev_release; in vduse_mgmtdev_init()
2062 ret = device_register(&vduse_mgmt->dev); in vduse_mgmtdev_init()
2068 vduse_mgmt->mgmt_dev.device = &vduse_mgmt->dev; in vduse_mgmtdev_init()
2071 device_unregister(&vduse_mgmt->dev); in vduse_mgmtdev_init()
2076 put_device(&vduse_mgmt->dev); in vduse_mgmtdev_init()
2083 device_unregister(&vduse_mgmt->dev); in vduse_mgmtdev_exit()
2089 struct device *dev; in vduse_init() local
2108 dev = device_create(vduse_class, NULL, vduse_major, NULL, "control"); in vduse_init()
2109 if (IS_ERR(dev)) { in vduse_init()
2110 ret = PTR_ERR(dev); in vduse_init()