1c9b9f5f8SMichael S. Tsirkin // SPDX-License-Identifier: GPL-2.0-only
2c9b9f5f8SMichael S. Tsirkin /*
3db1e8bb6SMax Gurtovoy * VDPA device simulator core.
4c9b9f5f8SMichael S. Tsirkin *
5c9b9f5f8SMichael S. Tsirkin * Copyright (c) 2020, Red Hat Inc. All rights reserved.
6c9b9f5f8SMichael S. Tsirkin * Author: Jason Wang <jasowang@redhat.com>
7c9b9f5f8SMichael S. Tsirkin *
8c9b9f5f8SMichael S. Tsirkin */
9c9b9f5f8SMichael S. Tsirkin
10c9b9f5f8SMichael S. Tsirkin #include <linux/init.h>
11c9b9f5f8SMichael S. Tsirkin #include <linux/module.h>
12c9b9f5f8SMichael S. Tsirkin #include <linux/device.h>
13c9b9f5f8SMichael S. Tsirkin #include <linux/kernel.h>
1476acfa7bSStefano Garzarella #include <linux/kthread.h>
15c9b9f5f8SMichael S. Tsirkin #include <linux/slab.h>
160a0f0d8bSChristoph Hellwig #include <linux/dma-map-ops.h>
17c9b9f5f8SMichael S. Tsirkin #include <linux/vringh.h>
18c9b9f5f8SMichael S. Tsirkin #include <linux/vdpa.h>
19c9b9f5f8SMichael S. Tsirkin #include <linux/vhost_iotlb.h>
20477f7197SJason Wang #include <uapi/linux/vdpa.h>
212c9c6371SEugenio Pérez #include <uapi/linux/vhost_types.h>
22db1e8bb6SMax Gurtovoy
23db1e8bb6SMax Gurtovoy #include "vdpa_sim.h"
24c9b9f5f8SMichael S. Tsirkin
25c9b9f5f8SMichael S. Tsirkin #define DRV_VERSION "0.1"
26c9b9f5f8SMichael S. Tsirkin #define DRV_AUTHOR "Jason Wang <jasowang@redhat.com>"
27db1e8bb6SMax Gurtovoy #define DRV_DESC "vDPA Device Simulator core"
28c9b9f5f8SMichael S. Tsirkin #define DRV_LICENSE "GPL v2"
29c9b9f5f8SMichael S. Tsirkin
30de91a4d0SJason Wang static int batch_mapping = 1;
31de91a4d0SJason Wang module_param(batch_mapping, int, 0444);
32de91a4d0SJason Wang MODULE_PARM_DESC(batch_mapping, "Batched mapping 1 -Enable; 0 - Disable");
33de91a4d0SJason Wang
342fc0ebfaSStefano Garzarella static int max_iotlb_entries = 2048;
352fc0ebfaSStefano Garzarella module_param(max_iotlb_entries, int, 0444);
362fc0ebfaSStefano Garzarella MODULE_PARM_DESC(max_iotlb_entries,
3767f8f10cSStefano Garzarella "Maximum number of iotlb entries for each address space. 0 means unlimited. (default: 2048)");
382fc0ebfaSStefano Garzarella
394bb94d2dSStefano Garzarella static bool use_va = true;
404bb94d2dSStefano Garzarella module_param(use_va, bool, 0444);
414bb94d2dSStefano Garzarella MODULE_PARM_DESC(use_va, "Enable/disable the device's ability to use VA");
424bb94d2dSStefano Garzarella
43c9b9f5f8SMichael S. Tsirkin #define VDPASIM_QUEUE_ALIGN PAGE_SIZE
44c9b9f5f8SMichael S. Tsirkin #define VDPASIM_QUEUE_MAX 256
45c9b9f5f8SMichael S. Tsirkin #define VDPASIM_VENDOR_ID 0
46c9b9f5f8SMichael S. Tsirkin
474bb94d2dSStefano Garzarella struct vdpasim_mm_work {
484bb94d2dSStefano Garzarella struct kthread_work work;
494bb94d2dSStefano Garzarella struct vdpasim *vdpasim;
504bb94d2dSStefano Garzarella struct mm_struct *mm_to_bind;
514bb94d2dSStefano Garzarella int ret;
524bb94d2dSStefano Garzarella };
534bb94d2dSStefano Garzarella
vdpasim_mm_work_fn(struct kthread_work * work)544bb94d2dSStefano Garzarella static void vdpasim_mm_work_fn(struct kthread_work *work)
554bb94d2dSStefano Garzarella {
564bb94d2dSStefano Garzarella struct vdpasim_mm_work *mm_work =
574bb94d2dSStefano Garzarella container_of(work, struct vdpasim_mm_work, work);
584bb94d2dSStefano Garzarella struct vdpasim *vdpasim = mm_work->vdpasim;
594bb94d2dSStefano Garzarella
604bb94d2dSStefano Garzarella mm_work->ret = 0;
614bb94d2dSStefano Garzarella
624bb94d2dSStefano Garzarella //TODO: should we attach the cgroup of the mm owner?
634bb94d2dSStefano Garzarella vdpasim->mm_bound = mm_work->mm_to_bind;
644bb94d2dSStefano Garzarella }
654bb94d2dSStefano Garzarella
vdpasim_worker_change_mm_sync(struct vdpasim * vdpasim,struct vdpasim_mm_work * mm_work)664bb94d2dSStefano Garzarella static void vdpasim_worker_change_mm_sync(struct vdpasim *vdpasim,
674bb94d2dSStefano Garzarella struct vdpasim_mm_work *mm_work)
684bb94d2dSStefano Garzarella {
694bb94d2dSStefano Garzarella struct kthread_work *work = &mm_work->work;
704bb94d2dSStefano Garzarella
714bb94d2dSStefano Garzarella kthread_init_work(work, vdpasim_mm_work_fn);
724bb94d2dSStefano Garzarella kthread_queue_work(vdpasim->worker, work);
734bb94d2dSStefano Garzarella
744bb94d2dSStefano Garzarella kthread_flush_work(work);
754bb94d2dSStefano Garzarella }
764bb94d2dSStefano Garzarella
vdpa_to_sim(struct vdpa_device * vdpa)77c9b9f5f8SMichael S. Tsirkin static struct vdpasim *vdpa_to_sim(struct vdpa_device *vdpa)
78c9b9f5f8SMichael S. Tsirkin {
79c9b9f5f8SMichael S. Tsirkin return container_of(vdpa, struct vdpasim, vdpa);
80c9b9f5f8SMichael S. Tsirkin }
81c9b9f5f8SMichael S. Tsirkin
vdpasim_vq_notify(struct vringh * vring)82b240491bSStefano Garzarella static void vdpasim_vq_notify(struct vringh *vring)
83b240491bSStefano Garzarella {
84b240491bSStefano Garzarella struct vdpasim_virtqueue *vq =
85b240491bSStefano Garzarella container_of(vring, struct vdpasim_virtqueue, vring);
86b240491bSStefano Garzarella
87b240491bSStefano Garzarella if (!vq->cb)
88b240491bSStefano Garzarella return;
89b240491bSStefano Garzarella
90b240491bSStefano Garzarella vq->cb(vq->private);
91b240491bSStefano Garzarella }
92b240491bSStefano Garzarella
vdpasim_queue_ready(struct vdpasim * vdpasim,unsigned int idx)93c9b9f5f8SMichael S. Tsirkin static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx)
94c9b9f5f8SMichael S. Tsirkin {
95c9b9f5f8SMichael S. Tsirkin struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
960e84f918SEugenio Pérez uint16_t last_avail_idx = vq->vring.last_avail_idx;
974bb94d2dSStefano Garzarella struct vring_desc *desc = (struct vring_desc *)
984bb94d2dSStefano Garzarella (uintptr_t)vq->desc_addr;
994bb94d2dSStefano Garzarella struct vring_avail *avail = (struct vring_avail *)
1004bb94d2dSStefano Garzarella (uintptr_t)vq->driver_addr;
1014bb94d2dSStefano Garzarella struct vring_used *used = (struct vring_used *)
1024bb94d2dSStefano Garzarella (uintptr_t)vq->device_addr;
103c9b9f5f8SMichael S. Tsirkin
1044bb94d2dSStefano Garzarella if (use_va && vdpasim->mm_bound) {
1054bb94d2dSStefano Garzarella vringh_init_iotlb_va(&vq->vring, vdpasim->features, vq->num,
1064bb94d2dSStefano Garzarella true, desc, avail, used);
1074bb94d2dSStefano Garzarella } else {
1084bb94d2dSStefano Garzarella vringh_init_iotlb(&vq->vring, vdpasim->features, vq->num,
1094bb94d2dSStefano Garzarella true, desc, avail, used);
1104bb94d2dSStefano Garzarella }
111b240491bSStefano Garzarella
1120e84f918SEugenio Pérez vq->vring.last_avail_idx = last_avail_idx;
113b4cca6d4SEugenio Pérez
114b4cca6d4SEugenio Pérez /*
115b4cca6d4SEugenio Pérez * Since vdpa_sim does not support receive inflight descriptors as a
116b4cca6d4SEugenio Pérez * destination of a migration, let's set both avail_idx and used_idx
117b4cca6d4SEugenio Pérez * the same at vq start. This is how vhost-user works in a
118b4cca6d4SEugenio Pérez * VHOST_SET_VRING_BASE call.
119b4cca6d4SEugenio Pérez *
120b4cca6d4SEugenio Pérez * Although the simple fix is to set last_used_idx at
121b4cca6d4SEugenio Pérez * vdpasim_set_vq_state, it would be reset at vdpasim_queue_ready.
122b4cca6d4SEugenio Pérez */
123b4cca6d4SEugenio Pérez vq->vring.last_used_idx = last_avail_idx;
124b240491bSStefano Garzarella vq->vring.notify = vdpasim_vq_notify;
125c9b9f5f8SMichael S. Tsirkin }
126c9b9f5f8SMichael S. Tsirkin
vdpasim_vq_reset(struct vdpasim * vdpasim,struct vdpasim_virtqueue * vq)127011c35baSStefano Garzarella static void vdpasim_vq_reset(struct vdpasim *vdpasim,
128011c35baSStefano Garzarella struct vdpasim_virtqueue *vq)
129c9b9f5f8SMichael S. Tsirkin {
13018e643cdSSamuel Zou vq->ready = false;
131c9b9f5f8SMichael S. Tsirkin vq->desc_addr = 0;
132c9b9f5f8SMichael S. Tsirkin vq->driver_addr = 0;
133c9b9f5f8SMichael S. Tsirkin vq->device_addr = 0;
134c9b9f5f8SMichael S. Tsirkin vq->cb = NULL;
135c9b9f5f8SMichael S. Tsirkin vq->private = NULL;
136011c35baSStefano Garzarella vringh_init_iotlb(&vq->vring, vdpasim->dev_attr.supported_features,
137011c35baSStefano Garzarella VDPASIM_QUEUE_MAX, false, NULL, NULL, NULL);
138b240491bSStefano Garzarella
139b240491bSStefano Garzarella vq->vring.notify = NULL;
140c9b9f5f8SMichael S. Tsirkin }
141c9b9f5f8SMichael S. Tsirkin
vdpasim_do_reset(struct vdpasim * vdpasim)1420686082dSXie Yongji static void vdpasim_do_reset(struct vdpasim *vdpasim)
143c9b9f5f8SMichael S. Tsirkin {
144c9b9f5f8SMichael S. Tsirkin int i;
145c9b9f5f8SMichael S. Tsirkin
1460ea9ee43SMax Gurtovoy spin_lock(&vdpasim->iommu_lock);
147bda324fdSGautam Dawar
148bda324fdSGautam Dawar for (i = 0; i < vdpasim->dev_attr.nvqs; i++) {
149bda324fdSGautam Dawar vdpasim_vq_reset(vdpasim, &vdpasim->vqs[i]);
150bda324fdSGautam Dawar vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0],
151bda324fdSGautam Dawar &vdpasim->iommu_lock);
152bda324fdSGautam Dawar }
153bda324fdSGautam Dawar
1546c3d329eSJason Wang for (i = 0; i < vdpasim->dev_attr.nas; i++) {
155bda324fdSGautam Dawar vhost_iotlb_reset(&vdpasim->iommu[i]);
1566c3d329eSJason Wang vhost_iotlb_add_range(&vdpasim->iommu[i], 0, ULONG_MAX,
1576c3d329eSJason Wang 0, VHOST_MAP_RW);
1586c3d329eSJason Wang vdpasim->iommu_pt[i] = true;
1596c3d329eSJason Wang }
160bda324fdSGautam Dawar
161*cddd85daSSteve Sistare vdpasim->running = false;
1620ea9ee43SMax Gurtovoy spin_unlock(&vdpasim->iommu_lock);
163c9b9f5f8SMichael S. Tsirkin
164c9b9f5f8SMichael S. Tsirkin vdpasim->features = 0;
165c9b9f5f8SMichael S. Tsirkin vdpasim->status = 0;
166c9b9f5f8SMichael S. Tsirkin ++vdpasim->generation;
167c9b9f5f8SMichael S. Tsirkin }
168c9b9f5f8SMichael S. Tsirkin
16936a9c306SStefano Garzarella static const struct vdpa_config_ops vdpasim_config_ops;
17036a9c306SStefano Garzarella static const struct vdpa_config_ops vdpasim_batch_config_ops;
171c9b9f5f8SMichael S. Tsirkin
vdpasim_work_fn(struct kthread_work * work)17276acfa7bSStefano Garzarella static void vdpasim_work_fn(struct kthread_work *work)
173e2a4f808SStefano Garzarella {
174e2a4f808SStefano Garzarella struct vdpasim *vdpasim = container_of(work, struct vdpasim, work);
1754bb94d2dSStefano Garzarella struct mm_struct *mm = vdpasim->mm_bound;
1764bb94d2dSStefano Garzarella
1774bb94d2dSStefano Garzarella if (use_va && mm) {
1784bb94d2dSStefano Garzarella if (!mmget_not_zero(mm))
1794bb94d2dSStefano Garzarella return;
1804bb94d2dSStefano Garzarella kthread_use_mm(mm);
1814bb94d2dSStefano Garzarella }
182e2a4f808SStefano Garzarella
183e2a4f808SStefano Garzarella vdpasim->dev_attr.work_fn(vdpasim);
1844bb94d2dSStefano Garzarella
1854bb94d2dSStefano Garzarella if (use_va && mm) {
1864bb94d2dSStefano Garzarella kthread_unuse_mm(mm);
1874bb94d2dSStefano Garzarella mmput(mm);
1884bb94d2dSStefano Garzarella }
189e2a4f808SStefano Garzarella }
190e2a4f808SStefano Garzarella
vdpasim_create(struct vdpasim_dev_attr * dev_attr,const struct vdpa_dev_set_config * config)191477f7197SJason Wang struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,
192477f7197SJason Wang const struct vdpa_dev_set_config *config)
193c9b9f5f8SMichael S. Tsirkin {
194de91a4d0SJason Wang const struct vdpa_config_ops *ops;
1950497f23eSJason Wang struct vdpa_device *vdpa;
196c9b9f5f8SMichael S. Tsirkin struct vdpasim *vdpasim;
197c9b9f5f8SMichael S. Tsirkin struct device *dev;
198423248d6SMax Gurtovoy int i, ret = -ENOMEM;
199c9b9f5f8SMichael S. Tsirkin
200bb105d51SJason Wang if (!dev_attr->alloc_size)
201bb105d51SJason Wang return ERR_PTR(-EINVAL);
202bb105d51SJason Wang
203477f7197SJason Wang if (config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {
204477f7197SJason Wang if (config->device_features &
205477f7197SJason Wang ~dev_attr->supported_features)
206477f7197SJason Wang return ERR_PTR(-EINVAL);
207477f7197SJason Wang dev_attr->supported_features =
208477f7197SJason Wang config->device_features;
209477f7197SJason Wang }
210477f7197SJason Wang
211de91a4d0SJason Wang if (batch_mapping)
21236a9c306SStefano Garzarella ops = &vdpasim_batch_config_ops;
213de91a4d0SJason Wang else
21436a9c306SStefano Garzarella ops = &vdpasim_config_ops;
215de91a4d0SJason Wang
2160497f23eSJason Wang vdpa = __vdpa_alloc_device(NULL, ops,
217bda324fdSGautam Dawar dev_attr->ngroups, dev_attr->nas,
218bb105d51SJason Wang dev_attr->alloc_size,
2194bb94d2dSStefano Garzarella dev_attr->name, use_va);
2200497f23eSJason Wang if (IS_ERR(vdpa)) {
2210497f23eSJason Wang ret = PTR_ERR(vdpa);
222c9b9f5f8SMichael S. Tsirkin goto err_alloc;
2232b847f21SXie Yongji }
224c9b9f5f8SMichael S. Tsirkin
2250497f23eSJason Wang vdpasim = vdpa_to_sim(vdpa);
2266c6e28feSStefano Garzarella vdpasim->dev_attr = *dev_attr;
22776acfa7bSStefano Garzarella dev = &vdpasim->vdpa.dev;
22876acfa7bSStefano Garzarella
22976acfa7bSStefano Garzarella kthread_init_work(&vdpasim->work, vdpasim_work_fn);
23076acfa7bSStefano Garzarella vdpasim->worker = kthread_create_worker(0, "vDPA sim worker: %s",
23176acfa7bSStefano Garzarella dev_attr->name);
23276acfa7bSStefano Garzarella if (IS_ERR(vdpasim->worker))
23376acfa7bSStefano Garzarella goto err_iommu;
23476acfa7bSStefano Garzarella
235d7621c28SStefano Garzarella mutex_init(&vdpasim->mutex);
2361e3e7926SMichael S. Tsirkin spin_lock_init(&vdpasim->iommu_lock);
237c9b9f5f8SMichael S. Tsirkin
2381eca16b2SLaurent Vivier dev->dma_mask = &dev->coherent_dma_mask;
2391eca16b2SLaurent Vivier if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)))
2401eca16b2SLaurent Vivier goto err_iommu;
241a3c06ae1SParav Pandit vdpasim->vdpa.mdev = dev_attr->mgmt_dev;
242c9b9f5f8SMichael S. Tsirkin
243f37cbbc6SStefano Garzarella vdpasim->config = kzalloc(dev_attr->config_size, GFP_KERNEL);
244f37cbbc6SStefano Garzarella if (!vdpasim->config)
245f37cbbc6SStefano Garzarella goto err_iommu;
246f37cbbc6SStefano Garzarella
2476c6e28feSStefano Garzarella vdpasim->vqs = kcalloc(dev_attr->nvqs, sizeof(struct vdpasim_virtqueue),
248423248d6SMax Gurtovoy GFP_KERNEL);
249423248d6SMax Gurtovoy if (!vdpasim->vqs)
250423248d6SMax Gurtovoy goto err_iommu;
251423248d6SMax Gurtovoy
252bda324fdSGautam Dawar vdpasim->iommu = kmalloc_array(vdpasim->dev_attr.nas,
253bda324fdSGautam Dawar sizeof(*vdpasim->iommu), GFP_KERNEL);
254c9b9f5f8SMichael S. Tsirkin if (!vdpasim->iommu)
255c9b9f5f8SMichael S. Tsirkin goto err_iommu;
256c9b9f5f8SMichael S. Tsirkin
2576c3d329eSJason Wang vdpasim->iommu_pt = kmalloc_array(vdpasim->dev_attr.nas,
2586c3d329eSJason Wang sizeof(*vdpasim->iommu_pt), GFP_KERNEL);
2596c3d329eSJason Wang if (!vdpasim->iommu_pt)
2606c3d329eSJason Wang goto err_iommu;
2616c3d329eSJason Wang
262bda324fdSGautam Dawar for (i = 0; i < vdpasim->dev_attr.nas; i++)
26367f8f10cSStefano Garzarella vhost_iotlb_init(&vdpasim->iommu[i], max_iotlb_entries, 0);
264bda324fdSGautam Dawar
2656c6e28feSStefano Garzarella for (i = 0; i < dev_attr->nvqs; i++)
266bda324fdSGautam Dawar vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0],
267f53d9910SStefano Garzarella &vdpasim->iommu_lock);
268c9b9f5f8SMichael S. Tsirkin
269c9b9f5f8SMichael S. Tsirkin vdpasim->vdpa.dma_dev = dev;
270c9b9f5f8SMichael S. Tsirkin
271c9b9f5f8SMichael S. Tsirkin return vdpasim;
272c9b9f5f8SMichael S. Tsirkin
273c9b9f5f8SMichael S. Tsirkin err_iommu:
274c9b9f5f8SMichael S. Tsirkin put_device(dev);
275c9b9f5f8SMichael S. Tsirkin err_alloc:
276c9b9f5f8SMichael S. Tsirkin return ERR_PTR(ret);
277c9b9f5f8SMichael S. Tsirkin }
278db1e8bb6SMax Gurtovoy EXPORT_SYMBOL_GPL(vdpasim_create);
279c9b9f5f8SMichael S. Tsirkin
vdpasim_schedule_work(struct vdpasim * vdpasim)280e2a4f808SStefano Garzarella void vdpasim_schedule_work(struct vdpasim *vdpasim)
281e2a4f808SStefano Garzarella {
28276acfa7bSStefano Garzarella kthread_queue_work(vdpasim->worker, &vdpasim->work);
283e2a4f808SStefano Garzarella }
284e2a4f808SStefano Garzarella EXPORT_SYMBOL_GPL(vdpasim_schedule_work);
285e2a4f808SStefano Garzarella
vdpasim_set_vq_address(struct vdpa_device * vdpa,u16 idx,u64 desc_area,u64 driver_area,u64 device_area)286c9b9f5f8SMichael S. Tsirkin static int vdpasim_set_vq_address(struct vdpa_device *vdpa, u16 idx,
287c9b9f5f8SMichael S. Tsirkin u64 desc_area, u64 driver_area,
288c9b9f5f8SMichael S. Tsirkin u64 device_area)
289c9b9f5f8SMichael S. Tsirkin {
290c9b9f5f8SMichael S. Tsirkin struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
291c9b9f5f8SMichael S. Tsirkin struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
292c9b9f5f8SMichael S. Tsirkin
293c9b9f5f8SMichael S. Tsirkin vq->desc_addr = desc_area;
294c9b9f5f8SMichael S. Tsirkin vq->driver_addr = driver_area;
295c9b9f5f8SMichael S. Tsirkin vq->device_addr = device_area;
296c9b9f5f8SMichael S. Tsirkin
297c9b9f5f8SMichael S. Tsirkin return 0;
298c9b9f5f8SMichael S. Tsirkin }
299c9b9f5f8SMichael S. Tsirkin
vdpasim_set_vq_num(struct vdpa_device * vdpa,u16 idx,u32 num)300c9b9f5f8SMichael S. Tsirkin static void vdpasim_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num)
301c9b9f5f8SMichael S. Tsirkin {
302c9b9f5f8SMichael S. Tsirkin struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
303c9b9f5f8SMichael S. Tsirkin struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
304c9b9f5f8SMichael S. Tsirkin
305c9b9f5f8SMichael S. Tsirkin vq->num = num;
306c9b9f5f8SMichael S. Tsirkin }
307c9b9f5f8SMichael S. Tsirkin
vdpasim_kick_vq(struct vdpa_device * vdpa,u16 idx)308c9b9f5f8SMichael S. Tsirkin static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
309c9b9f5f8SMichael S. Tsirkin {
310c9b9f5f8SMichael S. Tsirkin struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
311c9b9f5f8SMichael S. Tsirkin struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
312c9b9f5f8SMichael S. Tsirkin
313f9d9f57eSSebastien Boeuf if (!vdpasim->running &&
314f9d9f57eSSebastien Boeuf (vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
315f9d9f57eSSebastien Boeuf vdpasim->pending_kick = true;
316f9d9f57eSSebastien Boeuf return;
317f9d9f57eSSebastien Boeuf }
318f9d9f57eSSebastien Boeuf
319c9b9f5f8SMichael S. Tsirkin if (vq->ready)
320e2a4f808SStefano Garzarella vdpasim_schedule_work(vdpasim);
321c9b9f5f8SMichael S. Tsirkin }
322c9b9f5f8SMichael S. Tsirkin
vdpasim_set_vq_cb(struct vdpa_device * vdpa,u16 idx,struct vdpa_callback * cb)323c9b9f5f8SMichael S. Tsirkin static void vdpasim_set_vq_cb(struct vdpa_device *vdpa, u16 idx,
324c9b9f5f8SMichael S. Tsirkin struct vdpa_callback *cb)
325c9b9f5f8SMichael S. Tsirkin {
326c9b9f5f8SMichael S. Tsirkin struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
327c9b9f5f8SMichael S. Tsirkin struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
328c9b9f5f8SMichael S. Tsirkin
329c9b9f5f8SMichael S. Tsirkin vq->cb = cb->callback;
330c9b9f5f8SMichael S. Tsirkin vq->private = cb->private;
331c9b9f5f8SMichael S. Tsirkin }
332c9b9f5f8SMichael S. Tsirkin
vdpasim_set_vq_ready(struct vdpa_device * vdpa,u16 idx,bool ready)333c9b9f5f8SMichael S. Tsirkin static void vdpasim_set_vq_ready(struct vdpa_device *vdpa, u16 idx, bool ready)
334c9b9f5f8SMichael S. Tsirkin {
335c9b9f5f8SMichael S. Tsirkin struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
336c9b9f5f8SMichael S. Tsirkin struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
33724243697SEugenio Pérez bool old_ready;
338c9b9f5f8SMichael S. Tsirkin
339d7621c28SStefano Garzarella mutex_lock(&vdpasim->mutex);
34024243697SEugenio Pérez old_ready = vq->ready;
341c9b9f5f8SMichael S. Tsirkin vq->ready = ready;
34224243697SEugenio Pérez if (vq->ready && !old_ready) {
343c9b9f5f8SMichael S. Tsirkin vdpasim_queue_ready(vdpasim, idx);
34424243697SEugenio Pérez }
345d7621c28SStefano Garzarella mutex_unlock(&vdpasim->mutex);
346c9b9f5f8SMichael S. Tsirkin }
347c9b9f5f8SMichael S. Tsirkin
vdpasim_get_vq_ready(struct vdpa_device * vdpa,u16 idx)348c9b9f5f8SMichael S. Tsirkin static bool vdpasim_get_vq_ready(struct vdpa_device *vdpa, u16 idx)
349c9b9f5f8SMichael S. Tsirkin {
350c9b9f5f8SMichael S. Tsirkin struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
351c9b9f5f8SMichael S. Tsirkin struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
352c9b9f5f8SMichael S. Tsirkin
353c9b9f5f8SMichael S. Tsirkin return vq->ready;
354c9b9f5f8SMichael S. Tsirkin }
355c9b9f5f8SMichael S. Tsirkin
vdpasim_set_vq_state(struct vdpa_device * vdpa,u16 idx,const struct vdpa_vq_state * state)356aac50c0bSEli Cohen static int vdpasim_set_vq_state(struct vdpa_device *vdpa, u16 idx,
357aac50c0bSEli Cohen const struct vdpa_vq_state *state)
358c9b9f5f8SMichael S. Tsirkin {
359c9b9f5f8SMichael S. Tsirkin struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
360c9b9f5f8SMichael S. Tsirkin struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
361c9b9f5f8SMichael S. Tsirkin struct vringh *vrh = &vq->vring;
362c9b9f5f8SMichael S. Tsirkin
363d7621c28SStefano Garzarella mutex_lock(&vdpasim->mutex);
364530a5678SJason Wang vrh->last_avail_idx = state->split.avail_index;
365d7621c28SStefano Garzarella mutex_unlock(&vdpasim->mutex);
366c9b9f5f8SMichael S. Tsirkin
367c9b9f5f8SMichael S. Tsirkin return 0;
368c9b9f5f8SMichael S. Tsirkin }
369c9b9f5f8SMichael S. Tsirkin
vdpasim_get_vq_state(struct vdpa_device * vdpa,u16 idx,struct vdpa_vq_state * state)37023750e39SEli Cohen static int vdpasim_get_vq_state(struct vdpa_device *vdpa, u16 idx,
371aac50c0bSEli Cohen struct vdpa_vq_state *state)
372c9b9f5f8SMichael S. Tsirkin {
373c9b9f5f8SMichael S. Tsirkin struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
374c9b9f5f8SMichael S. Tsirkin struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
375c9b9f5f8SMichael S. Tsirkin struct vringh *vrh = &vq->vring;
376c9b9f5f8SMichael S. Tsirkin
377530a5678SJason Wang state->split.avail_index = vrh->last_avail_idx;
37823750e39SEli Cohen return 0;
379c9b9f5f8SMichael S. Tsirkin }
380c9b9f5f8SMichael S. Tsirkin
vdpasim_get_vq_stats(struct vdpa_device * vdpa,u16 idx,struct sk_buff * msg,struct netlink_ext_ack * extack)3815dbb063aSJason Wang static int vdpasim_get_vq_stats(struct vdpa_device *vdpa, u16 idx,
3825dbb063aSJason Wang struct sk_buff *msg,
3835dbb063aSJason Wang struct netlink_ext_ack *extack)
3845dbb063aSJason Wang {
3855dbb063aSJason Wang struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
3865dbb063aSJason Wang
3875dbb063aSJason Wang if (vdpasim->dev_attr.get_stats)
3885dbb063aSJason Wang return vdpasim->dev_attr.get_stats(vdpasim, idx,
3895dbb063aSJason Wang msg, extack);
3905dbb063aSJason Wang return -EOPNOTSUPP;
3915dbb063aSJason Wang }
3925dbb063aSJason Wang
vdpasim_get_vq_align(struct vdpa_device * vdpa)393425a5070SMichael S. Tsirkin static u32 vdpasim_get_vq_align(struct vdpa_device *vdpa)
394c9b9f5f8SMichael S. Tsirkin {
395c9b9f5f8SMichael S. Tsirkin return VDPASIM_QUEUE_ALIGN;
396c9b9f5f8SMichael S. Tsirkin }
397c9b9f5f8SMichael S. Tsirkin
vdpasim_get_vq_group(struct vdpa_device * vdpa,u16 idx)398d4821902SGautam Dawar static u32 vdpasim_get_vq_group(struct vdpa_device *vdpa, u16 idx)
399d4821902SGautam Dawar {
400bda324fdSGautam Dawar /* RX and TX belongs to group 0, CVQ belongs to group 1 */
401bda324fdSGautam Dawar if (idx == 2)
402bda324fdSGautam Dawar return 1;
403bda324fdSGautam Dawar else
404d4821902SGautam Dawar return 0;
405d4821902SGautam Dawar }
406d4821902SGautam Dawar
vdpasim_get_device_features(struct vdpa_device * vdpa)407a64917bcSEli Cohen static u64 vdpasim_get_device_features(struct vdpa_device *vdpa)
408c9b9f5f8SMichael S. Tsirkin {
409011c35baSStefano Garzarella struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
410011c35baSStefano Garzarella
411011c35baSStefano Garzarella return vdpasim->dev_attr.supported_features;
412c9b9f5f8SMichael S. Tsirkin }
413c9b9f5f8SMichael S. Tsirkin
vdpasim_get_backend_features(const struct vdpa_device * vdpa)4142c9c6371SEugenio Pérez static u64 vdpasim_get_backend_features(const struct vdpa_device *vdpa)
4152c9c6371SEugenio Pérez {
4162c9c6371SEugenio Pérez return BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK);
4172c9c6371SEugenio Pérez }
4182c9c6371SEugenio Pérez
vdpasim_set_driver_features(struct vdpa_device * vdpa,u64 features)419a64917bcSEli Cohen static int vdpasim_set_driver_features(struct vdpa_device *vdpa, u64 features)
420c9b9f5f8SMichael S. Tsirkin {
421c9b9f5f8SMichael S. Tsirkin struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
422c9b9f5f8SMichael S. Tsirkin
423c9b9f5f8SMichael S. Tsirkin /* DMA mapping must be done by driver */
424321bd212SMichael S. Tsirkin if (!(features & (1ULL << VIRTIO_F_ACCESS_PLATFORM)))
425c9b9f5f8SMichael S. Tsirkin return -EINVAL;
426c9b9f5f8SMichael S. Tsirkin
427011c35baSStefano Garzarella vdpasim->features = features & vdpasim->dev_attr.supported_features;
428c9b9f5f8SMichael S. Tsirkin
429c9b9f5f8SMichael S. Tsirkin return 0;
430c9b9f5f8SMichael S. Tsirkin }
431c9b9f5f8SMichael S. Tsirkin
vdpasim_get_driver_features(struct vdpa_device * vdpa)432a64917bcSEli Cohen static u64 vdpasim_get_driver_features(struct vdpa_device *vdpa)
433a64917bcSEli Cohen {
434a64917bcSEli Cohen struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
435a64917bcSEli Cohen
436a64917bcSEli Cohen return vdpasim->features;
437a64917bcSEli Cohen }
438a64917bcSEli Cohen
vdpasim_set_config_cb(struct vdpa_device * vdpa,struct vdpa_callback * cb)439c9b9f5f8SMichael S. Tsirkin static void vdpasim_set_config_cb(struct vdpa_device *vdpa,
440c9b9f5f8SMichael S. Tsirkin struct vdpa_callback *cb)
441c9b9f5f8SMichael S. Tsirkin {
442c9b9f5f8SMichael S. Tsirkin /* We don't support config interrupt */
443c9b9f5f8SMichael S. Tsirkin }
444c9b9f5f8SMichael S. Tsirkin
vdpasim_get_vq_num_max(struct vdpa_device * vdpa)445c9b9f5f8SMichael S. Tsirkin static u16 vdpasim_get_vq_num_max(struct vdpa_device *vdpa)
446c9b9f5f8SMichael S. Tsirkin {
447c9b9f5f8SMichael S. Tsirkin return VDPASIM_QUEUE_MAX;
448c9b9f5f8SMichael S. Tsirkin }
449c9b9f5f8SMichael S. Tsirkin
vdpasim_get_device_id(struct vdpa_device * vdpa)450c9b9f5f8SMichael S. Tsirkin static u32 vdpasim_get_device_id(struct vdpa_device *vdpa)
451c9b9f5f8SMichael S. Tsirkin {
4522f8f4618SStefano Garzarella struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
4532f8f4618SStefano Garzarella
4542f8f4618SStefano Garzarella return vdpasim->dev_attr.id;
455c9b9f5f8SMichael S. Tsirkin }
456c9b9f5f8SMichael S. Tsirkin
vdpasim_get_vendor_id(struct vdpa_device * vdpa)457c9b9f5f8SMichael S. Tsirkin static u32 vdpasim_get_vendor_id(struct vdpa_device *vdpa)
458c9b9f5f8SMichael S. Tsirkin {
459c9b9f5f8SMichael S. Tsirkin return VDPASIM_VENDOR_ID;
460c9b9f5f8SMichael S. Tsirkin }
461c9b9f5f8SMichael S. Tsirkin
vdpasim_get_status(struct vdpa_device * vdpa)462c9b9f5f8SMichael S. Tsirkin static u8 vdpasim_get_status(struct vdpa_device *vdpa)
463c9b9f5f8SMichael S. Tsirkin {
464c9b9f5f8SMichael S. Tsirkin struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
465c9b9f5f8SMichael S. Tsirkin u8 status;
466c9b9f5f8SMichael S. Tsirkin
467d7621c28SStefano Garzarella mutex_lock(&vdpasim->mutex);
468c9b9f5f8SMichael S. Tsirkin status = vdpasim->status;
469d7621c28SStefano Garzarella mutex_unlock(&vdpasim->mutex);
470c9b9f5f8SMichael S. Tsirkin
47121818ed0SYueHaibing return status;
472c9b9f5f8SMichael S. Tsirkin }
473c9b9f5f8SMichael S. Tsirkin
vdpasim_set_status(struct vdpa_device * vdpa,u8 status)474c9b9f5f8SMichael S. Tsirkin static void vdpasim_set_status(struct vdpa_device *vdpa, u8 status)
475c9b9f5f8SMichael S. Tsirkin {
476c9b9f5f8SMichael S. Tsirkin struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
477c9b9f5f8SMichael S. Tsirkin
478d7621c28SStefano Garzarella mutex_lock(&vdpasim->mutex);
479c9b9f5f8SMichael S. Tsirkin vdpasim->status = status;
480*cddd85daSSteve Sistare vdpasim->running = (status & VIRTIO_CONFIG_S_DRIVER_OK) != 0;
481d7621c28SStefano Garzarella mutex_unlock(&vdpasim->mutex);
482c9b9f5f8SMichael S. Tsirkin }
483c9b9f5f8SMichael S. Tsirkin
vdpasim_reset(struct vdpa_device * vdpa)4840686082dSXie Yongji static int vdpasim_reset(struct vdpa_device *vdpa)
4850686082dSXie Yongji {
4860686082dSXie Yongji struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
4870686082dSXie Yongji
488d7621c28SStefano Garzarella mutex_lock(&vdpasim->mutex);
4890686082dSXie Yongji vdpasim->status = 0;
4900686082dSXie Yongji vdpasim_do_reset(vdpasim);
491d7621c28SStefano Garzarella mutex_unlock(&vdpasim->mutex);
4920686082dSXie Yongji
4930686082dSXie Yongji return 0;
4940686082dSXie Yongji }
4950686082dSXie Yongji
vdpasim_suspend(struct vdpa_device * vdpa)4960c89e2a3SEugenio Pérez static int vdpasim_suspend(struct vdpa_device *vdpa)
4970c89e2a3SEugenio Pérez {
4980c89e2a3SEugenio Pérez struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
4990c89e2a3SEugenio Pérez
500d7621c28SStefano Garzarella mutex_lock(&vdpasim->mutex);
5010c89e2a3SEugenio Pérez vdpasim->running = false;
502d7621c28SStefano Garzarella mutex_unlock(&vdpasim->mutex);
5030c89e2a3SEugenio Pérez
5040c89e2a3SEugenio Pérez return 0;
5050c89e2a3SEugenio Pérez }
5060c89e2a3SEugenio Pérez
vdpasim_resume(struct vdpa_device * vdpa)507f9d9f57eSSebastien Boeuf static int vdpasim_resume(struct vdpa_device *vdpa)
508f9d9f57eSSebastien Boeuf {
509f9d9f57eSSebastien Boeuf struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
510f9d9f57eSSebastien Boeuf int i;
511f9d9f57eSSebastien Boeuf
512d7621c28SStefano Garzarella mutex_lock(&vdpasim->mutex);
513f9d9f57eSSebastien Boeuf vdpasim->running = true;
514f9d9f57eSSebastien Boeuf
515f9d9f57eSSebastien Boeuf if (vdpasim->pending_kick) {
516f9d9f57eSSebastien Boeuf /* Process pending descriptors */
517f9d9f57eSSebastien Boeuf for (i = 0; i < vdpasim->dev_attr.nvqs; ++i)
518f9d9f57eSSebastien Boeuf vdpasim_kick_vq(vdpa, i);
519f9d9f57eSSebastien Boeuf
520f9d9f57eSSebastien Boeuf vdpasim->pending_kick = false;
521f9d9f57eSSebastien Boeuf }
522f9d9f57eSSebastien Boeuf
523d7621c28SStefano Garzarella mutex_unlock(&vdpasim->mutex);
524f9d9f57eSSebastien Boeuf
525f9d9f57eSSebastien Boeuf return 0;
526f9d9f57eSSebastien Boeuf }
527f9d9f57eSSebastien Boeuf
vdpasim_get_config_size(struct vdpa_device * vdpa)528442706f9SStefano Garzarella static size_t vdpasim_get_config_size(struct vdpa_device *vdpa)
529442706f9SStefano Garzarella {
530442706f9SStefano Garzarella struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
531442706f9SStefano Garzarella
532442706f9SStefano Garzarella return vdpasim->dev_attr.config_size;
533442706f9SStefano Garzarella }
534442706f9SStefano Garzarella
vdpasim_get_config(struct vdpa_device * vdpa,unsigned int offset,void * buf,unsigned int len)535c9b9f5f8SMichael S. Tsirkin static void vdpasim_get_config(struct vdpa_device *vdpa, unsigned int offset,
536c9b9f5f8SMichael S. Tsirkin void *buf, unsigned int len)
537c9b9f5f8SMichael S. Tsirkin {
538c9b9f5f8SMichael S. Tsirkin struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
539c9b9f5f8SMichael S. Tsirkin
54065b70958SStefano Garzarella if (offset + len > vdpasim->dev_attr.config_size)
54165b70958SStefano Garzarella return;
54265b70958SStefano Garzarella
54365b70958SStefano Garzarella if (vdpasim->dev_attr.get_config)
54465b70958SStefano Garzarella vdpasim->dev_attr.get_config(vdpasim, vdpasim->config);
54565b70958SStefano Garzarella
546f37cbbc6SStefano Garzarella memcpy(buf, vdpasim->config + offset, len);
547c9b9f5f8SMichael S. Tsirkin }
548c9b9f5f8SMichael S. Tsirkin
vdpasim_set_config(struct vdpa_device * vdpa,unsigned int offset,const void * buf,unsigned int len)549c9b9f5f8SMichael S. Tsirkin static void vdpasim_set_config(struct vdpa_device *vdpa, unsigned int offset,
550c9b9f5f8SMichael S. Tsirkin const void *buf, unsigned int len)
551c9b9f5f8SMichael S. Tsirkin {
552c124a95eSStefano Garzarella struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
553c124a95eSStefano Garzarella
554c124a95eSStefano Garzarella if (offset + len > vdpasim->dev_attr.config_size)
555c124a95eSStefano Garzarella return;
556c124a95eSStefano Garzarella
557c124a95eSStefano Garzarella memcpy(vdpasim->config + offset, buf, len);
558c124a95eSStefano Garzarella
559c124a95eSStefano Garzarella if (vdpasim->dev_attr.set_config)
560c124a95eSStefano Garzarella vdpasim->dev_attr.set_config(vdpasim, vdpasim->config);
561c9b9f5f8SMichael S. Tsirkin }
562c9b9f5f8SMichael S. Tsirkin
vdpasim_get_generation(struct vdpa_device * vdpa)563c9b9f5f8SMichael S. Tsirkin static u32 vdpasim_get_generation(struct vdpa_device *vdpa)
564c9b9f5f8SMichael S. Tsirkin {
565c9b9f5f8SMichael S. Tsirkin struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
566c9b9f5f8SMichael S. Tsirkin
567c9b9f5f8SMichael S. Tsirkin return vdpasim->generation;
568c9b9f5f8SMichael S. Tsirkin }
569c9b9f5f8SMichael S. Tsirkin
vdpasim_get_iova_range(struct vdpa_device * vdpa)57070a62fceSJason Wang static struct vdpa_iova_range vdpasim_get_iova_range(struct vdpa_device *vdpa)
57170a62fceSJason Wang {
57270a62fceSJason Wang struct vdpa_iova_range range = {
57370a62fceSJason Wang .first = 0ULL,
57470a62fceSJason Wang .last = ULLONG_MAX,
57570a62fceSJason Wang };
57670a62fceSJason Wang
57770a62fceSJason Wang return range;
57870a62fceSJason Wang }
57970a62fceSJason Wang
vdpasim_set_group_asid(struct vdpa_device * vdpa,unsigned int group,unsigned int asid)580bda324fdSGautam Dawar static int vdpasim_set_group_asid(struct vdpa_device *vdpa, unsigned int group,
581bda324fdSGautam Dawar unsigned int asid)
582bda324fdSGautam Dawar {
583bda324fdSGautam Dawar struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
584bda324fdSGautam Dawar struct vhost_iotlb *iommu;
585bda324fdSGautam Dawar int i;
586bda324fdSGautam Dawar
587bda324fdSGautam Dawar if (group > vdpasim->dev_attr.ngroups)
588bda324fdSGautam Dawar return -EINVAL;
589bda324fdSGautam Dawar
5901f97b978SDan Carpenter if (asid >= vdpasim->dev_attr.nas)
591bda324fdSGautam Dawar return -EINVAL;
592bda324fdSGautam Dawar
593bda324fdSGautam Dawar iommu = &vdpasim->iommu[asid];
594bda324fdSGautam Dawar
595d7621c28SStefano Garzarella mutex_lock(&vdpasim->mutex);
596bda324fdSGautam Dawar
597bda324fdSGautam Dawar for (i = 0; i < vdpasim->dev_attr.nvqs; i++)
598bda324fdSGautam Dawar if (vdpasim_get_vq_group(vdpa, i) == group)
599bda324fdSGautam Dawar vringh_set_iotlb(&vdpasim->vqs[i].vring, iommu,
600bda324fdSGautam Dawar &vdpasim->iommu_lock);
601bda324fdSGautam Dawar
602d7621c28SStefano Garzarella mutex_unlock(&vdpasim->mutex);
603bda324fdSGautam Dawar
604bda324fdSGautam Dawar return 0;
605bda324fdSGautam Dawar }
606bda324fdSGautam Dawar
vdpasim_set_map(struct vdpa_device * vdpa,unsigned int asid,struct vhost_iotlb * iotlb)607db9adcbfSGautam Dawar static int vdpasim_set_map(struct vdpa_device *vdpa, unsigned int asid,
608c9b9f5f8SMichael S. Tsirkin struct vhost_iotlb *iotlb)
609c9b9f5f8SMichael S. Tsirkin {
610c9b9f5f8SMichael S. Tsirkin struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
611c9b9f5f8SMichael S. Tsirkin struct vhost_iotlb_map *map;
612bda324fdSGautam Dawar struct vhost_iotlb *iommu;
613c9b9f5f8SMichael S. Tsirkin u64 start = 0ULL, last = 0ULL - 1;
614c9b9f5f8SMichael S. Tsirkin int ret;
615c9b9f5f8SMichael S. Tsirkin
616bda324fdSGautam Dawar if (asid >= vdpasim->dev_attr.nas)
617bda324fdSGautam Dawar return -EINVAL;
618bda324fdSGautam Dawar
6190ea9ee43SMax Gurtovoy spin_lock(&vdpasim->iommu_lock);
620bda324fdSGautam Dawar
621bda324fdSGautam Dawar iommu = &vdpasim->iommu[asid];
622bda324fdSGautam Dawar vhost_iotlb_reset(iommu);
6236c3d329eSJason Wang vdpasim->iommu_pt[asid] = false;
624c9b9f5f8SMichael S. Tsirkin
625c9b9f5f8SMichael S. Tsirkin for (map = vhost_iotlb_itree_first(iotlb, start, last); map;
626c9b9f5f8SMichael S. Tsirkin map = vhost_iotlb_itree_next(map, start, last)) {
627bda324fdSGautam Dawar ret = vhost_iotlb_add_range(iommu, map->start,
628c9b9f5f8SMichael S. Tsirkin map->last, map->addr, map->perm);
629c9b9f5f8SMichael S. Tsirkin if (ret)
630c9b9f5f8SMichael S. Tsirkin goto err;
631c9b9f5f8SMichael S. Tsirkin }
6320ea9ee43SMax Gurtovoy spin_unlock(&vdpasim->iommu_lock);
633c9b9f5f8SMichael S. Tsirkin return 0;
634c9b9f5f8SMichael S. Tsirkin
635c9b9f5f8SMichael S. Tsirkin err:
636bda324fdSGautam Dawar vhost_iotlb_reset(iommu);
6370ea9ee43SMax Gurtovoy spin_unlock(&vdpasim->iommu_lock);
638c9b9f5f8SMichael S. Tsirkin return ret;
639c9b9f5f8SMichael S. Tsirkin }
640c9b9f5f8SMichael S. Tsirkin
vdpasim_bind_mm(struct vdpa_device * vdpa,struct mm_struct * mm)6414bb94d2dSStefano Garzarella static int vdpasim_bind_mm(struct vdpa_device *vdpa, struct mm_struct *mm)
6424bb94d2dSStefano Garzarella {
6434bb94d2dSStefano Garzarella struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
6444bb94d2dSStefano Garzarella struct vdpasim_mm_work mm_work;
6454bb94d2dSStefano Garzarella
6464bb94d2dSStefano Garzarella mm_work.vdpasim = vdpasim;
6474bb94d2dSStefano Garzarella mm_work.mm_to_bind = mm;
6484bb94d2dSStefano Garzarella
6494bb94d2dSStefano Garzarella vdpasim_worker_change_mm_sync(vdpasim, &mm_work);
6504bb94d2dSStefano Garzarella
6514bb94d2dSStefano Garzarella return mm_work.ret;
6524bb94d2dSStefano Garzarella }
6534bb94d2dSStefano Garzarella
vdpasim_unbind_mm(struct vdpa_device * vdpa)6544bb94d2dSStefano Garzarella static void vdpasim_unbind_mm(struct vdpa_device *vdpa)
6554bb94d2dSStefano Garzarella {
6564bb94d2dSStefano Garzarella struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
6574bb94d2dSStefano Garzarella struct vdpasim_mm_work mm_work;
6584bb94d2dSStefano Garzarella
6594bb94d2dSStefano Garzarella mm_work.vdpasim = vdpasim;
6604bb94d2dSStefano Garzarella mm_work.mm_to_bind = NULL;
6614bb94d2dSStefano Garzarella
6624bb94d2dSStefano Garzarella vdpasim_worker_change_mm_sync(vdpasim, &mm_work);
6634bb94d2dSStefano Garzarella }
6644bb94d2dSStefano Garzarella
vdpasim_dma_map(struct vdpa_device * vdpa,unsigned int asid,u64 iova,u64 size,u64 pa,u32 perm,void * opaque)665db9adcbfSGautam Dawar static int vdpasim_dma_map(struct vdpa_device *vdpa, unsigned int asid,
666db9adcbfSGautam Dawar u64 iova, u64 size,
667c10fb945SXie Yongji u64 pa, u32 perm, void *opaque)
668c9b9f5f8SMichael S. Tsirkin {
669c9b9f5f8SMichael S. Tsirkin struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
6700ea9ee43SMax Gurtovoy int ret;
671c9b9f5f8SMichael S. Tsirkin
672bda324fdSGautam Dawar if (asid >= vdpasim->dev_attr.nas)
673bda324fdSGautam Dawar return -EINVAL;
674bda324fdSGautam Dawar
6750ea9ee43SMax Gurtovoy spin_lock(&vdpasim->iommu_lock);
6766c3d329eSJason Wang if (vdpasim->iommu_pt[asid]) {
6776c3d329eSJason Wang vhost_iotlb_reset(&vdpasim->iommu[asid]);
6786c3d329eSJason Wang vdpasim->iommu_pt[asid] = false;
6796c3d329eSJason Wang }
680bda324fdSGautam Dawar ret = vhost_iotlb_add_range_ctx(&vdpasim->iommu[asid], iova,
681bda324fdSGautam Dawar iova + size - 1, pa, perm, opaque);
6820ea9ee43SMax Gurtovoy spin_unlock(&vdpasim->iommu_lock);
6830ea9ee43SMax Gurtovoy
6840ea9ee43SMax Gurtovoy return ret;
685c9b9f5f8SMichael S. Tsirkin }
686c9b9f5f8SMichael S. Tsirkin
vdpasim_dma_unmap(struct vdpa_device * vdpa,unsigned int asid,u64 iova,u64 size)687db9adcbfSGautam Dawar static int vdpasim_dma_unmap(struct vdpa_device *vdpa, unsigned int asid,
688db9adcbfSGautam Dawar u64 iova, u64 size)
689c9b9f5f8SMichael S. Tsirkin {
690c9b9f5f8SMichael S. Tsirkin struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
691c9b9f5f8SMichael S. Tsirkin
692bda324fdSGautam Dawar if (asid >= vdpasim->dev_attr.nas)
693bda324fdSGautam Dawar return -EINVAL;
694bda324fdSGautam Dawar
6956c3d329eSJason Wang if (vdpasim->iommu_pt[asid]) {
6966c3d329eSJason Wang vhost_iotlb_reset(&vdpasim->iommu[asid]);
6976c3d329eSJason Wang vdpasim->iommu_pt[asid] = false;
6986c3d329eSJason Wang }
6996c3d329eSJason Wang
7000ea9ee43SMax Gurtovoy spin_lock(&vdpasim->iommu_lock);
701bda324fdSGautam Dawar vhost_iotlb_del_range(&vdpasim->iommu[asid], iova, iova + size - 1);
7020ea9ee43SMax Gurtovoy spin_unlock(&vdpasim->iommu_lock);
703c9b9f5f8SMichael S. Tsirkin
704c9b9f5f8SMichael S. Tsirkin return 0;
705c9b9f5f8SMichael S. Tsirkin }
706c9b9f5f8SMichael S. Tsirkin
vdpasim_free(struct vdpa_device * vdpa)707c9b9f5f8SMichael S. Tsirkin static void vdpasim_free(struct vdpa_device *vdpa)
708c9b9f5f8SMichael S. Tsirkin {
709c9b9f5f8SMichael S. Tsirkin struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
710bc433e5eSStefano Garzarella int i;
711c9b9f5f8SMichael S. Tsirkin
71276acfa7bSStefano Garzarella kthread_cancel_work_sync(&vdpasim->work);
71376acfa7bSStefano Garzarella kthread_destroy_worker(vdpasim->worker);
714bc433e5eSStefano Garzarella
715bc433e5eSStefano Garzarella for (i = 0; i < vdpasim->dev_attr.nvqs; i++) {
716bc433e5eSStefano Garzarella vringh_kiov_cleanup(&vdpasim->vqs[i].out_iov);
717bc433e5eSStefano Garzarella vringh_kiov_cleanup(&vdpasim->vqs[i].in_iov);
718bc433e5eSStefano Garzarella }
719bc433e5eSStefano Garzarella
720112f23cdSStefano Garzarella vdpasim->dev_attr.free(vdpasim);
721112f23cdSStefano Garzarella
7220b7a04a3SJason Wang for (i = 0; i < vdpasim->dev_attr.nas; i++)
7230b7a04a3SJason Wang vhost_iotlb_reset(&vdpasim->iommu[i]);
7240b7a04a3SJason Wang kfree(vdpasim->iommu);
7256c3d329eSJason Wang kfree(vdpasim->iommu_pt);
726423248d6SMax Gurtovoy kfree(vdpasim->vqs);
727f37cbbc6SStefano Garzarella kfree(vdpasim->config);
728c9b9f5f8SMichael S. Tsirkin }
729c9b9f5f8SMichael S. Tsirkin
73036a9c306SStefano Garzarella static const struct vdpa_config_ops vdpasim_config_ops = {
731c9b9f5f8SMichael S. Tsirkin .set_vq_address = vdpasim_set_vq_address,
732c9b9f5f8SMichael S. Tsirkin .set_vq_num = vdpasim_set_vq_num,
733c9b9f5f8SMichael S. Tsirkin .kick_vq = vdpasim_kick_vq,
734c9b9f5f8SMichael S. Tsirkin .set_vq_cb = vdpasim_set_vq_cb,
735c9b9f5f8SMichael S. Tsirkin .set_vq_ready = vdpasim_set_vq_ready,
736c9b9f5f8SMichael S. Tsirkin .get_vq_ready = vdpasim_get_vq_ready,
737c9b9f5f8SMichael S. Tsirkin .set_vq_state = vdpasim_set_vq_state,
7385dbb063aSJason Wang .get_vendor_vq_stats = vdpasim_get_vq_stats,
739c9b9f5f8SMichael S. Tsirkin .get_vq_state = vdpasim_get_vq_state,
740c9b9f5f8SMichael S. Tsirkin .get_vq_align = vdpasim_get_vq_align,
741d4821902SGautam Dawar .get_vq_group = vdpasim_get_vq_group,
742a64917bcSEli Cohen .get_device_features = vdpasim_get_device_features,
7432c9c6371SEugenio Pérez .get_backend_features = vdpasim_get_backend_features,
744a64917bcSEli Cohen .set_driver_features = vdpasim_set_driver_features,
745a64917bcSEli Cohen .get_driver_features = vdpasim_get_driver_features,
746c9b9f5f8SMichael S. Tsirkin .set_config_cb = vdpasim_set_config_cb,
747c9b9f5f8SMichael S. Tsirkin .get_vq_num_max = vdpasim_get_vq_num_max,
748c9b9f5f8SMichael S. Tsirkin .get_device_id = vdpasim_get_device_id,
749c9b9f5f8SMichael S. Tsirkin .get_vendor_id = vdpasim_get_vendor_id,
750c9b9f5f8SMichael S. Tsirkin .get_status = vdpasim_get_status,
751c9b9f5f8SMichael S. Tsirkin .set_status = vdpasim_set_status,
7520686082dSXie Yongji .reset = vdpasim_reset,
7530c89e2a3SEugenio Pérez .suspend = vdpasim_suspend,
754f9d9f57eSSebastien Boeuf .resume = vdpasim_resume,
755442706f9SStefano Garzarella .get_config_size = vdpasim_get_config_size,
756c9b9f5f8SMichael S. Tsirkin .get_config = vdpasim_get_config,
757c9b9f5f8SMichael S. Tsirkin .set_config = vdpasim_set_config,
758c9b9f5f8SMichael S. Tsirkin .get_generation = vdpasim_get_generation,
75970a62fceSJason Wang .get_iova_range = vdpasim_get_iova_range,
760bda324fdSGautam Dawar .set_group_asid = vdpasim_set_group_asid,
761c9b9f5f8SMichael S. Tsirkin .dma_map = vdpasim_dma_map,
762c9b9f5f8SMichael S. Tsirkin .dma_unmap = vdpasim_dma_unmap,
7634bb94d2dSStefano Garzarella .bind_mm = vdpasim_bind_mm,
7644bb94d2dSStefano Garzarella .unbind_mm = vdpasim_unbind_mm,
765c9b9f5f8SMichael S. Tsirkin .free = vdpasim_free,
766c9b9f5f8SMichael S. Tsirkin };
767c9b9f5f8SMichael S. Tsirkin
76836a9c306SStefano Garzarella static const struct vdpa_config_ops vdpasim_batch_config_ops = {
769de91a4d0SJason Wang .set_vq_address = vdpasim_set_vq_address,
770de91a4d0SJason Wang .set_vq_num = vdpasim_set_vq_num,
771de91a4d0SJason Wang .kick_vq = vdpasim_kick_vq,
772de91a4d0SJason Wang .set_vq_cb = vdpasim_set_vq_cb,
773de91a4d0SJason Wang .set_vq_ready = vdpasim_set_vq_ready,
774de91a4d0SJason Wang .get_vq_ready = vdpasim_get_vq_ready,
775de91a4d0SJason Wang .set_vq_state = vdpasim_set_vq_state,
7765dbb063aSJason Wang .get_vendor_vq_stats = vdpasim_get_vq_stats,
777de91a4d0SJason Wang .get_vq_state = vdpasim_get_vq_state,
778de91a4d0SJason Wang .get_vq_align = vdpasim_get_vq_align,
779d4821902SGautam Dawar .get_vq_group = vdpasim_get_vq_group,
780a64917bcSEli Cohen .get_device_features = vdpasim_get_device_features,
7812c9c6371SEugenio Pérez .get_backend_features = vdpasim_get_backend_features,
782a64917bcSEli Cohen .set_driver_features = vdpasim_set_driver_features,
783a64917bcSEli Cohen .get_driver_features = vdpasim_get_driver_features,
784de91a4d0SJason Wang .set_config_cb = vdpasim_set_config_cb,
785de91a4d0SJason Wang .get_vq_num_max = vdpasim_get_vq_num_max,
786de91a4d0SJason Wang .get_device_id = vdpasim_get_device_id,
787de91a4d0SJason Wang .get_vendor_id = vdpasim_get_vendor_id,
788de91a4d0SJason Wang .get_status = vdpasim_get_status,
789de91a4d0SJason Wang .set_status = vdpasim_set_status,
7900686082dSXie Yongji .reset = vdpasim_reset,
7910c89e2a3SEugenio Pérez .suspend = vdpasim_suspend,
792f9d9f57eSSebastien Boeuf .resume = vdpasim_resume,
793442706f9SStefano Garzarella .get_config_size = vdpasim_get_config_size,
794de91a4d0SJason Wang .get_config = vdpasim_get_config,
795de91a4d0SJason Wang .set_config = vdpasim_set_config,
796de91a4d0SJason Wang .get_generation = vdpasim_get_generation,
79770a62fceSJason Wang .get_iova_range = vdpasim_get_iova_range,
798bda324fdSGautam Dawar .set_group_asid = vdpasim_set_group_asid,
799de91a4d0SJason Wang .set_map = vdpasim_set_map,
8004bb94d2dSStefano Garzarella .bind_mm = vdpasim_bind_mm,
8014bb94d2dSStefano Garzarella .unbind_mm = vdpasim_unbind_mm,
802de91a4d0SJason Wang .free = vdpasim_free,
803de91a4d0SJason Wang };
804de91a4d0SJason Wang
805c9b9f5f8SMichael S. Tsirkin MODULE_VERSION(DRV_VERSION);
806c9b9f5f8SMichael S. Tsirkin MODULE_LICENSE(DRV_LICENSE);
807c9b9f5f8SMichael S. Tsirkin MODULE_AUTHOR(DRV_AUTHOR);
808c9b9f5f8SMichael S. Tsirkin MODULE_DESCRIPTION(DRV_DESC);
809