1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VIRTIO based driver for vDPA device 4 * 5 * Copyright (c) 2020, Red Hat. All rights reserved. 6 * Author: Jason Wang <jasowang@redhat.com> 7 * 8 */ 9 10 #include <linux/init.h> 11 #include <linux/module.h> 12 #include <linux/device.h> 13 #include <linux/kernel.h> 14 #include <linux/slab.h> 15 #include <linux/uuid.h> 16 #include <linux/virtio.h> 17 #include <linux/vdpa.h> 18 #include <linux/virtio_config.h> 19 #include <linux/virtio_ring.h> 20 21 #define MOD_VERSION "0.1" 22 #define MOD_AUTHOR "Jason Wang <jasowang@redhat.com>" 23 #define MOD_DESC "vDPA bus driver for virtio devices" 24 #define MOD_LICENSE "GPL v2" 25 26 struct virtio_vdpa_device { 27 struct virtio_device vdev; 28 struct vdpa_device *vdpa; 29 u64 features; 30 31 /* The lock to protect virtqueue list */ 32 spinlock_t lock; 33 /* List of virtio_vdpa_vq_info */ 34 struct list_head virtqueues; 35 }; 36 37 struct virtio_vdpa_vq_info { 38 /* the actual virtqueue */ 39 struct virtqueue *vq; 40 41 /* the list node for the virtqueues list */ 42 struct list_head node; 43 }; 44 45 static inline struct virtio_vdpa_device * 46 to_virtio_vdpa_device(struct virtio_device *dev) 47 { 48 return container_of(dev, struct virtio_vdpa_device, vdev); 49 } 50 51 static struct vdpa_device *vd_get_vdpa(struct virtio_device *vdev) 52 { 53 return to_virtio_vdpa_device(vdev)->vdpa; 54 } 55 56 static void virtio_vdpa_get(struct virtio_device *vdev, unsigned offset, 57 void *buf, unsigned len) 58 { 59 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 60 61 vdpa_get_config(vdpa, offset, buf, len); 62 } 63 64 static void virtio_vdpa_set(struct virtio_device *vdev, unsigned offset, 65 const void *buf, unsigned len) 66 { 67 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 68 const struct vdpa_config_ops *ops = vdpa->config; 69 70 ops->set_config(vdpa, offset, buf, len); 71 } 72 73 static u32 virtio_vdpa_generation(struct virtio_device *vdev) 74 { 75 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 76 const struct vdpa_config_ops *ops = vdpa->config; 77 78 if (ops->get_generation) 79 return ops->get_generation(vdpa); 80 81 return 0; 82 } 83 84 static u8 virtio_vdpa_get_status(struct virtio_device *vdev) 85 { 86 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 87 const struct vdpa_config_ops *ops = vdpa->config; 88 89 return ops->get_status(vdpa); 90 } 91 92 static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status) 93 { 94 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 95 const struct vdpa_config_ops *ops = vdpa->config; 96 97 return ops->set_status(vdpa, status); 98 } 99 100 static void virtio_vdpa_reset(struct virtio_device *vdev) 101 { 102 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 103 104 vdpa_reset(vdpa); 105 } 106 107 static bool virtio_vdpa_notify(struct virtqueue *vq) 108 { 109 struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev); 110 const struct vdpa_config_ops *ops = vdpa->config; 111 112 ops->kick_vq(vdpa, vq->index); 113 114 return true; 115 } 116 117 static irqreturn_t virtio_vdpa_config_cb(void *private) 118 { 119 struct virtio_vdpa_device *vd_dev = private; 120 121 virtio_config_changed(&vd_dev->vdev); 122 123 return IRQ_HANDLED; 124 } 125 126 static irqreturn_t virtio_vdpa_virtqueue_cb(void *private) 127 { 128 struct virtio_vdpa_vq_info *info = private; 129 130 return vring_interrupt(0, info->vq); 131 } 132 133 static struct virtqueue * 134 virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, 135 void (*callback)(struct virtqueue *vq), 136 const char *name, bool ctx) 137 { 138 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); 139 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 140 const struct vdpa_config_ops *ops = vdpa->config; 141 struct virtio_vdpa_vq_info *info; 142 struct vdpa_callback cb; 143 struct virtqueue *vq; 144 u64 desc_addr, driver_addr, device_addr; 145 unsigned long flags; 146 u32 align, num; 147 int err; 148 149 if (!name) 150 return NULL; 151 152 /* Queue shouldn't already be set up. */ 153 if (ops->get_vq_ready(vdpa, index)) 154 return ERR_PTR(-ENOENT); 155 156 /* Allocate and fill out our active queue description */ 157 info = kmalloc(sizeof(*info), GFP_KERNEL); 158 if (!info) 159 return ERR_PTR(-ENOMEM); 160 161 num = ops->get_vq_num_max(vdpa); 162 if (num == 0) { 163 err = -ENOENT; 164 goto error_new_virtqueue; 165 } 166 167 /* Create the vring */ 168 align = ops->get_vq_align(vdpa); 169 vq = vring_create_virtqueue(index, num, align, vdev, 170 true, true, ctx, 171 virtio_vdpa_notify, callback, name); 172 if (!vq) { 173 err = -ENOMEM; 174 goto error_new_virtqueue; 175 } 176 177 /* Setup virtqueue callback */ 178 cb.callback = virtio_vdpa_virtqueue_cb; 179 cb.private = info; 180 ops->set_vq_cb(vdpa, index, &cb); 181 ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq)); 182 183 desc_addr = virtqueue_get_desc_addr(vq); 184 driver_addr = virtqueue_get_avail_addr(vq); 185 device_addr = virtqueue_get_used_addr(vq); 186 187 if (ops->set_vq_address(vdpa, index, 188 desc_addr, driver_addr, 189 device_addr)) { 190 err = -EINVAL; 191 goto err_vq; 192 } 193 194 ops->set_vq_ready(vdpa, index, 1); 195 196 vq->priv = info; 197 info->vq = vq; 198 199 spin_lock_irqsave(&vd_dev->lock, flags); 200 list_add(&info->node, &vd_dev->virtqueues); 201 spin_unlock_irqrestore(&vd_dev->lock, flags); 202 203 return vq; 204 205 err_vq: 206 vring_del_virtqueue(vq); 207 error_new_virtqueue: 208 ops->set_vq_ready(vdpa, index, 0); 209 /* VDPA driver should make sure vq is stopeed here */ 210 WARN_ON(ops->get_vq_ready(vdpa, index)); 211 kfree(info); 212 return ERR_PTR(err); 213 } 214 215 static void virtio_vdpa_del_vq(struct virtqueue *vq) 216 { 217 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev); 218 struct vdpa_device *vdpa = vd_dev->vdpa; 219 const struct vdpa_config_ops *ops = vdpa->config; 220 struct virtio_vdpa_vq_info *info = vq->priv; 221 unsigned int index = vq->index; 222 unsigned long flags; 223 224 spin_lock_irqsave(&vd_dev->lock, flags); 225 list_del(&info->node); 226 spin_unlock_irqrestore(&vd_dev->lock, flags); 227 228 /* Select and deactivate the queue */ 229 ops->set_vq_ready(vdpa, index, 0); 230 WARN_ON(ops->get_vq_ready(vdpa, index)); 231 232 vring_del_virtqueue(vq); 233 234 kfree(info); 235 } 236 237 static void virtio_vdpa_del_vqs(struct virtio_device *vdev) 238 { 239 struct virtqueue *vq, *n; 240 241 list_for_each_entry_safe(vq, n, &vdev->vqs, list) 242 virtio_vdpa_del_vq(vq); 243 } 244 245 static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned nvqs, 246 struct virtqueue *vqs[], 247 vq_callback_t *callbacks[], 248 const char * const names[], 249 const bool *ctx, 250 struct irq_affinity *desc) 251 { 252 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); 253 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 254 const struct vdpa_config_ops *ops = vdpa->config; 255 struct vdpa_callback cb; 256 int i, err, queue_idx = 0; 257 258 for (i = 0; i < nvqs; ++i) { 259 if (!names[i]) { 260 vqs[i] = NULL; 261 continue; 262 } 263 264 vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++, 265 callbacks[i], names[i], ctx ? 266 ctx[i] : false); 267 if (IS_ERR(vqs[i])) { 268 err = PTR_ERR(vqs[i]); 269 goto err_setup_vq; 270 } 271 } 272 273 cb.callback = virtio_vdpa_config_cb; 274 cb.private = vd_dev; 275 ops->set_config_cb(vdpa, &cb); 276 277 return 0; 278 279 err_setup_vq: 280 virtio_vdpa_del_vqs(vdev); 281 return err; 282 } 283 284 static u64 virtio_vdpa_get_features(struct virtio_device *vdev) 285 { 286 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 287 const struct vdpa_config_ops *ops = vdpa->config; 288 289 return ops->get_features(vdpa); 290 } 291 292 static int virtio_vdpa_finalize_features(struct virtio_device *vdev) 293 { 294 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 295 296 /* Give virtio_ring a chance to accept features. */ 297 vring_transport_features(vdev); 298 299 return vdpa_set_features(vdpa, vdev->features); 300 } 301 302 static const char *virtio_vdpa_bus_name(struct virtio_device *vdev) 303 { 304 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); 305 struct vdpa_device *vdpa = vd_dev->vdpa; 306 307 return dev_name(&vdpa->dev); 308 } 309 310 static const struct virtio_config_ops virtio_vdpa_config_ops = { 311 .get = virtio_vdpa_get, 312 .set = virtio_vdpa_set, 313 .generation = virtio_vdpa_generation, 314 .get_status = virtio_vdpa_get_status, 315 .set_status = virtio_vdpa_set_status, 316 .reset = virtio_vdpa_reset, 317 .find_vqs = virtio_vdpa_find_vqs, 318 .del_vqs = virtio_vdpa_del_vqs, 319 .get_features = virtio_vdpa_get_features, 320 .finalize_features = virtio_vdpa_finalize_features, 321 .bus_name = virtio_vdpa_bus_name, 322 }; 323 324 static void virtio_vdpa_release_dev(struct device *_d) 325 { 326 struct virtio_device *vdev = 327 container_of(_d, struct virtio_device, dev); 328 struct virtio_vdpa_device *vd_dev = 329 container_of(vdev, struct virtio_vdpa_device, vdev); 330 331 kfree(vd_dev); 332 } 333 334 static int virtio_vdpa_probe(struct vdpa_device *vdpa) 335 { 336 const struct vdpa_config_ops *ops = vdpa->config; 337 struct virtio_vdpa_device *vd_dev, *reg_dev = NULL; 338 int ret = -EINVAL; 339 340 vd_dev = kzalloc(sizeof(*vd_dev), GFP_KERNEL); 341 if (!vd_dev) 342 return -ENOMEM; 343 344 vd_dev->vdev.dev.parent = vdpa_get_dma_dev(vdpa); 345 vd_dev->vdev.dev.release = virtio_vdpa_release_dev; 346 vd_dev->vdev.config = &virtio_vdpa_config_ops; 347 vd_dev->vdpa = vdpa; 348 INIT_LIST_HEAD(&vd_dev->virtqueues); 349 spin_lock_init(&vd_dev->lock); 350 351 vd_dev->vdev.id.device = ops->get_device_id(vdpa); 352 if (vd_dev->vdev.id.device == 0) 353 goto err; 354 355 vd_dev->vdev.id.vendor = ops->get_vendor_id(vdpa); 356 ret = register_virtio_device(&vd_dev->vdev); 357 reg_dev = vd_dev; 358 if (ret) 359 goto err; 360 361 vdpa_set_drvdata(vdpa, vd_dev); 362 363 return 0; 364 365 err: 366 if (reg_dev) 367 put_device(&vd_dev->vdev.dev); 368 else 369 kfree(vd_dev); 370 return ret; 371 } 372 373 static void virtio_vdpa_remove(struct vdpa_device *vdpa) 374 { 375 struct virtio_vdpa_device *vd_dev = vdpa_get_drvdata(vdpa); 376 377 unregister_virtio_device(&vd_dev->vdev); 378 } 379 380 static struct vdpa_driver virtio_vdpa_driver = { 381 .driver = { 382 .name = "virtio_vdpa", 383 }, 384 .probe = virtio_vdpa_probe, 385 .remove = virtio_vdpa_remove, 386 }; 387 388 module_vdpa_driver(virtio_vdpa_driver); 389 390 MODULE_VERSION(MOD_VERSION); 391 MODULE_LICENSE(MOD_LICENSE); 392 MODULE_AUTHOR(MOD_AUTHOR); 393 MODULE_DESCRIPTION(MOD_DESC); 394