1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VIRTIO based driver for vDPA device 4 * 5 * Copyright (c) 2020, Red Hat. All rights reserved. 6 * Author: Jason Wang <jasowang@redhat.com> 7 * 8 */ 9 10 #include <linux/init.h> 11 #include <linux/module.h> 12 #include <linux/device.h> 13 #include <linux/kernel.h> 14 #include <linux/slab.h> 15 #include <linux/uuid.h> 16 #include <linux/virtio.h> 17 #include <linux/vdpa.h> 18 #include <linux/virtio_config.h> 19 #include <linux/virtio_ring.h> 20 21 #define MOD_VERSION "0.1" 22 #define MOD_AUTHOR "Jason Wang <jasowang@redhat.com>" 23 #define MOD_DESC "vDPA bus driver for virtio devices" 24 #define MOD_LICENSE "GPL v2" 25 26 struct virtio_vdpa_device { 27 struct virtio_device vdev; 28 struct vdpa_device *vdpa; 29 u64 features; 30 31 /* The lock to protect virtqueue list */ 32 spinlock_t lock; 33 /* List of virtio_vdpa_vq_info */ 34 struct list_head virtqueues; 35 }; 36 37 struct virtio_vdpa_vq_info { 38 /* the actual virtqueue */ 39 struct virtqueue *vq; 40 41 /* the list node for the virtqueues list */ 42 struct list_head node; 43 }; 44 45 static inline struct virtio_vdpa_device * 46 to_virtio_vdpa_device(struct virtio_device *dev) 47 { 48 return container_of(dev, struct virtio_vdpa_device, vdev); 49 } 50 51 static struct vdpa_device *vd_get_vdpa(struct virtio_device *vdev) 52 { 53 return to_virtio_vdpa_device(vdev)->vdpa; 54 } 55 56 static void virtio_vdpa_get(struct virtio_device *vdev, unsigned offset, 57 void *buf, unsigned len) 58 { 59 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 60 const struct vdpa_config_ops *ops = vdpa->config; 61 62 ops->get_config(vdpa, offset, buf, len); 63 } 64 65 static void virtio_vdpa_set(struct virtio_device *vdev, unsigned offset, 66 const void *buf, unsigned len) 67 { 68 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 69 const struct vdpa_config_ops *ops = vdpa->config; 70 71 ops->set_config(vdpa, offset, buf, len); 72 } 73 74 static u32 virtio_vdpa_generation(struct virtio_device *vdev) 75 { 76 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 77 const struct vdpa_config_ops *ops = vdpa->config; 78 79 if (ops->get_generation) 80 return ops->get_generation(vdpa); 81 82 return 0; 83 } 84 85 static u8 virtio_vdpa_get_status(struct virtio_device *vdev) 86 { 87 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 88 const struct vdpa_config_ops *ops = vdpa->config; 89 90 return ops->get_status(vdpa); 91 } 92 93 static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status) 94 { 95 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 96 const struct vdpa_config_ops *ops = vdpa->config; 97 98 return ops->set_status(vdpa, status); 99 } 100 101 static void virtio_vdpa_reset(struct virtio_device *vdev) 102 { 103 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 104 const struct vdpa_config_ops *ops = vdpa->config; 105 106 return ops->set_status(vdpa, 0); 107 } 108 109 static bool virtio_vdpa_notify(struct virtqueue *vq) 110 { 111 struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev); 112 const struct vdpa_config_ops *ops = vdpa->config; 113 114 ops->kick_vq(vdpa, vq->index); 115 116 return true; 117 } 118 119 static irqreturn_t virtio_vdpa_config_cb(void *private) 120 { 121 struct virtio_vdpa_device *vd_dev = private; 122 123 virtio_config_changed(&vd_dev->vdev); 124 125 return IRQ_HANDLED; 126 } 127 128 static irqreturn_t virtio_vdpa_virtqueue_cb(void *private) 129 { 130 struct virtio_vdpa_vq_info *info = private; 131 132 return vring_interrupt(0, info->vq); 133 } 134 135 static struct virtqueue * 136 virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, 137 void (*callback)(struct virtqueue *vq), 138 const char *name, bool ctx) 139 { 140 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); 141 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 142 const struct vdpa_config_ops *ops = vdpa->config; 143 struct virtio_vdpa_vq_info *info; 144 struct vdpa_callback cb; 145 struct virtqueue *vq; 146 u64 desc_addr, driver_addr, device_addr; 147 unsigned long flags; 148 u32 align, num; 149 int err; 150 151 if (!name) 152 return NULL; 153 154 /* Queue shouldn't already be set up. */ 155 if (ops->get_vq_ready(vdpa, index)) 156 return ERR_PTR(-ENOENT); 157 158 /* Allocate and fill out our active queue description */ 159 info = kmalloc(sizeof(*info), GFP_KERNEL); 160 if (!info) 161 return ERR_PTR(-ENOMEM); 162 163 num = ops->get_vq_num_max(vdpa); 164 if (num == 0) { 165 err = -ENOENT; 166 goto error_new_virtqueue; 167 } 168 169 /* Create the vring */ 170 align = ops->get_vq_align(vdpa); 171 vq = vring_create_virtqueue(index, num, align, vdev, 172 true, true, ctx, 173 virtio_vdpa_notify, callback, name); 174 if (!vq) { 175 err = -ENOMEM; 176 goto error_new_virtqueue; 177 } 178 179 /* Setup virtqueue callback */ 180 cb.callback = virtio_vdpa_virtqueue_cb; 181 cb.private = info; 182 ops->set_vq_cb(vdpa, index, &cb); 183 ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq)); 184 185 desc_addr = virtqueue_get_desc_addr(vq); 186 driver_addr = virtqueue_get_avail_addr(vq); 187 device_addr = virtqueue_get_used_addr(vq); 188 189 if (ops->set_vq_address(vdpa, index, 190 desc_addr, driver_addr, 191 device_addr)) { 192 err = -EINVAL; 193 goto err_vq; 194 } 195 196 ops->set_vq_ready(vdpa, index, 1); 197 198 vq->priv = info; 199 info->vq = vq; 200 201 spin_lock_irqsave(&vd_dev->lock, flags); 202 list_add(&info->node, &vd_dev->virtqueues); 203 spin_unlock_irqrestore(&vd_dev->lock, flags); 204 205 return vq; 206 207 err_vq: 208 vring_del_virtqueue(vq); 209 error_new_virtqueue: 210 ops->set_vq_ready(vdpa, index, 0); 211 /* VDPA driver should make sure vq is stopeed here */ 212 WARN_ON(ops->get_vq_ready(vdpa, index)); 213 kfree(info); 214 return ERR_PTR(err); 215 } 216 217 static void virtio_vdpa_del_vq(struct virtqueue *vq) 218 { 219 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev); 220 struct vdpa_device *vdpa = vd_dev->vdpa; 221 const struct vdpa_config_ops *ops = vdpa->config; 222 struct virtio_vdpa_vq_info *info = vq->priv; 223 unsigned int index = vq->index; 224 unsigned long flags; 225 226 spin_lock_irqsave(&vd_dev->lock, flags); 227 list_del(&info->node); 228 spin_unlock_irqrestore(&vd_dev->lock, flags); 229 230 /* Select and deactivate the queue */ 231 ops->set_vq_ready(vdpa, index, 0); 232 WARN_ON(ops->get_vq_ready(vdpa, index)); 233 234 vring_del_virtqueue(vq); 235 236 kfree(info); 237 } 238 239 static void virtio_vdpa_del_vqs(struct virtio_device *vdev) 240 { 241 struct virtqueue *vq, *n; 242 243 list_for_each_entry_safe(vq, n, &vdev->vqs, list) 244 virtio_vdpa_del_vq(vq); 245 } 246 247 static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned nvqs, 248 struct virtqueue *vqs[], 249 vq_callback_t *callbacks[], 250 const char * const names[], 251 const bool *ctx, 252 struct irq_affinity *desc) 253 { 254 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); 255 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 256 const struct vdpa_config_ops *ops = vdpa->config; 257 struct vdpa_callback cb; 258 int i, err, queue_idx = 0; 259 260 for (i = 0; i < nvqs; ++i) { 261 if (!names[i]) { 262 vqs[i] = NULL; 263 continue; 264 } 265 266 vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++, 267 callbacks[i], names[i], ctx ? 268 ctx[i] : false); 269 if (IS_ERR(vqs[i])) { 270 err = PTR_ERR(vqs[i]); 271 goto err_setup_vq; 272 } 273 } 274 275 cb.callback = virtio_vdpa_config_cb; 276 cb.private = vd_dev; 277 ops->set_config_cb(vdpa, &cb); 278 279 return 0; 280 281 err_setup_vq: 282 virtio_vdpa_del_vqs(vdev); 283 return err; 284 } 285 286 static u64 virtio_vdpa_get_features(struct virtio_device *vdev) 287 { 288 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 289 const struct vdpa_config_ops *ops = vdpa->config; 290 291 return ops->get_features(vdpa); 292 } 293 294 static int virtio_vdpa_finalize_features(struct virtio_device *vdev) 295 { 296 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 297 const struct vdpa_config_ops *ops = vdpa->config; 298 299 /* Give virtio_ring a chance to accept features. */ 300 vring_transport_features(vdev); 301 302 return ops->set_features(vdpa, vdev->features); 303 } 304 305 static const char *virtio_vdpa_bus_name(struct virtio_device *vdev) 306 { 307 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); 308 struct vdpa_device *vdpa = vd_dev->vdpa; 309 310 return dev_name(&vdpa->dev); 311 } 312 313 static const struct virtio_config_ops virtio_vdpa_config_ops = { 314 .get = virtio_vdpa_get, 315 .set = virtio_vdpa_set, 316 .generation = virtio_vdpa_generation, 317 .get_status = virtio_vdpa_get_status, 318 .set_status = virtio_vdpa_set_status, 319 .reset = virtio_vdpa_reset, 320 .find_vqs = virtio_vdpa_find_vqs, 321 .del_vqs = virtio_vdpa_del_vqs, 322 .get_features = virtio_vdpa_get_features, 323 .finalize_features = virtio_vdpa_finalize_features, 324 .bus_name = virtio_vdpa_bus_name, 325 }; 326 327 static void virtio_vdpa_release_dev(struct device *_d) 328 { 329 struct virtio_device *vdev = 330 container_of(_d, struct virtio_device, dev); 331 struct virtio_vdpa_device *vd_dev = 332 container_of(vdev, struct virtio_vdpa_device, vdev); 333 334 kfree(vd_dev); 335 } 336 337 static int virtio_vdpa_probe(struct vdpa_device *vdpa) 338 { 339 const struct vdpa_config_ops *ops = vdpa->config; 340 struct virtio_vdpa_device *vd_dev, *reg_dev = NULL; 341 int ret = -EINVAL; 342 343 vd_dev = kzalloc(sizeof(*vd_dev), GFP_KERNEL); 344 if (!vd_dev) 345 return -ENOMEM; 346 347 vd_dev->vdev.dev.parent = vdpa_get_dma_dev(vdpa); 348 vd_dev->vdev.dev.release = virtio_vdpa_release_dev; 349 vd_dev->vdev.config = &virtio_vdpa_config_ops; 350 vd_dev->vdpa = vdpa; 351 INIT_LIST_HEAD(&vd_dev->virtqueues); 352 spin_lock_init(&vd_dev->lock); 353 354 vd_dev->vdev.id.device = ops->get_device_id(vdpa); 355 if (vd_dev->vdev.id.device == 0) 356 goto err; 357 358 vd_dev->vdev.id.vendor = ops->get_vendor_id(vdpa); 359 ret = register_virtio_device(&vd_dev->vdev); 360 reg_dev = vd_dev; 361 if (ret) 362 goto err; 363 364 vdpa_set_drvdata(vdpa, vd_dev); 365 366 return 0; 367 368 err: 369 if (reg_dev) 370 put_device(&vd_dev->vdev.dev); 371 else 372 kfree(vd_dev); 373 return ret; 374 } 375 376 static void virtio_vdpa_remove(struct vdpa_device *vdpa) 377 { 378 struct virtio_vdpa_device *vd_dev = vdpa_get_drvdata(vdpa); 379 380 unregister_virtio_device(&vd_dev->vdev); 381 } 382 383 static struct vdpa_driver virtio_vdpa_driver = { 384 .driver = { 385 .name = "virtio_vdpa", 386 }, 387 .probe = virtio_vdpa_probe, 388 .remove = virtio_vdpa_remove, 389 }; 390 391 module_vdpa_driver(virtio_vdpa_driver); 392 393 MODULE_VERSION(MOD_VERSION); 394 MODULE_LICENSE(MOD_LICENSE); 395 MODULE_AUTHOR(MOD_AUTHOR); 396 MODULE_DESCRIPTION(MOD_DESC); 397