1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VIRTIO based driver for vDPA device 4 * 5 * Copyright (c) 2020, Red Hat. All rights reserved. 6 * Author: Jason Wang <jasowang@redhat.com> 7 * 8 */ 9 10 #include <linux/init.h> 11 #include <linux/module.h> 12 #include <linux/device.h> 13 #include <linux/kernel.h> 14 #include <linux/slab.h> 15 #include <linux/uuid.h> 16 #include <linux/virtio.h> 17 #include <linux/vdpa.h> 18 #include <linux/virtio_config.h> 19 #include <linux/virtio_ring.h> 20 21 #define MOD_VERSION "0.1" 22 #define MOD_AUTHOR "Jason Wang <jasowang@redhat.com>" 23 #define MOD_DESC "vDPA bus driver for virtio devices" 24 #define MOD_LICENSE "GPL v2" 25 26 struct virtio_vdpa_device { 27 struct virtio_device vdev; 28 struct vdpa_device *vdpa; 29 u64 features; 30 31 /* The lock to protect virtqueue list */ 32 spinlock_t lock; 33 /* List of virtio_vdpa_vq_info */ 34 struct list_head virtqueues; 35 }; 36 37 struct virtio_vdpa_vq_info { 38 /* the actual virtqueue */ 39 struct virtqueue *vq; 40 41 /* the list node for the virtqueues list */ 42 struct list_head node; 43 }; 44 45 static inline struct virtio_vdpa_device * 46 to_virtio_vdpa_device(struct virtio_device *dev) 47 { 48 return container_of(dev, struct virtio_vdpa_device, vdev); 49 } 50 51 static struct vdpa_device *vd_get_vdpa(struct virtio_device *vdev) 52 { 53 return to_virtio_vdpa_device(vdev)->vdpa; 54 } 55 56 static void virtio_vdpa_get(struct virtio_device *vdev, unsigned int offset, 57 void *buf, unsigned int len) 58 { 59 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 60 61 vdpa_get_config(vdpa, offset, buf, len); 62 } 63 64 static void virtio_vdpa_set(struct virtio_device *vdev, unsigned int offset, 65 const void *buf, unsigned int len) 66 { 67 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 68 69 vdpa_set_config(vdpa, offset, buf, len); 70 } 71 72 static u32 virtio_vdpa_generation(struct virtio_device *vdev) 73 { 74 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 75 const struct vdpa_config_ops *ops = vdpa->config; 76 77 if (ops->get_generation) 78 return ops->get_generation(vdpa); 79 80 return 0; 81 } 82 83 static u8 virtio_vdpa_get_status(struct virtio_device *vdev) 84 { 85 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 86 const struct vdpa_config_ops *ops = vdpa->config; 87 88 return ops->get_status(vdpa); 89 } 90 91 static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status) 92 { 93 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 94 95 return vdpa_set_status(vdpa, status); 96 } 97 98 static void virtio_vdpa_reset(struct virtio_device *vdev) 99 { 100 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 101 102 vdpa_reset(vdpa); 103 } 104 105 static bool virtio_vdpa_notify(struct virtqueue *vq) 106 { 107 struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev); 108 const struct vdpa_config_ops *ops = vdpa->config; 109 110 ops->kick_vq(vdpa, vq->index); 111 112 return true; 113 } 114 115 static irqreturn_t virtio_vdpa_config_cb(void *private) 116 { 117 struct virtio_vdpa_device *vd_dev = private; 118 119 virtio_config_changed(&vd_dev->vdev); 120 121 return IRQ_HANDLED; 122 } 123 124 static irqreturn_t virtio_vdpa_virtqueue_cb(void *private) 125 { 126 struct virtio_vdpa_vq_info *info = private; 127 128 return vring_interrupt(0, info->vq); 129 } 130 131 static struct virtqueue * 132 virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, 133 void (*callback)(struct virtqueue *vq), 134 const char *name, u32 size, bool ctx) 135 { 136 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); 137 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 138 const struct vdpa_config_ops *ops = vdpa->config; 139 struct virtio_vdpa_vq_info *info; 140 struct vdpa_callback cb; 141 struct virtqueue *vq; 142 u64 desc_addr, driver_addr, device_addr; 143 /* Assume split virtqueue, switch to packed if necessary */ 144 struct vdpa_vq_state state = {0}; 145 unsigned long flags; 146 u32 align, max_num, min_num = 1; 147 bool may_reduce_num = true; 148 int err; 149 150 if (!name) 151 return NULL; 152 153 if (index >= vdpa->nvqs) 154 return ERR_PTR(-ENOENT); 155 156 /* Queue shouldn't already be set up. */ 157 if (ops->get_vq_ready(vdpa, index)) 158 return ERR_PTR(-ENOENT); 159 160 /* Allocate and fill out our active queue description */ 161 info = kmalloc(sizeof(*info), GFP_KERNEL); 162 if (!info) 163 return ERR_PTR(-ENOMEM); 164 165 max_num = ops->get_vq_num_max(vdpa); 166 if (max_num == 0) { 167 err = -ENOENT; 168 goto error_new_virtqueue; 169 } 170 171 if (!size || size > max_num) 172 size = max_num; 173 174 if (ops->get_vq_num_min) 175 min_num = ops->get_vq_num_min(vdpa); 176 177 may_reduce_num = (size == min_num) ? false : true; 178 179 /* Create the vring */ 180 align = ops->get_vq_align(vdpa); 181 vq = vring_create_virtqueue(index, size, align, vdev, 182 true, may_reduce_num, ctx, 183 virtio_vdpa_notify, callback, name); 184 if (!vq) { 185 err = -ENOMEM; 186 goto error_new_virtqueue; 187 } 188 189 vq->num_max = max_num; 190 191 /* Setup virtqueue callback */ 192 cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL; 193 cb.private = info; 194 ops->set_vq_cb(vdpa, index, &cb); 195 ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq)); 196 197 desc_addr = virtqueue_get_desc_addr(vq); 198 driver_addr = virtqueue_get_avail_addr(vq); 199 device_addr = virtqueue_get_used_addr(vq); 200 201 if (ops->set_vq_address(vdpa, index, 202 desc_addr, driver_addr, 203 device_addr)) { 204 err = -EINVAL; 205 goto err_vq; 206 } 207 208 /* reset virtqueue state index */ 209 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 210 struct vdpa_vq_state_packed *s = &state.packed; 211 212 s->last_avail_counter = 1; 213 s->last_avail_idx = 0; 214 s->last_used_counter = 1; 215 s->last_used_idx = 0; 216 } 217 err = ops->set_vq_state(vdpa, index, &state); 218 if (err) 219 goto err_vq; 220 221 ops->set_vq_ready(vdpa, index, 1); 222 223 vq->priv = info; 224 info->vq = vq; 225 226 spin_lock_irqsave(&vd_dev->lock, flags); 227 list_add(&info->node, &vd_dev->virtqueues); 228 spin_unlock_irqrestore(&vd_dev->lock, flags); 229 230 return vq; 231 232 err_vq: 233 vring_del_virtqueue(vq); 234 error_new_virtqueue: 235 ops->set_vq_ready(vdpa, index, 0); 236 /* VDPA driver should make sure vq is stopeed here */ 237 WARN_ON(ops->get_vq_ready(vdpa, index)); 238 kfree(info); 239 return ERR_PTR(err); 240 } 241 242 static void virtio_vdpa_del_vq(struct virtqueue *vq) 243 { 244 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev); 245 struct vdpa_device *vdpa = vd_dev->vdpa; 246 const struct vdpa_config_ops *ops = vdpa->config; 247 struct virtio_vdpa_vq_info *info = vq->priv; 248 unsigned int index = vq->index; 249 unsigned long flags; 250 251 spin_lock_irqsave(&vd_dev->lock, flags); 252 list_del(&info->node); 253 spin_unlock_irqrestore(&vd_dev->lock, flags); 254 255 /* Select and deactivate the queue (best effort) */ 256 ops->set_vq_ready(vdpa, index, 0); 257 258 vring_del_virtqueue(vq); 259 260 kfree(info); 261 } 262 263 static void virtio_vdpa_del_vqs(struct virtio_device *vdev) 264 { 265 struct virtqueue *vq, *n; 266 267 list_for_each_entry_safe(vq, n, &vdev->vqs, list) 268 virtio_vdpa_del_vq(vq); 269 } 270 271 static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs, 272 struct virtqueue *vqs[], 273 vq_callback_t *callbacks[], 274 const char * const names[], 275 u32 sizes[], 276 const bool *ctx, 277 struct irq_affinity *desc) 278 { 279 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); 280 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 281 const struct vdpa_config_ops *ops = vdpa->config; 282 struct vdpa_callback cb; 283 int i, err, queue_idx = 0; 284 285 for (i = 0; i < nvqs; ++i) { 286 if (!names[i]) { 287 vqs[i] = NULL; 288 continue; 289 } 290 291 vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++, callbacks[i], 292 names[i], sizes ? sizes[i] : 0, 293 ctx ? ctx[i] : false); 294 if (IS_ERR(vqs[i])) { 295 err = PTR_ERR(vqs[i]); 296 goto err_setup_vq; 297 } 298 } 299 300 cb.callback = virtio_vdpa_config_cb; 301 cb.private = vd_dev; 302 ops->set_config_cb(vdpa, &cb); 303 304 return 0; 305 306 err_setup_vq: 307 virtio_vdpa_del_vqs(vdev); 308 return err; 309 } 310 311 static u64 virtio_vdpa_get_features(struct virtio_device *vdev) 312 { 313 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 314 const struct vdpa_config_ops *ops = vdpa->config; 315 316 return ops->get_device_features(vdpa); 317 } 318 319 static int virtio_vdpa_finalize_features(struct virtio_device *vdev) 320 { 321 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 322 323 /* Give virtio_ring a chance to accept features. */ 324 vring_transport_features(vdev); 325 326 return vdpa_set_features(vdpa, vdev->features); 327 } 328 329 static const char *virtio_vdpa_bus_name(struct virtio_device *vdev) 330 { 331 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); 332 struct vdpa_device *vdpa = vd_dev->vdpa; 333 334 return dev_name(&vdpa->dev); 335 } 336 337 static const struct virtio_config_ops virtio_vdpa_config_ops = { 338 .get = virtio_vdpa_get, 339 .set = virtio_vdpa_set, 340 .generation = virtio_vdpa_generation, 341 .get_status = virtio_vdpa_get_status, 342 .set_status = virtio_vdpa_set_status, 343 .reset = virtio_vdpa_reset, 344 .find_vqs = virtio_vdpa_find_vqs, 345 .del_vqs = virtio_vdpa_del_vqs, 346 .get_features = virtio_vdpa_get_features, 347 .finalize_features = virtio_vdpa_finalize_features, 348 .bus_name = virtio_vdpa_bus_name, 349 }; 350 351 static void virtio_vdpa_release_dev(struct device *_d) 352 { 353 struct virtio_device *vdev = 354 container_of(_d, struct virtio_device, dev); 355 struct virtio_vdpa_device *vd_dev = 356 container_of(vdev, struct virtio_vdpa_device, vdev); 357 358 kfree(vd_dev); 359 } 360 361 static int virtio_vdpa_probe(struct vdpa_device *vdpa) 362 { 363 const struct vdpa_config_ops *ops = vdpa->config; 364 struct virtio_vdpa_device *vd_dev, *reg_dev = NULL; 365 int ret = -EINVAL; 366 367 vd_dev = kzalloc(sizeof(*vd_dev), GFP_KERNEL); 368 if (!vd_dev) 369 return -ENOMEM; 370 371 vd_dev->vdev.dev.parent = vdpa_get_dma_dev(vdpa); 372 vd_dev->vdev.dev.release = virtio_vdpa_release_dev; 373 vd_dev->vdev.config = &virtio_vdpa_config_ops; 374 vd_dev->vdpa = vdpa; 375 INIT_LIST_HEAD(&vd_dev->virtqueues); 376 spin_lock_init(&vd_dev->lock); 377 378 vd_dev->vdev.id.device = ops->get_device_id(vdpa); 379 if (vd_dev->vdev.id.device == 0) 380 goto err; 381 382 vd_dev->vdev.id.vendor = ops->get_vendor_id(vdpa); 383 ret = register_virtio_device(&vd_dev->vdev); 384 reg_dev = vd_dev; 385 if (ret) 386 goto err; 387 388 vdpa_set_drvdata(vdpa, vd_dev); 389 390 return 0; 391 392 err: 393 if (reg_dev) 394 put_device(&vd_dev->vdev.dev); 395 else 396 kfree(vd_dev); 397 return ret; 398 } 399 400 static void virtio_vdpa_remove(struct vdpa_device *vdpa) 401 { 402 struct virtio_vdpa_device *vd_dev = vdpa_get_drvdata(vdpa); 403 404 unregister_virtio_device(&vd_dev->vdev); 405 } 406 407 static struct vdpa_driver virtio_vdpa_driver = { 408 .driver = { 409 .name = "virtio_vdpa", 410 }, 411 .probe = virtio_vdpa_probe, 412 .remove = virtio_vdpa_remove, 413 }; 414 415 module_vdpa_driver(virtio_vdpa_driver); 416 417 MODULE_VERSION(MOD_VERSION); 418 MODULE_LICENSE(MOD_LICENSE); 419 MODULE_AUTHOR(MOD_AUTHOR); 420 MODULE_DESCRIPTION(MOD_DESC); 421