1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VDPA device simulator core. 4 * 5 * Copyright (c) 2020, Red Hat Inc. All rights reserved. 6 * Author: Jason Wang <jasowang@redhat.com> 7 * 8 */ 9 10 #include <linux/init.h> 11 #include <linux/module.h> 12 #include <linux/device.h> 13 #include <linux/kernel.h> 14 #include <linux/slab.h> 15 #include <linux/sched.h> 16 #include <linux/dma-map-ops.h> 17 #include <linux/vringh.h> 18 #include <linux/vdpa.h> 19 #include <linux/vhost_iotlb.h> 20 #include <linux/iova.h> 21 22 #include "vdpa_sim.h" 23 24 #define DRV_VERSION "0.1" 25 #define DRV_AUTHOR "Jason Wang <jasowang@redhat.com>" 26 #define DRV_DESC "vDPA Device Simulator core" 27 #define DRV_LICENSE "GPL v2" 28 29 static int batch_mapping = 1; 30 module_param(batch_mapping, int, 0444); 31 MODULE_PARM_DESC(batch_mapping, "Batched mapping 1 -Enable; 0 - Disable"); 32 33 static int max_iotlb_entries = 2048; 34 module_param(max_iotlb_entries, int, 0444); 35 MODULE_PARM_DESC(max_iotlb_entries, 36 "Maximum number of iotlb entries. 0 means unlimited. (default: 2048)"); 37 38 #define VDPASIM_QUEUE_ALIGN PAGE_SIZE 39 #define VDPASIM_QUEUE_MAX 256 40 #define VDPASIM_VENDOR_ID 0 41 42 static struct vdpasim *vdpa_to_sim(struct vdpa_device *vdpa) 43 { 44 return container_of(vdpa, struct vdpasim, vdpa); 45 } 46 47 static struct vdpasim *dev_to_sim(struct device *dev) 48 { 49 struct vdpa_device *vdpa = dev_to_vdpa(dev); 50 51 return vdpa_to_sim(vdpa); 52 } 53 54 static void vdpasim_vq_notify(struct vringh *vring) 55 { 56 struct vdpasim_virtqueue *vq = 57 container_of(vring, struct vdpasim_virtqueue, vring); 58 59 if (!vq->cb) 60 return; 61 62 vq->cb(vq->private); 63 } 64 65 static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx) 66 { 67 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 68 69 vringh_init_iotlb(&vq->vring, vdpasim->dev_attr.supported_features, 70 VDPASIM_QUEUE_MAX, false, 71 (struct vring_desc *)(uintptr_t)vq->desc_addr, 72 (struct vring_avail *) 73 (uintptr_t)vq->driver_addr, 74 (struct vring_used *) 75 (uintptr_t)vq->device_addr); 76 77 vq->vring.notify = vdpasim_vq_notify; 78 } 79 80 static void vdpasim_vq_reset(struct vdpasim *vdpasim, 81 struct vdpasim_virtqueue *vq) 82 { 83 vq->ready = false; 84 vq->desc_addr = 0; 85 vq->driver_addr = 0; 86 vq->device_addr = 0; 87 vq->cb = NULL; 88 vq->private = NULL; 89 vringh_init_iotlb(&vq->vring, vdpasim->dev_attr.supported_features, 90 VDPASIM_QUEUE_MAX, false, NULL, NULL, NULL); 91 92 vq->vring.notify = NULL; 93 } 94 95 static void vdpasim_reset(struct vdpasim *vdpasim) 96 { 97 int i; 98 99 for (i = 0; i < vdpasim->dev_attr.nvqs; i++) 100 vdpasim_vq_reset(vdpasim, &vdpasim->vqs[i]); 101 102 spin_lock(&vdpasim->iommu_lock); 103 vhost_iotlb_reset(vdpasim->iommu); 104 spin_unlock(&vdpasim->iommu_lock); 105 106 vdpasim->features = 0; 107 vdpasim->status = 0; 108 ++vdpasim->generation; 109 } 110 111 static int dir_to_perm(enum dma_data_direction dir) 112 { 113 int perm = -EFAULT; 114 115 switch (dir) { 116 case DMA_FROM_DEVICE: 117 perm = VHOST_MAP_WO; 118 break; 119 case DMA_TO_DEVICE: 120 perm = VHOST_MAP_RO; 121 break; 122 case DMA_BIDIRECTIONAL: 123 perm = VHOST_MAP_RW; 124 break; 125 default: 126 break; 127 } 128 129 return perm; 130 } 131 132 static dma_addr_t vdpasim_map_range(struct vdpasim *vdpasim, phys_addr_t paddr, 133 size_t size, unsigned int perm) 134 { 135 struct iova *iova; 136 dma_addr_t dma_addr; 137 int ret; 138 139 /* We set the limit_pfn to the maximum (ULONG_MAX - 1) */ 140 iova = alloc_iova(&vdpasim->iova, size, ULONG_MAX - 1, true); 141 if (!iova) 142 return DMA_MAPPING_ERROR; 143 144 dma_addr = iova_dma_addr(&vdpasim->iova, iova); 145 146 spin_lock(&vdpasim->iommu_lock); 147 ret = vhost_iotlb_add_range(vdpasim->iommu, (u64)dma_addr, 148 (u64)dma_addr + size - 1, (u64)paddr, perm); 149 spin_unlock(&vdpasim->iommu_lock); 150 151 if (ret) { 152 __free_iova(&vdpasim->iova, iova); 153 return DMA_MAPPING_ERROR; 154 } 155 156 return dma_addr; 157 } 158 159 static void vdpasim_unmap_range(struct vdpasim *vdpasim, dma_addr_t dma_addr, 160 size_t size) 161 { 162 spin_lock(&vdpasim->iommu_lock); 163 vhost_iotlb_del_range(vdpasim->iommu, (u64)dma_addr, 164 (u64)dma_addr + size - 1); 165 spin_unlock(&vdpasim->iommu_lock); 166 167 free_iova(&vdpasim->iova, iova_pfn(&vdpasim->iova, dma_addr)); 168 } 169 170 static dma_addr_t vdpasim_map_page(struct device *dev, struct page *page, 171 unsigned long offset, size_t size, 172 enum dma_data_direction dir, 173 unsigned long attrs) 174 { 175 struct vdpasim *vdpasim = dev_to_sim(dev); 176 phys_addr_t paddr = page_to_phys(page) + offset; 177 int perm = dir_to_perm(dir); 178 179 if (perm < 0) 180 return DMA_MAPPING_ERROR; 181 182 return vdpasim_map_range(vdpasim, paddr, size, perm); 183 } 184 185 static void vdpasim_unmap_page(struct device *dev, dma_addr_t dma_addr, 186 size_t size, enum dma_data_direction dir, 187 unsigned long attrs) 188 { 189 struct vdpasim *vdpasim = dev_to_sim(dev); 190 191 vdpasim_unmap_range(vdpasim, dma_addr, size); 192 } 193 194 static void *vdpasim_alloc_coherent(struct device *dev, size_t size, 195 dma_addr_t *dma_addr, gfp_t flag, 196 unsigned long attrs) 197 { 198 struct vdpasim *vdpasim = dev_to_sim(dev); 199 phys_addr_t paddr; 200 void *addr; 201 202 addr = kmalloc(size, flag); 203 if (!addr) { 204 *dma_addr = DMA_MAPPING_ERROR; 205 return NULL; 206 } 207 208 paddr = virt_to_phys(addr); 209 210 *dma_addr = vdpasim_map_range(vdpasim, paddr, size, VHOST_MAP_RW); 211 if (*dma_addr == DMA_MAPPING_ERROR) { 212 kfree(addr); 213 return NULL; 214 } 215 216 return addr; 217 } 218 219 static void vdpasim_free_coherent(struct device *dev, size_t size, 220 void *vaddr, dma_addr_t dma_addr, 221 unsigned long attrs) 222 { 223 struct vdpasim *vdpasim = dev_to_sim(dev); 224 225 vdpasim_unmap_range(vdpasim, dma_addr, size); 226 227 kfree(vaddr); 228 } 229 230 static const struct dma_map_ops vdpasim_dma_ops = { 231 .map_page = vdpasim_map_page, 232 .unmap_page = vdpasim_unmap_page, 233 .alloc = vdpasim_alloc_coherent, 234 .free = vdpasim_free_coherent, 235 }; 236 237 static const struct vdpa_config_ops vdpasim_config_ops; 238 static const struct vdpa_config_ops vdpasim_batch_config_ops; 239 240 struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr) 241 { 242 const struct vdpa_config_ops *ops; 243 struct vdpasim *vdpasim; 244 struct device *dev; 245 int i, ret = -ENOMEM; 246 247 if (batch_mapping) 248 ops = &vdpasim_batch_config_ops; 249 else 250 ops = &vdpasim_config_ops; 251 252 vdpasim = vdpa_alloc_device(struct vdpasim, vdpa, NULL, ops, 253 dev_attr->name); 254 if (!vdpasim) 255 goto err_alloc; 256 257 vdpasim->dev_attr = *dev_attr; 258 INIT_WORK(&vdpasim->work, dev_attr->work_fn); 259 spin_lock_init(&vdpasim->lock); 260 spin_lock_init(&vdpasim->iommu_lock); 261 262 dev = &vdpasim->vdpa.dev; 263 dev->dma_mask = &dev->coherent_dma_mask; 264 if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64))) 265 goto err_iommu; 266 set_dma_ops(dev, &vdpasim_dma_ops); 267 vdpasim->vdpa.mdev = dev_attr->mgmt_dev; 268 269 vdpasim->config = kzalloc(dev_attr->config_size, GFP_KERNEL); 270 if (!vdpasim->config) 271 goto err_iommu; 272 273 vdpasim->vqs = kcalloc(dev_attr->nvqs, sizeof(struct vdpasim_virtqueue), 274 GFP_KERNEL); 275 if (!vdpasim->vqs) 276 goto err_iommu; 277 278 vdpasim->iommu = vhost_iotlb_alloc(max_iotlb_entries, 0); 279 if (!vdpasim->iommu) 280 goto err_iommu; 281 282 vdpasim->buffer = kvmalloc(dev_attr->buffer_size, GFP_KERNEL); 283 if (!vdpasim->buffer) 284 goto err_iommu; 285 286 for (i = 0; i < dev_attr->nvqs; i++) 287 vringh_set_iotlb(&vdpasim->vqs[i].vring, vdpasim->iommu, 288 &vdpasim->iommu_lock); 289 290 ret = iova_cache_get(); 291 if (ret) 292 goto err_iommu; 293 294 /* For simplicity we use an IOVA allocator with byte granularity */ 295 init_iova_domain(&vdpasim->iova, 1, 0); 296 297 vdpasim->vdpa.dma_dev = dev; 298 299 return vdpasim; 300 301 err_iommu: 302 put_device(dev); 303 err_alloc: 304 return ERR_PTR(ret); 305 } 306 EXPORT_SYMBOL_GPL(vdpasim_create); 307 308 static int vdpasim_set_vq_address(struct vdpa_device *vdpa, u16 idx, 309 u64 desc_area, u64 driver_area, 310 u64 device_area) 311 { 312 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 313 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 314 315 vq->desc_addr = desc_area; 316 vq->driver_addr = driver_area; 317 vq->device_addr = device_area; 318 319 return 0; 320 } 321 322 static void vdpasim_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num) 323 { 324 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 325 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 326 327 vq->num = num; 328 } 329 330 static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx) 331 { 332 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 333 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 334 335 if (vq->ready) 336 schedule_work(&vdpasim->work); 337 } 338 339 static void vdpasim_set_vq_cb(struct vdpa_device *vdpa, u16 idx, 340 struct vdpa_callback *cb) 341 { 342 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 343 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 344 345 vq->cb = cb->callback; 346 vq->private = cb->private; 347 } 348 349 static void vdpasim_set_vq_ready(struct vdpa_device *vdpa, u16 idx, bool ready) 350 { 351 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 352 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 353 354 spin_lock(&vdpasim->lock); 355 vq->ready = ready; 356 if (vq->ready) 357 vdpasim_queue_ready(vdpasim, idx); 358 spin_unlock(&vdpasim->lock); 359 } 360 361 static bool vdpasim_get_vq_ready(struct vdpa_device *vdpa, u16 idx) 362 { 363 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 364 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 365 366 return vq->ready; 367 } 368 369 static int vdpasim_set_vq_state(struct vdpa_device *vdpa, u16 idx, 370 const struct vdpa_vq_state *state) 371 { 372 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 373 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 374 struct vringh *vrh = &vq->vring; 375 376 spin_lock(&vdpasim->lock); 377 vrh->last_avail_idx = state->avail_index; 378 spin_unlock(&vdpasim->lock); 379 380 return 0; 381 } 382 383 static int vdpasim_get_vq_state(struct vdpa_device *vdpa, u16 idx, 384 struct vdpa_vq_state *state) 385 { 386 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 387 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 388 struct vringh *vrh = &vq->vring; 389 390 state->avail_index = vrh->last_avail_idx; 391 return 0; 392 } 393 394 static u32 vdpasim_get_vq_align(struct vdpa_device *vdpa) 395 { 396 return VDPASIM_QUEUE_ALIGN; 397 } 398 399 static u64 vdpasim_get_features(struct vdpa_device *vdpa) 400 { 401 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 402 403 return vdpasim->dev_attr.supported_features; 404 } 405 406 static int vdpasim_set_features(struct vdpa_device *vdpa, u64 features) 407 { 408 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 409 410 /* DMA mapping must be done by driver */ 411 if (!(features & (1ULL << VIRTIO_F_ACCESS_PLATFORM))) 412 return -EINVAL; 413 414 vdpasim->features = features & vdpasim->dev_attr.supported_features; 415 416 return 0; 417 } 418 419 static void vdpasim_set_config_cb(struct vdpa_device *vdpa, 420 struct vdpa_callback *cb) 421 { 422 /* We don't support config interrupt */ 423 } 424 425 static u16 vdpasim_get_vq_num_max(struct vdpa_device *vdpa) 426 { 427 return VDPASIM_QUEUE_MAX; 428 } 429 430 static u32 vdpasim_get_device_id(struct vdpa_device *vdpa) 431 { 432 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 433 434 return vdpasim->dev_attr.id; 435 } 436 437 static u32 vdpasim_get_vendor_id(struct vdpa_device *vdpa) 438 { 439 return VDPASIM_VENDOR_ID; 440 } 441 442 static u8 vdpasim_get_status(struct vdpa_device *vdpa) 443 { 444 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 445 u8 status; 446 447 spin_lock(&vdpasim->lock); 448 status = vdpasim->status; 449 spin_unlock(&vdpasim->lock); 450 451 return status; 452 } 453 454 static void vdpasim_set_status(struct vdpa_device *vdpa, u8 status) 455 { 456 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 457 458 spin_lock(&vdpasim->lock); 459 vdpasim->status = status; 460 if (status == 0) 461 vdpasim_reset(vdpasim); 462 spin_unlock(&vdpasim->lock); 463 } 464 465 static size_t vdpasim_get_config_size(struct vdpa_device *vdpa) 466 { 467 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 468 469 return vdpasim->dev_attr.config_size; 470 } 471 472 static void vdpasim_get_config(struct vdpa_device *vdpa, unsigned int offset, 473 void *buf, unsigned int len) 474 { 475 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 476 477 if (offset + len > vdpasim->dev_attr.config_size) 478 return; 479 480 if (vdpasim->dev_attr.get_config) 481 vdpasim->dev_attr.get_config(vdpasim, vdpasim->config); 482 483 memcpy(buf, vdpasim->config + offset, len); 484 } 485 486 static void vdpasim_set_config(struct vdpa_device *vdpa, unsigned int offset, 487 const void *buf, unsigned int len) 488 { 489 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 490 491 if (offset + len > vdpasim->dev_attr.config_size) 492 return; 493 494 memcpy(vdpasim->config + offset, buf, len); 495 496 if (vdpasim->dev_attr.set_config) 497 vdpasim->dev_attr.set_config(vdpasim, vdpasim->config); 498 } 499 500 static u32 vdpasim_get_generation(struct vdpa_device *vdpa) 501 { 502 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 503 504 return vdpasim->generation; 505 } 506 507 static struct vdpa_iova_range vdpasim_get_iova_range(struct vdpa_device *vdpa) 508 { 509 struct vdpa_iova_range range = { 510 .first = 0ULL, 511 .last = ULLONG_MAX, 512 }; 513 514 return range; 515 } 516 517 static int vdpasim_set_map(struct vdpa_device *vdpa, 518 struct vhost_iotlb *iotlb) 519 { 520 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 521 struct vhost_iotlb_map *map; 522 u64 start = 0ULL, last = 0ULL - 1; 523 int ret; 524 525 spin_lock(&vdpasim->iommu_lock); 526 vhost_iotlb_reset(vdpasim->iommu); 527 528 for (map = vhost_iotlb_itree_first(iotlb, start, last); map; 529 map = vhost_iotlb_itree_next(map, start, last)) { 530 ret = vhost_iotlb_add_range(vdpasim->iommu, map->start, 531 map->last, map->addr, map->perm); 532 if (ret) 533 goto err; 534 } 535 spin_unlock(&vdpasim->iommu_lock); 536 return 0; 537 538 err: 539 vhost_iotlb_reset(vdpasim->iommu); 540 spin_unlock(&vdpasim->iommu_lock); 541 return ret; 542 } 543 544 static int vdpasim_dma_map(struct vdpa_device *vdpa, u64 iova, u64 size, 545 u64 pa, u32 perm) 546 { 547 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 548 int ret; 549 550 spin_lock(&vdpasim->iommu_lock); 551 ret = vhost_iotlb_add_range(vdpasim->iommu, iova, iova + size - 1, pa, 552 perm); 553 spin_unlock(&vdpasim->iommu_lock); 554 555 return ret; 556 } 557 558 static int vdpasim_dma_unmap(struct vdpa_device *vdpa, u64 iova, u64 size) 559 { 560 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 561 562 spin_lock(&vdpasim->iommu_lock); 563 vhost_iotlb_del_range(vdpasim->iommu, iova, iova + size - 1); 564 spin_unlock(&vdpasim->iommu_lock); 565 566 return 0; 567 } 568 569 static void vdpasim_free(struct vdpa_device *vdpa) 570 { 571 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 572 int i; 573 574 cancel_work_sync(&vdpasim->work); 575 576 for (i = 0; i < vdpasim->dev_attr.nvqs; i++) { 577 vringh_kiov_cleanup(&vdpasim->vqs[i].out_iov); 578 vringh_kiov_cleanup(&vdpasim->vqs[i].in_iov); 579 } 580 581 put_iova_domain(&vdpasim->iova); 582 iova_cache_put(); 583 kvfree(vdpasim->buffer); 584 if (vdpasim->iommu) 585 vhost_iotlb_free(vdpasim->iommu); 586 kfree(vdpasim->vqs); 587 kfree(vdpasim->config); 588 } 589 590 static const struct vdpa_config_ops vdpasim_config_ops = { 591 .set_vq_address = vdpasim_set_vq_address, 592 .set_vq_num = vdpasim_set_vq_num, 593 .kick_vq = vdpasim_kick_vq, 594 .set_vq_cb = vdpasim_set_vq_cb, 595 .set_vq_ready = vdpasim_set_vq_ready, 596 .get_vq_ready = vdpasim_get_vq_ready, 597 .set_vq_state = vdpasim_set_vq_state, 598 .get_vq_state = vdpasim_get_vq_state, 599 .get_vq_align = vdpasim_get_vq_align, 600 .get_features = vdpasim_get_features, 601 .set_features = vdpasim_set_features, 602 .set_config_cb = vdpasim_set_config_cb, 603 .get_vq_num_max = vdpasim_get_vq_num_max, 604 .get_device_id = vdpasim_get_device_id, 605 .get_vendor_id = vdpasim_get_vendor_id, 606 .get_status = vdpasim_get_status, 607 .set_status = vdpasim_set_status, 608 .get_config_size = vdpasim_get_config_size, 609 .get_config = vdpasim_get_config, 610 .set_config = vdpasim_set_config, 611 .get_generation = vdpasim_get_generation, 612 .get_iova_range = vdpasim_get_iova_range, 613 .dma_map = vdpasim_dma_map, 614 .dma_unmap = vdpasim_dma_unmap, 615 .free = vdpasim_free, 616 }; 617 618 static const struct vdpa_config_ops vdpasim_batch_config_ops = { 619 .set_vq_address = vdpasim_set_vq_address, 620 .set_vq_num = vdpasim_set_vq_num, 621 .kick_vq = vdpasim_kick_vq, 622 .set_vq_cb = vdpasim_set_vq_cb, 623 .set_vq_ready = vdpasim_set_vq_ready, 624 .get_vq_ready = vdpasim_get_vq_ready, 625 .set_vq_state = vdpasim_set_vq_state, 626 .get_vq_state = vdpasim_get_vq_state, 627 .get_vq_align = vdpasim_get_vq_align, 628 .get_features = vdpasim_get_features, 629 .set_features = vdpasim_set_features, 630 .set_config_cb = vdpasim_set_config_cb, 631 .get_vq_num_max = vdpasim_get_vq_num_max, 632 .get_device_id = vdpasim_get_device_id, 633 .get_vendor_id = vdpasim_get_vendor_id, 634 .get_status = vdpasim_get_status, 635 .set_status = vdpasim_set_status, 636 .get_config_size = vdpasim_get_config_size, 637 .get_config = vdpasim_get_config, 638 .set_config = vdpasim_set_config, 639 .get_generation = vdpasim_get_generation, 640 .get_iova_range = vdpasim_get_iova_range, 641 .set_map = vdpasim_set_map, 642 .free = vdpasim_free, 643 }; 644 645 MODULE_VERSION(DRV_VERSION); 646 MODULE_LICENSE(DRV_LICENSE); 647 MODULE_AUTHOR(DRV_AUTHOR); 648 MODULE_DESCRIPTION(DRV_DESC); 649