1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VDPA device simulator core. 4 * 5 * Copyright (c) 2020, Red Hat Inc. All rights reserved. 6 * Author: Jason Wang <jasowang@redhat.com> 7 * 8 */ 9 10 #include <linux/init.h> 11 #include <linux/module.h> 12 #include <linux/device.h> 13 #include <linux/kernel.h> 14 #include <linux/slab.h> 15 #include <linux/sched.h> 16 #include <linux/dma-map-ops.h> 17 #include <linux/vringh.h> 18 #include <linux/vdpa.h> 19 #include <linux/vhost_iotlb.h> 20 21 #include "vdpa_sim.h" 22 23 #define DRV_VERSION "0.1" 24 #define DRV_AUTHOR "Jason Wang <jasowang@redhat.com>" 25 #define DRV_DESC "vDPA Device Simulator core" 26 #define DRV_LICENSE "GPL v2" 27 28 static int batch_mapping = 1; 29 module_param(batch_mapping, int, 0444); 30 MODULE_PARM_DESC(batch_mapping, "Batched mapping 1 -Enable; 0 - Disable"); 31 32 static int max_iotlb_entries = 2048; 33 module_param(max_iotlb_entries, int, 0444); 34 MODULE_PARM_DESC(max_iotlb_entries, 35 "Maximum number of iotlb entries. 0 means unlimited. (default: 2048)"); 36 37 #define VDPASIM_QUEUE_ALIGN PAGE_SIZE 38 #define VDPASIM_QUEUE_MAX 256 39 #define VDPASIM_VENDOR_ID 0 40 41 static struct vdpasim *vdpa_to_sim(struct vdpa_device *vdpa) 42 { 43 return container_of(vdpa, struct vdpasim, vdpa); 44 } 45 46 static struct vdpasim *dev_to_sim(struct device *dev) 47 { 48 struct vdpa_device *vdpa = dev_to_vdpa(dev); 49 50 return vdpa_to_sim(vdpa); 51 } 52 53 static void vdpasim_vq_notify(struct vringh *vring) 54 { 55 struct vdpasim_virtqueue *vq = 56 container_of(vring, struct vdpasim_virtqueue, vring); 57 58 if (!vq->cb) 59 return; 60 61 vq->cb(vq->private); 62 } 63 64 static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx) 65 { 66 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 67 68 vringh_init_iotlb(&vq->vring, vdpasim->dev_attr.supported_features, 69 VDPASIM_QUEUE_MAX, false, 70 (struct vring_desc *)(uintptr_t)vq->desc_addr, 71 (struct vring_avail *) 72 (uintptr_t)vq->driver_addr, 73 (struct vring_used *) 74 (uintptr_t)vq->device_addr); 75 76 vq->vring.notify = vdpasim_vq_notify; 77 } 78 79 static void vdpasim_vq_reset(struct vdpasim *vdpasim, 80 struct vdpasim_virtqueue *vq) 81 { 82 vq->ready = false; 83 vq->desc_addr = 0; 84 vq->driver_addr = 0; 85 vq->device_addr = 0; 86 vq->cb = NULL; 87 vq->private = NULL; 88 vringh_init_iotlb(&vq->vring, vdpasim->dev_attr.supported_features, 89 VDPASIM_QUEUE_MAX, false, NULL, NULL, NULL); 90 91 vq->vring.notify = NULL; 92 } 93 94 static void vdpasim_reset(struct vdpasim *vdpasim) 95 { 96 int i; 97 98 for (i = 0; i < vdpasim->dev_attr.nvqs; i++) 99 vdpasim_vq_reset(vdpasim, &vdpasim->vqs[i]); 100 101 spin_lock(&vdpasim->iommu_lock); 102 vhost_iotlb_reset(vdpasim->iommu); 103 spin_unlock(&vdpasim->iommu_lock); 104 105 vdpasim->features = 0; 106 vdpasim->status = 0; 107 ++vdpasim->generation; 108 } 109 110 static int dir_to_perm(enum dma_data_direction dir) 111 { 112 int perm = -EFAULT; 113 114 switch (dir) { 115 case DMA_FROM_DEVICE: 116 perm = VHOST_MAP_WO; 117 break; 118 case DMA_TO_DEVICE: 119 perm = VHOST_MAP_RO; 120 break; 121 case DMA_BIDIRECTIONAL: 122 perm = VHOST_MAP_RW; 123 break; 124 default: 125 break; 126 } 127 128 return perm; 129 } 130 131 static dma_addr_t vdpasim_map_page(struct device *dev, struct page *page, 132 unsigned long offset, size_t size, 133 enum dma_data_direction dir, 134 unsigned long attrs) 135 { 136 struct vdpasim *vdpasim = dev_to_sim(dev); 137 struct vhost_iotlb *iommu = vdpasim->iommu; 138 u64 pa = (page_to_pfn(page) << PAGE_SHIFT) + offset; 139 int ret, perm = dir_to_perm(dir); 140 141 if (perm < 0) 142 return DMA_MAPPING_ERROR; 143 144 /* For simplicity, use identical mapping to avoid e.g iova 145 * allocator. 146 */ 147 spin_lock(&vdpasim->iommu_lock); 148 ret = vhost_iotlb_add_range(iommu, pa, pa + size - 1, 149 pa, dir_to_perm(dir)); 150 spin_unlock(&vdpasim->iommu_lock); 151 if (ret) 152 return DMA_MAPPING_ERROR; 153 154 return (dma_addr_t)(pa); 155 } 156 157 static void vdpasim_unmap_page(struct device *dev, dma_addr_t dma_addr, 158 size_t size, enum dma_data_direction dir, 159 unsigned long attrs) 160 { 161 struct vdpasim *vdpasim = dev_to_sim(dev); 162 struct vhost_iotlb *iommu = vdpasim->iommu; 163 164 spin_lock(&vdpasim->iommu_lock); 165 vhost_iotlb_del_range(iommu, (u64)dma_addr, 166 (u64)dma_addr + size - 1); 167 spin_unlock(&vdpasim->iommu_lock); 168 } 169 170 static void *vdpasim_alloc_coherent(struct device *dev, size_t size, 171 dma_addr_t *dma_addr, gfp_t flag, 172 unsigned long attrs) 173 { 174 struct vdpasim *vdpasim = dev_to_sim(dev); 175 struct vhost_iotlb *iommu = vdpasim->iommu; 176 void *addr = kmalloc(size, flag); 177 int ret; 178 179 spin_lock(&vdpasim->iommu_lock); 180 if (!addr) { 181 *dma_addr = DMA_MAPPING_ERROR; 182 } else { 183 u64 pa = virt_to_phys(addr); 184 185 ret = vhost_iotlb_add_range(iommu, (u64)pa, 186 (u64)pa + size - 1, 187 pa, VHOST_MAP_RW); 188 if (ret) { 189 *dma_addr = DMA_MAPPING_ERROR; 190 kfree(addr); 191 addr = NULL; 192 } else 193 *dma_addr = (dma_addr_t)pa; 194 } 195 spin_unlock(&vdpasim->iommu_lock); 196 197 return addr; 198 } 199 200 static void vdpasim_free_coherent(struct device *dev, size_t size, 201 void *vaddr, dma_addr_t dma_addr, 202 unsigned long attrs) 203 { 204 struct vdpasim *vdpasim = dev_to_sim(dev); 205 struct vhost_iotlb *iommu = vdpasim->iommu; 206 207 spin_lock(&vdpasim->iommu_lock); 208 vhost_iotlb_del_range(iommu, (u64)dma_addr, 209 (u64)dma_addr + size - 1); 210 spin_unlock(&vdpasim->iommu_lock); 211 212 kfree(phys_to_virt((uintptr_t)dma_addr)); 213 } 214 215 static const struct dma_map_ops vdpasim_dma_ops = { 216 .map_page = vdpasim_map_page, 217 .unmap_page = vdpasim_unmap_page, 218 .alloc = vdpasim_alloc_coherent, 219 .free = vdpasim_free_coherent, 220 }; 221 222 static const struct vdpa_config_ops vdpasim_config_ops; 223 static const struct vdpa_config_ops vdpasim_batch_config_ops; 224 225 struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr) 226 { 227 const struct vdpa_config_ops *ops; 228 struct vdpasim *vdpasim; 229 struct device *dev; 230 int i, ret = -ENOMEM; 231 232 if (batch_mapping) 233 ops = &vdpasim_batch_config_ops; 234 else 235 ops = &vdpasim_config_ops; 236 237 vdpasim = vdpa_alloc_device(struct vdpasim, vdpa, NULL, ops, 238 dev_attr->nvqs, dev_attr->name); 239 if (!vdpasim) 240 goto err_alloc; 241 242 vdpasim->dev_attr = *dev_attr; 243 INIT_WORK(&vdpasim->work, dev_attr->work_fn); 244 spin_lock_init(&vdpasim->lock); 245 spin_lock_init(&vdpasim->iommu_lock); 246 247 dev = &vdpasim->vdpa.dev; 248 dev->dma_mask = &dev->coherent_dma_mask; 249 if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64))) 250 goto err_iommu; 251 set_dma_ops(dev, &vdpasim_dma_ops); 252 vdpasim->vdpa.mdev = dev_attr->mgmt_dev; 253 254 vdpasim->config = kzalloc(dev_attr->config_size, GFP_KERNEL); 255 if (!vdpasim->config) 256 goto err_iommu; 257 258 vdpasim->vqs = kcalloc(dev_attr->nvqs, sizeof(struct vdpasim_virtqueue), 259 GFP_KERNEL); 260 if (!vdpasim->vqs) 261 goto err_iommu; 262 263 vdpasim->iommu = vhost_iotlb_alloc(max_iotlb_entries, 0); 264 if (!vdpasim->iommu) 265 goto err_iommu; 266 267 vdpasim->buffer = kvmalloc(dev_attr->buffer_size, GFP_KERNEL); 268 if (!vdpasim->buffer) 269 goto err_iommu; 270 271 for (i = 0; i < dev_attr->nvqs; i++) 272 vringh_set_iotlb(&vdpasim->vqs[i].vring, vdpasim->iommu); 273 274 vdpasim->vdpa.dma_dev = dev; 275 276 return vdpasim; 277 278 err_iommu: 279 put_device(dev); 280 err_alloc: 281 return ERR_PTR(ret); 282 } 283 EXPORT_SYMBOL_GPL(vdpasim_create); 284 285 static int vdpasim_set_vq_address(struct vdpa_device *vdpa, u16 idx, 286 u64 desc_area, u64 driver_area, 287 u64 device_area) 288 { 289 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 290 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 291 292 vq->desc_addr = desc_area; 293 vq->driver_addr = driver_area; 294 vq->device_addr = device_area; 295 296 return 0; 297 } 298 299 static void vdpasim_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num) 300 { 301 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 302 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 303 304 vq->num = num; 305 } 306 307 static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx) 308 { 309 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 310 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 311 312 if (vq->ready) 313 schedule_work(&vdpasim->work); 314 } 315 316 static void vdpasim_set_vq_cb(struct vdpa_device *vdpa, u16 idx, 317 struct vdpa_callback *cb) 318 { 319 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 320 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 321 322 vq->cb = cb->callback; 323 vq->private = cb->private; 324 } 325 326 static void vdpasim_set_vq_ready(struct vdpa_device *vdpa, u16 idx, bool ready) 327 { 328 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 329 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 330 331 spin_lock(&vdpasim->lock); 332 vq->ready = ready; 333 if (vq->ready) 334 vdpasim_queue_ready(vdpasim, idx); 335 spin_unlock(&vdpasim->lock); 336 } 337 338 static bool vdpasim_get_vq_ready(struct vdpa_device *vdpa, u16 idx) 339 { 340 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 341 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 342 343 return vq->ready; 344 } 345 346 static int vdpasim_set_vq_state(struct vdpa_device *vdpa, u16 idx, 347 const struct vdpa_vq_state *state) 348 { 349 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 350 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 351 struct vringh *vrh = &vq->vring; 352 353 spin_lock(&vdpasim->lock); 354 vrh->last_avail_idx = state->avail_index; 355 spin_unlock(&vdpasim->lock); 356 357 return 0; 358 } 359 360 static int vdpasim_get_vq_state(struct vdpa_device *vdpa, u16 idx, 361 struct vdpa_vq_state *state) 362 { 363 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 364 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 365 struct vringh *vrh = &vq->vring; 366 367 state->avail_index = vrh->last_avail_idx; 368 return 0; 369 } 370 371 static u32 vdpasim_get_vq_align(struct vdpa_device *vdpa) 372 { 373 return VDPASIM_QUEUE_ALIGN; 374 } 375 376 static u64 vdpasim_get_features(struct vdpa_device *vdpa) 377 { 378 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 379 380 return vdpasim->dev_attr.supported_features; 381 } 382 383 static int vdpasim_set_features(struct vdpa_device *vdpa, u64 features) 384 { 385 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 386 387 /* DMA mapping must be done by driver */ 388 if (!(features & (1ULL << VIRTIO_F_ACCESS_PLATFORM))) 389 return -EINVAL; 390 391 vdpasim->features = features & vdpasim->dev_attr.supported_features; 392 393 return 0; 394 } 395 396 static void vdpasim_set_config_cb(struct vdpa_device *vdpa, 397 struct vdpa_callback *cb) 398 { 399 /* We don't support config interrupt */ 400 } 401 402 static u16 vdpasim_get_vq_num_max(struct vdpa_device *vdpa) 403 { 404 return VDPASIM_QUEUE_MAX; 405 } 406 407 static u32 vdpasim_get_device_id(struct vdpa_device *vdpa) 408 { 409 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 410 411 return vdpasim->dev_attr.id; 412 } 413 414 static u32 vdpasim_get_vendor_id(struct vdpa_device *vdpa) 415 { 416 return VDPASIM_VENDOR_ID; 417 } 418 419 static u8 vdpasim_get_status(struct vdpa_device *vdpa) 420 { 421 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 422 u8 status; 423 424 spin_lock(&vdpasim->lock); 425 status = vdpasim->status; 426 spin_unlock(&vdpasim->lock); 427 428 return status; 429 } 430 431 static void vdpasim_set_status(struct vdpa_device *vdpa, u8 status) 432 { 433 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 434 435 spin_lock(&vdpasim->lock); 436 vdpasim->status = status; 437 if (status == 0) 438 vdpasim_reset(vdpasim); 439 spin_unlock(&vdpasim->lock); 440 } 441 442 static void vdpasim_get_config(struct vdpa_device *vdpa, unsigned int offset, 443 void *buf, unsigned int len) 444 { 445 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 446 447 if (offset + len > vdpasim->dev_attr.config_size) 448 return; 449 450 if (vdpasim->dev_attr.get_config) 451 vdpasim->dev_attr.get_config(vdpasim, vdpasim->config); 452 453 memcpy(buf, vdpasim->config + offset, len); 454 } 455 456 static void vdpasim_set_config(struct vdpa_device *vdpa, unsigned int offset, 457 const void *buf, unsigned int len) 458 { 459 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 460 461 if (offset + len > vdpasim->dev_attr.config_size) 462 return; 463 464 memcpy(vdpasim->config + offset, buf, len); 465 466 if (vdpasim->dev_attr.set_config) 467 vdpasim->dev_attr.set_config(vdpasim, vdpasim->config); 468 } 469 470 static u32 vdpasim_get_generation(struct vdpa_device *vdpa) 471 { 472 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 473 474 return vdpasim->generation; 475 } 476 477 static struct vdpa_iova_range vdpasim_get_iova_range(struct vdpa_device *vdpa) 478 { 479 struct vdpa_iova_range range = { 480 .first = 0ULL, 481 .last = ULLONG_MAX, 482 }; 483 484 return range; 485 } 486 487 static int vdpasim_set_map(struct vdpa_device *vdpa, 488 struct vhost_iotlb *iotlb) 489 { 490 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 491 struct vhost_iotlb_map *map; 492 u64 start = 0ULL, last = 0ULL - 1; 493 int ret; 494 495 spin_lock(&vdpasim->iommu_lock); 496 vhost_iotlb_reset(vdpasim->iommu); 497 498 for (map = vhost_iotlb_itree_first(iotlb, start, last); map; 499 map = vhost_iotlb_itree_next(map, start, last)) { 500 ret = vhost_iotlb_add_range(vdpasim->iommu, map->start, 501 map->last, map->addr, map->perm); 502 if (ret) 503 goto err; 504 } 505 spin_unlock(&vdpasim->iommu_lock); 506 return 0; 507 508 err: 509 vhost_iotlb_reset(vdpasim->iommu); 510 spin_unlock(&vdpasim->iommu_lock); 511 return ret; 512 } 513 514 static int vdpasim_dma_map(struct vdpa_device *vdpa, u64 iova, u64 size, 515 u64 pa, u32 perm) 516 { 517 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 518 int ret; 519 520 spin_lock(&vdpasim->iommu_lock); 521 ret = vhost_iotlb_add_range(vdpasim->iommu, iova, iova + size - 1, pa, 522 perm); 523 spin_unlock(&vdpasim->iommu_lock); 524 525 return ret; 526 } 527 528 static int vdpasim_dma_unmap(struct vdpa_device *vdpa, u64 iova, u64 size) 529 { 530 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 531 532 spin_lock(&vdpasim->iommu_lock); 533 vhost_iotlb_del_range(vdpasim->iommu, iova, iova + size - 1); 534 spin_unlock(&vdpasim->iommu_lock); 535 536 return 0; 537 } 538 539 static void vdpasim_free(struct vdpa_device *vdpa) 540 { 541 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 542 543 cancel_work_sync(&vdpasim->work); 544 kvfree(vdpasim->buffer); 545 if (vdpasim->iommu) 546 vhost_iotlb_free(vdpasim->iommu); 547 kfree(vdpasim->vqs); 548 kfree(vdpasim->config); 549 } 550 551 static const struct vdpa_config_ops vdpasim_config_ops = { 552 .set_vq_address = vdpasim_set_vq_address, 553 .set_vq_num = vdpasim_set_vq_num, 554 .kick_vq = vdpasim_kick_vq, 555 .set_vq_cb = vdpasim_set_vq_cb, 556 .set_vq_ready = vdpasim_set_vq_ready, 557 .get_vq_ready = vdpasim_get_vq_ready, 558 .set_vq_state = vdpasim_set_vq_state, 559 .get_vq_state = vdpasim_get_vq_state, 560 .get_vq_align = vdpasim_get_vq_align, 561 .get_features = vdpasim_get_features, 562 .set_features = vdpasim_set_features, 563 .set_config_cb = vdpasim_set_config_cb, 564 .get_vq_num_max = vdpasim_get_vq_num_max, 565 .get_device_id = vdpasim_get_device_id, 566 .get_vendor_id = vdpasim_get_vendor_id, 567 .get_status = vdpasim_get_status, 568 .set_status = vdpasim_set_status, 569 .get_config = vdpasim_get_config, 570 .set_config = vdpasim_set_config, 571 .get_generation = vdpasim_get_generation, 572 .get_iova_range = vdpasim_get_iova_range, 573 .dma_map = vdpasim_dma_map, 574 .dma_unmap = vdpasim_dma_unmap, 575 .free = vdpasim_free, 576 }; 577 578 static const struct vdpa_config_ops vdpasim_batch_config_ops = { 579 .set_vq_address = vdpasim_set_vq_address, 580 .set_vq_num = vdpasim_set_vq_num, 581 .kick_vq = vdpasim_kick_vq, 582 .set_vq_cb = vdpasim_set_vq_cb, 583 .set_vq_ready = vdpasim_set_vq_ready, 584 .get_vq_ready = vdpasim_get_vq_ready, 585 .set_vq_state = vdpasim_set_vq_state, 586 .get_vq_state = vdpasim_get_vq_state, 587 .get_vq_align = vdpasim_get_vq_align, 588 .get_features = vdpasim_get_features, 589 .set_features = vdpasim_set_features, 590 .set_config_cb = vdpasim_set_config_cb, 591 .get_vq_num_max = vdpasim_get_vq_num_max, 592 .get_device_id = vdpasim_get_device_id, 593 .get_vendor_id = vdpasim_get_vendor_id, 594 .get_status = vdpasim_get_status, 595 .set_status = vdpasim_set_status, 596 .get_config = vdpasim_get_config, 597 .set_config = vdpasim_set_config, 598 .get_generation = vdpasim_get_generation, 599 .get_iova_range = vdpasim_get_iova_range, 600 .set_map = vdpasim_set_map, 601 .free = vdpasim_free, 602 }; 603 604 MODULE_VERSION(DRV_VERSION); 605 MODULE_LICENSE(DRV_LICENSE); 606 MODULE_AUTHOR(DRV_AUTHOR); 607 MODULE_DESCRIPTION(DRV_DESC); 608