1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* 3 * Copyright 2013-2016 Freescale Semiconductor Inc. 4 * Copyright 2016-2017,2019-2020 NXP 5 */ 6 7 #include <linux/device.h> 8 #include <linux/iommu.h> 9 #include <linux/module.h> 10 #include <linux/mutex.h> 11 #include <linux/slab.h> 12 #include <linux/types.h> 13 #include <linux/vfio.h> 14 #include <linux/fsl/mc.h> 15 #include <linux/delay.h> 16 #include <linux/io-64-nonatomic-hi-lo.h> 17 18 #include "vfio_fsl_mc_private.h" 19 20 static struct fsl_mc_driver vfio_fsl_mc_driver; 21 22 static int vfio_fsl_mc_open_device(struct vfio_device *core_vdev) 23 { 24 struct vfio_fsl_mc_device *vdev = 25 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 26 struct fsl_mc_device *mc_dev = vdev->mc_dev; 27 int count = mc_dev->obj_desc.region_count; 28 int i; 29 30 vdev->regions = kcalloc(count, sizeof(struct vfio_fsl_mc_region), 31 GFP_KERNEL); 32 if (!vdev->regions) 33 return -ENOMEM; 34 35 for (i = 0; i < count; i++) { 36 struct resource *res = &mc_dev->regions[i]; 37 int no_mmap = is_fsl_mc_bus_dprc(mc_dev); 38 39 vdev->regions[i].addr = res->start; 40 vdev->regions[i].size = resource_size(res); 41 vdev->regions[i].type = mc_dev->regions[i].flags & IORESOURCE_BITS; 42 /* 43 * Only regions addressed with PAGE granularity may be 44 * MMAPed securely. 45 */ 46 if (!no_mmap && !(vdev->regions[i].addr & ~PAGE_MASK) && 47 !(vdev->regions[i].size & ~PAGE_MASK)) 48 vdev->regions[i].flags |= 49 VFIO_REGION_INFO_FLAG_MMAP; 50 vdev->regions[i].flags |= VFIO_REGION_INFO_FLAG_READ; 51 if (!(mc_dev->regions[i].flags & IORESOURCE_READONLY)) 52 vdev->regions[i].flags |= VFIO_REGION_INFO_FLAG_WRITE; 53 } 54 55 return 0; 56 } 57 58 static void vfio_fsl_mc_regions_cleanup(struct vfio_fsl_mc_device *vdev) 59 { 60 struct fsl_mc_device *mc_dev = vdev->mc_dev; 61 int i; 62 63 for (i = 0; i < mc_dev->obj_desc.region_count; i++) 64 iounmap(vdev->regions[i].ioaddr); 65 kfree(vdev->regions); 66 } 67 68 69 static void vfio_fsl_mc_close_device(struct vfio_device *core_vdev) 70 { 71 struct vfio_fsl_mc_device *vdev = 72 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 73 struct fsl_mc_device *mc_dev = vdev->mc_dev; 74 struct device *cont_dev = fsl_mc_cont_dev(&mc_dev->dev); 75 struct fsl_mc_device *mc_cont = to_fsl_mc_device(cont_dev); 76 int ret; 77 78 vfio_fsl_mc_regions_cleanup(vdev); 79 80 /* reset the device before cleaning up the interrupts */ 81 ret = dprc_reset_container(mc_cont->mc_io, 0, mc_cont->mc_handle, 82 mc_cont->obj_desc.id, 83 DPRC_RESET_OPTION_NON_RECURSIVE); 84 85 if (WARN_ON(ret)) 86 dev_warn(&mc_cont->dev, 87 "VFIO_FLS_MC: reset device has failed (%d)\n", ret); 88 89 vfio_fsl_mc_irqs_cleanup(vdev); 90 91 fsl_mc_cleanup_irq_pool(mc_cont); 92 } 93 94 static long vfio_fsl_mc_ioctl(struct vfio_device *core_vdev, 95 unsigned int cmd, unsigned long arg) 96 { 97 unsigned long minsz; 98 struct vfio_fsl_mc_device *vdev = 99 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 100 struct fsl_mc_device *mc_dev = vdev->mc_dev; 101 102 switch (cmd) { 103 case VFIO_DEVICE_GET_INFO: 104 { 105 struct vfio_device_info info; 106 107 minsz = offsetofend(struct vfio_device_info, num_irqs); 108 109 if (copy_from_user(&info, (void __user *)arg, minsz)) 110 return -EFAULT; 111 112 if (info.argsz < minsz) 113 return -EINVAL; 114 115 info.flags = VFIO_DEVICE_FLAGS_FSL_MC; 116 117 if (is_fsl_mc_bus_dprc(mc_dev)) 118 info.flags |= VFIO_DEVICE_FLAGS_RESET; 119 120 info.num_regions = mc_dev->obj_desc.region_count; 121 info.num_irqs = mc_dev->obj_desc.irq_count; 122 123 return copy_to_user((void __user *)arg, &info, minsz) ? 124 -EFAULT : 0; 125 } 126 case VFIO_DEVICE_GET_REGION_INFO: 127 { 128 struct vfio_region_info info; 129 130 minsz = offsetofend(struct vfio_region_info, offset); 131 132 if (copy_from_user(&info, (void __user *)arg, minsz)) 133 return -EFAULT; 134 135 if (info.argsz < minsz) 136 return -EINVAL; 137 138 if (info.index >= mc_dev->obj_desc.region_count) 139 return -EINVAL; 140 141 /* map offset to the physical address */ 142 info.offset = VFIO_FSL_MC_INDEX_TO_OFFSET(info.index); 143 info.size = vdev->regions[info.index].size; 144 info.flags = vdev->regions[info.index].flags; 145 146 if (copy_to_user((void __user *)arg, &info, minsz)) 147 return -EFAULT; 148 return 0; 149 } 150 case VFIO_DEVICE_GET_IRQ_INFO: 151 { 152 struct vfio_irq_info info; 153 154 minsz = offsetofend(struct vfio_irq_info, count); 155 if (copy_from_user(&info, (void __user *)arg, minsz)) 156 return -EFAULT; 157 158 if (info.argsz < minsz) 159 return -EINVAL; 160 161 if (info.index >= mc_dev->obj_desc.irq_count) 162 return -EINVAL; 163 164 info.flags = VFIO_IRQ_INFO_EVENTFD; 165 info.count = 1; 166 167 if (copy_to_user((void __user *)arg, &info, minsz)) 168 return -EFAULT; 169 return 0; 170 } 171 case VFIO_DEVICE_SET_IRQS: 172 { 173 struct vfio_irq_set hdr; 174 u8 *data = NULL; 175 int ret = 0; 176 size_t data_size = 0; 177 178 minsz = offsetofend(struct vfio_irq_set, count); 179 180 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 181 return -EFAULT; 182 183 ret = vfio_set_irqs_validate_and_prepare(&hdr, mc_dev->obj_desc.irq_count, 184 mc_dev->obj_desc.irq_count, &data_size); 185 if (ret) 186 return ret; 187 188 if (data_size) { 189 data = memdup_user((void __user *)(arg + minsz), 190 data_size); 191 if (IS_ERR(data)) 192 return PTR_ERR(data); 193 } 194 195 mutex_lock(&vdev->igate); 196 ret = vfio_fsl_mc_set_irqs_ioctl(vdev, hdr.flags, 197 hdr.index, hdr.start, 198 hdr.count, data); 199 mutex_unlock(&vdev->igate); 200 kfree(data); 201 202 return ret; 203 } 204 case VFIO_DEVICE_RESET: 205 { 206 int ret; 207 struct fsl_mc_device *mc_dev = vdev->mc_dev; 208 209 /* reset is supported only for the DPRC */ 210 if (!is_fsl_mc_bus_dprc(mc_dev)) 211 return -ENOTTY; 212 213 ret = dprc_reset_container(mc_dev->mc_io, 0, 214 mc_dev->mc_handle, 215 mc_dev->obj_desc.id, 216 DPRC_RESET_OPTION_NON_RECURSIVE); 217 return ret; 218 219 } 220 default: 221 return -ENOTTY; 222 } 223 } 224 225 static ssize_t vfio_fsl_mc_read(struct vfio_device *core_vdev, char __user *buf, 226 size_t count, loff_t *ppos) 227 { 228 struct vfio_fsl_mc_device *vdev = 229 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 230 unsigned int index = VFIO_FSL_MC_OFFSET_TO_INDEX(*ppos); 231 loff_t off = *ppos & VFIO_FSL_MC_OFFSET_MASK; 232 struct fsl_mc_device *mc_dev = vdev->mc_dev; 233 struct vfio_fsl_mc_region *region; 234 u64 data[8]; 235 int i; 236 237 if (index >= mc_dev->obj_desc.region_count) 238 return -EINVAL; 239 240 region = &vdev->regions[index]; 241 242 if (!(region->flags & VFIO_REGION_INFO_FLAG_READ)) 243 return -EINVAL; 244 245 if (!region->ioaddr) { 246 region->ioaddr = ioremap(region->addr, region->size); 247 if (!region->ioaddr) 248 return -ENOMEM; 249 } 250 251 if (count != 64 || off != 0) 252 return -EINVAL; 253 254 for (i = 7; i >= 0; i--) 255 data[i] = readq(region->ioaddr + i * sizeof(uint64_t)); 256 257 if (copy_to_user(buf, data, 64)) 258 return -EFAULT; 259 260 return count; 261 } 262 263 #define MC_CMD_COMPLETION_TIMEOUT_MS 5000 264 #define MC_CMD_COMPLETION_POLLING_MAX_SLEEP_USECS 500 265 266 static int vfio_fsl_mc_send_command(void __iomem *ioaddr, uint64_t *cmd_data) 267 { 268 int i; 269 enum mc_cmd_status status; 270 unsigned long timeout_usecs = MC_CMD_COMPLETION_TIMEOUT_MS * 1000; 271 272 /* Write at command parameter into portal */ 273 for (i = 7; i >= 1; i--) 274 writeq_relaxed(cmd_data[i], ioaddr + i * sizeof(uint64_t)); 275 276 /* Write command header in the end */ 277 writeq(cmd_data[0], ioaddr); 278 279 /* Wait for response before returning to user-space 280 * This can be optimized in future to even prepare response 281 * before returning to user-space and avoid read ioctl. 282 */ 283 for (;;) { 284 u64 header; 285 struct mc_cmd_header *resp_hdr; 286 287 header = cpu_to_le64(readq_relaxed(ioaddr)); 288 289 resp_hdr = (struct mc_cmd_header *)&header; 290 status = (enum mc_cmd_status)resp_hdr->status; 291 if (status != MC_CMD_STATUS_READY) 292 break; 293 294 udelay(MC_CMD_COMPLETION_POLLING_MAX_SLEEP_USECS); 295 timeout_usecs -= MC_CMD_COMPLETION_POLLING_MAX_SLEEP_USECS; 296 if (timeout_usecs == 0) 297 return -ETIMEDOUT; 298 } 299 300 return 0; 301 } 302 303 static ssize_t vfio_fsl_mc_write(struct vfio_device *core_vdev, 304 const char __user *buf, size_t count, 305 loff_t *ppos) 306 { 307 struct vfio_fsl_mc_device *vdev = 308 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 309 unsigned int index = VFIO_FSL_MC_OFFSET_TO_INDEX(*ppos); 310 loff_t off = *ppos & VFIO_FSL_MC_OFFSET_MASK; 311 struct fsl_mc_device *mc_dev = vdev->mc_dev; 312 struct vfio_fsl_mc_region *region; 313 u64 data[8]; 314 int ret; 315 316 if (index >= mc_dev->obj_desc.region_count) 317 return -EINVAL; 318 319 region = &vdev->regions[index]; 320 321 if (!(region->flags & VFIO_REGION_INFO_FLAG_WRITE)) 322 return -EINVAL; 323 324 if (!region->ioaddr) { 325 region->ioaddr = ioremap(region->addr, region->size); 326 if (!region->ioaddr) 327 return -ENOMEM; 328 } 329 330 if (count != 64 || off != 0) 331 return -EINVAL; 332 333 if (copy_from_user(&data, buf, 64)) 334 return -EFAULT; 335 336 ret = vfio_fsl_mc_send_command(region->ioaddr, data); 337 if (ret) 338 return ret; 339 340 return count; 341 342 } 343 344 static int vfio_fsl_mc_mmap_mmio(struct vfio_fsl_mc_region region, 345 struct vm_area_struct *vma) 346 { 347 u64 size = vma->vm_end - vma->vm_start; 348 u64 pgoff, base; 349 u8 region_cacheable; 350 351 pgoff = vma->vm_pgoff & 352 ((1U << (VFIO_FSL_MC_OFFSET_SHIFT - PAGE_SHIFT)) - 1); 353 base = pgoff << PAGE_SHIFT; 354 355 if (region.size < PAGE_SIZE || base + size > region.size) 356 return -EINVAL; 357 358 region_cacheable = (region.type & FSL_MC_REGION_CACHEABLE) && 359 (region.type & FSL_MC_REGION_SHAREABLE); 360 if (!region_cacheable) 361 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 362 363 vma->vm_pgoff = (region.addr >> PAGE_SHIFT) + pgoff; 364 365 return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 366 size, vma->vm_page_prot); 367 } 368 369 static int vfio_fsl_mc_mmap(struct vfio_device *core_vdev, 370 struct vm_area_struct *vma) 371 { 372 struct vfio_fsl_mc_device *vdev = 373 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 374 struct fsl_mc_device *mc_dev = vdev->mc_dev; 375 unsigned int index; 376 377 index = vma->vm_pgoff >> (VFIO_FSL_MC_OFFSET_SHIFT - PAGE_SHIFT); 378 379 if (vma->vm_end < vma->vm_start) 380 return -EINVAL; 381 if (vma->vm_start & ~PAGE_MASK) 382 return -EINVAL; 383 if (vma->vm_end & ~PAGE_MASK) 384 return -EINVAL; 385 if (!(vma->vm_flags & VM_SHARED)) 386 return -EINVAL; 387 if (index >= mc_dev->obj_desc.region_count) 388 return -EINVAL; 389 390 if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_MMAP)) 391 return -EINVAL; 392 393 if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_READ) 394 && (vma->vm_flags & VM_READ)) 395 return -EINVAL; 396 397 if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE) 398 && (vma->vm_flags & VM_WRITE)) 399 return -EINVAL; 400 401 vma->vm_private_data = mc_dev; 402 403 return vfio_fsl_mc_mmap_mmio(vdev->regions[index], vma); 404 } 405 406 static const struct vfio_device_ops vfio_fsl_mc_ops = { 407 .name = "vfio-fsl-mc", 408 .open_device = vfio_fsl_mc_open_device, 409 .close_device = vfio_fsl_mc_close_device, 410 .ioctl = vfio_fsl_mc_ioctl, 411 .read = vfio_fsl_mc_read, 412 .write = vfio_fsl_mc_write, 413 .mmap = vfio_fsl_mc_mmap, 414 }; 415 416 static int vfio_fsl_mc_bus_notifier(struct notifier_block *nb, 417 unsigned long action, void *data) 418 { 419 struct vfio_fsl_mc_device *vdev = container_of(nb, 420 struct vfio_fsl_mc_device, nb); 421 struct device *dev = data; 422 struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev); 423 struct fsl_mc_device *mc_cont = to_fsl_mc_device(mc_dev->dev.parent); 424 425 if (action == BUS_NOTIFY_ADD_DEVICE && 426 vdev->mc_dev == mc_cont) { 427 mc_dev->driver_override = kasprintf(GFP_KERNEL, "%s", 428 vfio_fsl_mc_ops.name); 429 if (!mc_dev->driver_override) 430 dev_warn(dev, "VFIO_FSL_MC: Setting driver override for device in dprc %s failed\n", 431 dev_name(&mc_cont->dev)); 432 else 433 dev_info(dev, "VFIO_FSL_MC: Setting driver override for device in dprc %s\n", 434 dev_name(&mc_cont->dev)); 435 } else if (action == BUS_NOTIFY_BOUND_DRIVER && 436 vdev->mc_dev == mc_cont) { 437 struct fsl_mc_driver *mc_drv = to_fsl_mc_driver(dev->driver); 438 439 if (mc_drv && mc_drv != &vfio_fsl_mc_driver) 440 dev_warn(dev, "VFIO_FSL_MC: Object %s bound to driver %s while DPRC bound to vfio-fsl-mc\n", 441 dev_name(dev), mc_drv->driver.name); 442 } 443 444 return 0; 445 } 446 447 static int vfio_fsl_mc_init_device(struct vfio_fsl_mc_device *vdev) 448 { 449 struct fsl_mc_device *mc_dev = vdev->mc_dev; 450 int ret; 451 452 /* Non-dprc devices share mc_io from parent */ 453 if (!is_fsl_mc_bus_dprc(mc_dev)) { 454 struct fsl_mc_device *mc_cont = to_fsl_mc_device(mc_dev->dev.parent); 455 456 mc_dev->mc_io = mc_cont->mc_io; 457 return 0; 458 } 459 460 vdev->nb.notifier_call = vfio_fsl_mc_bus_notifier; 461 ret = bus_register_notifier(&fsl_mc_bus_type, &vdev->nb); 462 if (ret) 463 return ret; 464 465 /* open DPRC, allocate a MC portal */ 466 ret = dprc_setup(mc_dev); 467 if (ret) { 468 dev_err(&mc_dev->dev, "VFIO_FSL_MC: Failed to setup DPRC (%d)\n", ret); 469 goto out_nc_unreg; 470 } 471 return 0; 472 473 out_nc_unreg: 474 bus_unregister_notifier(&fsl_mc_bus_type, &vdev->nb); 475 return ret; 476 } 477 478 static int vfio_fsl_mc_scan_container(struct fsl_mc_device *mc_dev) 479 { 480 int ret; 481 482 /* non dprc devices do not scan for other devices */ 483 if (!is_fsl_mc_bus_dprc(mc_dev)) 484 return 0; 485 ret = dprc_scan_container(mc_dev, false); 486 if (ret) { 487 dev_err(&mc_dev->dev, 488 "VFIO_FSL_MC: Container scanning failed (%d)\n", ret); 489 dprc_remove_devices(mc_dev, NULL, 0); 490 return ret; 491 } 492 return 0; 493 } 494 495 static void vfio_fsl_uninit_device(struct vfio_fsl_mc_device *vdev) 496 { 497 struct fsl_mc_device *mc_dev = vdev->mc_dev; 498 499 if (!is_fsl_mc_bus_dprc(mc_dev)) 500 return; 501 502 dprc_cleanup(mc_dev); 503 bus_unregister_notifier(&fsl_mc_bus_type, &vdev->nb); 504 } 505 506 static int vfio_fsl_mc_probe(struct fsl_mc_device *mc_dev) 507 { 508 struct iommu_group *group; 509 struct vfio_fsl_mc_device *vdev; 510 struct device *dev = &mc_dev->dev; 511 int ret; 512 513 group = vfio_iommu_group_get(dev); 514 if (!group) { 515 dev_err(dev, "VFIO_FSL_MC: No IOMMU group\n"); 516 return -EINVAL; 517 } 518 519 vdev = kzalloc(sizeof(*vdev), GFP_KERNEL); 520 if (!vdev) { 521 ret = -ENOMEM; 522 goto out_group_put; 523 } 524 525 vfio_init_group_dev(&vdev->vdev, dev, &vfio_fsl_mc_ops); 526 vdev->mc_dev = mc_dev; 527 mutex_init(&vdev->igate); 528 529 if (is_fsl_mc_bus_dprc(mc_dev)) 530 ret = vfio_assign_device_set(&vdev->vdev, &mc_dev->dev); 531 else 532 ret = vfio_assign_device_set(&vdev->vdev, mc_dev->dev.parent); 533 if (ret) 534 goto out_uninit; 535 536 ret = vfio_fsl_mc_init_device(vdev); 537 if (ret) 538 goto out_uninit; 539 540 ret = vfio_register_group_dev(&vdev->vdev); 541 if (ret) { 542 dev_err(dev, "VFIO_FSL_MC: Failed to add to vfio group\n"); 543 goto out_device; 544 } 545 546 ret = vfio_fsl_mc_scan_container(mc_dev); 547 if (ret) 548 goto out_group_dev; 549 dev_set_drvdata(dev, vdev); 550 return 0; 551 552 out_group_dev: 553 vfio_unregister_group_dev(&vdev->vdev); 554 out_device: 555 vfio_fsl_uninit_device(vdev); 556 out_uninit: 557 vfio_uninit_group_dev(&vdev->vdev); 558 kfree(vdev); 559 out_group_put: 560 vfio_iommu_group_put(group, dev); 561 return ret; 562 } 563 564 static int vfio_fsl_mc_remove(struct fsl_mc_device *mc_dev) 565 { 566 struct device *dev = &mc_dev->dev; 567 struct vfio_fsl_mc_device *vdev = dev_get_drvdata(dev); 568 569 vfio_unregister_group_dev(&vdev->vdev); 570 mutex_destroy(&vdev->igate); 571 572 dprc_remove_devices(mc_dev, NULL, 0); 573 vfio_fsl_uninit_device(vdev); 574 575 vfio_uninit_group_dev(&vdev->vdev); 576 kfree(vdev); 577 vfio_iommu_group_put(mc_dev->dev.iommu_group, dev); 578 579 return 0; 580 } 581 582 static struct fsl_mc_driver vfio_fsl_mc_driver = { 583 .probe = vfio_fsl_mc_probe, 584 .remove = vfio_fsl_mc_remove, 585 .driver = { 586 .name = "vfio-fsl-mc", 587 .owner = THIS_MODULE, 588 }, 589 }; 590 591 static int __init vfio_fsl_mc_driver_init(void) 592 { 593 return fsl_mc_driver_register(&vfio_fsl_mc_driver); 594 } 595 596 static void __exit vfio_fsl_mc_driver_exit(void) 597 { 598 fsl_mc_driver_unregister(&vfio_fsl_mc_driver); 599 } 600 601 module_init(vfio_fsl_mc_driver_init); 602 module_exit(vfio_fsl_mc_driver_exit); 603 604 MODULE_LICENSE("Dual BSD/GPL"); 605 MODULE_DESCRIPTION("VFIO for FSL-MC devices - User Level meta-driver"); 606