1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include <linux/device.h> 8 #include <linux/sched/task.h> 9 #include <linux/io-64-nonatomic-lo-hi.h> 10 #include <linux/cdev.h> 11 #include <linux/fs.h> 12 #include <linux/poll.h> 13 #include <linux/iommu.h> 14 #include <uapi/linux/idxd.h> 15 #include "registers.h" 16 #include "idxd.h" 17 18 struct idxd_cdev_context { 19 const char *name; 20 dev_t devt; 21 struct ida minor_ida; 22 }; 23 24 /* 25 * ictx is an array based off of accelerator types. enum idxd_type 26 * is used as index 27 */ 28 static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = { 29 { .name = "dsa" }, 30 { .name = "iax" } 31 }; 32 33 struct idxd_user_context { 34 struct idxd_wq *wq; 35 struct task_struct *task; 36 unsigned int pasid; 37 unsigned int flags; 38 struct iommu_sva *sva; 39 }; 40 41 static void idxd_cdev_dev_release(struct device *dev) 42 { 43 struct idxd_cdev *idxd_cdev = dev_to_cdev(dev); 44 struct idxd_cdev_context *cdev_ctx; 45 struct idxd_wq *wq = idxd_cdev->wq; 46 47 cdev_ctx = &ictx[wq->idxd->data->type]; 48 ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor); 49 kfree(idxd_cdev); 50 } 51 52 static struct device_type idxd_cdev_device_type = { 53 .name = "idxd_cdev", 54 .release = idxd_cdev_dev_release, 55 }; 56 57 static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode) 58 { 59 struct cdev *cdev = inode->i_cdev; 60 61 return container_of(cdev, struct idxd_cdev, cdev); 62 } 63 64 static inline struct idxd_wq *inode_wq(struct inode *inode) 65 { 66 struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode); 67 68 return idxd_cdev->wq; 69 } 70 71 static int idxd_cdev_open(struct inode *inode, struct file *filp) 72 { 73 struct idxd_user_context *ctx; 74 struct idxd_device *idxd; 75 struct idxd_wq *wq; 76 struct device *dev; 77 int rc = 0; 78 struct iommu_sva *sva; 79 unsigned int pasid; 80 81 wq = inode_wq(inode); 82 idxd = wq->idxd; 83 dev = &idxd->pdev->dev; 84 85 dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq)); 86 87 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 88 if (!ctx) 89 return -ENOMEM; 90 91 mutex_lock(&wq->wq_lock); 92 93 if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) { 94 rc = -EBUSY; 95 goto failed; 96 } 97 98 ctx->wq = wq; 99 filp->private_data = ctx; 100 101 if (device_user_pasid_enabled(idxd)) { 102 sva = iommu_sva_bind_device(dev, current->mm); 103 if (IS_ERR(sva)) { 104 rc = PTR_ERR(sva); 105 dev_err(dev, "pasid allocation failed: %d\n", rc); 106 goto failed; 107 } 108 109 pasid = iommu_sva_get_pasid(sva); 110 if (pasid == IOMMU_PASID_INVALID) { 111 iommu_sva_unbind_device(sva); 112 rc = -EINVAL; 113 goto failed; 114 } 115 116 ctx->sva = sva; 117 ctx->pasid = pasid; 118 119 if (wq_dedicated(wq)) { 120 rc = idxd_wq_set_pasid(wq, pasid); 121 if (rc < 0) { 122 iommu_sva_unbind_device(sva); 123 dev_err(dev, "wq set pasid failed: %d\n", rc); 124 goto failed; 125 } 126 } 127 } 128 129 idxd_wq_get(wq); 130 mutex_unlock(&wq->wq_lock); 131 return 0; 132 133 failed: 134 mutex_unlock(&wq->wq_lock); 135 kfree(ctx); 136 return rc; 137 } 138 139 static int idxd_cdev_release(struct inode *node, struct file *filep) 140 { 141 struct idxd_user_context *ctx = filep->private_data; 142 struct idxd_wq *wq = ctx->wq; 143 struct idxd_device *idxd = wq->idxd; 144 struct device *dev = &idxd->pdev->dev; 145 int rc; 146 147 dev_dbg(dev, "%s called\n", __func__); 148 filep->private_data = NULL; 149 150 /* Wait for in-flight operations to complete. */ 151 if (wq_shared(wq)) { 152 idxd_device_drain_pasid(idxd, ctx->pasid); 153 } else { 154 if (device_user_pasid_enabled(idxd)) { 155 /* The wq disable in the disable pasid function will drain the wq */ 156 rc = idxd_wq_disable_pasid(wq); 157 if (rc < 0) 158 dev_err(dev, "wq disable pasid failed.\n"); 159 } else { 160 idxd_wq_drain(wq); 161 } 162 } 163 164 if (ctx->sva) 165 iommu_sva_unbind_device(ctx->sva); 166 kfree(ctx); 167 mutex_lock(&wq->wq_lock); 168 idxd_wq_put(wq); 169 mutex_unlock(&wq->wq_lock); 170 return 0; 171 } 172 173 static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma, 174 const char *func) 175 { 176 struct device *dev = &wq->idxd->pdev->dev; 177 178 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) { 179 dev_info_ratelimited(dev, 180 "%s: %s: mapping too large: %lu\n", 181 current->comm, func, 182 vma->vm_end - vma->vm_start); 183 return -EINVAL; 184 } 185 186 return 0; 187 } 188 189 static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma) 190 { 191 struct idxd_user_context *ctx = filp->private_data; 192 struct idxd_wq *wq = ctx->wq; 193 struct idxd_device *idxd = wq->idxd; 194 struct pci_dev *pdev = idxd->pdev; 195 phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR); 196 unsigned long pfn; 197 int rc; 198 199 dev_dbg(&pdev->dev, "%s called\n", __func__); 200 rc = check_vma(wq, vma, __func__); 201 if (rc < 0) 202 return rc; 203 204 vma->vm_flags |= VM_DONTCOPY; 205 pfn = (base + idxd_get_wq_portal_full_offset(wq->id, 206 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT; 207 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 208 vma->vm_private_data = ctx; 209 210 return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE, 211 vma->vm_page_prot); 212 } 213 214 static __poll_t idxd_cdev_poll(struct file *filp, 215 struct poll_table_struct *wait) 216 { 217 struct idxd_user_context *ctx = filp->private_data; 218 struct idxd_wq *wq = ctx->wq; 219 struct idxd_device *idxd = wq->idxd; 220 __poll_t out = 0; 221 222 poll_wait(filp, &wq->err_queue, wait); 223 spin_lock(&idxd->dev_lock); 224 if (idxd->sw_err.valid) 225 out = EPOLLIN | EPOLLRDNORM; 226 spin_unlock(&idxd->dev_lock); 227 228 return out; 229 } 230 231 static const struct file_operations idxd_cdev_fops = { 232 .owner = THIS_MODULE, 233 .open = idxd_cdev_open, 234 .release = idxd_cdev_release, 235 .mmap = idxd_cdev_mmap, 236 .poll = idxd_cdev_poll, 237 }; 238 239 int idxd_cdev_get_major(struct idxd_device *idxd) 240 { 241 return MAJOR(ictx[idxd->data->type].devt); 242 } 243 244 int idxd_wq_add_cdev(struct idxd_wq *wq) 245 { 246 struct idxd_device *idxd = wq->idxd; 247 struct idxd_cdev *idxd_cdev; 248 struct cdev *cdev; 249 struct device *dev; 250 struct idxd_cdev_context *cdev_ctx; 251 int rc, minor; 252 253 idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL); 254 if (!idxd_cdev) 255 return -ENOMEM; 256 257 idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV; 258 idxd_cdev->wq = wq; 259 cdev = &idxd_cdev->cdev; 260 dev = cdev_dev(idxd_cdev); 261 cdev_ctx = &ictx[wq->idxd->data->type]; 262 minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL); 263 if (minor < 0) { 264 kfree(idxd_cdev); 265 return minor; 266 } 267 idxd_cdev->minor = minor; 268 269 device_initialize(dev); 270 dev->parent = wq_confdev(wq); 271 dev->bus = &dsa_bus_type; 272 dev->type = &idxd_cdev_device_type; 273 dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor); 274 275 rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id); 276 if (rc < 0) 277 goto err; 278 279 wq->idxd_cdev = idxd_cdev; 280 cdev_init(cdev, &idxd_cdev_fops); 281 rc = cdev_device_add(cdev, dev); 282 if (rc) { 283 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc); 284 goto err; 285 } 286 287 return 0; 288 289 err: 290 put_device(dev); 291 wq->idxd_cdev = NULL; 292 return rc; 293 } 294 295 void idxd_wq_del_cdev(struct idxd_wq *wq) 296 { 297 struct idxd_cdev *idxd_cdev; 298 299 idxd_cdev = wq->idxd_cdev; 300 wq->idxd_cdev = NULL; 301 cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev)); 302 put_device(cdev_dev(idxd_cdev)); 303 } 304 305 static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) 306 { 307 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); 308 struct idxd_device *idxd = wq->idxd; 309 int rc; 310 311 if (idxd->state != IDXD_DEV_ENABLED) 312 return -ENXIO; 313 314 /* 315 * User type WQ is enabled only when SVA is enabled for two reasons: 316 * - If no IOMMU or IOMMU Passthrough without SVA, userspace 317 * can directly access physical address through the WQ. 318 * - The IDXD cdev driver does not provide any ways to pin 319 * user pages and translate the address from user VA to IOVA or 320 * PA without IOMMU SVA. Therefore the application has no way 321 * to instruct the device to perform DMA function. This makes 322 * the cdev not usable for normal application usage. 323 */ 324 if (!device_user_pasid_enabled(idxd)) { 325 idxd->cmd_status = IDXD_SCMD_WQ_USER_NO_IOMMU; 326 dev_dbg(&idxd->pdev->dev, 327 "User type WQ cannot be enabled without SVA.\n"); 328 329 return -EOPNOTSUPP; 330 } 331 332 mutex_lock(&wq->wq_lock); 333 wq->type = IDXD_WQT_USER; 334 rc = drv_enable_wq(wq); 335 if (rc < 0) 336 goto err; 337 338 rc = idxd_wq_add_cdev(wq); 339 if (rc < 0) { 340 idxd->cmd_status = IDXD_SCMD_CDEV_ERR; 341 goto err_cdev; 342 } 343 344 idxd->cmd_status = 0; 345 mutex_unlock(&wq->wq_lock); 346 return 0; 347 348 err_cdev: 349 drv_disable_wq(wq); 350 err: 351 wq->type = IDXD_WQT_NONE; 352 mutex_unlock(&wq->wq_lock); 353 return rc; 354 } 355 356 static void idxd_user_drv_remove(struct idxd_dev *idxd_dev) 357 { 358 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); 359 360 mutex_lock(&wq->wq_lock); 361 idxd_wq_del_cdev(wq); 362 drv_disable_wq(wq); 363 wq->type = IDXD_WQT_NONE; 364 mutex_unlock(&wq->wq_lock); 365 } 366 367 static enum idxd_dev_type dev_types[] = { 368 IDXD_DEV_WQ, 369 IDXD_DEV_NONE, 370 }; 371 372 struct idxd_device_driver idxd_user_drv = { 373 .probe = idxd_user_drv_probe, 374 .remove = idxd_user_drv_remove, 375 .name = "user", 376 .type = dev_types, 377 }; 378 EXPORT_SYMBOL_GPL(idxd_user_drv); 379 380 int idxd_cdev_register(void) 381 { 382 int rc, i; 383 384 for (i = 0; i < IDXD_TYPE_MAX; i++) { 385 ida_init(&ictx[i].minor_ida); 386 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK, 387 ictx[i].name); 388 if (rc) 389 goto err_free_chrdev_region; 390 } 391 392 return 0; 393 394 err_free_chrdev_region: 395 for (i--; i >= 0; i--) 396 unregister_chrdev_region(ictx[i].devt, MINORMASK); 397 398 return rc; 399 } 400 401 void idxd_cdev_remove(void) 402 { 403 int i; 404 405 for (i = 0; i < IDXD_TYPE_MAX; i++) { 406 unregister_chrdev_region(ictx[i].devt, MINORMASK); 407 ida_destroy(&ictx[i].minor_ida); 408 } 409 } 410