1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include <linux/device.h> 8 #include <linux/sched/task.h> 9 #include <linux/intel-svm.h> 10 #include <linux/io-64-nonatomic-lo-hi.h> 11 #include <linux/cdev.h> 12 #include <linux/fs.h> 13 #include <linux/poll.h> 14 #include <linux/iommu.h> 15 #include <uapi/linux/idxd.h> 16 #include "registers.h" 17 #include "idxd.h" 18 19 struct idxd_cdev_context { 20 const char *name; 21 dev_t devt; 22 struct ida minor_ida; 23 }; 24 25 /* 26 * ictx is an array based off of accelerator types. enum idxd_type 27 * is used as index 28 */ 29 static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = { 30 { .name = "dsa" }, 31 { .name = "iax" } 32 }; 33 34 struct idxd_user_context { 35 struct idxd_wq *wq; 36 struct task_struct *task; 37 unsigned int pasid; 38 unsigned int flags; 39 struct iommu_sva *sva; 40 }; 41 42 static void idxd_cdev_dev_release(struct device *dev) 43 { 44 struct idxd_cdev *idxd_cdev = dev_to_cdev(dev); 45 struct idxd_cdev_context *cdev_ctx; 46 struct idxd_wq *wq = idxd_cdev->wq; 47 48 cdev_ctx = &ictx[wq->idxd->data->type]; 49 ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor); 50 kfree(idxd_cdev); 51 } 52 53 static struct device_type idxd_cdev_device_type = { 54 .name = "idxd_cdev", 55 .release = idxd_cdev_dev_release, 56 }; 57 58 static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode) 59 { 60 struct cdev *cdev = inode->i_cdev; 61 62 return container_of(cdev, struct idxd_cdev, cdev); 63 } 64 65 static inline struct idxd_wq *inode_wq(struct inode *inode) 66 { 67 struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode); 68 69 return idxd_cdev->wq; 70 } 71 72 static int idxd_cdev_open(struct inode *inode, struct file *filp) 73 { 74 struct idxd_user_context *ctx; 75 struct idxd_device *idxd; 76 struct idxd_wq *wq; 77 struct device *dev; 78 int rc = 0; 79 struct iommu_sva *sva; 80 unsigned int pasid; 81 82 wq = inode_wq(inode); 83 idxd = wq->idxd; 84 dev = &idxd->pdev->dev; 85 86 dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq)); 87 88 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 89 if (!ctx) 90 return -ENOMEM; 91 92 mutex_lock(&wq->wq_lock); 93 94 if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) { 95 rc = -EBUSY; 96 goto failed; 97 } 98 99 ctx->wq = wq; 100 filp->private_data = ctx; 101 102 if (device_user_pasid_enabled(idxd)) { 103 sva = iommu_sva_bind_device(dev, current->mm, NULL); 104 if (IS_ERR(sva)) { 105 rc = PTR_ERR(sva); 106 dev_err(dev, "pasid allocation failed: %d\n", rc); 107 goto failed; 108 } 109 110 pasid = iommu_sva_get_pasid(sva); 111 if (pasid == IOMMU_PASID_INVALID) { 112 iommu_sva_unbind_device(sva); 113 rc = -EINVAL; 114 goto failed; 115 } 116 117 ctx->sva = sva; 118 ctx->pasid = pasid; 119 120 if (wq_dedicated(wq)) { 121 rc = idxd_wq_set_pasid(wq, pasid); 122 if (rc < 0) { 123 iommu_sva_unbind_device(sva); 124 dev_err(dev, "wq set pasid failed: %d\n", rc); 125 goto failed; 126 } 127 } 128 } 129 130 idxd_wq_get(wq); 131 mutex_unlock(&wq->wq_lock); 132 return 0; 133 134 failed: 135 mutex_unlock(&wq->wq_lock); 136 kfree(ctx); 137 return rc; 138 } 139 140 static int idxd_cdev_release(struct inode *node, struct file *filep) 141 { 142 struct idxd_user_context *ctx = filep->private_data; 143 struct idxd_wq *wq = ctx->wq; 144 struct idxd_device *idxd = wq->idxd; 145 struct device *dev = &idxd->pdev->dev; 146 int rc; 147 148 dev_dbg(dev, "%s called\n", __func__); 149 filep->private_data = NULL; 150 151 /* Wait for in-flight operations to complete. */ 152 if (wq_shared(wq)) { 153 idxd_device_drain_pasid(idxd, ctx->pasid); 154 } else { 155 if (device_user_pasid_enabled(idxd)) { 156 /* The wq disable in the disable pasid function will drain the wq */ 157 rc = idxd_wq_disable_pasid(wq); 158 if (rc < 0) 159 dev_err(dev, "wq disable pasid failed.\n"); 160 } else { 161 idxd_wq_drain(wq); 162 } 163 } 164 165 if (ctx->sva) 166 iommu_sva_unbind_device(ctx->sva); 167 kfree(ctx); 168 mutex_lock(&wq->wq_lock); 169 idxd_wq_put(wq); 170 mutex_unlock(&wq->wq_lock); 171 return 0; 172 } 173 174 static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma, 175 const char *func) 176 { 177 struct device *dev = &wq->idxd->pdev->dev; 178 179 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) { 180 dev_info_ratelimited(dev, 181 "%s: %s: mapping too large: %lu\n", 182 current->comm, func, 183 vma->vm_end - vma->vm_start); 184 return -EINVAL; 185 } 186 187 return 0; 188 } 189 190 static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma) 191 { 192 struct idxd_user_context *ctx = filp->private_data; 193 struct idxd_wq *wq = ctx->wq; 194 struct idxd_device *idxd = wq->idxd; 195 struct pci_dev *pdev = idxd->pdev; 196 phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR); 197 unsigned long pfn; 198 int rc; 199 200 dev_dbg(&pdev->dev, "%s called\n", __func__); 201 rc = check_vma(wq, vma, __func__); 202 if (rc < 0) 203 return rc; 204 205 vma->vm_flags |= VM_DONTCOPY; 206 pfn = (base + idxd_get_wq_portal_full_offset(wq->id, 207 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT; 208 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 209 vma->vm_private_data = ctx; 210 211 return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE, 212 vma->vm_page_prot); 213 } 214 215 static __poll_t idxd_cdev_poll(struct file *filp, 216 struct poll_table_struct *wait) 217 { 218 struct idxd_user_context *ctx = filp->private_data; 219 struct idxd_wq *wq = ctx->wq; 220 struct idxd_device *idxd = wq->idxd; 221 __poll_t out = 0; 222 223 poll_wait(filp, &wq->err_queue, wait); 224 spin_lock(&idxd->dev_lock); 225 if (idxd->sw_err.valid) 226 out = EPOLLIN | EPOLLRDNORM; 227 spin_unlock(&idxd->dev_lock); 228 229 return out; 230 } 231 232 static const struct file_operations idxd_cdev_fops = { 233 .owner = THIS_MODULE, 234 .open = idxd_cdev_open, 235 .release = idxd_cdev_release, 236 .mmap = idxd_cdev_mmap, 237 .poll = idxd_cdev_poll, 238 }; 239 240 int idxd_cdev_get_major(struct idxd_device *idxd) 241 { 242 return MAJOR(ictx[idxd->data->type].devt); 243 } 244 245 int idxd_wq_add_cdev(struct idxd_wq *wq) 246 { 247 struct idxd_device *idxd = wq->idxd; 248 struct idxd_cdev *idxd_cdev; 249 struct cdev *cdev; 250 struct device *dev; 251 struct idxd_cdev_context *cdev_ctx; 252 int rc, minor; 253 254 idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL); 255 if (!idxd_cdev) 256 return -ENOMEM; 257 258 idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV; 259 idxd_cdev->wq = wq; 260 cdev = &idxd_cdev->cdev; 261 dev = cdev_dev(idxd_cdev); 262 cdev_ctx = &ictx[wq->idxd->data->type]; 263 minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL); 264 if (minor < 0) { 265 kfree(idxd_cdev); 266 return minor; 267 } 268 idxd_cdev->minor = minor; 269 270 device_initialize(dev); 271 dev->parent = wq_confdev(wq); 272 dev->bus = &dsa_bus_type; 273 dev->type = &idxd_cdev_device_type; 274 dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor); 275 276 rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id); 277 if (rc < 0) 278 goto err; 279 280 wq->idxd_cdev = idxd_cdev; 281 cdev_init(cdev, &idxd_cdev_fops); 282 rc = cdev_device_add(cdev, dev); 283 if (rc) { 284 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc); 285 goto err; 286 } 287 288 return 0; 289 290 err: 291 put_device(dev); 292 wq->idxd_cdev = NULL; 293 return rc; 294 } 295 296 void idxd_wq_del_cdev(struct idxd_wq *wq) 297 { 298 struct idxd_cdev *idxd_cdev; 299 300 idxd_cdev = wq->idxd_cdev; 301 wq->idxd_cdev = NULL; 302 cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev)); 303 put_device(cdev_dev(idxd_cdev)); 304 } 305 306 static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) 307 { 308 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); 309 struct idxd_device *idxd = wq->idxd; 310 int rc; 311 312 if (idxd->state != IDXD_DEV_ENABLED) 313 return -ENXIO; 314 315 /* 316 * User type WQ is enabled only when SVA is enabled for two reasons: 317 * - If no IOMMU or IOMMU Passthrough without SVA, userspace 318 * can directly access physical address through the WQ. 319 * - The IDXD cdev driver does not provide any ways to pin 320 * user pages and translate the address from user VA to IOVA or 321 * PA without IOMMU SVA. Therefore the application has no way 322 * to instruct the device to perform DMA function. This makes 323 * the cdev not usable for normal application usage. 324 */ 325 if (!device_user_pasid_enabled(idxd)) { 326 idxd->cmd_status = IDXD_SCMD_WQ_USER_NO_IOMMU; 327 dev_dbg(&idxd->pdev->dev, 328 "User type WQ cannot be enabled without SVA.\n"); 329 330 return -EOPNOTSUPP; 331 } 332 333 mutex_lock(&wq->wq_lock); 334 wq->type = IDXD_WQT_USER; 335 rc = drv_enable_wq(wq); 336 if (rc < 0) 337 goto err; 338 339 rc = idxd_wq_add_cdev(wq); 340 if (rc < 0) { 341 idxd->cmd_status = IDXD_SCMD_CDEV_ERR; 342 goto err_cdev; 343 } 344 345 idxd->cmd_status = 0; 346 mutex_unlock(&wq->wq_lock); 347 return 0; 348 349 err_cdev: 350 drv_disable_wq(wq); 351 err: 352 wq->type = IDXD_WQT_NONE; 353 mutex_unlock(&wq->wq_lock); 354 return rc; 355 } 356 357 static void idxd_user_drv_remove(struct idxd_dev *idxd_dev) 358 { 359 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); 360 361 mutex_lock(&wq->wq_lock); 362 idxd_wq_del_cdev(wq); 363 drv_disable_wq(wq); 364 wq->type = IDXD_WQT_NONE; 365 mutex_unlock(&wq->wq_lock); 366 } 367 368 static enum idxd_dev_type dev_types[] = { 369 IDXD_DEV_WQ, 370 IDXD_DEV_NONE, 371 }; 372 373 struct idxd_device_driver idxd_user_drv = { 374 .probe = idxd_user_drv_probe, 375 .remove = idxd_user_drv_remove, 376 .name = "user", 377 .type = dev_types, 378 }; 379 EXPORT_SYMBOL_GPL(idxd_user_drv); 380 381 int idxd_cdev_register(void) 382 { 383 int rc, i; 384 385 for (i = 0; i < IDXD_TYPE_MAX; i++) { 386 ida_init(&ictx[i].minor_ida); 387 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK, 388 ictx[i].name); 389 if (rc) 390 goto err_free_chrdev_region; 391 } 392 393 return 0; 394 395 err_free_chrdev_region: 396 for (i--; i >= 0; i--) 397 unregister_chrdev_region(ictx[i].devt, MINORMASK); 398 399 return rc; 400 } 401 402 void idxd_cdev_remove(void) 403 { 404 int i; 405 406 for (i = 0; i < IDXD_TYPE_MAX; i++) { 407 unregister_chrdev_region(ictx[i].devt, MINORMASK); 408 ida_destroy(&ictx[i].minor_ida); 409 } 410 } 411