1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include <linux/device.h> 8 #include <linux/sched/task.h> 9 #include <linux/intel-svm.h> 10 #include <linux/io-64-nonatomic-lo-hi.h> 11 #include <linux/cdev.h> 12 #include <linux/fs.h> 13 #include <linux/poll.h> 14 #include <linux/iommu.h> 15 #include <uapi/linux/idxd.h> 16 #include "registers.h" 17 #include "idxd.h" 18 19 struct idxd_cdev_context { 20 const char *name; 21 dev_t devt; 22 struct ida minor_ida; 23 }; 24 25 /* 26 * ictx is an array based off of accelerator types. enum idxd_type 27 * is used as index 28 */ 29 static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = { 30 { .name = "dsa" }, 31 { .name = "iax" } 32 }; 33 34 struct idxd_user_context { 35 struct idxd_wq *wq; 36 struct task_struct *task; 37 unsigned int pasid; 38 unsigned int flags; 39 struct iommu_sva *sva; 40 }; 41 42 enum idxd_cdev_cleanup { 43 CDEV_NORMAL = 0, 44 CDEV_FAILED, 45 }; 46 47 static void idxd_cdev_dev_release(struct device *dev) 48 { 49 dev_dbg(dev, "releasing cdev device\n"); 50 kfree(dev); 51 } 52 53 static struct device_type idxd_cdev_device_type = { 54 .name = "idxd_cdev", 55 .release = idxd_cdev_dev_release, 56 }; 57 58 static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode) 59 { 60 struct cdev *cdev = inode->i_cdev; 61 62 return container_of(cdev, struct idxd_cdev, cdev); 63 } 64 65 static inline struct idxd_wq *idxd_cdev_wq(struct idxd_cdev *idxd_cdev) 66 { 67 return container_of(idxd_cdev, struct idxd_wq, idxd_cdev); 68 } 69 70 static inline struct idxd_wq *inode_wq(struct inode *inode) 71 { 72 return idxd_cdev_wq(inode_idxd_cdev(inode)); 73 } 74 75 static int idxd_cdev_open(struct inode *inode, struct file *filp) 76 { 77 struct idxd_user_context *ctx; 78 struct idxd_device *idxd; 79 struct idxd_wq *wq; 80 struct device *dev; 81 int rc = 0; 82 struct iommu_sva *sva; 83 unsigned int pasid; 84 85 wq = inode_wq(inode); 86 idxd = wq->idxd; 87 dev = &idxd->pdev->dev; 88 89 dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq)); 90 91 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 92 if (!ctx) 93 return -ENOMEM; 94 95 mutex_lock(&wq->wq_lock); 96 97 if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) { 98 rc = -EBUSY; 99 goto failed; 100 } 101 102 ctx->wq = wq; 103 filp->private_data = ctx; 104 105 if (device_pasid_enabled(idxd)) { 106 sva = iommu_sva_bind_device(dev, current->mm, NULL); 107 if (IS_ERR(sva)) { 108 rc = PTR_ERR(sva); 109 dev_err(dev, "pasid allocation failed: %d\n", rc); 110 goto failed; 111 } 112 113 pasid = iommu_sva_get_pasid(sva); 114 if (pasid == IOMMU_PASID_INVALID) { 115 iommu_sva_unbind_device(sva); 116 goto failed; 117 } 118 119 ctx->sva = sva; 120 ctx->pasid = pasid; 121 122 if (wq_dedicated(wq)) { 123 rc = idxd_wq_set_pasid(wq, pasid); 124 if (rc < 0) { 125 iommu_sva_unbind_device(sva); 126 dev_err(dev, "wq set pasid failed: %d\n", rc); 127 goto failed; 128 } 129 } 130 } 131 132 idxd_wq_get(wq); 133 mutex_unlock(&wq->wq_lock); 134 return 0; 135 136 failed: 137 mutex_unlock(&wq->wq_lock); 138 kfree(ctx); 139 return rc; 140 } 141 142 static int idxd_cdev_release(struct inode *node, struct file *filep) 143 { 144 struct idxd_user_context *ctx = filep->private_data; 145 struct idxd_wq *wq = ctx->wq; 146 struct idxd_device *idxd = wq->idxd; 147 struct device *dev = &idxd->pdev->dev; 148 int rc; 149 150 dev_dbg(dev, "%s called\n", __func__); 151 filep->private_data = NULL; 152 153 /* Wait for in-flight operations to complete. */ 154 if (wq_shared(wq)) { 155 idxd_device_drain_pasid(idxd, ctx->pasid); 156 } else { 157 if (device_pasid_enabled(idxd)) { 158 /* The wq disable in the disable pasid function will drain the wq */ 159 rc = idxd_wq_disable_pasid(wq); 160 if (rc < 0) 161 dev_err(dev, "wq disable pasid failed.\n"); 162 } else { 163 idxd_wq_drain(wq); 164 } 165 } 166 167 if (ctx->sva) 168 iommu_sva_unbind_device(ctx->sva); 169 kfree(ctx); 170 mutex_lock(&wq->wq_lock); 171 idxd_wq_put(wq); 172 mutex_unlock(&wq->wq_lock); 173 return 0; 174 } 175 176 static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma, 177 const char *func) 178 { 179 struct device *dev = &wq->idxd->pdev->dev; 180 181 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) { 182 dev_info_ratelimited(dev, 183 "%s: %s: mapping too large: %lu\n", 184 current->comm, func, 185 vma->vm_end - vma->vm_start); 186 return -EINVAL; 187 } 188 189 return 0; 190 } 191 192 static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma) 193 { 194 struct idxd_user_context *ctx = filp->private_data; 195 struct idxd_wq *wq = ctx->wq; 196 struct idxd_device *idxd = wq->idxd; 197 struct pci_dev *pdev = idxd->pdev; 198 phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR); 199 unsigned long pfn; 200 int rc; 201 202 dev_dbg(&pdev->dev, "%s called\n", __func__); 203 rc = check_vma(wq, vma, __func__); 204 if (rc < 0) 205 return rc; 206 207 vma->vm_flags |= VM_DONTCOPY; 208 pfn = (base + idxd_get_wq_portal_full_offset(wq->id, 209 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT; 210 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 211 vma->vm_private_data = ctx; 212 213 return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE, 214 vma->vm_page_prot); 215 } 216 217 static __poll_t idxd_cdev_poll(struct file *filp, 218 struct poll_table_struct *wait) 219 { 220 struct idxd_user_context *ctx = filp->private_data; 221 struct idxd_wq *wq = ctx->wq; 222 struct idxd_device *idxd = wq->idxd; 223 struct idxd_cdev *idxd_cdev = &wq->idxd_cdev; 224 unsigned long flags; 225 __poll_t out = 0; 226 227 poll_wait(filp, &idxd_cdev->err_queue, wait); 228 spin_lock_irqsave(&idxd->dev_lock, flags); 229 if (idxd->sw_err.valid) 230 out = EPOLLIN | EPOLLRDNORM; 231 spin_unlock_irqrestore(&idxd->dev_lock, flags); 232 233 return out; 234 } 235 236 static const struct file_operations idxd_cdev_fops = { 237 .owner = THIS_MODULE, 238 .open = idxd_cdev_open, 239 .release = idxd_cdev_release, 240 .mmap = idxd_cdev_mmap, 241 .poll = idxd_cdev_poll, 242 }; 243 244 int idxd_cdev_get_major(struct idxd_device *idxd) 245 { 246 return MAJOR(ictx[idxd->type].devt); 247 } 248 249 static int idxd_wq_cdev_dev_setup(struct idxd_wq *wq) 250 { 251 struct idxd_device *idxd = wq->idxd; 252 struct idxd_cdev *idxd_cdev = &wq->idxd_cdev; 253 struct idxd_cdev_context *cdev_ctx; 254 struct device *dev; 255 int minor, rc; 256 257 idxd_cdev->dev = kzalloc(sizeof(*idxd_cdev->dev), GFP_KERNEL); 258 if (!idxd_cdev->dev) 259 return -ENOMEM; 260 261 dev = idxd_cdev->dev; 262 dev->parent = &idxd->pdev->dev; 263 dev_set_name(dev, "%s/wq%u.%u", idxd_get_dev_name(idxd), 264 idxd->id, wq->id); 265 dev->bus = idxd_get_bus_type(idxd); 266 267 cdev_ctx = &ictx[wq->idxd->type]; 268 minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL); 269 if (minor < 0) { 270 rc = minor; 271 kfree(dev); 272 goto ida_err; 273 } 274 275 dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor); 276 dev->type = &idxd_cdev_device_type; 277 rc = device_register(dev); 278 if (rc < 0) { 279 dev_err(&idxd->pdev->dev, "device register failed\n"); 280 goto dev_reg_err; 281 } 282 idxd_cdev->minor = minor; 283 284 return 0; 285 286 dev_reg_err: 287 ida_simple_remove(&cdev_ctx->minor_ida, MINOR(dev->devt)); 288 put_device(dev); 289 ida_err: 290 idxd_cdev->dev = NULL; 291 return rc; 292 } 293 294 static void idxd_wq_cdev_cleanup(struct idxd_wq *wq, 295 enum idxd_cdev_cleanup cdev_state) 296 { 297 struct idxd_cdev *idxd_cdev = &wq->idxd_cdev; 298 struct idxd_cdev_context *cdev_ctx; 299 300 cdev_ctx = &ictx[wq->idxd->type]; 301 if (cdev_state == CDEV_NORMAL) 302 cdev_del(&idxd_cdev->cdev); 303 device_unregister(idxd_cdev->dev); 304 /* 305 * The device_type->release() will be called on the device and free 306 * the allocated struct device. We can just forget it. 307 */ 308 ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor); 309 idxd_cdev->dev = NULL; 310 idxd_cdev->minor = -1; 311 } 312 313 int idxd_wq_add_cdev(struct idxd_wq *wq) 314 { 315 struct idxd_cdev *idxd_cdev = &wq->idxd_cdev; 316 struct cdev *cdev = &idxd_cdev->cdev; 317 struct device *dev; 318 int rc; 319 320 rc = idxd_wq_cdev_dev_setup(wq); 321 if (rc < 0) 322 return rc; 323 324 dev = idxd_cdev->dev; 325 cdev_init(cdev, &idxd_cdev_fops); 326 cdev_set_parent(cdev, &dev->kobj); 327 rc = cdev_add(cdev, dev->devt, 1); 328 if (rc) { 329 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc); 330 idxd_wq_cdev_cleanup(wq, CDEV_FAILED); 331 return rc; 332 } 333 334 init_waitqueue_head(&idxd_cdev->err_queue); 335 return 0; 336 } 337 338 void idxd_wq_del_cdev(struct idxd_wq *wq) 339 { 340 idxd_wq_cdev_cleanup(wq, CDEV_NORMAL); 341 } 342 343 int idxd_cdev_register(void) 344 { 345 int rc, i; 346 347 for (i = 0; i < IDXD_TYPE_MAX; i++) { 348 ida_init(&ictx[i].minor_ida); 349 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK, 350 ictx[i].name); 351 if (rc) 352 return rc; 353 } 354 355 return 0; 356 } 357 358 void idxd_cdev_remove(void) 359 { 360 int i; 361 362 for (i = 0; i < IDXD_TYPE_MAX; i++) { 363 unregister_chrdev_region(ictx[i].devt, MINORMASK); 364 ida_destroy(&ictx[i].minor_ida); 365 } 366 } 367