1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include <linux/device.h> 8 #include <linux/sched/task.h> 9 #include <linux/intel-svm.h> 10 #include <linux/io-64-nonatomic-lo-hi.h> 11 #include <linux/cdev.h> 12 #include <linux/fs.h> 13 #include <linux/poll.h> 14 #include <linux/iommu.h> 15 #include <uapi/linux/idxd.h> 16 #include "registers.h" 17 #include "idxd.h" 18 19 struct idxd_cdev_context { 20 const char *name; 21 dev_t devt; 22 struct ida minor_ida; 23 }; 24 25 /* 26 * ictx is an array based off of accelerator types. enum idxd_type 27 * is used as index 28 */ 29 static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = { 30 { .name = "dsa" }, 31 }; 32 33 struct idxd_user_context { 34 struct idxd_wq *wq; 35 struct task_struct *task; 36 unsigned int pasid; 37 unsigned int flags; 38 struct iommu_sva *sva; 39 }; 40 41 enum idxd_cdev_cleanup { 42 CDEV_NORMAL = 0, 43 CDEV_FAILED, 44 }; 45 46 static void idxd_cdev_dev_release(struct device *dev) 47 { 48 dev_dbg(dev, "releasing cdev device\n"); 49 kfree(dev); 50 } 51 52 static struct device_type idxd_cdev_device_type = { 53 .name = "idxd_cdev", 54 .release = idxd_cdev_dev_release, 55 }; 56 57 static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode) 58 { 59 struct cdev *cdev = inode->i_cdev; 60 61 return container_of(cdev, struct idxd_cdev, cdev); 62 } 63 64 static inline struct idxd_wq *idxd_cdev_wq(struct idxd_cdev *idxd_cdev) 65 { 66 return container_of(idxd_cdev, struct idxd_wq, idxd_cdev); 67 } 68 69 static inline struct idxd_wq *inode_wq(struct inode *inode) 70 { 71 return idxd_cdev_wq(inode_idxd_cdev(inode)); 72 } 73 74 static int idxd_cdev_open(struct inode *inode, struct file *filp) 75 { 76 struct idxd_user_context *ctx; 77 struct idxd_device *idxd; 78 struct idxd_wq *wq; 79 struct device *dev; 80 int rc = 0; 81 struct iommu_sva *sva; 82 unsigned int pasid; 83 84 wq = inode_wq(inode); 85 idxd = wq->idxd; 86 dev = &idxd->pdev->dev; 87 88 dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq)); 89 90 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 91 if (!ctx) 92 return -ENOMEM; 93 94 mutex_lock(&wq->wq_lock); 95 96 if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) { 97 rc = -EBUSY; 98 goto failed; 99 } 100 101 ctx->wq = wq; 102 filp->private_data = ctx; 103 104 if (device_pasid_enabled(idxd)) { 105 sva = iommu_sva_bind_device(dev, current->mm, NULL); 106 if (IS_ERR(sva)) { 107 rc = PTR_ERR(sva); 108 dev_err(dev, "pasid allocation failed: %d\n", rc); 109 goto failed; 110 } 111 112 pasid = iommu_sva_get_pasid(sva); 113 if (pasid == IOMMU_PASID_INVALID) { 114 iommu_sva_unbind_device(sva); 115 goto failed; 116 } 117 118 ctx->sva = sva; 119 ctx->pasid = pasid; 120 121 if (wq_dedicated(wq)) { 122 rc = idxd_wq_set_pasid(wq, pasid); 123 if (rc < 0) { 124 iommu_sva_unbind_device(sva); 125 dev_err(dev, "wq set pasid failed: %d\n", rc); 126 goto failed; 127 } 128 } 129 } 130 131 idxd_wq_get(wq); 132 mutex_unlock(&wq->wq_lock); 133 return 0; 134 135 failed: 136 mutex_unlock(&wq->wq_lock); 137 kfree(ctx); 138 return rc; 139 } 140 141 static int idxd_cdev_release(struct inode *node, struct file *filep) 142 { 143 struct idxd_user_context *ctx = filep->private_data; 144 struct idxd_wq *wq = ctx->wq; 145 struct idxd_device *idxd = wq->idxd; 146 struct device *dev = &idxd->pdev->dev; 147 int rc; 148 149 dev_dbg(dev, "%s called\n", __func__); 150 filep->private_data = NULL; 151 152 /* Wait for in-flight operations to complete. */ 153 if (wq_shared(wq)) { 154 idxd_device_drain_pasid(idxd, ctx->pasid); 155 } else { 156 if (device_pasid_enabled(idxd)) { 157 /* The wq disable in the disable pasid function will drain the wq */ 158 rc = idxd_wq_disable_pasid(wq); 159 if (rc < 0) 160 dev_err(dev, "wq disable pasid failed.\n"); 161 } else { 162 idxd_wq_drain(wq); 163 } 164 } 165 166 if (ctx->sva) 167 iommu_sva_unbind_device(ctx->sva); 168 kfree(ctx); 169 mutex_lock(&wq->wq_lock); 170 idxd_wq_put(wq); 171 mutex_unlock(&wq->wq_lock); 172 return 0; 173 } 174 175 static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma, 176 const char *func) 177 { 178 struct device *dev = &wq->idxd->pdev->dev; 179 180 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) { 181 dev_info_ratelimited(dev, 182 "%s: %s: mapping too large: %lu\n", 183 current->comm, func, 184 vma->vm_end - vma->vm_start); 185 return -EINVAL; 186 } 187 188 return 0; 189 } 190 191 static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma) 192 { 193 struct idxd_user_context *ctx = filp->private_data; 194 struct idxd_wq *wq = ctx->wq; 195 struct idxd_device *idxd = wq->idxd; 196 struct pci_dev *pdev = idxd->pdev; 197 phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR); 198 unsigned long pfn; 199 int rc; 200 201 dev_dbg(&pdev->dev, "%s called\n", __func__); 202 rc = check_vma(wq, vma, __func__); 203 if (rc < 0) 204 return rc; 205 206 vma->vm_flags |= VM_DONTCOPY; 207 pfn = (base + idxd_get_wq_portal_full_offset(wq->id, 208 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT; 209 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 210 vma->vm_private_data = ctx; 211 212 return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE, 213 vma->vm_page_prot); 214 } 215 216 static __poll_t idxd_cdev_poll(struct file *filp, 217 struct poll_table_struct *wait) 218 { 219 struct idxd_user_context *ctx = filp->private_data; 220 struct idxd_wq *wq = ctx->wq; 221 struct idxd_device *idxd = wq->idxd; 222 struct idxd_cdev *idxd_cdev = &wq->idxd_cdev; 223 unsigned long flags; 224 __poll_t out = 0; 225 226 poll_wait(filp, &idxd_cdev->err_queue, wait); 227 spin_lock_irqsave(&idxd->dev_lock, flags); 228 if (idxd->sw_err.valid) 229 out = EPOLLIN | EPOLLRDNORM; 230 spin_unlock_irqrestore(&idxd->dev_lock, flags); 231 232 return out; 233 } 234 235 static const struct file_operations idxd_cdev_fops = { 236 .owner = THIS_MODULE, 237 .open = idxd_cdev_open, 238 .release = idxd_cdev_release, 239 .mmap = idxd_cdev_mmap, 240 .poll = idxd_cdev_poll, 241 }; 242 243 int idxd_cdev_get_major(struct idxd_device *idxd) 244 { 245 return MAJOR(ictx[idxd->type].devt); 246 } 247 248 static int idxd_wq_cdev_dev_setup(struct idxd_wq *wq) 249 { 250 struct idxd_device *idxd = wq->idxd; 251 struct idxd_cdev *idxd_cdev = &wq->idxd_cdev; 252 struct idxd_cdev_context *cdev_ctx; 253 struct device *dev; 254 int minor, rc; 255 256 idxd_cdev->dev = kzalloc(sizeof(*idxd_cdev->dev), GFP_KERNEL); 257 if (!idxd_cdev->dev) 258 return -ENOMEM; 259 260 dev = idxd_cdev->dev; 261 dev->parent = &idxd->pdev->dev; 262 dev_set_name(dev, "%s/wq%u.%u", idxd_get_dev_name(idxd), 263 idxd->id, wq->id); 264 dev->bus = idxd_get_bus_type(idxd); 265 266 cdev_ctx = &ictx[wq->idxd->type]; 267 minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL); 268 if (minor < 0) { 269 rc = minor; 270 kfree(dev); 271 goto ida_err; 272 } 273 274 dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor); 275 dev->type = &idxd_cdev_device_type; 276 rc = device_register(dev); 277 if (rc < 0) { 278 dev_err(&idxd->pdev->dev, "device register failed\n"); 279 goto dev_reg_err; 280 } 281 idxd_cdev->minor = minor; 282 283 return 0; 284 285 dev_reg_err: 286 ida_simple_remove(&cdev_ctx->minor_ida, MINOR(dev->devt)); 287 put_device(dev); 288 ida_err: 289 idxd_cdev->dev = NULL; 290 return rc; 291 } 292 293 static void idxd_wq_cdev_cleanup(struct idxd_wq *wq, 294 enum idxd_cdev_cleanup cdev_state) 295 { 296 struct idxd_cdev *idxd_cdev = &wq->idxd_cdev; 297 struct idxd_cdev_context *cdev_ctx; 298 299 cdev_ctx = &ictx[wq->idxd->type]; 300 if (cdev_state == CDEV_NORMAL) 301 cdev_del(&idxd_cdev->cdev); 302 device_unregister(idxd_cdev->dev); 303 /* 304 * The device_type->release() will be called on the device and free 305 * the allocated struct device. We can just forget it. 306 */ 307 ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor); 308 idxd_cdev->dev = NULL; 309 idxd_cdev->minor = -1; 310 } 311 312 int idxd_wq_add_cdev(struct idxd_wq *wq) 313 { 314 struct idxd_cdev *idxd_cdev = &wq->idxd_cdev; 315 struct cdev *cdev = &idxd_cdev->cdev; 316 struct device *dev; 317 int rc; 318 319 rc = idxd_wq_cdev_dev_setup(wq); 320 if (rc < 0) 321 return rc; 322 323 dev = idxd_cdev->dev; 324 cdev_init(cdev, &idxd_cdev_fops); 325 cdev_set_parent(cdev, &dev->kobj); 326 rc = cdev_add(cdev, dev->devt, 1); 327 if (rc) { 328 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc); 329 idxd_wq_cdev_cleanup(wq, CDEV_FAILED); 330 return rc; 331 } 332 333 init_waitqueue_head(&idxd_cdev->err_queue); 334 return 0; 335 } 336 337 void idxd_wq_del_cdev(struct idxd_wq *wq) 338 { 339 idxd_wq_cdev_cleanup(wq, CDEV_NORMAL); 340 } 341 342 int idxd_cdev_register(void) 343 { 344 int rc, i; 345 346 for (i = 0; i < IDXD_TYPE_MAX; i++) { 347 ida_init(&ictx[i].minor_ida); 348 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK, 349 ictx[i].name); 350 if (rc) 351 return rc; 352 } 353 354 return 0; 355 } 356 357 void idxd_cdev_remove(void) 358 { 359 int i; 360 361 for (i = 0; i < IDXD_TYPE_MAX; i++) { 362 unregister_chrdev_region(ictx[i].devt, MINORMASK); 363 ida_destroy(&ictx[i].minor_ida); 364 } 365 } 366