1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include <linux/device.h> 8 #include <linux/sched/task.h> 9 #include <linux/intel-svm.h> 10 #include <linux/io-64-nonatomic-lo-hi.h> 11 #include <linux/cdev.h> 12 #include <linux/fs.h> 13 #include <linux/poll.h> 14 #include <uapi/linux/idxd.h> 15 #include "registers.h" 16 #include "idxd.h" 17 18 struct idxd_cdev_context { 19 const char *name; 20 dev_t devt; 21 struct ida minor_ida; 22 }; 23 24 /* 25 * ictx is an array based off of accelerator types. enum idxd_type 26 * is used as index 27 */ 28 static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = { 29 { .name = "dsa" }, 30 }; 31 32 struct idxd_user_context { 33 struct idxd_wq *wq; 34 struct task_struct *task; 35 unsigned int flags; 36 }; 37 38 enum idxd_cdev_cleanup { 39 CDEV_NORMAL = 0, 40 CDEV_FAILED, 41 }; 42 43 static void idxd_cdev_dev_release(struct device *dev) 44 { 45 dev_dbg(dev, "releasing cdev device\n"); 46 kfree(dev); 47 } 48 49 static struct device_type idxd_cdev_device_type = { 50 .name = "idxd_cdev", 51 .release = idxd_cdev_dev_release, 52 }; 53 54 static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode) 55 { 56 struct cdev *cdev = inode->i_cdev; 57 58 return container_of(cdev, struct idxd_cdev, cdev); 59 } 60 61 static inline struct idxd_wq *idxd_cdev_wq(struct idxd_cdev *idxd_cdev) 62 { 63 return container_of(idxd_cdev, struct idxd_wq, idxd_cdev); 64 } 65 66 static inline struct idxd_wq *inode_wq(struct inode *inode) 67 { 68 return idxd_cdev_wq(inode_idxd_cdev(inode)); 69 } 70 71 static int idxd_cdev_open(struct inode *inode, struct file *filp) 72 { 73 struct idxd_user_context *ctx; 74 struct idxd_device *idxd; 75 struct idxd_wq *wq; 76 struct device *dev; 77 int rc = 0; 78 79 wq = inode_wq(inode); 80 idxd = wq->idxd; 81 dev = &idxd->pdev->dev; 82 83 dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq)); 84 85 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 86 if (!ctx) 87 return -ENOMEM; 88 89 mutex_lock(&wq->wq_lock); 90 91 if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) { 92 rc = -EBUSY; 93 goto failed; 94 } 95 96 ctx->wq = wq; 97 filp->private_data = ctx; 98 idxd_wq_get(wq); 99 mutex_unlock(&wq->wq_lock); 100 return 0; 101 102 failed: 103 mutex_unlock(&wq->wq_lock); 104 kfree(ctx); 105 return rc; 106 } 107 108 static int idxd_cdev_release(struct inode *node, struct file *filep) 109 { 110 struct idxd_user_context *ctx = filep->private_data; 111 struct idxd_wq *wq = ctx->wq; 112 struct idxd_device *idxd = wq->idxd; 113 struct device *dev = &idxd->pdev->dev; 114 115 dev_dbg(dev, "%s called\n", __func__); 116 filep->private_data = NULL; 117 118 /* Wait for in-flight operations to complete. */ 119 idxd_wq_drain(wq); 120 121 kfree(ctx); 122 mutex_lock(&wq->wq_lock); 123 idxd_wq_put(wq); 124 mutex_unlock(&wq->wq_lock); 125 return 0; 126 } 127 128 static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma, 129 const char *func) 130 { 131 struct device *dev = &wq->idxd->pdev->dev; 132 133 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) { 134 dev_info_ratelimited(dev, 135 "%s: %s: mapping too large: %lu\n", 136 current->comm, func, 137 vma->vm_end - vma->vm_start); 138 return -EINVAL; 139 } 140 141 return 0; 142 } 143 144 static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma) 145 { 146 struct idxd_user_context *ctx = filp->private_data; 147 struct idxd_wq *wq = ctx->wq; 148 struct idxd_device *idxd = wq->idxd; 149 struct pci_dev *pdev = idxd->pdev; 150 phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR); 151 unsigned long pfn; 152 int rc; 153 154 dev_dbg(&pdev->dev, "%s called\n", __func__); 155 rc = check_vma(wq, vma, __func__); 156 if (rc < 0) 157 return rc; 158 159 vma->vm_flags |= VM_DONTCOPY; 160 pfn = (base + idxd_get_wq_portal_full_offset(wq->id, 161 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT; 162 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 163 vma->vm_private_data = ctx; 164 165 return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE, 166 vma->vm_page_prot); 167 } 168 169 static __poll_t idxd_cdev_poll(struct file *filp, 170 struct poll_table_struct *wait) 171 { 172 struct idxd_user_context *ctx = filp->private_data; 173 struct idxd_wq *wq = ctx->wq; 174 struct idxd_device *idxd = wq->idxd; 175 struct idxd_cdev *idxd_cdev = &wq->idxd_cdev; 176 unsigned long flags; 177 __poll_t out = 0; 178 179 poll_wait(filp, &idxd_cdev->err_queue, wait); 180 spin_lock_irqsave(&idxd->dev_lock, flags); 181 if (idxd->sw_err.valid) 182 out = EPOLLIN | EPOLLRDNORM; 183 spin_unlock_irqrestore(&idxd->dev_lock, flags); 184 185 return out; 186 } 187 188 static const struct file_operations idxd_cdev_fops = { 189 .owner = THIS_MODULE, 190 .open = idxd_cdev_open, 191 .release = idxd_cdev_release, 192 .mmap = idxd_cdev_mmap, 193 .poll = idxd_cdev_poll, 194 }; 195 196 int idxd_cdev_get_major(struct idxd_device *idxd) 197 { 198 return MAJOR(ictx[idxd->type].devt); 199 } 200 201 static int idxd_wq_cdev_dev_setup(struct idxd_wq *wq) 202 { 203 struct idxd_device *idxd = wq->idxd; 204 struct idxd_cdev *idxd_cdev = &wq->idxd_cdev; 205 struct idxd_cdev_context *cdev_ctx; 206 struct device *dev; 207 int minor, rc; 208 209 idxd_cdev->dev = kzalloc(sizeof(*idxd_cdev->dev), GFP_KERNEL); 210 if (!idxd_cdev->dev) 211 return -ENOMEM; 212 213 dev = idxd_cdev->dev; 214 dev->parent = &idxd->pdev->dev; 215 dev_set_name(dev, "%s/wq%u.%u", idxd_get_dev_name(idxd), 216 idxd->id, wq->id); 217 dev->bus = idxd_get_bus_type(idxd); 218 219 cdev_ctx = &ictx[wq->idxd->type]; 220 minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL); 221 if (minor < 0) { 222 rc = minor; 223 kfree(dev); 224 goto ida_err; 225 } 226 227 dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor); 228 dev->type = &idxd_cdev_device_type; 229 rc = device_register(dev); 230 if (rc < 0) { 231 dev_err(&idxd->pdev->dev, "device register failed\n"); 232 goto dev_reg_err; 233 } 234 idxd_cdev->minor = minor; 235 236 return 0; 237 238 dev_reg_err: 239 ida_simple_remove(&cdev_ctx->minor_ida, MINOR(dev->devt)); 240 put_device(dev); 241 ida_err: 242 idxd_cdev->dev = NULL; 243 return rc; 244 } 245 246 static void idxd_wq_cdev_cleanup(struct idxd_wq *wq, 247 enum idxd_cdev_cleanup cdev_state) 248 { 249 struct idxd_cdev *idxd_cdev = &wq->idxd_cdev; 250 struct idxd_cdev_context *cdev_ctx; 251 252 cdev_ctx = &ictx[wq->idxd->type]; 253 if (cdev_state == CDEV_NORMAL) 254 cdev_del(&idxd_cdev->cdev); 255 device_unregister(idxd_cdev->dev); 256 /* 257 * The device_type->release() will be called on the device and free 258 * the allocated struct device. We can just forget it. 259 */ 260 ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor); 261 idxd_cdev->dev = NULL; 262 idxd_cdev->minor = -1; 263 } 264 265 int idxd_wq_add_cdev(struct idxd_wq *wq) 266 { 267 struct idxd_cdev *idxd_cdev = &wq->idxd_cdev; 268 struct cdev *cdev = &idxd_cdev->cdev; 269 struct device *dev; 270 int rc; 271 272 rc = idxd_wq_cdev_dev_setup(wq); 273 if (rc < 0) 274 return rc; 275 276 dev = idxd_cdev->dev; 277 cdev_init(cdev, &idxd_cdev_fops); 278 cdev_set_parent(cdev, &dev->kobj); 279 rc = cdev_add(cdev, dev->devt, 1); 280 if (rc) { 281 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc); 282 idxd_wq_cdev_cleanup(wq, CDEV_FAILED); 283 return rc; 284 } 285 286 init_waitqueue_head(&idxd_cdev->err_queue); 287 return 0; 288 } 289 290 void idxd_wq_del_cdev(struct idxd_wq *wq) 291 { 292 idxd_wq_cdev_cleanup(wq, CDEV_NORMAL); 293 } 294 295 int idxd_cdev_register(void) 296 { 297 int rc, i; 298 299 for (i = 0; i < IDXD_TYPE_MAX; i++) { 300 ida_init(&ictx[i].minor_ida); 301 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK, 302 ictx[i].name); 303 if (rc) 304 return rc; 305 } 306 307 return 0; 308 } 309 310 void idxd_cdev_remove(void) 311 { 312 int i; 313 314 for (i = 0; i < IDXD_TYPE_MAX; i++) { 315 unregister_chrdev_region(ictx[i].devt, MINORMASK); 316 ida_destroy(&ictx[i].minor_ida); 317 } 318 } 319