43 	struct idxd_wq *wq;  member
55 static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid);
101 	struct idxd_wq *wq = ctx->wq;  in cdev_file_attr_visible()  local
103 	if (!wq_pasid_enabled(wq))  in cdev_file_attr_visible()
122 	struct idxd_wq *wq = ctx->wq;  in idxd_file_dev_release()  local
123 	struct idxd_device *idxd = wq->idxd;  in idxd_file_dev_release()
131 	if (wq_shared(wq)) {  in idxd_file_dev_release()
136 			rc = idxd_wq_disable_pasid(wq);  in idxd_file_dev_release()
140 			idxd_wq_drain(wq);  in idxd_file_dev_release()
145 		idxd_cdev_evl_drain_pasid(wq, ctx->pasid);  in idxd_file_dev_release()
150 	mutex_lock(&wq->wq_lock);  in idxd_file_dev_release()
151 	idxd_wq_put(wq);  in idxd_file_dev_release()
152 	mutex_unlock(&wq->wq_lock);  in idxd_file_dev_release()
165 	struct idxd_wq *wq = idxd_cdev->wq;  in idxd_cdev_dev_release()  local
167 	cdev_ctx = &ictx[wq->idxd->data->type];  in idxd_cdev_dev_release()
188 	return idxd_cdev->wq;  in inode_wq()
193 	struct idxd_wq *wq = ctx->wq;  in idxd_xa_pasid_remove()  local
196 	mutex_lock(&wq->uc_lock);  in idxd_xa_pasid_remove()
197 	ptr = xa_cmpxchg(&wq->upasid_xa, ctx->pasid, ctx, NULL, GFP_KERNEL);  in idxd_xa_pasid_remove()
199 		dev_warn(&wq->idxd->pdev->dev, "xarray cmpxchg failed for pasid %u\n",  in idxd_xa_pasid_remove()
201 	mutex_unlock(&wq->uc_lock);  in idxd_xa_pasid_remove()
204 void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index)  in idxd_user_counter_increment()  argument
211 	mutex_lock(&wq->uc_lock);  in idxd_user_counter_increment()
212 	ctx = xa_load(&wq->upasid_xa, pasid);  in idxd_user_counter_increment()
214 		mutex_unlock(&wq->uc_lock);  in idxd_user_counter_increment()
218 	mutex_unlock(&wq->uc_lock);  in idxd_user_counter_increment()
225 	struct idxd_wq *wq;  in idxd_cdev_open()  local
232 	wq = inode_wq(inode);  in idxd_cdev_open()
233 	idxd = wq->idxd;  in idxd_cdev_open()
236 	dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq));  in idxd_cdev_open()
242 	mutex_lock(&wq->wq_lock);  in idxd_cdev_open()
244 	if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) {  in idxd_cdev_open()
249 	ctx->wq = wq;  in idxd_cdev_open()
271 		mutex_lock(&wq->uc_lock);  in idxd_cdev_open()
272 		rc = xa_insert(&wq->upasid_xa, pasid, ctx, GFP_KERNEL);  in idxd_cdev_open()
273 		mutex_unlock(&wq->uc_lock);  in idxd_cdev_open()
277 		if (wq_dedicated(wq)) {  in idxd_cdev_open()
278 			rc = idxd_wq_set_pasid(wq, pasid);  in idxd_cdev_open()
286 	idxd_cdev = wq->idxd_cdev;  in idxd_cdev_open()
313 	idxd_wq_get(wq);  in idxd_cdev_open()
314 	mutex_unlock(&wq->wq_lock);  in idxd_cdev_open()
328 	mutex_unlock(&wq->wq_lock);  in idxd_cdev_open()
333 static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid)  in idxd_cdev_evl_drain_pasid()  argument
335 	struct idxd_device *idxd = wq->idxd;  in idxd_cdev_evl_drain_pasid()
353 		if (entry_head->pasid == pasid && entry_head->wq_idx == wq->id)  in idxd_cdev_evl_drain_pasid()
357 	drain_workqueue(wq->wq);  in idxd_cdev_evl_drain_pasid()
364 	struct idxd_wq *wq = ctx->wq;  in idxd_cdev_release()  local
365 	struct idxd_device *idxd = wq->idxd;  in idxd_cdev_release()
376 static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma,  in check_vma()  argument
379 	struct device *dev = &wq->idxd->pdev->dev;  in check_vma()
395 	struct idxd_wq *wq = ctx->wq;  in idxd_cdev_mmap()  local
396 	struct idxd_device *idxd = wq->idxd;  in idxd_cdev_mmap()
415 	rc = check_vma(wq, vma, __func__);  in idxd_cdev_mmap()
420 	pfn = (base + idxd_get_wq_portal_full_offset(wq->id,  in idxd_cdev_mmap()
432 	struct idxd_wq *wq = ctx->wq;  in idxd_submit_user_descriptor()  local
433 	struct idxd_dev *idxd_dev = &wq->idxd->idxd_dev;  in idxd_submit_user_descriptor()
435 	void __iomem *portal = idxd_wq_portal_addr(wq);  in idxd_submit_user_descriptor()
450 		!wq->idxd->user_submission_safe)  in idxd_submit_user_descriptor()
460 	if (wq_dedicated(wq))  in idxd_submit_user_descriptor()
465 		rc = idxd_enqcmds(wq, portal, &descriptor);  in idxd_submit_user_descriptor()
497 	struct idxd_wq *wq = ctx->wq;  in idxd_cdev_poll()  local
498 	struct idxd_device *idxd = wq->idxd;  in idxd_cdev_poll()
501 	poll_wait(filp, &wq->err_queue, wait);  in idxd_cdev_poll()
524 int idxd_wq_add_cdev(struct idxd_wq *wq)  in idxd_wq_add_cdev()  argument
526 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_add_cdev()
538 	idxd_cdev->wq = wq;  in idxd_wq_add_cdev()
541 	cdev_ctx = &ictx[wq->idxd->data->type];  in idxd_wq_add_cdev()
550 	dev->parent = wq_confdev(wq);  in idxd_wq_add_cdev()
555 	rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id);  in idxd_wq_add_cdev()
559 	wq->idxd_cdev = idxd_cdev;  in idxd_wq_add_cdev()
563 		dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);  in idxd_wq_add_cdev()
571 	wq->idxd_cdev = NULL;  in idxd_wq_add_cdev()
575 void idxd_wq_del_cdev(struct idxd_wq *wq)  in idxd_wq_del_cdev()  argument
579 	idxd_cdev = wq->idxd_cdev;  in idxd_wq_del_cdev()
580 	wq->idxd_cdev = NULL;  in idxd_wq_del_cdev()
587 	struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);  in idxd_user_drv_probe()  local
588 	struct idxd_device *idxd = wq->idxd;  in idxd_user_drv_probe()
612 	mutex_lock(&wq->wq_lock);  in idxd_user_drv_probe()
614 	wq->wq = create_workqueue(dev_name(wq_confdev(wq)));  in idxd_user_drv_probe()
615 	if (!wq->wq) {  in idxd_user_drv_probe()
620 	wq->type = IDXD_WQT_USER;  in idxd_user_drv_probe()
621 	rc = drv_enable_wq(wq);  in idxd_user_drv_probe()
625 	rc = idxd_wq_add_cdev(wq);  in idxd_user_drv_probe()
632 	mutex_unlock(&wq->wq_lock);  in idxd_user_drv_probe()
636 	drv_disable_wq(wq);  in idxd_user_drv_probe()
638 	destroy_workqueue(wq->wq);  in idxd_user_drv_probe()
639 	wq->type = IDXD_WQT_NONE;  in idxd_user_drv_probe()
641 	mutex_unlock(&wq->wq_lock);  in idxd_user_drv_probe()
647 	struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);  in idxd_user_drv_remove()  local
649 	mutex_lock(&wq->wq_lock);  in idxd_user_drv_remove()
650 	idxd_wq_del_cdev(wq);  in idxd_user_drv_remove()
651 	drv_disable_wq(wq);  in idxd_user_drv_remove()
652 	wq->type = IDXD_WQT_NONE;  in idxd_user_drv_remove()
653 	destroy_workqueue(wq->wq);  in idxd_user_drv_remove()
654 	wq->wq = NULL;  in idxd_user_drv_remove()
655 	mutex_unlock(&wq->wq_lock);  in idxd_user_drv_remove()
715 int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr,  in idxd_copy_cr()  argument
718 	struct device *dev = &wq->idxd->pdev->dev;  in idxd_copy_cr()
723 	mutex_lock(&wq->uc_lock);  in idxd_copy_cr()
725 	ctx = xa_load(&wq->upasid_xa, pasid);  in idxd_copy_cr()
764 	mutex_unlock(&wq->uc_lock);  in idxd_copy_cr()