18 static void idxd_wq_disable_cleanup(struct idxd_wq *wq);
41 static void free_hw_descs(struct idxd_wq *wq)  in free_hw_descs()  argument
45 	for (i = 0; i < wq->num_descs; i++)  in free_hw_descs()
46 		kfree(wq->hw_descs[i]);  in free_hw_descs()
48 	kfree(wq->hw_descs);  in free_hw_descs()
51 static int alloc_hw_descs(struct idxd_wq *wq, int num)  in alloc_hw_descs()  argument
53 	struct device *dev = &wq->idxd->pdev->dev;  in alloc_hw_descs()
57 	wq->hw_descs = kcalloc_node(num, sizeof(struct dsa_hw_desc *),  in alloc_hw_descs()
59 	if (!wq->hw_descs)  in alloc_hw_descs()
63 		wq->hw_descs[i] = kzalloc_node(sizeof(*wq->hw_descs[i]),  in alloc_hw_descs()
65 		if (!wq->hw_descs[i]) {  in alloc_hw_descs()
66 			free_hw_descs(wq);  in alloc_hw_descs()
74 static void free_descs(struct idxd_wq *wq)  in free_descs()  argument
78 	for (i = 0; i < wq->num_descs; i++)  in free_descs()
79 		kfree(wq->descs[i]);  in free_descs()
81 	kfree(wq->descs);  in free_descs()
84 static int alloc_descs(struct idxd_wq *wq, int num)  in alloc_descs()  argument
86 	struct device *dev = &wq->idxd->pdev->dev;  in alloc_descs()
90 	wq->descs = kcalloc_node(num, sizeof(struct idxd_desc *),  in alloc_descs()
92 	if (!wq->descs)  in alloc_descs()
96 		wq->descs[i] = kzalloc_node(sizeof(*wq->descs[i]),  in alloc_descs()
98 		if (!wq->descs[i]) {  in alloc_descs()
99 			free_descs(wq);  in alloc_descs()
108 int idxd_wq_alloc_resources(struct idxd_wq *wq)  in idxd_wq_alloc_resources()  argument
110 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_alloc_resources()
114 	if (wq->type != IDXD_WQT_KERNEL)  in idxd_wq_alloc_resources()
117 	num_descs = wq_dedicated(wq) ? wq->size : wq->threshold;  in idxd_wq_alloc_resources()
118 	wq->num_descs = num_descs;  in idxd_wq_alloc_resources()
120 	rc = alloc_hw_descs(wq, num_descs);  in idxd_wq_alloc_resources()
124 	wq->compls_size = num_descs * idxd->data->compl_size;  in idxd_wq_alloc_resources()
125 	wq->compls = dma_alloc_coherent(dev, wq->compls_size, &wq->compls_addr, GFP_KERNEL);  in idxd_wq_alloc_resources()
126 	if (!wq->compls) {  in idxd_wq_alloc_resources()
131 	rc = alloc_descs(wq, num_descs);  in idxd_wq_alloc_resources()
135 	rc = sbitmap_queue_init_node(&wq->sbq, num_descs, -1, false, GFP_KERNEL,  in idxd_wq_alloc_resources()
141 		struct idxd_desc *desc = wq->descs[i];  in idxd_wq_alloc_resources()
143 		desc->hw = wq->hw_descs[i];  in idxd_wq_alloc_resources()
145 			desc->completion = &wq->compls[i];  in idxd_wq_alloc_resources()
147 			desc->iax_completion = &wq->iax_compls[i];  in idxd_wq_alloc_resources()
148 		desc->compl_dma = wq->compls_addr + idxd->data->compl_size * i;  in idxd_wq_alloc_resources()
150 		desc->wq = wq;  in idxd_wq_alloc_resources()
157 	free_descs(wq);  in idxd_wq_alloc_resources()
159 	dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);  in idxd_wq_alloc_resources()
161 	free_hw_descs(wq);  in idxd_wq_alloc_resources()
165 void idxd_wq_free_resources(struct idxd_wq *wq)  in idxd_wq_free_resources()  argument
167 	struct device *dev = &wq->idxd->pdev->dev;  in idxd_wq_free_resources()
169 	if (wq->type != IDXD_WQT_KERNEL)  in idxd_wq_free_resources()
172 	free_hw_descs(wq);  in idxd_wq_free_resources()
173 	free_descs(wq);  in idxd_wq_free_resources()
174 	dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);  in idxd_wq_free_resources()
175 	sbitmap_queue_free(&wq->sbq);  in idxd_wq_free_resources()
178 int idxd_wq_enable(struct idxd_wq *wq)  in idxd_wq_enable()  argument
180 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_enable()
184 	if (wq->state == IDXD_WQ_ENABLED) {  in idxd_wq_enable()
185 		dev_dbg(dev, "WQ %d already enabled\n", wq->id);  in idxd_wq_enable()
189 	idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_WQ, wq->id, &status);  in idxd_wq_enable()
197 	wq->state = IDXD_WQ_ENABLED;  in idxd_wq_enable()
198 	set_bit(wq->id, idxd->wq_enable_map);  in idxd_wq_enable()
199 	dev_dbg(dev, "WQ %d enabled\n", wq->id);  in idxd_wq_enable()
203 int idxd_wq_disable(struct idxd_wq *wq, bool reset_config)  in idxd_wq_disable()  argument
205 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_disable()
209 	dev_dbg(dev, "Disabling WQ %d\n", wq->id);  in idxd_wq_disable()
211 	if (wq->state != IDXD_WQ_ENABLED) {  in idxd_wq_disable()
212 		dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);  in idxd_wq_disable()
216 	operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);  in idxd_wq_disable()
225 		idxd_wq_disable_cleanup(wq);  in idxd_wq_disable()
226 	clear_bit(wq->id, idxd->wq_enable_map);  in idxd_wq_disable()
227 	wq->state = IDXD_WQ_DISABLED;  in idxd_wq_disable()
228 	dev_dbg(dev, "WQ %d disabled\n", wq->id);  in idxd_wq_disable()
232 void idxd_wq_drain(struct idxd_wq *wq)  in idxd_wq_drain()  argument
234 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_drain()
238 	if (wq->state != IDXD_WQ_ENABLED) {  in idxd_wq_drain()
239 		dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);  in idxd_wq_drain()
243 	dev_dbg(dev, "Draining WQ %d\n", wq->id);  in idxd_wq_drain()
244 	operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);  in idxd_wq_drain()
248 void idxd_wq_reset(struct idxd_wq *wq)  in idxd_wq_reset()  argument
250 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_reset()
254 	if (wq->state != IDXD_WQ_ENABLED) {  in idxd_wq_reset()
255 		dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);  in idxd_wq_reset()
259 	operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);  in idxd_wq_reset()
261 	idxd_wq_disable_cleanup(wq);  in idxd_wq_reset()
264 int idxd_wq_map_portal(struct idxd_wq *wq)  in idxd_wq_map_portal()  argument
266 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_map_portal()
272 	start += idxd_get_wq_portal_full_offset(wq->id, IDXD_PORTAL_LIMITED);  in idxd_wq_map_portal()
274 	wq->portal = devm_ioremap(dev, start, IDXD_PORTAL_SIZE);  in idxd_wq_map_portal()
275 	if (!wq->portal)  in idxd_wq_map_portal()
281 void idxd_wq_unmap_portal(struct idxd_wq *wq)  in idxd_wq_unmap_portal()  argument
283 	struct device *dev = &wq->idxd->pdev->dev;  in idxd_wq_unmap_portal()
285 	devm_iounmap(dev, wq->portal);  in idxd_wq_unmap_portal()
286 	wq->portal = NULL;  in idxd_wq_unmap_portal()
287 	wq->portal_offset = 0;  in idxd_wq_unmap_portal()
295 		struct idxd_wq *wq = idxd->wqs[i];  in idxd_wqs_unmap_portal()  local
297 		if (wq->portal)  in idxd_wqs_unmap_portal()
298 			idxd_wq_unmap_portal(wq);  in idxd_wqs_unmap_portal()
302 static void __idxd_wq_set_pasid_locked(struct idxd_wq *wq, int pasid)  in __idxd_wq_set_pasid_locked()  argument
304 	struct idxd_device *idxd = wq->idxd;  in __idxd_wq_set_pasid_locked()
308 	offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);  in __idxd_wq_set_pasid_locked()
313 	wq->wqcfg->bits[WQCFG_PASID_IDX] = wqcfg.bits[WQCFG_PASID_IDX];  in __idxd_wq_set_pasid_locked()
318 int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid)  in idxd_wq_set_pasid()  argument
322 	rc = idxd_wq_disable(wq, false);  in idxd_wq_set_pasid()
326 	__idxd_wq_set_pasid_locked(wq, pasid);  in idxd_wq_set_pasid()
328 	rc = idxd_wq_enable(wq);  in idxd_wq_set_pasid()
335 int idxd_wq_disable_pasid(struct idxd_wq *wq)  in idxd_wq_disable_pasid()  argument
337 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_disable_pasid()
342 	rc = idxd_wq_disable(wq, false);  in idxd_wq_disable_pasid()
346 	offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);  in idxd_wq_disable_pasid()
354 	rc = idxd_wq_enable(wq);  in idxd_wq_disable_pasid()
361 static void idxd_wq_disable_cleanup(struct idxd_wq *wq)  in idxd_wq_disable_cleanup()  argument
363 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_disable_cleanup()
365 	lockdep_assert_held(&wq->wq_lock);  in idxd_wq_disable_cleanup()
366 	wq->state = IDXD_WQ_DISABLED;  in idxd_wq_disable_cleanup()
367 	memset(wq->wqcfg, 0, idxd->wqcfg_size);  in idxd_wq_disable_cleanup()
368 	wq->type = IDXD_WQT_NONE;  in idxd_wq_disable_cleanup()
369 	wq->threshold = 0;  in idxd_wq_disable_cleanup()
370 	wq->priority = 0;  in idxd_wq_disable_cleanup()
371 	wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES;  in idxd_wq_disable_cleanup()
372 	wq->flags = 0;  in idxd_wq_disable_cleanup()
373 	memset(wq->name, 0, WQ_NAME_SIZE);  in idxd_wq_disable_cleanup()
374 	wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER;  in idxd_wq_disable_cleanup()
375 	idxd_wq_set_max_batch_size(idxd->data->type, wq, WQ_DEFAULT_MAX_BATCH);  in idxd_wq_disable_cleanup()
376 	if (wq->opcap_bmap)  in idxd_wq_disable_cleanup()
377 		bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS);  in idxd_wq_disable_cleanup()
380 static void idxd_wq_device_reset_cleanup(struct idxd_wq *wq)  in idxd_wq_device_reset_cleanup()  argument
382 	lockdep_assert_held(&wq->wq_lock);  in idxd_wq_device_reset_cleanup()
384 	wq->size = 0;  in idxd_wq_device_reset_cleanup()
385 	wq->group = NULL;  in idxd_wq_device_reset_cleanup()
390 	struct idxd_wq *wq = container_of(ref, struct idxd_wq, wq_active);  in idxd_wq_ref_release()  local
392 	complete(&wq->wq_dead);  in idxd_wq_ref_release()
395 int idxd_wq_init_percpu_ref(struct idxd_wq *wq)  in idxd_wq_init_percpu_ref()  argument
399 	memset(&wq->wq_active, 0, sizeof(wq->wq_active));  in idxd_wq_init_percpu_ref()
400 	rc = percpu_ref_init(&wq->wq_active, idxd_wq_ref_release,  in idxd_wq_init_percpu_ref()
404 	reinit_completion(&wq->wq_dead);  in idxd_wq_init_percpu_ref()
405 	reinit_completion(&wq->wq_resurrect);  in idxd_wq_init_percpu_ref()
409 void __idxd_wq_quiesce(struct idxd_wq *wq)  in __idxd_wq_quiesce()  argument
411 	lockdep_assert_held(&wq->wq_lock);  in __idxd_wq_quiesce()
412 	reinit_completion(&wq->wq_resurrect);  in __idxd_wq_quiesce()
413 	percpu_ref_kill(&wq->wq_active);  in __idxd_wq_quiesce()
414 	complete_all(&wq->wq_resurrect);  in __idxd_wq_quiesce()
415 	wait_for_completion(&wq->wq_dead);  in __idxd_wq_quiesce()
418 void idxd_wq_quiesce(struct idxd_wq *wq)  in idxd_wq_quiesce()  argument
420 	mutex_lock(&wq->wq_lock);  in idxd_wq_quiesce()
421 	__idxd_wq_quiesce(wq);  in idxd_wq_quiesce()
422 	mutex_unlock(&wq->wq_lock);  in idxd_wq_quiesce()
709 		struct idxd_wq *wq = idxd->wqs[i];  in idxd_device_wqs_clear_state()  local
711 		mutex_lock(&wq->wq_lock);  in idxd_device_wqs_clear_state()
712 		idxd_wq_disable_cleanup(wq);  in idxd_device_wqs_clear_state()
713 		idxd_wq_device_reset_cleanup(wq);  in idxd_device_wqs_clear_state()
714 		mutex_unlock(&wq->wq_lock);  in idxd_device_wqs_clear_state()
906 static int idxd_wq_config_write(struct idxd_wq *wq)  in idxd_wq_config_write()  argument
908 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_config_write()
913 	if (!wq->group)  in idxd_wq_config_write()
921 		wq_offset = WQCFG_OFFSET(idxd, wq->id, i);  in idxd_wq_config_write()
922 		wq->wqcfg->bits[i] |= ioread32(idxd->reg_base + wq_offset);  in idxd_wq_config_write()
925 	if (wq->size == 0 && wq->type != IDXD_WQT_NONE)  in idxd_wq_config_write()
926 		wq->size = WQ_DEFAULT_QUEUE_DEPTH;  in idxd_wq_config_write()
929 	wq->wqcfg->wq_size = wq->size;  in idxd_wq_config_write()
932 	wq->wqcfg->wq_thresh = wq->threshold;  in idxd_wq_config_write()
935 	if (wq_dedicated(wq))  in idxd_wq_config_write()
936 		wq->wqcfg->mode = 1;  in idxd_wq_config_write()
949 	if (wq_dedicated(wq) && wq->wqcfg->pasid_en &&  in idxd_wq_config_write()
951 	    wq->type == IDXD_WQT_KERNEL) {  in idxd_wq_config_write()
956 	wq->wqcfg->priority = wq->priority;  in idxd_wq_config_write()
959 	    test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags) &&  in idxd_wq_config_write()
960 	    !test_bit(WQ_FLAG_PRS_DISABLE, &wq->flags))  in idxd_wq_config_write()
961 		wq->wqcfg->bof = 1;  in idxd_wq_config_write()
964 		wq->wqcfg->wq_ats_disable = test_bit(WQ_FLAG_ATS_DISABLE, &wq->flags);  in idxd_wq_config_write()
967 		wq->wqcfg->wq_prs_disable = test_bit(WQ_FLAG_PRS_DISABLE, &wq->flags);  in idxd_wq_config_write()
970 	wq->wqcfg->max_xfer_shift = ilog2(wq->max_xfer_bytes);  in idxd_wq_config_write()
971 	idxd_wqcfg_set_max_batch_shift(idxd->data->type, wq->wqcfg, ilog2(wq->max_batch_size));  in idxd_wq_config_write()
974 	if (idxd->hw.wq_cap.op_config && wq->opcap_bmap) {  in idxd_wq_config_write()
975 		memset(wq->wqcfg->op_config, 0, IDXD_MAX_OPCAP_BITS / 8);  in idxd_wq_config_write()
976 		for_each_set_bit(n, wq->opcap_bmap, IDXD_MAX_OPCAP_BITS) {  in idxd_wq_config_write()
980 			wq->wqcfg->op_config[idx] |= BIT(pos);  in idxd_wq_config_write()
984 	dev_dbg(dev, "WQ %d CFGs\n", wq->id);  in idxd_wq_config_write()
986 		wq_offset = WQCFG_OFFSET(idxd, wq->id, i);  in idxd_wq_config_write()
987 		iowrite32(wq->wqcfg->bits[i], idxd->reg_base + wq_offset);  in idxd_wq_config_write()
989 			wq->id, i, wq_offset,  in idxd_wq_config_write()
1001 		struct idxd_wq *wq = idxd->wqs[i];  in idxd_wqs_config_write()  local
1003 		rc = idxd_wq_config_write(wq);  in idxd_wqs_config_write()
1065 	struct idxd_wq *wq;  in idxd_wqs_setup()  local
1077 		wq = idxd->wqs[i];  in idxd_wqs_setup()
1078 		group = wq->group;  in idxd_wqs_setup()
1080 		if (!wq->group)  in idxd_wqs_setup()
1083 		if (wq_shared(wq) && !wq_shared_supported(wq)) {  in idxd_wqs_setup()
1089 		group->grpcfg.wqs[wq->id / 64] |= BIT(wq->id % 64);  in idxd_wqs_setup()
1127 static int idxd_wq_load_config(struct idxd_wq *wq)  in idxd_wq_load_config()  argument
1129 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_load_config()
1134 	wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, 0);  in idxd_wq_load_config()
1135 	memcpy_fromio(wq->wqcfg, idxd->reg_base + wqcfg_offset, idxd->wqcfg_size);  in idxd_wq_load_config()
1137 	wq->size = wq->wqcfg->wq_size;  in idxd_wq_load_config()
1138 	wq->threshold = wq->wqcfg->wq_thresh;  in idxd_wq_load_config()
1141 	if (wq->wqcfg->mode == 0 || wq->wqcfg->pasid_en)  in idxd_wq_load_config()
1144 	set_bit(WQ_FLAG_DEDICATED, &wq->flags);  in idxd_wq_load_config()
1146 	wq->priority = wq->wqcfg->priority;  in idxd_wq_load_config()
1148 	wq->max_xfer_bytes = 1ULL << wq->wqcfg->max_xfer_shift;  in idxd_wq_load_config()
1149 	idxd_wq_set_max_batch_size(idxd->data->type, wq, 1U << wq->wqcfg->max_batch_shift);  in idxd_wq_load_config()
1152 		wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, i);  in idxd_wq_load_config()
1153 		dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n", wq->id, i, wqcfg_offset, wq->wqcfg->bits[i]);  in idxd_wq_load_config()
1170 		struct idxd_wq *wq;  in idxd_group_load_config()  local
1190 				wq = idxd->wqs[id];  in idxd_group_load_config()
1191 				wq->group = group;  in idxd_group_load_config()
1234 		struct idxd_wq *wq = idxd->wqs[i];  in idxd_device_load_config()  local
1236 		rc = idxd_wq_load_config(wq);  in idxd_device_load_config()
1300 void idxd_wq_free_irq(struct idxd_wq *wq)  in idxd_wq_free_irq()  argument
1302 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_free_irq()
1303 	struct idxd_irq_entry *ie = &wq->ie;  in idxd_wq_free_irq()
1305 	if (wq->type != IDXD_WQT_KERNEL)  in idxd_wq_free_irq()
1318 int idxd_wq_request_irq(struct idxd_wq *wq)  in idxd_wq_request_irq()  argument
1320 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_request_irq()
1326 	if (wq->type != IDXD_WQT_KERNEL)  in idxd_wq_request_irq()
1329 	ie = &wq->ie;  in idxd_wq_request_irq()
1360 int drv_enable_wq(struct idxd_wq *wq)  in drv_enable_wq()  argument
1362 	struct idxd_device *idxd = wq->idxd;  in drv_enable_wq()
1366 	lockdep_assert_held(&wq->wq_lock);  in drv_enable_wq()
1373 	if (wq->state != IDXD_WQ_DISABLED) {  in drv_enable_wq()
1374 		dev_dbg(dev, "wq %d already enabled.\n", wq->id);  in drv_enable_wq()
1380 	if (!wq->group) {  in drv_enable_wq()
1381 		dev_dbg(dev, "wq %d not attached to group.\n", wq->id);  in drv_enable_wq()
1386 	if (strlen(wq->name) == 0) {  in drv_enable_wq()
1388 		dev_dbg(dev, "wq %d name not set.\n", wq->id);  in drv_enable_wq()
1393 	if (wq_shared(wq)) {  in drv_enable_wq()
1394 		if (!wq_shared_supported(wq)) {  in drv_enable_wq()
1407 		if (wq->threshold == 0) {  in drv_enable_wq()
1423 		if (wq_pasid_enabled(wq)) {  in drv_enable_wq()
1424 			if (is_idxd_wq_kernel(wq) || wq_shared(wq)) {  in drv_enable_wq()
1425 				u32 pasid = wq_dedicated(wq) ? idxd->pasid : 0;  in drv_enable_wq()
1427 				__idxd_wq_set_pasid_locked(wq, pasid);  in drv_enable_wq()
1438 		dev_dbg(dev, "Writing wq %d config failed: %d\n", wq->id, rc);  in drv_enable_wq()
1442 	rc = idxd_wq_enable(wq);  in drv_enable_wq()
1444 		dev_dbg(dev, "wq %d enabling failed: %d\n", wq->id, rc);  in drv_enable_wq()
1448 	rc = idxd_wq_map_portal(wq);  in drv_enable_wq()
1451 		dev_dbg(dev, "wq %d portal mapping failed: %d\n", wq->id, rc);  in drv_enable_wq()
1455 	wq->client_count = 0;  in drv_enable_wq()
1457 	rc = idxd_wq_request_irq(wq);  in drv_enable_wq()
1460 		dev_dbg(dev, "WQ %d irq setup failed: %d\n", wq->id, rc);  in drv_enable_wq()
1464 	rc = idxd_wq_alloc_resources(wq);  in drv_enable_wq()
1471 	rc = idxd_wq_init_percpu_ref(wq);  in drv_enable_wq()
1481 	idxd_wq_free_resources(wq);  in drv_enable_wq()
1483 	idxd_wq_free_irq(wq);  in drv_enable_wq()
1485 	idxd_wq_unmap_portal(wq);  in drv_enable_wq()
1487 	if (idxd_wq_disable(wq, false))  in drv_enable_wq()
1488 		dev_dbg(dev, "wq %s disable failed\n", dev_name(wq_confdev(wq)));  in drv_enable_wq()
1493 void drv_disable_wq(struct idxd_wq *wq)  in drv_disable_wq()  argument
1495 	struct idxd_device *idxd = wq->idxd;  in drv_disable_wq()
1498 	lockdep_assert_held(&wq->wq_lock);  in drv_disable_wq()
1500 	if (idxd_wq_refcount(wq))  in drv_disable_wq()
1502 			 wq->id, idxd_wq_refcount(wq));  in drv_disable_wq()
1504 	idxd_wq_unmap_portal(wq);  in drv_disable_wq()
1505 	idxd_wq_drain(wq);  in drv_disable_wq()
1506 	idxd_wq_free_irq(wq);  in drv_disable_wq()
1507 	idxd_wq_reset(wq);  in drv_disable_wq()
1508 	idxd_wq_free_resources(wq);  in drv_disable_wq()
1509 	percpu_ref_exit(&wq->wq_active);  in drv_disable_wq()
1510 	wq->type = IDXD_WQT_NONE;  in drv_disable_wq()
1511 	wq->client_count = 0;  in drv_disable_wq()
1579 		struct idxd_wq *wq = idxd->wqs[i];  in idxd_device_drv_remove()  local
1580 		struct device *wq_dev = wq_confdev(wq);  in idxd_device_drv_remove()
1582 		if (wq->state == IDXD_WQ_DISABLED)  in idxd_device_drv_remove()