xref: /openbmc/linux/drivers/dma/idxd/idxd.h (revision 8cacaaa4)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3 #ifndef _IDXD_H_
4 #define _IDXD_H_
5 
6 #include <linux/sbitmap.h>
7 #include <linux/dmaengine.h>
8 #include <linux/percpu-rwsem.h>
9 #include <linux/wait.h>
10 #include <linux/cdev.h>
11 #include <linux/idr.h>
12 #include <linux/pci.h>
13 #include <linux/bitmap.h>
14 #include <linux/perf_event.h>
15 #include <linux/iommu.h>
16 #include <uapi/linux/idxd.h>
17 #include "registers.h"
18 
19 #define IDXD_DRIVER_VERSION	"1.00"
20 
21 extern struct kmem_cache *idxd_desc_pool;
22 extern bool tc_override;
23 
24 struct idxd_wq;
25 struct idxd_dev;
26 
27 enum idxd_dev_type {
28 	IDXD_DEV_NONE = -1,
29 	IDXD_DEV_DSA = 0,
30 	IDXD_DEV_IAX,
31 	IDXD_DEV_WQ,
32 	IDXD_DEV_GROUP,
33 	IDXD_DEV_ENGINE,
34 	IDXD_DEV_CDEV,
35 	IDXD_DEV_CDEV_FILE,
36 	IDXD_DEV_MAX_TYPE,
37 };
38 
39 struct idxd_dev {
40 	struct device conf_dev;
41 	enum idxd_dev_type type;
42 };
43 
44 #define IDXD_REG_TIMEOUT	50
45 #define IDXD_DRAIN_TIMEOUT	5000
46 
47 enum idxd_type {
48 	IDXD_TYPE_UNKNOWN = -1,
49 	IDXD_TYPE_DSA = 0,
50 	IDXD_TYPE_IAX,
51 	IDXD_TYPE_MAX,
52 };
53 
54 #define IDXD_NAME_SIZE		128
55 #define IDXD_PMU_EVENT_MAX	64
56 
57 #define IDXD_ENQCMDS_RETRIES		32
58 #define IDXD_ENQCMDS_MAX_RETRIES	64
59 
60 struct idxd_device_driver {
61 	const char *name;
62 	enum idxd_dev_type *type;
63 	int (*probe)(struct idxd_dev *idxd_dev);
64 	void (*remove)(struct idxd_dev *idxd_dev);
65 	struct device_driver drv;
66 };
67 
68 extern struct idxd_device_driver dsa_drv;
69 extern struct idxd_device_driver idxd_drv;
70 extern struct idxd_device_driver idxd_dmaengine_drv;
71 extern struct idxd_device_driver idxd_user_drv;
72 
73 #define INVALID_INT_HANDLE	-1
74 struct idxd_irq_entry {
75 	int id;
76 	int vector;
77 	struct llist_head pending_llist;
78 	struct list_head work_list;
79 	/*
80 	 * Lock to protect access between irq thread process descriptor
81 	 * and irq thread processing error descriptor.
82 	 */
83 	spinlock_t list_lock;
84 	int int_handle;
85 	ioasid_t pasid;
86 };
87 
88 struct idxd_group {
89 	struct idxd_dev idxd_dev;
90 	struct idxd_device *idxd;
91 	struct grpcfg grpcfg;
92 	int id;
93 	int num_engines;
94 	int num_wqs;
95 	bool use_rdbuf_limit;
96 	u8 rdbufs_allowed;
97 	u8 rdbufs_reserved;
98 	int tc_a;
99 	int tc_b;
100 	int desc_progress_limit;
101 	int batch_progress_limit;
102 };
103 
104 struct idxd_pmu {
105 	struct idxd_device *idxd;
106 
107 	struct perf_event *event_list[IDXD_PMU_EVENT_MAX];
108 	int n_events;
109 
110 	DECLARE_BITMAP(used_mask, IDXD_PMU_EVENT_MAX);
111 
112 	struct pmu pmu;
113 	char name[IDXD_NAME_SIZE];
114 	int cpu;
115 
116 	int n_counters;
117 	int counter_width;
118 	int n_event_categories;
119 
120 	bool per_counter_caps_supported;
121 	unsigned long supported_event_categories;
122 
123 	unsigned long supported_filters;
124 	int n_filters;
125 
126 	struct hlist_node cpuhp_node;
127 };
128 
129 #define IDXD_MAX_PRIORITY	0xf
130 
131 enum {
132 	COUNTER_FAULTS = 0,
133 	COUNTER_FAULT_FAILS,
134 	COUNTER_MAX
135 };
136 
137 enum idxd_wq_state {
138 	IDXD_WQ_DISABLED = 0,
139 	IDXD_WQ_ENABLED,
140 };
141 
142 enum idxd_wq_flag {
143 	WQ_FLAG_DEDICATED = 0,
144 	WQ_FLAG_BLOCK_ON_FAULT,
145 	WQ_FLAG_ATS_DISABLE,
146 	WQ_FLAG_PRS_DISABLE,
147 };
148 
149 enum idxd_wq_type {
150 	IDXD_WQT_NONE = 0,
151 	IDXD_WQT_KERNEL,
152 	IDXD_WQT_USER,
153 };
154 
155 struct idxd_cdev {
156 	struct idxd_wq *wq;
157 	struct cdev cdev;
158 	struct idxd_dev idxd_dev;
159 	int minor;
160 };
161 
162 #define IDXD_ALLOCATED_BATCH_SIZE	128U
163 #define WQ_NAME_SIZE   1024
164 #define WQ_TYPE_SIZE   10
165 
166 #define WQ_DEFAULT_QUEUE_DEPTH		16
167 #define WQ_DEFAULT_MAX_XFER		SZ_2M
168 #define WQ_DEFAULT_MAX_BATCH		32
169 
170 enum idxd_op_type {
171 	IDXD_OP_BLOCK = 0,
172 	IDXD_OP_NONBLOCK = 1,
173 };
174 
175 enum idxd_complete_type {
176 	IDXD_COMPLETE_NORMAL = 0,
177 	IDXD_COMPLETE_ABORT,
178 	IDXD_COMPLETE_DEV_FAIL,
179 };
180 
181 struct idxd_dma_chan {
182 	struct dma_chan chan;
183 	struct idxd_wq *wq;
184 };
185 
186 struct idxd_wq {
187 	void __iomem *portal;
188 	u32 portal_offset;
189 	unsigned int enqcmds_retries;
190 	struct percpu_ref wq_active;
191 	struct completion wq_dead;
192 	struct completion wq_resurrect;
193 	struct idxd_dev idxd_dev;
194 	struct idxd_cdev *idxd_cdev;
195 	struct wait_queue_head err_queue;
196 	struct workqueue_struct *wq;
197 	struct idxd_device *idxd;
198 	int id;
199 	struct idxd_irq_entry ie;
200 	enum idxd_wq_type type;
201 	struct idxd_group *group;
202 	int client_count;
203 	struct mutex wq_lock;	/* mutex for workqueue */
204 	u32 size;
205 	u32 threshold;
206 	u32 priority;
207 	enum idxd_wq_state state;
208 	unsigned long flags;
209 	union wqcfg *wqcfg;
210 	unsigned long *opcap_bmap;
211 
212 	struct dsa_hw_desc **hw_descs;
213 	int num_descs;
214 	union {
215 		struct dsa_completion_record *compls;
216 		struct iax_completion_record *iax_compls;
217 	};
218 	dma_addr_t compls_addr;
219 	int compls_size;
220 	struct idxd_desc **descs;
221 	struct sbitmap_queue sbq;
222 	struct idxd_dma_chan *idxd_chan;
223 	char name[WQ_NAME_SIZE + 1];
224 	u64 max_xfer_bytes;
225 	u32 max_batch_size;
226 
227 	/* Lock to protect upasid_xa access. */
228 	struct mutex uc_lock;
229 	struct xarray upasid_xa;
230 };
231 
232 struct idxd_engine {
233 	struct idxd_dev idxd_dev;
234 	int id;
235 	struct idxd_group *group;
236 	struct idxd_device *idxd;
237 };
238 
239 /* shadow registers */
240 struct idxd_hw {
241 	u32 version;
242 	union gen_cap_reg gen_cap;
243 	union wq_cap_reg wq_cap;
244 	union group_cap_reg group_cap;
245 	union engine_cap_reg engine_cap;
246 	struct opcap opcap;
247 	u32 cmd_cap;
248 	union iaa_cap_reg iaa_cap;
249 };
250 
251 enum idxd_device_state {
252 	IDXD_DEV_HALTED = -1,
253 	IDXD_DEV_DISABLED = 0,
254 	IDXD_DEV_ENABLED,
255 };
256 
257 enum idxd_device_flag {
258 	IDXD_FLAG_CONFIGURABLE = 0,
259 	IDXD_FLAG_CMD_RUNNING,
260 	IDXD_FLAG_PASID_ENABLED,
261 	IDXD_FLAG_USER_PASID_ENABLED,
262 };
263 
264 struct idxd_dma_dev {
265 	struct idxd_device *idxd;
266 	struct dma_device dma;
267 };
268 
269 struct idxd_driver_data {
270 	const char *name_prefix;
271 	enum idxd_type type;
272 	struct device_type *dev_type;
273 	int compl_size;
274 	int align;
275 	int evl_cr_off;
276 	int cr_status_off;
277 	int cr_result_off;
278 	bool user_submission_safe;
279 };
280 
281 struct idxd_evl {
282 	/* Lock to protect event log access. */
283 	struct mutex lock;
284 	void *log;
285 	dma_addr_t dma;
286 	/* Total size of event log = number of entries * entry size. */
287 	unsigned int log_size;
288 	/* The number of entries in the event log. */
289 	u16 size;
290 	unsigned long *bmap;
291 	bool batch_fail[IDXD_MAX_BATCH_IDENT];
292 };
293 
294 struct idxd_evl_fault {
295 	struct work_struct work;
296 	struct idxd_wq *wq;
297 	u8 status;
298 
299 	/* make this last member always */
300 	struct __evl_entry entry[];
301 };
302 
303 struct idxd_device {
304 	struct idxd_dev idxd_dev;
305 	struct idxd_driver_data *data;
306 	struct list_head list;
307 	struct idxd_hw hw;
308 	enum idxd_device_state state;
309 	unsigned long flags;
310 	int id;
311 	int major;
312 	u32 cmd_status;
313 	struct idxd_irq_entry ie;	/* misc irq, msix 0 */
314 
315 	struct pci_dev *pdev;
316 	void __iomem *reg_base;
317 
318 	spinlock_t dev_lock;	/* spinlock for device */
319 	spinlock_t cmd_lock;	/* spinlock for device commands */
320 	struct completion *cmd_done;
321 	struct idxd_group **groups;
322 	struct idxd_wq **wqs;
323 	struct idxd_engine **engines;
324 
325 	struct iommu_sva *sva;
326 	unsigned int pasid;
327 
328 	int num_groups;
329 	int irq_cnt;
330 	bool request_int_handles;
331 
332 	u32 msix_perm_offset;
333 	u32 wqcfg_offset;
334 	u32 grpcfg_offset;
335 	u32 perfmon_offset;
336 
337 	u64 max_xfer_bytes;
338 	u32 max_batch_size;
339 	int max_groups;
340 	int max_engines;
341 	int max_rdbufs;
342 	int max_wqs;
343 	int max_wq_size;
344 	int rdbuf_limit;
345 	int nr_rdbufs;		/* non-reserved read buffers */
346 	unsigned int wqcfg_size;
347 	unsigned long *wq_enable_map;
348 
349 	union sw_err_reg sw_err;
350 	wait_queue_head_t cmd_waitq;
351 
352 	struct idxd_dma_dev *idxd_dma;
353 	struct workqueue_struct *wq;
354 	struct work_struct work;
355 
356 	struct idxd_pmu *idxd_pmu;
357 
358 	unsigned long *opcap_bmap;
359 	struct idxd_evl *evl;
360 	struct kmem_cache *evl_cache;
361 
362 	struct dentry *dbgfs_dir;
363 	struct dentry *dbgfs_evl_file;
364 
365 	bool user_submission_safe;
366 };
367 
evl_ent_size(struct idxd_device * idxd)368 static inline unsigned int evl_ent_size(struct idxd_device *idxd)
369 {
370 	return idxd->hw.gen_cap.evl_support ?
371 	       (32 * (1 << idxd->hw.gen_cap.evl_support)) : 0;
372 }
373 
evl_size(struct idxd_device * idxd)374 static inline unsigned int evl_size(struct idxd_device *idxd)
375 {
376 	return idxd->evl->size * evl_ent_size(idxd);
377 }
378 
379 /* IDXD software descriptor */
380 struct idxd_desc {
381 	union {
382 		struct dsa_hw_desc *hw;
383 		struct iax_hw_desc *iax_hw;
384 	};
385 	dma_addr_t desc_dma;
386 	union {
387 		struct dsa_completion_record *completion;
388 		struct iax_completion_record *iax_completion;
389 	};
390 	dma_addr_t compl_dma;
391 	struct dma_async_tx_descriptor txd;
392 	struct llist_node llnode;
393 	struct list_head list;
394 	int id;
395 	int cpu;
396 	struct idxd_wq *wq;
397 };
398 
399 /*
400  * This is software defined error for the completion status. We overload the error code
401  * that will never appear in completion status and only SWERR register.
402  */
403 enum idxd_completion_status {
404 	IDXD_COMP_DESC_ABORT = 0xff,
405 };
406 
407 #define idxd_confdev(idxd) &idxd->idxd_dev.conf_dev
408 #define wq_confdev(wq) &wq->idxd_dev.conf_dev
409 #define engine_confdev(engine) &engine->idxd_dev.conf_dev
410 #define group_confdev(group) &group->idxd_dev.conf_dev
411 #define cdev_dev(cdev) &cdev->idxd_dev.conf_dev
412 #define user_ctx_dev(ctx) (&(ctx)->idxd_dev.conf_dev)
413 
414 #define confdev_to_idxd_dev(dev) container_of(dev, struct idxd_dev, conf_dev)
415 #define idxd_dev_to_idxd(idxd_dev) container_of(idxd_dev, struct idxd_device, idxd_dev)
416 #define idxd_dev_to_wq(idxd_dev) container_of(idxd_dev, struct idxd_wq, idxd_dev)
417 
confdev_to_idxd(struct device * dev)418 static inline struct idxd_device *confdev_to_idxd(struct device *dev)
419 {
420 	struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
421 
422 	return idxd_dev_to_idxd(idxd_dev);
423 }
424 
confdev_to_wq(struct device * dev)425 static inline struct idxd_wq *confdev_to_wq(struct device *dev)
426 {
427 	struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
428 
429 	return idxd_dev_to_wq(idxd_dev);
430 }
431 
confdev_to_engine(struct device * dev)432 static inline struct idxd_engine *confdev_to_engine(struct device *dev)
433 {
434 	struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
435 
436 	return container_of(idxd_dev, struct idxd_engine, idxd_dev);
437 }
438 
confdev_to_group(struct device * dev)439 static inline struct idxd_group *confdev_to_group(struct device *dev)
440 {
441 	struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
442 
443 	return container_of(idxd_dev, struct idxd_group, idxd_dev);
444 }
445 
dev_to_cdev(struct device * dev)446 static inline struct idxd_cdev *dev_to_cdev(struct device *dev)
447 {
448 	struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
449 
450 	return container_of(idxd_dev, struct idxd_cdev, idxd_dev);
451 }
452 
idxd_dev_set_type(struct idxd_dev * idev,int type)453 static inline void idxd_dev_set_type(struct idxd_dev *idev, int type)
454 {
455 	if (type >= IDXD_DEV_MAX_TYPE) {
456 		idev->type = IDXD_DEV_NONE;
457 		return;
458 	}
459 
460 	idev->type = type;
461 }
462 
idxd_get_ie(struct idxd_device * idxd,int idx)463 static inline struct idxd_irq_entry *idxd_get_ie(struct idxd_device *idxd, int idx)
464 {
465 	return (idx == 0) ? &idxd->ie : &idxd->wqs[idx - 1]->ie;
466 }
467 
ie_to_wq(struct idxd_irq_entry * ie)468 static inline struct idxd_wq *ie_to_wq(struct idxd_irq_entry *ie)
469 {
470 	return container_of(ie, struct idxd_wq, ie);
471 }
472 
ie_to_idxd(struct idxd_irq_entry * ie)473 static inline struct idxd_device *ie_to_idxd(struct idxd_irq_entry *ie)
474 {
475 	return container_of(ie, struct idxd_device, ie);
476 }
477 
idxd_set_user_intr(struct idxd_device * idxd,bool enable)478 static inline void idxd_set_user_intr(struct idxd_device *idxd, bool enable)
479 {
480 	union gencfg_reg reg;
481 
482 	reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET);
483 	reg.user_int_en = enable;
484 	iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET);
485 }
486 
487 extern struct bus_type dsa_bus_type;
488 
489 extern bool support_enqcmd;
490 extern struct ida idxd_ida;
491 extern struct device_type dsa_device_type;
492 extern struct device_type iax_device_type;
493 extern struct device_type idxd_wq_device_type;
494 extern struct device_type idxd_engine_device_type;
495 extern struct device_type idxd_group_device_type;
496 
is_dsa_dev(struct idxd_dev * idxd_dev)497 static inline bool is_dsa_dev(struct idxd_dev *idxd_dev)
498 {
499 	return idxd_dev->type == IDXD_DEV_DSA;
500 }
501 
is_iax_dev(struct idxd_dev * idxd_dev)502 static inline bool is_iax_dev(struct idxd_dev *idxd_dev)
503 {
504 	return idxd_dev->type == IDXD_DEV_IAX;
505 }
506 
is_idxd_dev(struct idxd_dev * idxd_dev)507 static inline bool is_idxd_dev(struct idxd_dev *idxd_dev)
508 {
509 	return is_dsa_dev(idxd_dev) || is_iax_dev(idxd_dev);
510 }
511 
is_idxd_wq_dev(struct idxd_dev * idxd_dev)512 static inline bool is_idxd_wq_dev(struct idxd_dev *idxd_dev)
513 {
514 	return idxd_dev->type == IDXD_DEV_WQ;
515 }
516 
is_idxd_wq_dmaengine(struct idxd_wq * wq)517 static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq)
518 {
519 	if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0)
520 		return true;
521 	return false;
522 }
523 
is_idxd_wq_user(struct idxd_wq * wq)524 static inline bool is_idxd_wq_user(struct idxd_wq *wq)
525 {
526 	return wq->type == IDXD_WQT_USER;
527 }
528 
is_idxd_wq_kernel(struct idxd_wq * wq)529 static inline bool is_idxd_wq_kernel(struct idxd_wq *wq)
530 {
531 	return wq->type == IDXD_WQT_KERNEL;
532 }
533 
wq_dedicated(struct idxd_wq * wq)534 static inline bool wq_dedicated(struct idxd_wq *wq)
535 {
536 	return test_bit(WQ_FLAG_DEDICATED, &wq->flags);
537 }
538 
wq_shared(struct idxd_wq * wq)539 static inline bool wq_shared(struct idxd_wq *wq)
540 {
541 	return !test_bit(WQ_FLAG_DEDICATED, &wq->flags);
542 }
543 
device_pasid_enabled(struct idxd_device * idxd)544 static inline bool device_pasid_enabled(struct idxd_device *idxd)
545 {
546 	return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
547 }
548 
device_user_pasid_enabled(struct idxd_device * idxd)549 static inline bool device_user_pasid_enabled(struct idxd_device *idxd)
550 {
551 	return test_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags);
552 }
553 
wq_pasid_enabled(struct idxd_wq * wq)554 static inline bool wq_pasid_enabled(struct idxd_wq *wq)
555 {
556 	return (is_idxd_wq_kernel(wq) && device_pasid_enabled(wq->idxd)) ||
557 	       (is_idxd_wq_user(wq) && device_user_pasid_enabled(wq->idxd));
558 }
559 
wq_shared_supported(struct idxd_wq * wq)560 static inline bool wq_shared_supported(struct idxd_wq *wq)
561 {
562 	return (support_enqcmd && wq_pasid_enabled(wq));
563 }
564 
565 enum idxd_portal_prot {
566 	IDXD_PORTAL_UNLIMITED = 0,
567 	IDXD_PORTAL_LIMITED,
568 };
569 
570 enum idxd_interrupt_type {
571 	IDXD_IRQ_MSIX = 0,
572 	IDXD_IRQ_IMS,
573 };
574 
idxd_get_wq_portal_offset(enum idxd_portal_prot prot)575 static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot)
576 {
577 	return prot * 0x1000;
578 }
579 
idxd_get_wq_portal_full_offset(int wq_id,enum idxd_portal_prot prot)580 static inline int idxd_get_wq_portal_full_offset(int wq_id,
581 						 enum idxd_portal_prot prot)
582 {
583 	return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot);
584 }
585 
586 #define IDXD_PORTAL_MASK	(PAGE_SIZE - 1)
587 
588 /*
589  * Even though this function can be accessed by multiple threads, it is safe to use.
590  * At worst the address gets used more than once before it gets incremented. We don't
591  * hit a threshold until iops becomes many million times a second. So the occasional
592  * reuse of the same address is tolerable compare to using an atomic variable. This is
593  * safe on a system that has atomic load/store for 32bit integers. Given that this is an
594  * Intel iEP device, that should not be a problem.
595  */
idxd_wq_portal_addr(struct idxd_wq * wq)596 static inline void __iomem *idxd_wq_portal_addr(struct idxd_wq *wq)
597 {
598 	int ofs = wq->portal_offset;
599 
600 	wq->portal_offset = (ofs + sizeof(struct dsa_raw_desc)) & IDXD_PORTAL_MASK;
601 	return wq->portal + ofs;
602 }
603 
idxd_wq_get(struct idxd_wq * wq)604 static inline void idxd_wq_get(struct idxd_wq *wq)
605 {
606 	wq->client_count++;
607 }
608 
idxd_wq_put(struct idxd_wq * wq)609 static inline void idxd_wq_put(struct idxd_wq *wq)
610 {
611 	wq->client_count--;
612 }
613 
idxd_wq_refcount(struct idxd_wq * wq)614 static inline int idxd_wq_refcount(struct idxd_wq *wq)
615 {
616 	return wq->client_count;
617 };
618 
619 /*
620  * Intel IAA does not support batch processing.
621  * The max batch size of device, max batch size of wq and
622  * max batch shift of wqcfg should be always 0 on IAA.
623  */
idxd_set_max_batch_size(int idxd_type,struct idxd_device * idxd,u32 max_batch_size)624 static inline void idxd_set_max_batch_size(int idxd_type, struct idxd_device *idxd,
625 					   u32 max_batch_size)
626 {
627 	if (idxd_type == IDXD_TYPE_IAX)
628 		idxd->max_batch_size = 0;
629 	else
630 		idxd->max_batch_size = max_batch_size;
631 }
632 
idxd_wq_set_max_batch_size(int idxd_type,struct idxd_wq * wq,u32 max_batch_size)633 static inline void idxd_wq_set_max_batch_size(int idxd_type, struct idxd_wq *wq,
634 					      u32 max_batch_size)
635 {
636 	if (idxd_type == IDXD_TYPE_IAX)
637 		wq->max_batch_size = 0;
638 	else
639 		wq->max_batch_size = max_batch_size;
640 }
641 
idxd_wqcfg_set_max_batch_shift(int idxd_type,union wqcfg * wqcfg,u32 max_batch_shift)642 static inline void idxd_wqcfg_set_max_batch_shift(int idxd_type, union wqcfg *wqcfg,
643 						  u32 max_batch_shift)
644 {
645 	if (idxd_type == IDXD_TYPE_IAX)
646 		wqcfg->max_batch_shift = 0;
647 	else
648 		wqcfg->max_batch_shift = max_batch_shift;
649 }
650 
651 int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv,
652 					struct module *module, const char *mod_name);
653 #define idxd_driver_register(driver) \
654 	__idxd_driver_register(driver, THIS_MODULE, KBUILD_MODNAME)
655 
656 void idxd_driver_unregister(struct idxd_device_driver *idxd_drv);
657 
658 #define module_idxd_driver(__idxd_driver) \
659 	module_driver(__idxd_driver, idxd_driver_register, idxd_driver_unregister)
660 
661 int idxd_register_bus_type(void);
662 void idxd_unregister_bus_type(void);
663 int idxd_register_devices(struct idxd_device *idxd);
664 void idxd_unregister_devices(struct idxd_device *idxd);
665 void idxd_wqs_quiesce(struct idxd_device *idxd);
666 bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc);
667 void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count);
668 
669 /* device interrupt control */
670 irqreturn_t idxd_misc_thread(int vec, void *data);
671 irqreturn_t idxd_wq_thread(int irq, void *data);
672 void idxd_mask_error_interrupts(struct idxd_device *idxd);
673 void idxd_unmask_error_interrupts(struct idxd_device *idxd);
674 
675 /* device control */
676 int idxd_device_drv_probe(struct idxd_dev *idxd_dev);
677 void idxd_device_drv_remove(struct idxd_dev *idxd_dev);
678 int drv_enable_wq(struct idxd_wq *wq);
679 void drv_disable_wq(struct idxd_wq *wq);
680 int idxd_device_init_reset(struct idxd_device *idxd);
681 int idxd_device_enable(struct idxd_device *idxd);
682 int idxd_device_disable(struct idxd_device *idxd);
683 void idxd_device_reset(struct idxd_device *idxd);
684 void idxd_device_clear_state(struct idxd_device *idxd);
685 int idxd_device_config(struct idxd_device *idxd);
686 void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid);
687 int idxd_device_load_config(struct idxd_device *idxd);
688 int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle,
689 				   enum idxd_interrupt_type irq_type);
690 int idxd_device_release_int_handle(struct idxd_device *idxd, int handle,
691 				   enum idxd_interrupt_type irq_type);
692 
693 /* work queue control */
694 void idxd_wqs_unmap_portal(struct idxd_device *idxd);
695 int idxd_wq_alloc_resources(struct idxd_wq *wq);
696 void idxd_wq_free_resources(struct idxd_wq *wq);
697 int idxd_wq_enable(struct idxd_wq *wq);
698 int idxd_wq_disable(struct idxd_wq *wq, bool reset_config);
699 void idxd_wq_drain(struct idxd_wq *wq);
700 void idxd_wq_reset(struct idxd_wq *wq);
701 int idxd_wq_map_portal(struct idxd_wq *wq);
702 void idxd_wq_unmap_portal(struct idxd_wq *wq);
703 int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid);
704 int idxd_wq_disable_pasid(struct idxd_wq *wq);
705 void __idxd_wq_quiesce(struct idxd_wq *wq);
706 void idxd_wq_quiesce(struct idxd_wq *wq);
707 int idxd_wq_init_percpu_ref(struct idxd_wq *wq);
708 void idxd_wq_free_irq(struct idxd_wq *wq);
709 int idxd_wq_request_irq(struct idxd_wq *wq);
710 
711 /* submission */
712 int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);
713 struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype);
714 void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc);
715 int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc);
716 
717 /* dmaengine */
718 int idxd_register_dma_device(struct idxd_device *idxd);
719 void idxd_unregister_dma_device(struct idxd_device *idxd);
720 void idxd_dma_complete_txd(struct idxd_desc *desc,
721 			   enum idxd_complete_type comp_type, bool free_desc);
722 
723 /* cdev */
724 int idxd_cdev_register(void);
725 void idxd_cdev_remove(void);
726 int idxd_cdev_get_major(struct idxd_device *idxd);
727 int idxd_wq_add_cdev(struct idxd_wq *wq);
728 void idxd_wq_del_cdev(struct idxd_wq *wq);
729 int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr,
730 		 void *buf, int len);
731 void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index);
732 
733 /* perfmon */
734 #if IS_ENABLED(CONFIG_INTEL_IDXD_PERFMON)
735 int perfmon_pmu_init(struct idxd_device *idxd);
736 void perfmon_pmu_remove(struct idxd_device *idxd);
737 void perfmon_counter_overflow(struct idxd_device *idxd);
738 void perfmon_init(void);
739 void perfmon_exit(void);
740 #else
perfmon_pmu_init(struct idxd_device * idxd)741 static inline int perfmon_pmu_init(struct idxd_device *idxd) { return 0; }
perfmon_pmu_remove(struct idxd_device * idxd)742 static inline void perfmon_pmu_remove(struct idxd_device *idxd) {}
perfmon_counter_overflow(struct idxd_device * idxd)743 static inline void perfmon_counter_overflow(struct idxd_device *idxd) {}
perfmon_init(void)744 static inline void perfmon_init(void) {}
perfmon_exit(void)745 static inline void perfmon_exit(void) {}
746 #endif
747 
748 /* debugfs */
749 int idxd_device_init_debugfs(struct idxd_device *idxd);
750 void idxd_device_remove_debugfs(struct idxd_device *idxd);
751 int idxd_init_debugfs(void);
752 void idxd_remove_debugfs(void);
753 
754 #endif
755