1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3 #ifndef _IDXD_H_
4 #define _IDXD_H_
5
6 #include <linux/sbitmap.h>
7 #include <linux/dmaengine.h>
8 #include <linux/percpu-rwsem.h>
9 #include <linux/wait.h>
10 #include <linux/cdev.h>
11 #include <linux/idr.h>
12 #include <linux/pci.h>
13 #include <linux/bitmap.h>
14 #include <linux/perf_event.h>
15 #include <linux/iommu.h>
16 #include <uapi/linux/idxd.h>
17 #include "registers.h"
18
19 #define IDXD_DRIVER_VERSION "1.00"
20
21 extern struct kmem_cache *idxd_desc_pool;
22 extern bool tc_override;
23
24 struct idxd_wq;
25 struct idxd_dev;
26
27 enum idxd_dev_type {
28 IDXD_DEV_NONE = -1,
29 IDXD_DEV_DSA = 0,
30 IDXD_DEV_IAX,
31 IDXD_DEV_WQ,
32 IDXD_DEV_GROUP,
33 IDXD_DEV_ENGINE,
34 IDXD_DEV_CDEV,
35 IDXD_DEV_CDEV_FILE,
36 IDXD_DEV_MAX_TYPE,
37 };
38
39 struct idxd_dev {
40 struct device conf_dev;
41 enum idxd_dev_type type;
42 };
43
44 #define IDXD_REG_TIMEOUT 50
45 #define IDXD_DRAIN_TIMEOUT 5000
46
47 enum idxd_type {
48 IDXD_TYPE_UNKNOWN = -1,
49 IDXD_TYPE_DSA = 0,
50 IDXD_TYPE_IAX,
51 IDXD_TYPE_MAX,
52 };
53
54 #define IDXD_NAME_SIZE 128
55 #define IDXD_PMU_EVENT_MAX 64
56
57 #define IDXD_ENQCMDS_RETRIES 32
58 #define IDXD_ENQCMDS_MAX_RETRIES 64
59
60 struct idxd_device_driver {
61 const char *name;
62 enum idxd_dev_type *type;
63 int (*probe)(struct idxd_dev *idxd_dev);
64 void (*remove)(struct idxd_dev *idxd_dev);
65 struct device_driver drv;
66 };
67
68 extern struct idxd_device_driver dsa_drv;
69 extern struct idxd_device_driver idxd_drv;
70 extern struct idxd_device_driver idxd_dmaengine_drv;
71 extern struct idxd_device_driver idxd_user_drv;
72
73 #define INVALID_INT_HANDLE -1
74 struct idxd_irq_entry {
75 int id;
76 int vector;
77 struct llist_head pending_llist;
78 struct list_head work_list;
79 /*
80 * Lock to protect access between irq thread process descriptor
81 * and irq thread processing error descriptor.
82 */
83 spinlock_t list_lock;
84 int int_handle;
85 ioasid_t pasid;
86 };
87
88 struct idxd_group {
89 struct idxd_dev idxd_dev;
90 struct idxd_device *idxd;
91 struct grpcfg grpcfg;
92 int id;
93 int num_engines;
94 int num_wqs;
95 bool use_rdbuf_limit;
96 u8 rdbufs_allowed;
97 u8 rdbufs_reserved;
98 int tc_a;
99 int tc_b;
100 int desc_progress_limit;
101 int batch_progress_limit;
102 };
103
104 struct idxd_pmu {
105 struct idxd_device *idxd;
106
107 struct perf_event *event_list[IDXD_PMU_EVENT_MAX];
108 int n_events;
109
110 DECLARE_BITMAP(used_mask, IDXD_PMU_EVENT_MAX);
111
112 struct pmu pmu;
113 char name[IDXD_NAME_SIZE];
114 int cpu;
115
116 int n_counters;
117 int counter_width;
118 int n_event_categories;
119
120 bool per_counter_caps_supported;
121 unsigned long supported_event_categories;
122
123 unsigned long supported_filters;
124 int n_filters;
125
126 struct hlist_node cpuhp_node;
127 };
128
129 #define IDXD_MAX_PRIORITY 0xf
130
131 enum {
132 COUNTER_FAULTS = 0,
133 COUNTER_FAULT_FAILS,
134 COUNTER_MAX
135 };
136
137 enum idxd_wq_state {
138 IDXD_WQ_DISABLED = 0,
139 IDXD_WQ_ENABLED,
140 };
141
142 enum idxd_wq_flag {
143 WQ_FLAG_DEDICATED = 0,
144 WQ_FLAG_BLOCK_ON_FAULT,
145 WQ_FLAG_ATS_DISABLE,
146 WQ_FLAG_PRS_DISABLE,
147 };
148
149 enum idxd_wq_type {
150 IDXD_WQT_NONE = 0,
151 IDXD_WQT_KERNEL,
152 IDXD_WQT_USER,
153 };
154
155 struct idxd_cdev {
156 struct idxd_wq *wq;
157 struct cdev cdev;
158 struct idxd_dev idxd_dev;
159 int minor;
160 };
161
162 #define DRIVER_NAME_SIZE 128
163
164 #define IDXD_ALLOCATED_BATCH_SIZE 128U
165 #define WQ_NAME_SIZE 1024
166 #define WQ_TYPE_SIZE 10
167
168 #define WQ_DEFAULT_QUEUE_DEPTH 16
169 #define WQ_DEFAULT_MAX_XFER SZ_2M
170 #define WQ_DEFAULT_MAX_BATCH 32
171
172 enum idxd_op_type {
173 IDXD_OP_BLOCK = 0,
174 IDXD_OP_NONBLOCK = 1,
175 };
176
177 enum idxd_complete_type {
178 IDXD_COMPLETE_NORMAL = 0,
179 IDXD_COMPLETE_ABORT,
180 IDXD_COMPLETE_DEV_FAIL,
181 };
182
183 struct idxd_dma_chan {
184 struct dma_chan chan;
185 struct idxd_wq *wq;
186 };
187
188 struct idxd_wq {
189 void __iomem *portal;
190 u32 portal_offset;
191 unsigned int enqcmds_retries;
192 struct percpu_ref wq_active;
193 struct completion wq_dead;
194 struct completion wq_resurrect;
195 struct idxd_dev idxd_dev;
196 struct idxd_cdev *idxd_cdev;
197 struct wait_queue_head err_queue;
198 struct workqueue_struct *wq;
199 struct idxd_device *idxd;
200 int id;
201 struct idxd_irq_entry ie;
202 enum idxd_wq_type type;
203 struct idxd_group *group;
204 int client_count;
205 struct mutex wq_lock; /* mutex for workqueue */
206 u32 size;
207 u32 threshold;
208 u32 priority;
209 enum idxd_wq_state state;
210 unsigned long flags;
211 union wqcfg *wqcfg;
212 unsigned long *opcap_bmap;
213
214 struct dsa_hw_desc **hw_descs;
215 int num_descs;
216 union {
217 struct dsa_completion_record *compls;
218 struct iax_completion_record *iax_compls;
219 };
220 dma_addr_t compls_addr;
221 int compls_size;
222 struct idxd_desc **descs;
223 struct sbitmap_queue sbq;
224 struct idxd_dma_chan *idxd_chan;
225 char name[WQ_NAME_SIZE + 1];
226 u64 max_xfer_bytes;
227 u32 max_batch_size;
228
229 /* Lock to protect upasid_xa access. */
230 struct mutex uc_lock;
231 struct xarray upasid_xa;
232
233 char driver_name[DRIVER_NAME_SIZE + 1];
234 };
235
236 struct idxd_engine {
237 struct idxd_dev idxd_dev;
238 int id;
239 struct idxd_group *group;
240 struct idxd_device *idxd;
241 };
242
243 /* shadow registers */
244 struct idxd_hw {
245 u32 version;
246 union gen_cap_reg gen_cap;
247 union wq_cap_reg wq_cap;
248 union group_cap_reg group_cap;
249 union engine_cap_reg engine_cap;
250 struct opcap opcap;
251 u32 cmd_cap;
252 union iaa_cap_reg iaa_cap;
253 };
254
255 enum idxd_device_state {
256 IDXD_DEV_HALTED = -1,
257 IDXD_DEV_DISABLED = 0,
258 IDXD_DEV_ENABLED,
259 };
260
261 enum idxd_device_flag {
262 IDXD_FLAG_CONFIGURABLE = 0,
263 IDXD_FLAG_CMD_RUNNING,
264 IDXD_FLAG_PASID_ENABLED,
265 IDXD_FLAG_USER_PASID_ENABLED,
266 };
267
268 struct idxd_dma_dev {
269 struct idxd_device *idxd;
270 struct dma_device dma;
271 };
272
273 struct idxd_driver_data {
274 const char *name_prefix;
275 enum idxd_type type;
276 struct device_type *dev_type;
277 int compl_size;
278 int align;
279 int evl_cr_off;
280 int cr_status_off;
281 int cr_result_off;
282 bool user_submission_safe;
283 };
284
285 struct idxd_evl {
286 /* Lock to protect event log access. */
287 struct mutex lock;
288 void *log;
289 dma_addr_t dma;
290 /* Total size of event log = number of entries * entry size. */
291 unsigned int log_size;
292 /* The number of entries in the event log. */
293 u16 size;
294 unsigned long *bmap;
295 bool batch_fail[IDXD_MAX_BATCH_IDENT];
296 };
297
298 struct idxd_evl_fault {
299 struct work_struct work;
300 struct idxd_wq *wq;
301 u8 status;
302
303 /* make this last member always */
304 struct __evl_entry entry[];
305 };
306
307 struct idxd_device {
308 struct idxd_dev idxd_dev;
309 struct idxd_driver_data *data;
310 struct list_head list;
311 struct idxd_hw hw;
312 enum idxd_device_state state;
313 unsigned long flags;
314 int id;
315 int major;
316 u32 cmd_status;
317 struct idxd_irq_entry ie; /* misc irq, msix 0 */
318
319 struct pci_dev *pdev;
320 void __iomem *reg_base;
321
322 spinlock_t dev_lock; /* spinlock for device */
323 spinlock_t cmd_lock; /* spinlock for device commands */
324 struct completion *cmd_done;
325 struct idxd_group **groups;
326 struct idxd_wq **wqs;
327 struct idxd_engine **engines;
328
329 struct iommu_sva *sva;
330 unsigned int pasid;
331
332 int num_groups;
333 int irq_cnt;
334 bool request_int_handles;
335
336 u32 msix_perm_offset;
337 u32 wqcfg_offset;
338 u32 grpcfg_offset;
339 u32 perfmon_offset;
340
341 u64 max_xfer_bytes;
342 u32 max_batch_size;
343 int max_groups;
344 int max_engines;
345 int max_rdbufs;
346 int max_wqs;
347 int max_wq_size;
348 int rdbuf_limit;
349 int nr_rdbufs; /* non-reserved read buffers */
350 unsigned int wqcfg_size;
351 unsigned long *wq_enable_map;
352
353 union sw_err_reg sw_err;
354 wait_queue_head_t cmd_waitq;
355
356 struct idxd_dma_dev *idxd_dma;
357 struct workqueue_struct *wq;
358 struct work_struct work;
359
360 struct idxd_pmu *idxd_pmu;
361
362 unsigned long *opcap_bmap;
363 struct idxd_evl *evl;
364 struct kmem_cache *evl_cache;
365
366 struct dentry *dbgfs_dir;
367 struct dentry *dbgfs_evl_file;
368
369 bool user_submission_safe;
370 };
371
evl_ent_size(struct idxd_device * idxd)372 static inline unsigned int evl_ent_size(struct idxd_device *idxd)
373 {
374 return idxd->hw.gen_cap.evl_support ?
375 (32 * (1 << idxd->hw.gen_cap.evl_support)) : 0;
376 }
377
evl_size(struct idxd_device * idxd)378 static inline unsigned int evl_size(struct idxd_device *idxd)
379 {
380 return idxd->evl->size * evl_ent_size(idxd);
381 }
382
383 /* IDXD software descriptor */
384 struct idxd_desc {
385 union {
386 struct dsa_hw_desc *hw;
387 struct iax_hw_desc *iax_hw;
388 };
389 dma_addr_t desc_dma;
390 union {
391 struct dsa_completion_record *completion;
392 struct iax_completion_record *iax_completion;
393 };
394 dma_addr_t compl_dma;
395 struct dma_async_tx_descriptor txd;
396 struct llist_node llnode;
397 struct list_head list;
398 int id;
399 int cpu;
400 struct idxd_wq *wq;
401 };
402
403 /*
404 * This is software defined error for the completion status. We overload the error code
405 * that will never appear in completion status and only SWERR register.
406 */
407 enum idxd_completion_status {
408 IDXD_COMP_DESC_ABORT = 0xff,
409 };
410
411 #define idxd_confdev(idxd) &idxd->idxd_dev.conf_dev
412 #define wq_confdev(wq) &wq->idxd_dev.conf_dev
413 #define engine_confdev(engine) &engine->idxd_dev.conf_dev
414 #define group_confdev(group) &group->idxd_dev.conf_dev
415 #define cdev_dev(cdev) &cdev->idxd_dev.conf_dev
416 #define user_ctx_dev(ctx) (&(ctx)->idxd_dev.conf_dev)
417
418 #define confdev_to_idxd_dev(dev) container_of(dev, struct idxd_dev, conf_dev)
419 #define idxd_dev_to_idxd(idxd_dev) container_of(idxd_dev, struct idxd_device, idxd_dev)
420 #define idxd_dev_to_wq(idxd_dev) container_of(idxd_dev, struct idxd_wq, idxd_dev)
421
confdev_to_idxd(struct device * dev)422 static inline struct idxd_device *confdev_to_idxd(struct device *dev)
423 {
424 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
425
426 return idxd_dev_to_idxd(idxd_dev);
427 }
428
confdev_to_wq(struct device * dev)429 static inline struct idxd_wq *confdev_to_wq(struct device *dev)
430 {
431 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
432
433 return idxd_dev_to_wq(idxd_dev);
434 }
435
confdev_to_engine(struct device * dev)436 static inline struct idxd_engine *confdev_to_engine(struct device *dev)
437 {
438 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
439
440 return container_of(idxd_dev, struct idxd_engine, idxd_dev);
441 }
442
confdev_to_group(struct device * dev)443 static inline struct idxd_group *confdev_to_group(struct device *dev)
444 {
445 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
446
447 return container_of(idxd_dev, struct idxd_group, idxd_dev);
448 }
449
dev_to_cdev(struct device * dev)450 static inline struct idxd_cdev *dev_to_cdev(struct device *dev)
451 {
452 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
453
454 return container_of(idxd_dev, struct idxd_cdev, idxd_dev);
455 }
456
idxd_dev_set_type(struct idxd_dev * idev,int type)457 static inline void idxd_dev_set_type(struct idxd_dev *idev, int type)
458 {
459 if (type >= IDXD_DEV_MAX_TYPE) {
460 idev->type = IDXD_DEV_NONE;
461 return;
462 }
463
464 idev->type = type;
465 }
466
idxd_get_ie(struct idxd_device * idxd,int idx)467 static inline struct idxd_irq_entry *idxd_get_ie(struct idxd_device *idxd, int idx)
468 {
469 return (idx == 0) ? &idxd->ie : &idxd->wqs[idx - 1]->ie;
470 }
471
ie_to_wq(struct idxd_irq_entry * ie)472 static inline struct idxd_wq *ie_to_wq(struct idxd_irq_entry *ie)
473 {
474 return container_of(ie, struct idxd_wq, ie);
475 }
476
ie_to_idxd(struct idxd_irq_entry * ie)477 static inline struct idxd_device *ie_to_idxd(struct idxd_irq_entry *ie)
478 {
479 return container_of(ie, struct idxd_device, ie);
480 }
481
idxd_set_user_intr(struct idxd_device * idxd,bool enable)482 static inline void idxd_set_user_intr(struct idxd_device *idxd, bool enable)
483 {
484 union gencfg_reg reg;
485
486 reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET);
487 reg.user_int_en = enable;
488 iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET);
489 }
490
491 extern struct bus_type dsa_bus_type;
492
493 extern bool support_enqcmd;
494 extern struct ida idxd_ida;
495 extern struct device_type dsa_device_type;
496 extern struct device_type iax_device_type;
497 extern struct device_type idxd_wq_device_type;
498 extern struct device_type idxd_engine_device_type;
499 extern struct device_type idxd_group_device_type;
500
is_dsa_dev(struct idxd_dev * idxd_dev)501 static inline bool is_dsa_dev(struct idxd_dev *idxd_dev)
502 {
503 return idxd_dev->type == IDXD_DEV_DSA;
504 }
505
is_iax_dev(struct idxd_dev * idxd_dev)506 static inline bool is_iax_dev(struct idxd_dev *idxd_dev)
507 {
508 return idxd_dev->type == IDXD_DEV_IAX;
509 }
510
is_idxd_dev(struct idxd_dev * idxd_dev)511 static inline bool is_idxd_dev(struct idxd_dev *idxd_dev)
512 {
513 return is_dsa_dev(idxd_dev) || is_iax_dev(idxd_dev);
514 }
515
is_idxd_wq_dev(struct idxd_dev * idxd_dev)516 static inline bool is_idxd_wq_dev(struct idxd_dev *idxd_dev)
517 {
518 return idxd_dev->type == IDXD_DEV_WQ;
519 }
520
is_idxd_wq_dmaengine(struct idxd_wq * wq)521 static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq)
522 {
523 if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0)
524 return true;
525 return false;
526 }
527
is_idxd_wq_user(struct idxd_wq * wq)528 static inline bool is_idxd_wq_user(struct idxd_wq *wq)
529 {
530 return wq->type == IDXD_WQT_USER;
531 }
532
is_idxd_wq_kernel(struct idxd_wq * wq)533 static inline bool is_idxd_wq_kernel(struct idxd_wq *wq)
534 {
535 return wq->type == IDXD_WQT_KERNEL;
536 }
537
wq_dedicated(struct idxd_wq * wq)538 static inline bool wq_dedicated(struct idxd_wq *wq)
539 {
540 return test_bit(WQ_FLAG_DEDICATED, &wq->flags);
541 }
542
wq_shared(struct idxd_wq * wq)543 static inline bool wq_shared(struct idxd_wq *wq)
544 {
545 return !test_bit(WQ_FLAG_DEDICATED, &wq->flags);
546 }
547
device_pasid_enabled(struct idxd_device * idxd)548 static inline bool device_pasid_enabled(struct idxd_device *idxd)
549 {
550 return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
551 }
552
device_user_pasid_enabled(struct idxd_device * idxd)553 static inline bool device_user_pasid_enabled(struct idxd_device *idxd)
554 {
555 return test_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags);
556 }
557
wq_pasid_enabled(struct idxd_wq * wq)558 static inline bool wq_pasid_enabled(struct idxd_wq *wq)
559 {
560 return (is_idxd_wq_kernel(wq) && device_pasid_enabled(wq->idxd)) ||
561 (is_idxd_wq_user(wq) && device_user_pasid_enabled(wq->idxd));
562 }
563
wq_shared_supported(struct idxd_wq * wq)564 static inline bool wq_shared_supported(struct idxd_wq *wq)
565 {
566 return (support_enqcmd && wq_pasid_enabled(wq));
567 }
568
569 enum idxd_portal_prot {
570 IDXD_PORTAL_UNLIMITED = 0,
571 IDXD_PORTAL_LIMITED,
572 };
573
574 enum idxd_interrupt_type {
575 IDXD_IRQ_MSIX = 0,
576 IDXD_IRQ_IMS,
577 };
578
idxd_get_wq_portal_offset(enum idxd_portal_prot prot)579 static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot)
580 {
581 return prot * 0x1000;
582 }
583
idxd_get_wq_portal_full_offset(int wq_id,enum idxd_portal_prot prot)584 static inline int idxd_get_wq_portal_full_offset(int wq_id,
585 enum idxd_portal_prot prot)
586 {
587 return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot);
588 }
589
590 #define IDXD_PORTAL_MASK (PAGE_SIZE - 1)
591
592 /*
593 * Even though this function can be accessed by multiple threads, it is safe to use.
594 * At worst the address gets used more than once before it gets incremented. We don't
595 * hit a threshold until iops becomes many million times a second. So the occasional
596 * reuse of the same address is tolerable compare to using an atomic variable. This is
597 * safe on a system that has atomic load/store for 32bit integers. Given that this is an
598 * Intel iEP device, that should not be a problem.
599 */
idxd_wq_portal_addr(struct idxd_wq * wq)600 static inline void __iomem *idxd_wq_portal_addr(struct idxd_wq *wq)
601 {
602 int ofs = wq->portal_offset;
603
604 wq->portal_offset = (ofs + sizeof(struct dsa_raw_desc)) & IDXD_PORTAL_MASK;
605 return wq->portal + ofs;
606 }
607
idxd_wq_get(struct idxd_wq * wq)608 static inline void idxd_wq_get(struct idxd_wq *wq)
609 {
610 wq->client_count++;
611 }
612
idxd_wq_put(struct idxd_wq * wq)613 static inline void idxd_wq_put(struct idxd_wq *wq)
614 {
615 wq->client_count--;
616 }
617
idxd_wq_refcount(struct idxd_wq * wq)618 static inline int idxd_wq_refcount(struct idxd_wq *wq)
619 {
620 return wq->client_count;
621 };
622
623 /*
624 * Intel IAA does not support batch processing.
625 * The max batch size of device, max batch size of wq and
626 * max batch shift of wqcfg should be always 0 on IAA.
627 */
idxd_set_max_batch_size(int idxd_type,struct idxd_device * idxd,u32 max_batch_size)628 static inline void idxd_set_max_batch_size(int idxd_type, struct idxd_device *idxd,
629 u32 max_batch_size)
630 {
631 if (idxd_type == IDXD_TYPE_IAX)
632 idxd->max_batch_size = 0;
633 else
634 idxd->max_batch_size = max_batch_size;
635 }
636
idxd_wq_set_max_batch_size(int idxd_type,struct idxd_wq * wq,u32 max_batch_size)637 static inline void idxd_wq_set_max_batch_size(int idxd_type, struct idxd_wq *wq,
638 u32 max_batch_size)
639 {
640 if (idxd_type == IDXD_TYPE_IAX)
641 wq->max_batch_size = 0;
642 else
643 wq->max_batch_size = max_batch_size;
644 }
645
idxd_wqcfg_set_max_batch_shift(int idxd_type,union wqcfg * wqcfg,u32 max_batch_shift)646 static inline void idxd_wqcfg_set_max_batch_shift(int idxd_type, union wqcfg *wqcfg,
647 u32 max_batch_shift)
648 {
649 if (idxd_type == IDXD_TYPE_IAX)
650 wqcfg->max_batch_shift = 0;
651 else
652 wqcfg->max_batch_shift = max_batch_shift;
653 }
654
idxd_wq_driver_name_match(struct idxd_wq * wq,struct device * dev)655 static inline int idxd_wq_driver_name_match(struct idxd_wq *wq, struct device *dev)
656 {
657 return (strncmp(wq->driver_name, dev->driver->name, strlen(dev->driver->name)) == 0);
658 }
659
660 int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv,
661 struct module *module, const char *mod_name);
662 #define idxd_driver_register(driver) \
663 __idxd_driver_register(driver, THIS_MODULE, KBUILD_MODNAME)
664
665 void idxd_driver_unregister(struct idxd_device_driver *idxd_drv);
666
667 #define module_idxd_driver(__idxd_driver) \
668 module_driver(__idxd_driver, idxd_driver_register, idxd_driver_unregister)
669
670 int idxd_register_bus_type(void);
671 void idxd_unregister_bus_type(void);
672 int idxd_register_devices(struct idxd_device *idxd);
673 void idxd_unregister_devices(struct idxd_device *idxd);
674 void idxd_wqs_quiesce(struct idxd_device *idxd);
675 bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc);
676 void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count);
677
678 /* device interrupt control */
679 irqreturn_t idxd_misc_thread(int vec, void *data);
680 irqreturn_t idxd_wq_thread(int irq, void *data);
681 void idxd_mask_error_interrupts(struct idxd_device *idxd);
682 void idxd_unmask_error_interrupts(struct idxd_device *idxd);
683
684 /* device control */
685 int idxd_device_drv_probe(struct idxd_dev *idxd_dev);
686 void idxd_device_drv_remove(struct idxd_dev *idxd_dev);
687 int drv_enable_wq(struct idxd_wq *wq);
688 void drv_disable_wq(struct idxd_wq *wq);
689 int idxd_device_init_reset(struct idxd_device *idxd);
690 int idxd_device_enable(struct idxd_device *idxd);
691 int idxd_device_disable(struct idxd_device *idxd);
692 void idxd_device_reset(struct idxd_device *idxd);
693 void idxd_device_clear_state(struct idxd_device *idxd);
694 int idxd_device_config(struct idxd_device *idxd);
695 void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid);
696 int idxd_device_load_config(struct idxd_device *idxd);
697 int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle,
698 enum idxd_interrupt_type irq_type);
699 int idxd_device_release_int_handle(struct idxd_device *idxd, int handle,
700 enum idxd_interrupt_type irq_type);
701
702 /* work queue control */
703 void idxd_wqs_unmap_portal(struct idxd_device *idxd);
704 int idxd_wq_alloc_resources(struct idxd_wq *wq);
705 void idxd_wq_free_resources(struct idxd_wq *wq);
706 int idxd_wq_enable(struct idxd_wq *wq);
707 int idxd_wq_disable(struct idxd_wq *wq, bool reset_config);
708 void idxd_wq_drain(struct idxd_wq *wq);
709 void idxd_wq_reset(struct idxd_wq *wq);
710 int idxd_wq_map_portal(struct idxd_wq *wq);
711 void idxd_wq_unmap_portal(struct idxd_wq *wq);
712 int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid);
713 int idxd_wq_disable_pasid(struct idxd_wq *wq);
714 void __idxd_wq_quiesce(struct idxd_wq *wq);
715 void idxd_wq_quiesce(struct idxd_wq *wq);
716 int idxd_wq_init_percpu_ref(struct idxd_wq *wq);
717 void idxd_wq_free_irq(struct idxd_wq *wq);
718 int idxd_wq_request_irq(struct idxd_wq *wq);
719
720 /* submission */
721 int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);
722 struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype);
723 void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc);
724 int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc);
725
726 /* dmaengine */
727 int idxd_register_dma_device(struct idxd_device *idxd);
728 void idxd_unregister_dma_device(struct idxd_device *idxd);
729 void idxd_dma_complete_txd(struct idxd_desc *desc,
730 enum idxd_complete_type comp_type, bool free_desc);
731
732 /* cdev */
733 int idxd_cdev_register(void);
734 void idxd_cdev_remove(void);
735 int idxd_cdev_get_major(struct idxd_device *idxd);
736 int idxd_wq_add_cdev(struct idxd_wq *wq);
737 void idxd_wq_del_cdev(struct idxd_wq *wq);
738 int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr,
739 void *buf, int len);
740 void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index);
741
742 /* perfmon */
743 #if IS_ENABLED(CONFIG_INTEL_IDXD_PERFMON)
744 int perfmon_pmu_init(struct idxd_device *idxd);
745 void perfmon_pmu_remove(struct idxd_device *idxd);
746 void perfmon_counter_overflow(struct idxd_device *idxd);
747 void perfmon_init(void);
748 void perfmon_exit(void);
749 #else
perfmon_pmu_init(struct idxd_device * idxd)750 static inline int perfmon_pmu_init(struct idxd_device *idxd) { return 0; }
perfmon_pmu_remove(struct idxd_device * idxd)751 static inline void perfmon_pmu_remove(struct idxd_device *idxd) {}
perfmon_counter_overflow(struct idxd_device * idxd)752 static inline void perfmon_counter_overflow(struct idxd_device *idxd) {}
perfmon_init(void)753 static inline void perfmon_init(void) {}
perfmon_exit(void)754 static inline void perfmon_exit(void) {}
755 #endif
756
757 /* debugfs */
758 int idxd_device_init_debugfs(struct idxd_device *idxd);
759 void idxd_device_remove_debugfs(struct idxd_device *idxd);
760 int idxd_init_debugfs(void);
761 void idxd_remove_debugfs(void);
762
763 #endif
764