xref: /openbmc/linux/drivers/dma/idxd/idxd.h (revision 7c8765308371be30f50c1b5b97618b731514b207)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3 #ifndef _IDXD_H_
4 #define _IDXD_H_
5 
6 #include <linux/sbitmap.h>
7 #include <linux/dmaengine.h>
8 #include <linux/percpu-rwsem.h>
9 #include <linux/wait.h>
10 #include <linux/cdev.h>
11 #include <linux/idr.h>
12 #include <linux/pci.h>
13 #include <linux/ioasid.h>
14 #include <linux/bitmap.h>
15 #include <linux/perf_event.h>
16 #include <uapi/linux/idxd.h>
17 #include "registers.h"
18 
19 #define IDXD_DRIVER_VERSION	"1.00"
20 
21 extern struct kmem_cache *idxd_desc_pool;
22 extern bool tc_override;
23 
24 struct idxd_wq;
25 struct idxd_dev;
26 
27 enum idxd_dev_type {
28 	IDXD_DEV_NONE = -1,
29 	IDXD_DEV_DSA = 0,
30 	IDXD_DEV_IAX,
31 	IDXD_DEV_WQ,
32 	IDXD_DEV_GROUP,
33 	IDXD_DEV_ENGINE,
34 	IDXD_DEV_CDEV,
35 	IDXD_DEV_MAX_TYPE,
36 };
37 
38 struct idxd_dev {
39 	struct device conf_dev;
40 	enum idxd_dev_type type;
41 };
42 
43 #define IDXD_REG_TIMEOUT	50
44 #define IDXD_DRAIN_TIMEOUT	5000
45 
46 enum idxd_type {
47 	IDXD_TYPE_UNKNOWN = -1,
48 	IDXD_TYPE_DSA = 0,
49 	IDXD_TYPE_IAX,
50 	IDXD_TYPE_MAX,
51 };
52 
53 #define IDXD_NAME_SIZE		128
54 #define IDXD_PMU_EVENT_MAX	64
55 
56 #define IDXD_ENQCMDS_RETRIES		32
57 #define IDXD_ENQCMDS_MAX_RETRIES	64
58 
59 struct idxd_device_driver {
60 	const char *name;
61 	enum idxd_dev_type *type;
62 	int (*probe)(struct idxd_dev *idxd_dev);
63 	void (*remove)(struct idxd_dev *idxd_dev);
64 	struct device_driver drv;
65 };
66 
67 extern struct idxd_device_driver dsa_drv;
68 extern struct idxd_device_driver idxd_drv;
69 extern struct idxd_device_driver idxd_dmaengine_drv;
70 extern struct idxd_device_driver idxd_user_drv;
71 
72 #define INVALID_INT_HANDLE	-1
73 struct idxd_irq_entry {
74 	int id;
75 	int vector;
76 	struct llist_head pending_llist;
77 	struct list_head work_list;
78 	/*
79 	 * Lock to protect access between irq thread process descriptor
80 	 * and irq thread processing error descriptor.
81 	 */
82 	spinlock_t list_lock;
83 	int int_handle;
84 	ioasid_t pasid;
85 };
86 
87 struct idxd_group {
88 	struct idxd_dev idxd_dev;
89 	struct idxd_device *idxd;
90 	struct grpcfg grpcfg;
91 	int id;
92 	int num_engines;
93 	int num_wqs;
94 	bool use_rdbuf_limit;
95 	u8 rdbufs_allowed;
96 	u8 rdbufs_reserved;
97 	int tc_a;
98 	int tc_b;
99 };
100 
101 struct idxd_pmu {
102 	struct idxd_device *idxd;
103 
104 	struct perf_event *event_list[IDXD_PMU_EVENT_MAX];
105 	int n_events;
106 
107 	DECLARE_BITMAP(used_mask, IDXD_PMU_EVENT_MAX);
108 
109 	struct pmu pmu;
110 	char name[IDXD_NAME_SIZE];
111 	int cpu;
112 
113 	int n_counters;
114 	int counter_width;
115 	int n_event_categories;
116 
117 	bool per_counter_caps_supported;
118 	unsigned long supported_event_categories;
119 
120 	unsigned long supported_filters;
121 	int n_filters;
122 
123 	struct hlist_node cpuhp_node;
124 };
125 
126 #define IDXD_MAX_PRIORITY	0xf
127 
128 enum idxd_wq_state {
129 	IDXD_WQ_DISABLED = 0,
130 	IDXD_WQ_ENABLED,
131 };
132 
133 enum idxd_wq_flag {
134 	WQ_FLAG_DEDICATED = 0,
135 	WQ_FLAG_BLOCK_ON_FAULT,
136 };
137 
138 enum idxd_wq_type {
139 	IDXD_WQT_NONE = 0,
140 	IDXD_WQT_KERNEL,
141 	IDXD_WQT_USER,
142 };
143 
144 struct idxd_cdev {
145 	struct idxd_wq *wq;
146 	struct cdev cdev;
147 	struct idxd_dev idxd_dev;
148 	int minor;
149 };
150 
151 #define IDXD_ALLOCATED_BATCH_SIZE	128U
152 #define WQ_NAME_SIZE   1024
153 #define WQ_TYPE_SIZE   10
154 
155 #define WQ_DEFAULT_QUEUE_DEPTH		16
156 #define WQ_DEFAULT_MAX_XFER		SZ_2M
157 #define WQ_DEFAULT_MAX_BATCH		32
158 
159 enum idxd_op_type {
160 	IDXD_OP_BLOCK = 0,
161 	IDXD_OP_NONBLOCK = 1,
162 };
163 
164 enum idxd_complete_type {
165 	IDXD_COMPLETE_NORMAL = 0,
166 	IDXD_COMPLETE_ABORT,
167 	IDXD_COMPLETE_DEV_FAIL,
168 };
169 
170 struct idxd_dma_chan {
171 	struct dma_chan chan;
172 	struct idxd_wq *wq;
173 };
174 
175 struct idxd_wq {
176 	void __iomem *portal;
177 	u32 portal_offset;
178 	unsigned int enqcmds_retries;
179 	struct percpu_ref wq_active;
180 	struct completion wq_dead;
181 	struct completion wq_resurrect;
182 	struct idxd_dev idxd_dev;
183 	struct idxd_cdev *idxd_cdev;
184 	struct wait_queue_head err_queue;
185 	struct idxd_device *idxd;
186 	int id;
187 	struct idxd_irq_entry ie;
188 	enum idxd_wq_type type;
189 	struct idxd_group *group;
190 	int client_count;
191 	struct mutex wq_lock;	/* mutex for workqueue */
192 	u32 size;
193 	u32 threshold;
194 	u32 priority;
195 	enum idxd_wq_state state;
196 	unsigned long flags;
197 	union wqcfg *wqcfg;
198 	struct dsa_hw_desc **hw_descs;
199 	int num_descs;
200 	union {
201 		struct dsa_completion_record *compls;
202 		struct iax_completion_record *iax_compls;
203 	};
204 	dma_addr_t compls_addr;
205 	int compls_size;
206 	struct idxd_desc **descs;
207 	struct sbitmap_queue sbq;
208 	struct idxd_dma_chan *idxd_chan;
209 	char name[WQ_NAME_SIZE + 1];
210 	u64 max_xfer_bytes;
211 	u32 max_batch_size;
212 	bool ats_dis;
213 };
214 
215 struct idxd_engine {
216 	struct idxd_dev idxd_dev;
217 	int id;
218 	struct idxd_group *group;
219 	struct idxd_device *idxd;
220 };
221 
222 /* shadow registers */
223 struct idxd_hw {
224 	u32 version;
225 	union gen_cap_reg gen_cap;
226 	union wq_cap_reg wq_cap;
227 	union group_cap_reg group_cap;
228 	union engine_cap_reg engine_cap;
229 	struct opcap opcap;
230 	u32 cmd_cap;
231 };
232 
233 enum idxd_device_state {
234 	IDXD_DEV_HALTED = -1,
235 	IDXD_DEV_DISABLED = 0,
236 	IDXD_DEV_ENABLED,
237 };
238 
239 enum idxd_device_flag {
240 	IDXD_FLAG_CONFIGURABLE = 0,
241 	IDXD_FLAG_CMD_RUNNING,
242 	IDXD_FLAG_PASID_ENABLED,
243 	IDXD_FLAG_USER_PASID_ENABLED,
244 };
245 
246 struct idxd_dma_dev {
247 	struct idxd_device *idxd;
248 	struct dma_device dma;
249 };
250 
251 struct idxd_driver_data {
252 	const char *name_prefix;
253 	enum idxd_type type;
254 	struct device_type *dev_type;
255 	int compl_size;
256 	int align;
257 };
258 
259 struct idxd_device {
260 	struct idxd_dev idxd_dev;
261 	struct idxd_driver_data *data;
262 	struct list_head list;
263 	struct idxd_hw hw;
264 	enum idxd_device_state state;
265 	unsigned long flags;
266 	int id;
267 	int major;
268 	u32 cmd_status;
269 	struct idxd_irq_entry ie;	/* misc irq, msix 0 */
270 
271 	struct pci_dev *pdev;
272 	void __iomem *reg_base;
273 
274 	spinlock_t dev_lock;	/* spinlock for device */
275 	spinlock_t cmd_lock;	/* spinlock for device commands */
276 	struct completion *cmd_done;
277 	struct idxd_group **groups;
278 	struct idxd_wq **wqs;
279 	struct idxd_engine **engines;
280 
281 	struct iommu_sva *sva;
282 	unsigned int pasid;
283 
284 	int num_groups;
285 	int irq_cnt;
286 	bool request_int_handles;
287 
288 	u32 msix_perm_offset;
289 	u32 wqcfg_offset;
290 	u32 grpcfg_offset;
291 	u32 perfmon_offset;
292 
293 	u64 max_xfer_bytes;
294 	u32 max_batch_size;
295 	int max_groups;
296 	int max_engines;
297 	int max_rdbufs;
298 	int max_wqs;
299 	int max_wq_size;
300 	int rdbuf_limit;
301 	int nr_rdbufs;		/* non-reserved read buffers */
302 	unsigned int wqcfg_size;
303 	unsigned long *wq_enable_map;
304 
305 	union sw_err_reg sw_err;
306 	wait_queue_head_t cmd_waitq;
307 
308 	struct idxd_dma_dev *idxd_dma;
309 	struct workqueue_struct *wq;
310 	struct work_struct work;
311 
312 	struct idxd_pmu *idxd_pmu;
313 };
314 
315 /* IDXD software descriptor */
316 struct idxd_desc {
317 	union {
318 		struct dsa_hw_desc *hw;
319 		struct iax_hw_desc *iax_hw;
320 	};
321 	dma_addr_t desc_dma;
322 	union {
323 		struct dsa_completion_record *completion;
324 		struct iax_completion_record *iax_completion;
325 	};
326 	dma_addr_t compl_dma;
327 	struct dma_async_tx_descriptor txd;
328 	struct llist_node llnode;
329 	struct list_head list;
330 	int id;
331 	int cpu;
332 	struct idxd_wq *wq;
333 };
334 
335 /*
336  * This is software defined error for the completion status. We overload the error code
337  * that will never appear in completion status and only SWERR register.
338  */
339 enum idxd_completion_status {
340 	IDXD_COMP_DESC_ABORT = 0xff,
341 };
342 
343 #define idxd_confdev(idxd) &idxd->idxd_dev.conf_dev
344 #define wq_confdev(wq) &wq->idxd_dev.conf_dev
345 #define engine_confdev(engine) &engine->idxd_dev.conf_dev
346 #define group_confdev(group) &group->idxd_dev.conf_dev
347 #define cdev_dev(cdev) &cdev->idxd_dev.conf_dev
348 
349 #define confdev_to_idxd_dev(dev) container_of(dev, struct idxd_dev, conf_dev)
350 #define idxd_dev_to_idxd(idxd_dev) container_of(idxd_dev, struct idxd_device, idxd_dev)
351 #define idxd_dev_to_wq(idxd_dev) container_of(idxd_dev, struct idxd_wq, idxd_dev)
352 
353 static inline struct idxd_device *confdev_to_idxd(struct device *dev)
354 {
355 	struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
356 
357 	return idxd_dev_to_idxd(idxd_dev);
358 }
359 
360 static inline struct idxd_wq *confdev_to_wq(struct device *dev)
361 {
362 	struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
363 
364 	return idxd_dev_to_wq(idxd_dev);
365 }
366 
367 static inline struct idxd_engine *confdev_to_engine(struct device *dev)
368 {
369 	struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
370 
371 	return container_of(idxd_dev, struct idxd_engine, idxd_dev);
372 }
373 
374 static inline struct idxd_group *confdev_to_group(struct device *dev)
375 {
376 	struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
377 
378 	return container_of(idxd_dev, struct idxd_group, idxd_dev);
379 }
380 
381 static inline struct idxd_cdev *dev_to_cdev(struct device *dev)
382 {
383 	struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
384 
385 	return container_of(idxd_dev, struct idxd_cdev, idxd_dev);
386 }
387 
388 static inline void idxd_dev_set_type(struct idxd_dev *idev, int type)
389 {
390 	if (type >= IDXD_DEV_MAX_TYPE) {
391 		idev->type = IDXD_DEV_NONE;
392 		return;
393 	}
394 
395 	idev->type = type;
396 }
397 
398 static inline struct idxd_irq_entry *idxd_get_ie(struct idxd_device *idxd, int idx)
399 {
400 	return (idx == 0) ? &idxd->ie : &idxd->wqs[idx - 1]->ie;
401 }
402 
403 static inline struct idxd_wq *ie_to_wq(struct idxd_irq_entry *ie)
404 {
405 	return container_of(ie, struct idxd_wq, ie);
406 }
407 
408 static inline struct idxd_device *ie_to_idxd(struct idxd_irq_entry *ie)
409 {
410 	return container_of(ie, struct idxd_device, ie);
411 }
412 
413 extern struct bus_type dsa_bus_type;
414 
415 extern bool support_enqcmd;
416 extern struct ida idxd_ida;
417 extern struct device_type dsa_device_type;
418 extern struct device_type iax_device_type;
419 extern struct device_type idxd_wq_device_type;
420 extern struct device_type idxd_engine_device_type;
421 extern struct device_type idxd_group_device_type;
422 
423 static inline bool is_dsa_dev(struct idxd_dev *idxd_dev)
424 {
425 	return idxd_dev->type == IDXD_DEV_DSA;
426 }
427 
428 static inline bool is_iax_dev(struct idxd_dev *idxd_dev)
429 {
430 	return idxd_dev->type == IDXD_DEV_IAX;
431 }
432 
433 static inline bool is_idxd_dev(struct idxd_dev *idxd_dev)
434 {
435 	return is_dsa_dev(idxd_dev) || is_iax_dev(idxd_dev);
436 }
437 
438 static inline bool is_idxd_wq_dev(struct idxd_dev *idxd_dev)
439 {
440 	return idxd_dev->type == IDXD_DEV_WQ;
441 }
442 
443 static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq)
444 {
445 	if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0)
446 		return true;
447 	return false;
448 }
449 
450 static inline bool is_idxd_wq_user(struct idxd_wq *wq)
451 {
452 	return wq->type == IDXD_WQT_USER;
453 }
454 
455 static inline bool is_idxd_wq_kernel(struct idxd_wq *wq)
456 {
457 	return wq->type == IDXD_WQT_KERNEL;
458 }
459 
460 static inline bool wq_dedicated(struct idxd_wq *wq)
461 {
462 	return test_bit(WQ_FLAG_DEDICATED, &wq->flags);
463 }
464 
465 static inline bool wq_shared(struct idxd_wq *wq)
466 {
467 	return !test_bit(WQ_FLAG_DEDICATED, &wq->flags);
468 }
469 
470 static inline bool device_pasid_enabled(struct idxd_device *idxd)
471 {
472 	return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
473 }
474 
475 static inline bool device_user_pasid_enabled(struct idxd_device *idxd)
476 {
477 	return test_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags);
478 }
479 
480 static inline bool wq_pasid_enabled(struct idxd_wq *wq)
481 {
482 	return (is_idxd_wq_kernel(wq) && device_pasid_enabled(wq->idxd)) ||
483 	       (is_idxd_wq_user(wq) && device_user_pasid_enabled(wq->idxd));
484 }
485 
486 static inline bool wq_shared_supported(struct idxd_wq *wq)
487 {
488 	return (support_enqcmd && wq_pasid_enabled(wq));
489 }
490 
491 enum idxd_portal_prot {
492 	IDXD_PORTAL_UNLIMITED = 0,
493 	IDXD_PORTAL_LIMITED,
494 };
495 
496 enum idxd_interrupt_type {
497 	IDXD_IRQ_MSIX = 0,
498 	IDXD_IRQ_IMS,
499 };
500 
501 static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot)
502 {
503 	return prot * 0x1000;
504 }
505 
506 static inline int idxd_get_wq_portal_full_offset(int wq_id,
507 						 enum idxd_portal_prot prot)
508 {
509 	return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot);
510 }
511 
512 #define IDXD_PORTAL_MASK	(PAGE_SIZE - 1)
513 
514 /*
515  * Even though this function can be accessed by multiple threads, it is safe to use.
516  * At worst the address gets used more than once before it gets incremented. We don't
517  * hit a threshold until iops becomes many million times a second. So the occasional
518  * reuse of the same address is tolerable compare to using an atomic variable. This is
519  * safe on a system that has atomic load/store for 32bit integers. Given that this is an
520  * Intel iEP device, that should not be a problem.
521  */
522 static inline void __iomem *idxd_wq_portal_addr(struct idxd_wq *wq)
523 {
524 	int ofs = wq->portal_offset;
525 
526 	wq->portal_offset = (ofs + sizeof(struct dsa_raw_desc)) & IDXD_PORTAL_MASK;
527 	return wq->portal + ofs;
528 }
529 
530 static inline void idxd_wq_get(struct idxd_wq *wq)
531 {
532 	wq->client_count++;
533 }
534 
535 static inline void idxd_wq_put(struct idxd_wq *wq)
536 {
537 	wq->client_count--;
538 }
539 
540 static inline int idxd_wq_refcount(struct idxd_wq *wq)
541 {
542 	return wq->client_count;
543 };
544 
545 int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv,
546 					struct module *module, const char *mod_name);
547 #define idxd_driver_register(driver) \
548 	__idxd_driver_register(driver, THIS_MODULE, KBUILD_MODNAME)
549 
550 void idxd_driver_unregister(struct idxd_device_driver *idxd_drv);
551 
552 #define module_idxd_driver(__idxd_driver) \
553 	module_driver(__idxd_driver, idxd_driver_register, idxd_driver_unregister)
554 
555 int idxd_register_bus_type(void);
556 void idxd_unregister_bus_type(void);
557 int idxd_register_devices(struct idxd_device *idxd);
558 void idxd_unregister_devices(struct idxd_device *idxd);
559 int idxd_register_driver(void);
560 void idxd_unregister_driver(void);
561 void idxd_wqs_quiesce(struct idxd_device *idxd);
562 bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc);
563 
564 /* device interrupt control */
565 irqreturn_t idxd_misc_thread(int vec, void *data);
566 irqreturn_t idxd_wq_thread(int irq, void *data);
567 void idxd_mask_error_interrupts(struct idxd_device *idxd);
568 void idxd_unmask_error_interrupts(struct idxd_device *idxd);
569 
570 /* device control */
571 int idxd_register_idxd_drv(void);
572 void idxd_unregister_idxd_drv(void);
573 int idxd_device_drv_probe(struct idxd_dev *idxd_dev);
574 void idxd_device_drv_remove(struct idxd_dev *idxd_dev);
575 int drv_enable_wq(struct idxd_wq *wq);
576 void drv_disable_wq(struct idxd_wq *wq);
577 int idxd_device_init_reset(struct idxd_device *idxd);
578 int idxd_device_enable(struct idxd_device *idxd);
579 int idxd_device_disable(struct idxd_device *idxd);
580 void idxd_device_reset(struct idxd_device *idxd);
581 void idxd_device_clear_state(struct idxd_device *idxd);
582 int idxd_device_config(struct idxd_device *idxd);
583 void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid);
584 int idxd_device_load_config(struct idxd_device *idxd);
585 int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle,
586 				   enum idxd_interrupt_type irq_type);
587 int idxd_device_release_int_handle(struct idxd_device *idxd, int handle,
588 				   enum idxd_interrupt_type irq_type);
589 
590 /* work queue control */
591 void idxd_wqs_unmap_portal(struct idxd_device *idxd);
592 int idxd_wq_alloc_resources(struct idxd_wq *wq);
593 void idxd_wq_free_resources(struct idxd_wq *wq);
594 int idxd_wq_enable(struct idxd_wq *wq);
595 int idxd_wq_disable(struct idxd_wq *wq, bool reset_config);
596 void idxd_wq_drain(struct idxd_wq *wq);
597 void idxd_wq_reset(struct idxd_wq *wq);
598 int idxd_wq_map_portal(struct idxd_wq *wq);
599 void idxd_wq_unmap_portal(struct idxd_wq *wq);
600 int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid);
601 int idxd_wq_disable_pasid(struct idxd_wq *wq);
602 void __idxd_wq_quiesce(struct idxd_wq *wq);
603 void idxd_wq_quiesce(struct idxd_wq *wq);
604 int idxd_wq_init_percpu_ref(struct idxd_wq *wq);
605 void idxd_wq_free_irq(struct idxd_wq *wq);
606 int idxd_wq_request_irq(struct idxd_wq *wq);
607 
608 /* submission */
609 int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);
610 struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype);
611 void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc);
612 int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc);
613 
614 /* dmaengine */
615 int idxd_register_dma_device(struct idxd_device *idxd);
616 void idxd_unregister_dma_device(struct idxd_device *idxd);
617 void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res);
618 void idxd_dma_complete_txd(struct idxd_desc *desc,
619 			   enum idxd_complete_type comp_type, bool free_desc);
620 
621 /* cdev */
622 int idxd_cdev_register(void);
623 void idxd_cdev_remove(void);
624 int idxd_cdev_get_major(struct idxd_device *idxd);
625 int idxd_wq_add_cdev(struct idxd_wq *wq);
626 void idxd_wq_del_cdev(struct idxd_wq *wq);
627 
628 /* perfmon */
629 #if IS_ENABLED(CONFIG_INTEL_IDXD_PERFMON)
630 int perfmon_pmu_init(struct idxd_device *idxd);
631 void perfmon_pmu_remove(struct idxd_device *idxd);
632 void perfmon_counter_overflow(struct idxd_device *idxd);
633 void perfmon_init(void);
634 void perfmon_exit(void);
635 #else
636 static inline int perfmon_pmu_init(struct idxd_device *idxd) { return 0; }
637 static inline void perfmon_pmu_remove(struct idxd_device *idxd) {}
638 static inline void perfmon_counter_overflow(struct idxd_device *idxd) {}
639 static inline void perfmon_init(void) {}
640 static inline void perfmon_exit(void) {}
641 #endif
642 
643 #endif
644