1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #ifndef _IDXD_H_ 4 #define _IDXD_H_ 5 6 #include <linux/sbitmap.h> 7 #include <linux/dmaengine.h> 8 #include <linux/percpu-rwsem.h> 9 #include <linux/wait.h> 10 #include <linux/cdev.h> 11 #include "registers.h" 12 13 #define IDXD_DRIVER_VERSION "1.00" 14 15 extern struct kmem_cache *idxd_desc_pool; 16 17 #define IDXD_REG_TIMEOUT 50 18 #define IDXD_DRAIN_TIMEOUT 5000 19 20 enum idxd_type { 21 IDXD_TYPE_UNKNOWN = -1, 22 IDXD_TYPE_DSA = 0, 23 IDXD_TYPE_MAX 24 }; 25 26 #define IDXD_NAME_SIZE 128 27 28 struct idxd_device_driver { 29 struct device_driver drv; 30 }; 31 32 struct idxd_irq_entry { 33 struct idxd_device *idxd; 34 int id; 35 struct llist_head pending_llist; 36 struct list_head work_list; 37 }; 38 39 struct idxd_group { 40 struct device conf_dev; 41 struct idxd_device *idxd; 42 struct grpcfg grpcfg; 43 int id; 44 int num_engines; 45 int num_wqs; 46 bool use_token_limit; 47 u8 tokens_allowed; 48 u8 tokens_reserved; 49 int tc_a; 50 int tc_b; 51 }; 52 53 #define IDXD_MAX_PRIORITY 0xf 54 55 enum idxd_wq_state { 56 IDXD_WQ_DISABLED = 0, 57 IDXD_WQ_ENABLED, 58 }; 59 60 enum idxd_wq_flag { 61 WQ_FLAG_DEDICATED = 0, 62 }; 63 64 enum idxd_wq_type { 65 IDXD_WQT_NONE = 0, 66 IDXD_WQT_KERNEL, 67 IDXD_WQT_USER, 68 }; 69 70 struct idxd_cdev { 71 struct cdev cdev; 72 struct device *dev; 73 int minor; 74 struct wait_queue_head err_queue; 75 }; 76 77 #define IDXD_ALLOCATED_BATCH_SIZE 128U 78 #define WQ_NAME_SIZE 1024 79 #define WQ_TYPE_SIZE 10 80 81 enum idxd_op_type { 82 IDXD_OP_BLOCK = 0, 83 IDXD_OP_NONBLOCK = 1, 84 }; 85 86 enum idxd_complete_type { 87 IDXD_COMPLETE_NORMAL = 0, 88 IDXD_COMPLETE_ABORT, 89 }; 90 91 struct idxd_wq { 92 void __iomem *dportal; 93 struct device conf_dev; 94 struct idxd_cdev idxd_cdev; 95 struct idxd_device *idxd; 96 int id; 97 enum idxd_wq_type type; 98 struct idxd_group *group; 99 int client_count; 100 struct mutex wq_lock; /* mutex for workqueue */ 101 u32 size; 102 u32 threshold; 103 u32 priority; 104 enum idxd_wq_state state; 105 unsigned long flags; 106 union wqcfg *wqcfg; 107 u32 vec_ptr; /* interrupt steering */ 108 struct dsa_hw_desc **hw_descs; 109 int num_descs; 110 struct dsa_completion_record *compls; 111 dma_addr_t compls_addr; 112 int compls_size; 113 struct idxd_desc **descs; 114 struct sbitmap_queue sbq; 115 struct dma_chan dma_chan; 116 char name[WQ_NAME_SIZE + 1]; 117 u64 max_xfer_bytes; 118 u32 max_batch_size; 119 }; 120 121 struct idxd_engine { 122 struct device conf_dev; 123 int id; 124 struct idxd_group *group; 125 struct idxd_device *idxd; 126 }; 127 128 /* shadow registers */ 129 struct idxd_hw { 130 u32 version; 131 union gen_cap_reg gen_cap; 132 union wq_cap_reg wq_cap; 133 union group_cap_reg group_cap; 134 union engine_cap_reg engine_cap; 135 struct opcap opcap; 136 }; 137 138 enum idxd_device_state { 139 IDXD_DEV_HALTED = -1, 140 IDXD_DEV_DISABLED = 0, 141 IDXD_DEV_CONF_READY, 142 IDXD_DEV_ENABLED, 143 }; 144 145 enum idxd_device_flag { 146 IDXD_FLAG_CONFIGURABLE = 0, 147 IDXD_FLAG_CMD_RUNNING, 148 }; 149 150 struct idxd_device { 151 enum idxd_type type; 152 struct device conf_dev; 153 struct list_head list; 154 struct idxd_hw hw; 155 enum idxd_device_state state; 156 unsigned long flags; 157 int id; 158 int major; 159 u8 cmd_status; 160 161 struct pci_dev *pdev; 162 void __iomem *reg_base; 163 164 spinlock_t dev_lock; /* spinlock for device */ 165 struct completion *cmd_done; 166 struct idxd_group *groups; 167 struct idxd_wq *wqs; 168 struct idxd_engine *engines; 169 170 int num_groups; 171 172 u32 msix_perm_offset; 173 u32 wqcfg_offset; 174 u32 grpcfg_offset; 175 u32 perfmon_offset; 176 177 u64 max_xfer_bytes; 178 u32 max_batch_size; 179 int max_groups; 180 int max_engines; 181 int max_tokens; 182 int max_wqs; 183 int max_wq_size; 184 int token_limit; 185 int nr_tokens; /* non-reserved tokens */ 186 unsigned int wqcfg_size; 187 188 union sw_err_reg sw_err; 189 wait_queue_head_t cmd_waitq; 190 struct msix_entry *msix_entries; 191 int num_wq_irqs; 192 struct idxd_irq_entry *irq_entries; 193 194 struct dma_device dma_dev; 195 struct workqueue_struct *wq; 196 struct work_struct work; 197 }; 198 199 /* IDXD software descriptor */ 200 struct idxd_desc { 201 struct dsa_hw_desc *hw; 202 dma_addr_t desc_dma; 203 struct dsa_completion_record *completion; 204 dma_addr_t compl_dma; 205 struct dma_async_tx_descriptor txd; 206 struct llist_node llnode; 207 struct list_head list; 208 int id; 209 int cpu; 210 struct idxd_wq *wq; 211 }; 212 213 #define confdev_to_idxd(dev) container_of(dev, struct idxd_device, conf_dev) 214 #define confdev_to_wq(dev) container_of(dev, struct idxd_wq, conf_dev) 215 216 extern struct bus_type dsa_bus_type; 217 218 static inline bool wq_dedicated(struct idxd_wq *wq) 219 { 220 return test_bit(WQ_FLAG_DEDICATED, &wq->flags); 221 } 222 223 enum idxd_portal_prot { 224 IDXD_PORTAL_UNLIMITED = 0, 225 IDXD_PORTAL_LIMITED, 226 }; 227 228 static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot) 229 { 230 return prot * 0x1000; 231 } 232 233 static inline int idxd_get_wq_portal_full_offset(int wq_id, 234 enum idxd_portal_prot prot) 235 { 236 return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot); 237 } 238 239 static inline void idxd_set_type(struct idxd_device *idxd) 240 { 241 struct pci_dev *pdev = idxd->pdev; 242 243 if (pdev->device == PCI_DEVICE_ID_INTEL_DSA_SPR0) 244 idxd->type = IDXD_TYPE_DSA; 245 else 246 idxd->type = IDXD_TYPE_UNKNOWN; 247 } 248 249 static inline void idxd_wq_get(struct idxd_wq *wq) 250 { 251 wq->client_count++; 252 } 253 254 static inline void idxd_wq_put(struct idxd_wq *wq) 255 { 256 wq->client_count--; 257 } 258 259 static inline int idxd_wq_refcount(struct idxd_wq *wq) 260 { 261 return wq->client_count; 262 }; 263 264 const char *idxd_get_dev_name(struct idxd_device *idxd); 265 int idxd_register_bus_type(void); 266 void idxd_unregister_bus_type(void); 267 int idxd_setup_sysfs(struct idxd_device *idxd); 268 void idxd_cleanup_sysfs(struct idxd_device *idxd); 269 int idxd_register_driver(void); 270 void idxd_unregister_driver(void); 271 struct bus_type *idxd_get_bus_type(struct idxd_device *idxd); 272 273 /* device interrupt control */ 274 irqreturn_t idxd_irq_handler(int vec, void *data); 275 irqreturn_t idxd_misc_thread(int vec, void *data); 276 irqreturn_t idxd_wq_thread(int irq, void *data); 277 void idxd_mask_error_interrupts(struct idxd_device *idxd); 278 void idxd_unmask_error_interrupts(struct idxd_device *idxd); 279 void idxd_mask_msix_vectors(struct idxd_device *idxd); 280 void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id); 281 void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id); 282 283 /* device control */ 284 void idxd_device_init_reset(struct idxd_device *idxd); 285 int idxd_device_enable(struct idxd_device *idxd); 286 int idxd_device_disable(struct idxd_device *idxd); 287 void idxd_device_reset(struct idxd_device *idxd); 288 void idxd_device_cleanup(struct idxd_device *idxd); 289 int idxd_device_config(struct idxd_device *idxd); 290 void idxd_device_wqs_clear_state(struct idxd_device *idxd); 291 292 /* work queue control */ 293 int idxd_wq_alloc_resources(struct idxd_wq *wq); 294 void idxd_wq_free_resources(struct idxd_wq *wq); 295 int idxd_wq_enable(struct idxd_wq *wq); 296 int idxd_wq_disable(struct idxd_wq *wq); 297 void idxd_wq_drain(struct idxd_wq *wq); 298 int idxd_wq_map_portal(struct idxd_wq *wq); 299 void idxd_wq_unmap_portal(struct idxd_wq *wq); 300 void idxd_wq_disable_cleanup(struct idxd_wq *wq); 301 302 /* submission */ 303 int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc); 304 struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype); 305 void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc); 306 307 /* dmaengine */ 308 int idxd_register_dma_device(struct idxd_device *idxd); 309 void idxd_unregister_dma_device(struct idxd_device *idxd); 310 int idxd_register_dma_channel(struct idxd_wq *wq); 311 void idxd_unregister_dma_channel(struct idxd_wq *wq); 312 void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res); 313 void idxd_dma_complete_txd(struct idxd_desc *desc, 314 enum idxd_complete_type comp_type); 315 dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx); 316 317 /* cdev */ 318 int idxd_cdev_register(void); 319 void idxd_cdev_remove(void); 320 int idxd_cdev_get_major(struct idxd_device *idxd); 321 int idxd_wq_add_cdev(struct idxd_wq *wq); 322 void idxd_wq_del_cdev(struct idxd_wq *wq); 323 324 #endif 325