1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #ifndef _IDXD_H_ 4 #define _IDXD_H_ 5 6 #include <linux/sbitmap.h> 7 #include <linux/dmaengine.h> 8 #include <linux/percpu-rwsem.h> 9 #include <linux/wait.h> 10 #include <linux/cdev.h> 11 #include <linux/idr.h> 12 #include <linux/pci.h> 13 #include <linux/ioasid.h> 14 #include <linux/bitmap.h> 15 #include <linux/perf_event.h> 16 #include <uapi/linux/idxd.h> 17 #include "registers.h" 18 19 #define IDXD_DRIVER_VERSION "1.00" 20 21 extern struct kmem_cache *idxd_desc_pool; 22 extern bool tc_override; 23 24 struct idxd_wq; 25 struct idxd_dev; 26 27 enum idxd_dev_type { 28 IDXD_DEV_NONE = -1, 29 IDXD_DEV_DSA = 0, 30 IDXD_DEV_IAX, 31 IDXD_DEV_WQ, 32 IDXD_DEV_GROUP, 33 IDXD_DEV_ENGINE, 34 IDXD_DEV_CDEV, 35 IDXD_DEV_MAX_TYPE, 36 }; 37 38 struct idxd_dev { 39 struct device conf_dev; 40 enum idxd_dev_type type; 41 }; 42 43 #define IDXD_REG_TIMEOUT 50 44 #define IDXD_DRAIN_TIMEOUT 5000 45 46 enum idxd_type { 47 IDXD_TYPE_UNKNOWN = -1, 48 IDXD_TYPE_DSA = 0, 49 IDXD_TYPE_IAX, 50 IDXD_TYPE_MAX, 51 }; 52 53 #define IDXD_NAME_SIZE 128 54 #define IDXD_PMU_EVENT_MAX 64 55 56 #define IDXD_ENQCMDS_RETRIES 32 57 #define IDXD_ENQCMDS_MAX_RETRIES 64 58 59 struct idxd_device_driver { 60 const char *name; 61 enum idxd_dev_type *type; 62 int (*probe)(struct idxd_dev *idxd_dev); 63 void (*remove)(struct idxd_dev *idxd_dev); 64 struct device_driver drv; 65 }; 66 67 extern struct idxd_device_driver dsa_drv; 68 extern struct idxd_device_driver idxd_drv; 69 extern struct idxd_device_driver idxd_dmaengine_drv; 70 extern struct idxd_device_driver idxd_user_drv; 71 72 #define INVALID_INT_HANDLE -1 73 struct idxd_irq_entry { 74 int id; 75 int vector; 76 struct llist_head pending_llist; 77 struct list_head work_list; 78 /* 79 * Lock to protect access between irq thread process descriptor 80 * and irq thread processing error descriptor. 81 */ 82 spinlock_t list_lock; 83 int int_handle; 84 ioasid_t pasid; 85 }; 86 87 struct idxd_group { 88 struct idxd_dev idxd_dev; 89 struct idxd_device *idxd; 90 struct grpcfg grpcfg; 91 int id; 92 int num_engines; 93 int num_wqs; 94 bool use_rdbuf_limit; 95 u8 rdbufs_allowed; 96 u8 rdbufs_reserved; 97 int tc_a; 98 int tc_b; 99 }; 100 101 struct idxd_pmu { 102 struct idxd_device *idxd; 103 104 struct perf_event *event_list[IDXD_PMU_EVENT_MAX]; 105 int n_events; 106 107 DECLARE_BITMAP(used_mask, IDXD_PMU_EVENT_MAX); 108 109 struct pmu pmu; 110 char name[IDXD_NAME_SIZE]; 111 int cpu; 112 113 int n_counters; 114 int counter_width; 115 int n_event_categories; 116 117 bool per_counter_caps_supported; 118 unsigned long supported_event_categories; 119 120 unsigned long supported_filters; 121 int n_filters; 122 123 struct hlist_node cpuhp_node; 124 }; 125 126 #define IDXD_MAX_PRIORITY 0xf 127 128 enum idxd_wq_state { 129 IDXD_WQ_DISABLED = 0, 130 IDXD_WQ_ENABLED, 131 }; 132 133 enum idxd_wq_flag { 134 WQ_FLAG_DEDICATED = 0, 135 WQ_FLAG_BLOCK_ON_FAULT, 136 }; 137 138 enum idxd_wq_type { 139 IDXD_WQT_NONE = 0, 140 IDXD_WQT_KERNEL, 141 IDXD_WQT_USER, 142 }; 143 144 struct idxd_cdev { 145 struct idxd_wq *wq; 146 struct cdev cdev; 147 struct idxd_dev idxd_dev; 148 int minor; 149 }; 150 151 #define IDXD_ALLOCATED_BATCH_SIZE 128U 152 #define WQ_NAME_SIZE 1024 153 #define WQ_TYPE_SIZE 10 154 155 #define WQ_DEFAULT_QUEUE_DEPTH 16 156 #define WQ_DEFAULT_MAX_XFER SZ_2M 157 #define WQ_DEFAULT_MAX_BATCH 32 158 159 enum idxd_op_type { 160 IDXD_OP_BLOCK = 0, 161 IDXD_OP_NONBLOCK = 1, 162 }; 163 164 enum idxd_complete_type { 165 IDXD_COMPLETE_NORMAL = 0, 166 IDXD_COMPLETE_ABORT, 167 IDXD_COMPLETE_DEV_FAIL, 168 }; 169 170 struct idxd_dma_chan { 171 struct dma_chan chan; 172 struct idxd_wq *wq; 173 }; 174 175 struct idxd_wq { 176 void __iomem *portal; 177 u32 portal_offset; 178 unsigned int enqcmds_retries; 179 struct percpu_ref wq_active; 180 struct completion wq_dead; 181 struct completion wq_resurrect; 182 struct idxd_dev idxd_dev; 183 struct idxd_cdev *idxd_cdev; 184 struct wait_queue_head err_queue; 185 struct idxd_device *idxd; 186 int id; 187 struct idxd_irq_entry ie; 188 enum idxd_wq_type type; 189 struct idxd_group *group; 190 int client_count; 191 struct mutex wq_lock; /* mutex for workqueue */ 192 u32 size; 193 u32 threshold; 194 u32 priority; 195 enum idxd_wq_state state; 196 unsigned long flags; 197 union wqcfg *wqcfg; 198 struct dsa_hw_desc **hw_descs; 199 int num_descs; 200 union { 201 struct dsa_completion_record *compls; 202 struct iax_completion_record *iax_compls; 203 }; 204 dma_addr_t compls_addr; 205 int compls_size; 206 struct idxd_desc **descs; 207 struct sbitmap_queue sbq; 208 struct idxd_dma_chan *idxd_chan; 209 char name[WQ_NAME_SIZE + 1]; 210 u64 max_xfer_bytes; 211 u32 max_batch_size; 212 bool ats_dis; 213 }; 214 215 struct idxd_engine { 216 struct idxd_dev idxd_dev; 217 int id; 218 struct idxd_group *group; 219 struct idxd_device *idxd; 220 }; 221 222 /* shadow registers */ 223 struct idxd_hw { 224 u32 version; 225 union gen_cap_reg gen_cap; 226 union wq_cap_reg wq_cap; 227 union group_cap_reg group_cap; 228 union engine_cap_reg engine_cap; 229 struct opcap opcap; 230 u32 cmd_cap; 231 }; 232 233 enum idxd_device_state { 234 IDXD_DEV_HALTED = -1, 235 IDXD_DEV_DISABLED = 0, 236 IDXD_DEV_ENABLED, 237 }; 238 239 enum idxd_device_flag { 240 IDXD_FLAG_CONFIGURABLE = 0, 241 IDXD_FLAG_CMD_RUNNING, 242 IDXD_FLAG_PASID_ENABLED, 243 IDXD_FLAG_USER_PASID_ENABLED, 244 }; 245 246 struct idxd_dma_dev { 247 struct idxd_device *idxd; 248 struct dma_device dma; 249 }; 250 251 struct idxd_driver_data { 252 const char *name_prefix; 253 enum idxd_type type; 254 struct device_type *dev_type; 255 int compl_size; 256 int align; 257 }; 258 259 struct idxd_device { 260 struct idxd_dev idxd_dev; 261 struct idxd_driver_data *data; 262 struct list_head list; 263 struct idxd_hw hw; 264 enum idxd_device_state state; 265 unsigned long flags; 266 int id; 267 int major; 268 u32 cmd_status; 269 struct idxd_irq_entry ie; /* misc irq, msix 0 */ 270 271 struct pci_dev *pdev; 272 void __iomem *reg_base; 273 274 spinlock_t dev_lock; /* spinlock for device */ 275 spinlock_t cmd_lock; /* spinlock for device commands */ 276 struct completion *cmd_done; 277 struct idxd_group **groups; 278 struct idxd_wq **wqs; 279 struct idxd_engine **engines; 280 281 struct iommu_sva *sva; 282 unsigned int pasid; 283 284 int num_groups; 285 int irq_cnt; 286 bool request_int_handles; 287 288 u32 msix_perm_offset; 289 u32 wqcfg_offset; 290 u32 grpcfg_offset; 291 u32 perfmon_offset; 292 293 u64 max_xfer_bytes; 294 u32 max_batch_size; 295 int max_groups; 296 int max_engines; 297 int max_rdbufs; 298 int max_wqs; 299 int max_wq_size; 300 int rdbuf_limit; 301 int nr_rdbufs; /* non-reserved read buffers */ 302 unsigned int wqcfg_size; 303 unsigned long *wq_enable_map; 304 305 union sw_err_reg sw_err; 306 wait_queue_head_t cmd_waitq; 307 308 struct idxd_dma_dev *idxd_dma; 309 struct workqueue_struct *wq; 310 struct work_struct work; 311 312 struct idxd_pmu *idxd_pmu; 313 }; 314 315 /* IDXD software descriptor */ 316 struct idxd_desc { 317 union { 318 struct dsa_hw_desc *hw; 319 struct iax_hw_desc *iax_hw; 320 }; 321 dma_addr_t desc_dma; 322 union { 323 struct dsa_completion_record *completion; 324 struct iax_completion_record *iax_completion; 325 }; 326 dma_addr_t compl_dma; 327 struct dma_async_tx_descriptor txd; 328 struct llist_node llnode; 329 struct list_head list; 330 int id; 331 int cpu; 332 struct idxd_wq *wq; 333 }; 334 335 /* 336 * This is software defined error for the completion status. We overload the error code 337 * that will never appear in completion status and only SWERR register. 338 */ 339 enum idxd_completion_status { 340 IDXD_COMP_DESC_ABORT = 0xff, 341 }; 342 343 #define idxd_confdev(idxd) &idxd->idxd_dev.conf_dev 344 #define wq_confdev(wq) &wq->idxd_dev.conf_dev 345 #define engine_confdev(engine) &engine->idxd_dev.conf_dev 346 #define group_confdev(group) &group->idxd_dev.conf_dev 347 #define cdev_dev(cdev) &cdev->idxd_dev.conf_dev 348 349 #define confdev_to_idxd_dev(dev) container_of(dev, struct idxd_dev, conf_dev) 350 #define idxd_dev_to_idxd(idxd_dev) container_of(idxd_dev, struct idxd_device, idxd_dev) 351 #define idxd_dev_to_wq(idxd_dev) container_of(idxd_dev, struct idxd_wq, idxd_dev) 352 353 static inline struct idxd_device *confdev_to_idxd(struct device *dev) 354 { 355 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 356 357 return idxd_dev_to_idxd(idxd_dev); 358 } 359 360 static inline struct idxd_wq *confdev_to_wq(struct device *dev) 361 { 362 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 363 364 return idxd_dev_to_wq(idxd_dev); 365 } 366 367 static inline struct idxd_engine *confdev_to_engine(struct device *dev) 368 { 369 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 370 371 return container_of(idxd_dev, struct idxd_engine, idxd_dev); 372 } 373 374 static inline struct idxd_group *confdev_to_group(struct device *dev) 375 { 376 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 377 378 return container_of(idxd_dev, struct idxd_group, idxd_dev); 379 } 380 381 static inline struct idxd_cdev *dev_to_cdev(struct device *dev) 382 { 383 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 384 385 return container_of(idxd_dev, struct idxd_cdev, idxd_dev); 386 } 387 388 static inline void idxd_dev_set_type(struct idxd_dev *idev, int type) 389 { 390 if (type >= IDXD_DEV_MAX_TYPE) { 391 idev->type = IDXD_DEV_NONE; 392 return; 393 } 394 395 idev->type = type; 396 } 397 398 static inline struct idxd_irq_entry *idxd_get_ie(struct idxd_device *idxd, int idx) 399 { 400 return (idx == 0) ? &idxd->ie : &idxd->wqs[idx - 1]->ie; 401 } 402 403 static inline struct idxd_wq *ie_to_wq(struct idxd_irq_entry *ie) 404 { 405 return container_of(ie, struct idxd_wq, ie); 406 } 407 408 static inline struct idxd_device *ie_to_idxd(struct idxd_irq_entry *ie) 409 { 410 return container_of(ie, struct idxd_device, ie); 411 } 412 413 extern struct bus_type dsa_bus_type; 414 415 extern bool support_enqcmd; 416 extern struct ida idxd_ida; 417 extern struct device_type dsa_device_type; 418 extern struct device_type iax_device_type; 419 extern struct device_type idxd_wq_device_type; 420 extern struct device_type idxd_engine_device_type; 421 extern struct device_type idxd_group_device_type; 422 423 static inline bool is_dsa_dev(struct idxd_dev *idxd_dev) 424 { 425 return idxd_dev->type == IDXD_DEV_DSA; 426 } 427 428 static inline bool is_iax_dev(struct idxd_dev *idxd_dev) 429 { 430 return idxd_dev->type == IDXD_DEV_IAX; 431 } 432 433 static inline bool is_idxd_dev(struct idxd_dev *idxd_dev) 434 { 435 return is_dsa_dev(idxd_dev) || is_iax_dev(idxd_dev); 436 } 437 438 static inline bool is_idxd_wq_dev(struct idxd_dev *idxd_dev) 439 { 440 return idxd_dev->type == IDXD_DEV_WQ; 441 } 442 443 static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq) 444 { 445 if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0) 446 return true; 447 return false; 448 } 449 450 static inline bool is_idxd_wq_user(struct idxd_wq *wq) 451 { 452 return wq->type == IDXD_WQT_USER; 453 } 454 455 static inline bool is_idxd_wq_kernel(struct idxd_wq *wq) 456 { 457 return wq->type == IDXD_WQT_KERNEL; 458 } 459 460 static inline bool wq_dedicated(struct idxd_wq *wq) 461 { 462 return test_bit(WQ_FLAG_DEDICATED, &wq->flags); 463 } 464 465 static inline bool wq_shared(struct idxd_wq *wq) 466 { 467 return !test_bit(WQ_FLAG_DEDICATED, &wq->flags); 468 } 469 470 static inline bool device_pasid_enabled(struct idxd_device *idxd) 471 { 472 return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 473 } 474 475 static inline bool device_user_pasid_enabled(struct idxd_device *idxd) 476 { 477 return test_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags); 478 } 479 480 static inline bool wq_pasid_enabled(struct idxd_wq *wq) 481 { 482 return (is_idxd_wq_kernel(wq) && device_pasid_enabled(wq->idxd)) || 483 (is_idxd_wq_user(wq) && device_user_pasid_enabled(wq->idxd)); 484 } 485 486 static inline bool wq_shared_supported(struct idxd_wq *wq) 487 { 488 return (support_enqcmd && wq_pasid_enabled(wq)); 489 } 490 491 enum idxd_portal_prot { 492 IDXD_PORTAL_UNLIMITED = 0, 493 IDXD_PORTAL_LIMITED, 494 }; 495 496 enum idxd_interrupt_type { 497 IDXD_IRQ_MSIX = 0, 498 IDXD_IRQ_IMS, 499 }; 500 501 static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot) 502 { 503 return prot * 0x1000; 504 } 505 506 static inline int idxd_get_wq_portal_full_offset(int wq_id, 507 enum idxd_portal_prot prot) 508 { 509 return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot); 510 } 511 512 #define IDXD_PORTAL_MASK (PAGE_SIZE - 1) 513 514 /* 515 * Even though this function can be accessed by multiple threads, it is safe to use. 516 * At worst the address gets used more than once before it gets incremented. We don't 517 * hit a threshold until iops becomes many million times a second. So the occasional 518 * reuse of the same address is tolerable compare to using an atomic variable. This is 519 * safe on a system that has atomic load/store for 32bit integers. Given that this is an 520 * Intel iEP device, that should not be a problem. 521 */ 522 static inline void __iomem *idxd_wq_portal_addr(struct idxd_wq *wq) 523 { 524 int ofs = wq->portal_offset; 525 526 wq->portal_offset = (ofs + sizeof(struct dsa_raw_desc)) & IDXD_PORTAL_MASK; 527 return wq->portal + ofs; 528 } 529 530 static inline void idxd_wq_get(struct idxd_wq *wq) 531 { 532 wq->client_count++; 533 } 534 535 static inline void idxd_wq_put(struct idxd_wq *wq) 536 { 537 wq->client_count--; 538 } 539 540 static inline int idxd_wq_refcount(struct idxd_wq *wq) 541 { 542 return wq->client_count; 543 }; 544 545 int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv, 546 struct module *module, const char *mod_name); 547 #define idxd_driver_register(driver) \ 548 __idxd_driver_register(driver, THIS_MODULE, KBUILD_MODNAME) 549 550 void idxd_driver_unregister(struct idxd_device_driver *idxd_drv); 551 552 #define module_idxd_driver(__idxd_driver) \ 553 module_driver(__idxd_driver, idxd_driver_register, idxd_driver_unregister) 554 555 int idxd_register_bus_type(void); 556 void idxd_unregister_bus_type(void); 557 int idxd_register_devices(struct idxd_device *idxd); 558 void idxd_unregister_devices(struct idxd_device *idxd); 559 int idxd_register_driver(void); 560 void idxd_unregister_driver(void); 561 void idxd_wqs_quiesce(struct idxd_device *idxd); 562 bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc); 563 564 /* device interrupt control */ 565 irqreturn_t idxd_misc_thread(int vec, void *data); 566 irqreturn_t idxd_wq_thread(int irq, void *data); 567 void idxd_mask_error_interrupts(struct idxd_device *idxd); 568 void idxd_unmask_error_interrupts(struct idxd_device *idxd); 569 570 /* device control */ 571 int idxd_register_idxd_drv(void); 572 void idxd_unregister_idxd_drv(void); 573 int idxd_device_drv_probe(struct idxd_dev *idxd_dev); 574 void idxd_device_drv_remove(struct idxd_dev *idxd_dev); 575 int drv_enable_wq(struct idxd_wq *wq); 576 void drv_disable_wq(struct idxd_wq *wq); 577 int idxd_device_init_reset(struct idxd_device *idxd); 578 int idxd_device_enable(struct idxd_device *idxd); 579 int idxd_device_disable(struct idxd_device *idxd); 580 void idxd_device_reset(struct idxd_device *idxd); 581 void idxd_device_clear_state(struct idxd_device *idxd); 582 int idxd_device_config(struct idxd_device *idxd); 583 void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid); 584 int idxd_device_load_config(struct idxd_device *idxd); 585 int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle, 586 enum idxd_interrupt_type irq_type); 587 int idxd_device_release_int_handle(struct idxd_device *idxd, int handle, 588 enum idxd_interrupt_type irq_type); 589 590 /* work queue control */ 591 void idxd_wqs_unmap_portal(struct idxd_device *idxd); 592 int idxd_wq_alloc_resources(struct idxd_wq *wq); 593 void idxd_wq_free_resources(struct idxd_wq *wq); 594 int idxd_wq_enable(struct idxd_wq *wq); 595 int idxd_wq_disable(struct idxd_wq *wq, bool reset_config); 596 void idxd_wq_drain(struct idxd_wq *wq); 597 void idxd_wq_reset(struct idxd_wq *wq); 598 int idxd_wq_map_portal(struct idxd_wq *wq); 599 void idxd_wq_unmap_portal(struct idxd_wq *wq); 600 int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid); 601 int idxd_wq_disable_pasid(struct idxd_wq *wq); 602 void __idxd_wq_quiesce(struct idxd_wq *wq); 603 void idxd_wq_quiesce(struct idxd_wq *wq); 604 int idxd_wq_init_percpu_ref(struct idxd_wq *wq); 605 void idxd_wq_free_irq(struct idxd_wq *wq); 606 int idxd_wq_request_irq(struct idxd_wq *wq); 607 608 /* submission */ 609 int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc); 610 struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype); 611 void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc); 612 int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc); 613 614 /* dmaengine */ 615 int idxd_register_dma_device(struct idxd_device *idxd); 616 void idxd_unregister_dma_device(struct idxd_device *idxd); 617 void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res); 618 void idxd_dma_complete_txd(struct idxd_desc *desc, 619 enum idxd_complete_type comp_type, bool free_desc); 620 621 /* cdev */ 622 int idxd_cdev_register(void); 623 void idxd_cdev_remove(void); 624 int idxd_cdev_get_major(struct idxd_device *idxd); 625 int idxd_wq_add_cdev(struct idxd_wq *wq); 626 void idxd_wq_del_cdev(struct idxd_wq *wq); 627 628 /* perfmon */ 629 #if IS_ENABLED(CONFIG_INTEL_IDXD_PERFMON) 630 int perfmon_pmu_init(struct idxd_device *idxd); 631 void perfmon_pmu_remove(struct idxd_device *idxd); 632 void perfmon_counter_overflow(struct idxd_device *idxd); 633 void perfmon_init(void); 634 void perfmon_exit(void); 635 #else 636 static inline int perfmon_pmu_init(struct idxd_device *idxd) { return 0; } 637 static inline void perfmon_pmu_remove(struct idxd_device *idxd) {} 638 static inline void perfmon_counter_overflow(struct idxd_device *idxd) {} 639 static inline void perfmon_init(void) {} 640 static inline void perfmon_exit(void) {} 641 #endif 642 643 #endif 644