1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #ifndef _IDXD_H_ 4 #define _IDXD_H_ 5 6 #include <linux/sbitmap.h> 7 #include <linux/dmaengine.h> 8 #include <linux/percpu-rwsem.h> 9 #include <linux/wait.h> 10 #include <linux/cdev.h> 11 #include <linux/idr.h> 12 #include <linux/pci.h> 13 #include <linux/ioasid.h> 14 #include <linux/bitmap.h> 15 #include <linux/perf_event.h> 16 #include <uapi/linux/idxd.h> 17 #include "registers.h" 18 19 #define IDXD_DRIVER_VERSION "1.00" 20 21 extern struct kmem_cache *idxd_desc_pool; 22 extern bool tc_override; 23 24 struct idxd_wq; 25 struct idxd_dev; 26 27 enum idxd_dev_type { 28 IDXD_DEV_NONE = -1, 29 IDXD_DEV_DSA = 0, 30 IDXD_DEV_IAX, 31 IDXD_DEV_WQ, 32 IDXD_DEV_GROUP, 33 IDXD_DEV_ENGINE, 34 IDXD_DEV_CDEV, 35 IDXD_DEV_MAX_TYPE, 36 }; 37 38 struct idxd_dev { 39 struct device conf_dev; 40 enum idxd_dev_type type; 41 }; 42 43 #define IDXD_REG_TIMEOUT 50 44 #define IDXD_DRAIN_TIMEOUT 5000 45 46 enum idxd_type { 47 IDXD_TYPE_UNKNOWN = -1, 48 IDXD_TYPE_DSA = 0, 49 IDXD_TYPE_IAX, 50 IDXD_TYPE_MAX, 51 }; 52 53 #define IDXD_NAME_SIZE 128 54 #define IDXD_PMU_EVENT_MAX 64 55 56 #define IDXD_ENQCMDS_RETRIES 32 57 #define IDXD_ENQCMDS_MAX_RETRIES 64 58 59 struct idxd_device_driver { 60 const char *name; 61 enum idxd_dev_type *type; 62 int (*probe)(struct idxd_dev *idxd_dev); 63 void (*remove)(struct idxd_dev *idxd_dev); 64 struct device_driver drv; 65 }; 66 67 extern struct idxd_device_driver dsa_drv; 68 extern struct idxd_device_driver idxd_drv; 69 extern struct idxd_device_driver idxd_dmaengine_drv; 70 extern struct idxd_device_driver idxd_user_drv; 71 72 #define INVALID_INT_HANDLE -1 73 struct idxd_irq_entry { 74 int id; 75 int vector; 76 struct llist_head pending_llist; 77 struct list_head work_list; 78 /* 79 * Lock to protect access between irq thread process descriptor 80 * and irq thread processing error descriptor. 81 */ 82 spinlock_t list_lock; 83 int int_handle; 84 ioasid_t pasid; 85 }; 86 87 struct idxd_group { 88 struct idxd_dev idxd_dev; 89 struct idxd_device *idxd; 90 struct grpcfg grpcfg; 91 int id; 92 int num_engines; 93 int num_wqs; 94 bool use_rdbuf_limit; 95 u8 rdbufs_allowed; 96 u8 rdbufs_reserved; 97 int tc_a; 98 int tc_b; 99 int desc_progress_limit; 100 int batch_progress_limit; 101 }; 102 103 struct idxd_pmu { 104 struct idxd_device *idxd; 105 106 struct perf_event *event_list[IDXD_PMU_EVENT_MAX]; 107 int n_events; 108 109 DECLARE_BITMAP(used_mask, IDXD_PMU_EVENT_MAX); 110 111 struct pmu pmu; 112 char name[IDXD_NAME_SIZE]; 113 int cpu; 114 115 int n_counters; 116 int counter_width; 117 int n_event_categories; 118 119 bool per_counter_caps_supported; 120 unsigned long supported_event_categories; 121 122 unsigned long supported_filters; 123 int n_filters; 124 125 struct hlist_node cpuhp_node; 126 }; 127 128 #define IDXD_MAX_PRIORITY 0xf 129 130 enum idxd_wq_state { 131 IDXD_WQ_DISABLED = 0, 132 IDXD_WQ_ENABLED, 133 }; 134 135 enum idxd_wq_flag { 136 WQ_FLAG_DEDICATED = 0, 137 WQ_FLAG_BLOCK_ON_FAULT, 138 WQ_FLAG_ATS_DISABLE, 139 }; 140 141 enum idxd_wq_type { 142 IDXD_WQT_NONE = 0, 143 IDXD_WQT_KERNEL, 144 IDXD_WQT_USER, 145 }; 146 147 struct idxd_cdev { 148 struct idxd_wq *wq; 149 struct cdev cdev; 150 struct idxd_dev idxd_dev; 151 int minor; 152 }; 153 154 #define IDXD_ALLOCATED_BATCH_SIZE 128U 155 #define WQ_NAME_SIZE 1024 156 #define WQ_TYPE_SIZE 10 157 158 #define WQ_DEFAULT_QUEUE_DEPTH 16 159 #define WQ_DEFAULT_MAX_XFER SZ_2M 160 #define WQ_DEFAULT_MAX_BATCH 32 161 162 enum idxd_op_type { 163 IDXD_OP_BLOCK = 0, 164 IDXD_OP_NONBLOCK = 1, 165 }; 166 167 enum idxd_complete_type { 168 IDXD_COMPLETE_NORMAL = 0, 169 IDXD_COMPLETE_ABORT, 170 IDXD_COMPLETE_DEV_FAIL, 171 }; 172 173 struct idxd_dma_chan { 174 struct dma_chan chan; 175 struct idxd_wq *wq; 176 }; 177 178 struct idxd_wq { 179 void __iomem *portal; 180 u32 portal_offset; 181 unsigned int enqcmds_retries; 182 struct percpu_ref wq_active; 183 struct completion wq_dead; 184 struct completion wq_resurrect; 185 struct idxd_dev idxd_dev; 186 struct idxd_cdev *idxd_cdev; 187 struct wait_queue_head err_queue; 188 struct idxd_device *idxd; 189 int id; 190 struct idxd_irq_entry ie; 191 enum idxd_wq_type type; 192 struct idxd_group *group; 193 int client_count; 194 struct mutex wq_lock; /* mutex for workqueue */ 195 u32 size; 196 u32 threshold; 197 u32 priority; 198 enum idxd_wq_state state; 199 unsigned long flags; 200 union wqcfg *wqcfg; 201 unsigned long *opcap_bmap; 202 203 struct dsa_hw_desc **hw_descs; 204 int num_descs; 205 union { 206 struct dsa_completion_record *compls; 207 struct iax_completion_record *iax_compls; 208 }; 209 dma_addr_t compls_addr; 210 int compls_size; 211 struct idxd_desc **descs; 212 struct sbitmap_queue sbq; 213 struct idxd_dma_chan *idxd_chan; 214 char name[WQ_NAME_SIZE + 1]; 215 u64 max_xfer_bytes; 216 u32 max_batch_size; 217 }; 218 219 struct idxd_engine { 220 struct idxd_dev idxd_dev; 221 int id; 222 struct idxd_group *group; 223 struct idxd_device *idxd; 224 }; 225 226 /* shadow registers */ 227 struct idxd_hw { 228 u32 version; 229 union gen_cap_reg gen_cap; 230 union wq_cap_reg wq_cap; 231 union group_cap_reg group_cap; 232 union engine_cap_reg engine_cap; 233 struct opcap opcap; 234 u32 cmd_cap; 235 }; 236 237 enum idxd_device_state { 238 IDXD_DEV_HALTED = -1, 239 IDXD_DEV_DISABLED = 0, 240 IDXD_DEV_ENABLED, 241 }; 242 243 enum idxd_device_flag { 244 IDXD_FLAG_CONFIGURABLE = 0, 245 IDXD_FLAG_CMD_RUNNING, 246 IDXD_FLAG_PASID_ENABLED, 247 IDXD_FLAG_USER_PASID_ENABLED, 248 }; 249 250 struct idxd_dma_dev { 251 struct idxd_device *idxd; 252 struct dma_device dma; 253 }; 254 255 struct idxd_driver_data { 256 const char *name_prefix; 257 enum idxd_type type; 258 struct device_type *dev_type; 259 int compl_size; 260 int align; 261 }; 262 263 struct idxd_device { 264 struct idxd_dev idxd_dev; 265 struct idxd_driver_data *data; 266 struct list_head list; 267 struct idxd_hw hw; 268 enum idxd_device_state state; 269 unsigned long flags; 270 int id; 271 int major; 272 u32 cmd_status; 273 struct idxd_irq_entry ie; /* misc irq, msix 0 */ 274 275 struct pci_dev *pdev; 276 void __iomem *reg_base; 277 278 spinlock_t dev_lock; /* spinlock for device */ 279 spinlock_t cmd_lock; /* spinlock for device commands */ 280 struct completion *cmd_done; 281 struct idxd_group **groups; 282 struct idxd_wq **wqs; 283 struct idxd_engine **engines; 284 285 struct iommu_sva *sva; 286 unsigned int pasid; 287 288 int num_groups; 289 int irq_cnt; 290 bool request_int_handles; 291 292 u32 msix_perm_offset; 293 u32 wqcfg_offset; 294 u32 grpcfg_offset; 295 u32 perfmon_offset; 296 297 u64 max_xfer_bytes; 298 u32 max_batch_size; 299 int max_groups; 300 int max_engines; 301 int max_rdbufs; 302 int max_wqs; 303 int max_wq_size; 304 int rdbuf_limit; 305 int nr_rdbufs; /* non-reserved read buffers */ 306 unsigned int wqcfg_size; 307 unsigned long *wq_enable_map; 308 309 union sw_err_reg sw_err; 310 wait_queue_head_t cmd_waitq; 311 312 struct idxd_dma_dev *idxd_dma; 313 struct workqueue_struct *wq; 314 struct work_struct work; 315 316 struct idxd_pmu *idxd_pmu; 317 318 unsigned long *opcap_bmap; 319 }; 320 321 /* IDXD software descriptor */ 322 struct idxd_desc { 323 union { 324 struct dsa_hw_desc *hw; 325 struct iax_hw_desc *iax_hw; 326 }; 327 dma_addr_t desc_dma; 328 union { 329 struct dsa_completion_record *completion; 330 struct iax_completion_record *iax_completion; 331 }; 332 dma_addr_t compl_dma; 333 struct dma_async_tx_descriptor txd; 334 struct llist_node llnode; 335 struct list_head list; 336 int id; 337 int cpu; 338 struct idxd_wq *wq; 339 }; 340 341 /* 342 * This is software defined error for the completion status. We overload the error code 343 * that will never appear in completion status and only SWERR register. 344 */ 345 enum idxd_completion_status { 346 IDXD_COMP_DESC_ABORT = 0xff, 347 }; 348 349 #define idxd_confdev(idxd) &idxd->idxd_dev.conf_dev 350 #define wq_confdev(wq) &wq->idxd_dev.conf_dev 351 #define engine_confdev(engine) &engine->idxd_dev.conf_dev 352 #define group_confdev(group) &group->idxd_dev.conf_dev 353 #define cdev_dev(cdev) &cdev->idxd_dev.conf_dev 354 355 #define confdev_to_idxd_dev(dev) container_of(dev, struct idxd_dev, conf_dev) 356 #define idxd_dev_to_idxd(idxd_dev) container_of(idxd_dev, struct idxd_device, idxd_dev) 357 #define idxd_dev_to_wq(idxd_dev) container_of(idxd_dev, struct idxd_wq, idxd_dev) 358 359 static inline struct idxd_device *confdev_to_idxd(struct device *dev) 360 { 361 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 362 363 return idxd_dev_to_idxd(idxd_dev); 364 } 365 366 static inline struct idxd_wq *confdev_to_wq(struct device *dev) 367 { 368 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 369 370 return idxd_dev_to_wq(idxd_dev); 371 } 372 373 static inline struct idxd_engine *confdev_to_engine(struct device *dev) 374 { 375 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 376 377 return container_of(idxd_dev, struct idxd_engine, idxd_dev); 378 } 379 380 static inline struct idxd_group *confdev_to_group(struct device *dev) 381 { 382 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 383 384 return container_of(idxd_dev, struct idxd_group, idxd_dev); 385 } 386 387 static inline struct idxd_cdev *dev_to_cdev(struct device *dev) 388 { 389 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 390 391 return container_of(idxd_dev, struct idxd_cdev, idxd_dev); 392 } 393 394 static inline void idxd_dev_set_type(struct idxd_dev *idev, int type) 395 { 396 if (type >= IDXD_DEV_MAX_TYPE) { 397 idev->type = IDXD_DEV_NONE; 398 return; 399 } 400 401 idev->type = type; 402 } 403 404 static inline struct idxd_irq_entry *idxd_get_ie(struct idxd_device *idxd, int idx) 405 { 406 return (idx == 0) ? &idxd->ie : &idxd->wqs[idx - 1]->ie; 407 } 408 409 static inline struct idxd_wq *ie_to_wq(struct idxd_irq_entry *ie) 410 { 411 return container_of(ie, struct idxd_wq, ie); 412 } 413 414 static inline struct idxd_device *ie_to_idxd(struct idxd_irq_entry *ie) 415 { 416 return container_of(ie, struct idxd_device, ie); 417 } 418 419 extern struct bus_type dsa_bus_type; 420 421 extern bool support_enqcmd; 422 extern struct ida idxd_ida; 423 extern struct device_type dsa_device_type; 424 extern struct device_type iax_device_type; 425 extern struct device_type idxd_wq_device_type; 426 extern struct device_type idxd_engine_device_type; 427 extern struct device_type idxd_group_device_type; 428 429 static inline bool is_dsa_dev(struct idxd_dev *idxd_dev) 430 { 431 return idxd_dev->type == IDXD_DEV_DSA; 432 } 433 434 static inline bool is_iax_dev(struct idxd_dev *idxd_dev) 435 { 436 return idxd_dev->type == IDXD_DEV_IAX; 437 } 438 439 static inline bool is_idxd_dev(struct idxd_dev *idxd_dev) 440 { 441 return is_dsa_dev(idxd_dev) || is_iax_dev(idxd_dev); 442 } 443 444 static inline bool is_idxd_wq_dev(struct idxd_dev *idxd_dev) 445 { 446 return idxd_dev->type == IDXD_DEV_WQ; 447 } 448 449 static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq) 450 { 451 if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0) 452 return true; 453 return false; 454 } 455 456 static inline bool is_idxd_wq_user(struct idxd_wq *wq) 457 { 458 return wq->type == IDXD_WQT_USER; 459 } 460 461 static inline bool is_idxd_wq_kernel(struct idxd_wq *wq) 462 { 463 return wq->type == IDXD_WQT_KERNEL; 464 } 465 466 static inline bool wq_dedicated(struct idxd_wq *wq) 467 { 468 return test_bit(WQ_FLAG_DEDICATED, &wq->flags); 469 } 470 471 static inline bool wq_shared(struct idxd_wq *wq) 472 { 473 return !test_bit(WQ_FLAG_DEDICATED, &wq->flags); 474 } 475 476 static inline bool device_pasid_enabled(struct idxd_device *idxd) 477 { 478 return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 479 } 480 481 static inline bool device_user_pasid_enabled(struct idxd_device *idxd) 482 { 483 return test_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags); 484 } 485 486 static inline bool wq_pasid_enabled(struct idxd_wq *wq) 487 { 488 return (is_idxd_wq_kernel(wq) && device_pasid_enabled(wq->idxd)) || 489 (is_idxd_wq_user(wq) && device_user_pasid_enabled(wq->idxd)); 490 } 491 492 static inline bool wq_shared_supported(struct idxd_wq *wq) 493 { 494 return (support_enqcmd && wq_pasid_enabled(wq)); 495 } 496 497 enum idxd_portal_prot { 498 IDXD_PORTAL_UNLIMITED = 0, 499 IDXD_PORTAL_LIMITED, 500 }; 501 502 enum idxd_interrupt_type { 503 IDXD_IRQ_MSIX = 0, 504 IDXD_IRQ_IMS, 505 }; 506 507 static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot) 508 { 509 return prot * 0x1000; 510 } 511 512 static inline int idxd_get_wq_portal_full_offset(int wq_id, 513 enum idxd_portal_prot prot) 514 { 515 return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot); 516 } 517 518 #define IDXD_PORTAL_MASK (PAGE_SIZE - 1) 519 520 /* 521 * Even though this function can be accessed by multiple threads, it is safe to use. 522 * At worst the address gets used more than once before it gets incremented. We don't 523 * hit a threshold until iops becomes many million times a second. So the occasional 524 * reuse of the same address is tolerable compare to using an atomic variable. This is 525 * safe on a system that has atomic load/store for 32bit integers. Given that this is an 526 * Intel iEP device, that should not be a problem. 527 */ 528 static inline void __iomem *idxd_wq_portal_addr(struct idxd_wq *wq) 529 { 530 int ofs = wq->portal_offset; 531 532 wq->portal_offset = (ofs + sizeof(struct dsa_raw_desc)) & IDXD_PORTAL_MASK; 533 return wq->portal + ofs; 534 } 535 536 static inline void idxd_wq_get(struct idxd_wq *wq) 537 { 538 wq->client_count++; 539 } 540 541 static inline void idxd_wq_put(struct idxd_wq *wq) 542 { 543 wq->client_count--; 544 } 545 546 static inline int idxd_wq_refcount(struct idxd_wq *wq) 547 { 548 return wq->client_count; 549 }; 550 551 int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv, 552 struct module *module, const char *mod_name); 553 #define idxd_driver_register(driver) \ 554 __idxd_driver_register(driver, THIS_MODULE, KBUILD_MODNAME) 555 556 void idxd_driver_unregister(struct idxd_device_driver *idxd_drv); 557 558 #define module_idxd_driver(__idxd_driver) \ 559 module_driver(__idxd_driver, idxd_driver_register, idxd_driver_unregister) 560 561 int idxd_register_bus_type(void); 562 void idxd_unregister_bus_type(void); 563 int idxd_register_devices(struct idxd_device *idxd); 564 void idxd_unregister_devices(struct idxd_device *idxd); 565 int idxd_register_driver(void); 566 void idxd_unregister_driver(void); 567 void idxd_wqs_quiesce(struct idxd_device *idxd); 568 bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc); 569 570 /* device interrupt control */ 571 irqreturn_t idxd_misc_thread(int vec, void *data); 572 irqreturn_t idxd_wq_thread(int irq, void *data); 573 void idxd_mask_error_interrupts(struct idxd_device *idxd); 574 void idxd_unmask_error_interrupts(struct idxd_device *idxd); 575 576 /* device control */ 577 int idxd_register_idxd_drv(void); 578 void idxd_unregister_idxd_drv(void); 579 int idxd_device_drv_probe(struct idxd_dev *idxd_dev); 580 void idxd_device_drv_remove(struct idxd_dev *idxd_dev); 581 int drv_enable_wq(struct idxd_wq *wq); 582 void drv_disable_wq(struct idxd_wq *wq); 583 int idxd_device_init_reset(struct idxd_device *idxd); 584 int idxd_device_enable(struct idxd_device *idxd); 585 int idxd_device_disable(struct idxd_device *idxd); 586 void idxd_device_reset(struct idxd_device *idxd); 587 void idxd_device_clear_state(struct idxd_device *idxd); 588 int idxd_device_config(struct idxd_device *idxd); 589 void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid); 590 int idxd_device_load_config(struct idxd_device *idxd); 591 int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle, 592 enum idxd_interrupt_type irq_type); 593 int idxd_device_release_int_handle(struct idxd_device *idxd, int handle, 594 enum idxd_interrupt_type irq_type); 595 596 /* work queue control */ 597 void idxd_wqs_unmap_portal(struct idxd_device *idxd); 598 int idxd_wq_alloc_resources(struct idxd_wq *wq); 599 void idxd_wq_free_resources(struct idxd_wq *wq); 600 int idxd_wq_enable(struct idxd_wq *wq); 601 int idxd_wq_disable(struct idxd_wq *wq, bool reset_config); 602 void idxd_wq_drain(struct idxd_wq *wq); 603 void idxd_wq_reset(struct idxd_wq *wq); 604 int idxd_wq_map_portal(struct idxd_wq *wq); 605 void idxd_wq_unmap_portal(struct idxd_wq *wq); 606 int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid); 607 int idxd_wq_disable_pasid(struct idxd_wq *wq); 608 void __idxd_wq_quiesce(struct idxd_wq *wq); 609 void idxd_wq_quiesce(struct idxd_wq *wq); 610 int idxd_wq_init_percpu_ref(struct idxd_wq *wq); 611 void idxd_wq_free_irq(struct idxd_wq *wq); 612 int idxd_wq_request_irq(struct idxd_wq *wq); 613 614 /* submission */ 615 int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc); 616 struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype); 617 void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc); 618 int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc); 619 620 /* dmaengine */ 621 int idxd_register_dma_device(struct idxd_device *idxd); 622 void idxd_unregister_dma_device(struct idxd_device *idxd); 623 void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res); 624 void idxd_dma_complete_txd(struct idxd_desc *desc, 625 enum idxd_complete_type comp_type, bool free_desc); 626 627 /* cdev */ 628 int idxd_cdev_register(void); 629 void idxd_cdev_remove(void); 630 int idxd_cdev_get_major(struct idxd_device *idxd); 631 int idxd_wq_add_cdev(struct idxd_wq *wq); 632 void idxd_wq_del_cdev(struct idxd_wq *wq); 633 634 /* perfmon */ 635 #if IS_ENABLED(CONFIG_INTEL_IDXD_PERFMON) 636 int perfmon_pmu_init(struct idxd_device *idxd); 637 void perfmon_pmu_remove(struct idxd_device *idxd); 638 void perfmon_counter_overflow(struct idxd_device *idxd); 639 void perfmon_init(void); 640 void perfmon_exit(void); 641 #else 642 static inline int perfmon_pmu_init(struct idxd_device *idxd) { return 0; } 643 static inline void perfmon_pmu_remove(struct idxd_device *idxd) {} 644 static inline void perfmon_counter_overflow(struct idxd_device *idxd) {} 645 static inline void perfmon_init(void) {} 646 static inline void perfmon_exit(void) {} 647 #endif 648 649 #endif 650