1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #ifndef _IDXD_H_ 4 #define _IDXD_H_ 5 6 #include <linux/sbitmap.h> 7 #include <linux/dmaengine.h> 8 #include <linux/percpu-rwsem.h> 9 #include <linux/wait.h> 10 #include <linux/cdev.h> 11 #include <linux/idr.h> 12 #include <linux/pci.h> 13 #include <linux/ioasid.h> 14 #include <linux/perf_event.h> 15 #include <uapi/linux/idxd.h> 16 #include "registers.h" 17 18 #define IDXD_DRIVER_VERSION "1.00" 19 20 extern struct kmem_cache *idxd_desc_pool; 21 extern bool tc_override; 22 23 struct idxd_wq; 24 struct idxd_dev; 25 26 enum idxd_dev_type { 27 IDXD_DEV_NONE = -1, 28 IDXD_DEV_DSA = 0, 29 IDXD_DEV_IAX, 30 IDXD_DEV_WQ, 31 IDXD_DEV_GROUP, 32 IDXD_DEV_ENGINE, 33 IDXD_DEV_CDEV, 34 IDXD_DEV_MAX_TYPE, 35 }; 36 37 struct idxd_dev { 38 struct device conf_dev; 39 enum idxd_dev_type type; 40 }; 41 42 #define IDXD_REG_TIMEOUT 50 43 #define IDXD_DRAIN_TIMEOUT 5000 44 45 enum idxd_type { 46 IDXD_TYPE_UNKNOWN = -1, 47 IDXD_TYPE_DSA = 0, 48 IDXD_TYPE_IAX, 49 IDXD_TYPE_MAX, 50 }; 51 52 #define IDXD_NAME_SIZE 128 53 #define IDXD_PMU_EVENT_MAX 64 54 55 #define IDXD_ENQCMDS_RETRIES 32 56 #define IDXD_ENQCMDS_MAX_RETRIES 64 57 58 struct idxd_device_driver { 59 const char *name; 60 enum idxd_dev_type *type; 61 int (*probe)(struct idxd_dev *idxd_dev); 62 void (*remove)(struct idxd_dev *idxd_dev); 63 struct device_driver drv; 64 }; 65 66 extern struct idxd_device_driver dsa_drv; 67 extern struct idxd_device_driver idxd_drv; 68 extern struct idxd_device_driver idxd_dmaengine_drv; 69 extern struct idxd_device_driver idxd_user_drv; 70 71 #define INVALID_INT_HANDLE -1 72 struct idxd_irq_entry { 73 int id; 74 int vector; 75 struct llist_head pending_llist; 76 struct list_head work_list; 77 /* 78 * Lock to protect access between irq thread process descriptor 79 * and irq thread processing error descriptor. 80 */ 81 spinlock_t list_lock; 82 int int_handle; 83 ioasid_t pasid; 84 }; 85 86 struct idxd_group { 87 struct idxd_dev idxd_dev; 88 struct idxd_device *idxd; 89 struct grpcfg grpcfg; 90 int id; 91 int num_engines; 92 int num_wqs; 93 bool use_rdbuf_limit; 94 u8 rdbufs_allowed; 95 u8 rdbufs_reserved; 96 int tc_a; 97 int tc_b; 98 }; 99 100 struct idxd_pmu { 101 struct idxd_device *idxd; 102 103 struct perf_event *event_list[IDXD_PMU_EVENT_MAX]; 104 int n_events; 105 106 DECLARE_BITMAP(used_mask, IDXD_PMU_EVENT_MAX); 107 108 struct pmu pmu; 109 char name[IDXD_NAME_SIZE]; 110 int cpu; 111 112 int n_counters; 113 int counter_width; 114 int n_event_categories; 115 116 bool per_counter_caps_supported; 117 unsigned long supported_event_categories; 118 119 unsigned long supported_filters; 120 int n_filters; 121 122 struct hlist_node cpuhp_node; 123 }; 124 125 #define IDXD_MAX_PRIORITY 0xf 126 127 enum idxd_wq_state { 128 IDXD_WQ_DISABLED = 0, 129 IDXD_WQ_ENABLED, 130 }; 131 132 enum idxd_wq_flag { 133 WQ_FLAG_DEDICATED = 0, 134 WQ_FLAG_BLOCK_ON_FAULT, 135 }; 136 137 enum idxd_wq_type { 138 IDXD_WQT_NONE = 0, 139 IDXD_WQT_KERNEL, 140 IDXD_WQT_USER, 141 }; 142 143 struct idxd_cdev { 144 struct idxd_wq *wq; 145 struct cdev cdev; 146 struct idxd_dev idxd_dev; 147 int minor; 148 }; 149 150 #define IDXD_ALLOCATED_BATCH_SIZE 128U 151 #define WQ_NAME_SIZE 1024 152 #define WQ_TYPE_SIZE 10 153 154 #define WQ_DEFAULT_QUEUE_DEPTH 16 155 #define WQ_DEFAULT_MAX_XFER SZ_2M 156 #define WQ_DEFAULT_MAX_BATCH 32 157 158 enum idxd_op_type { 159 IDXD_OP_BLOCK = 0, 160 IDXD_OP_NONBLOCK = 1, 161 }; 162 163 enum idxd_complete_type { 164 IDXD_COMPLETE_NORMAL = 0, 165 IDXD_COMPLETE_ABORT, 166 IDXD_COMPLETE_DEV_FAIL, 167 }; 168 169 struct idxd_dma_chan { 170 struct dma_chan chan; 171 struct idxd_wq *wq; 172 }; 173 174 struct idxd_wq { 175 void __iomem *portal; 176 u32 portal_offset; 177 unsigned int enqcmds_retries; 178 struct percpu_ref wq_active; 179 struct completion wq_dead; 180 struct completion wq_resurrect; 181 struct idxd_dev idxd_dev; 182 struct idxd_cdev *idxd_cdev; 183 struct wait_queue_head err_queue; 184 struct idxd_device *idxd; 185 int id; 186 struct idxd_irq_entry ie; 187 enum idxd_wq_type type; 188 struct idxd_group *group; 189 int client_count; 190 struct mutex wq_lock; /* mutex for workqueue */ 191 u32 size; 192 u32 threshold; 193 u32 priority; 194 enum idxd_wq_state state; 195 unsigned long flags; 196 union wqcfg *wqcfg; 197 struct dsa_hw_desc **hw_descs; 198 int num_descs; 199 union { 200 struct dsa_completion_record *compls; 201 struct iax_completion_record *iax_compls; 202 }; 203 dma_addr_t compls_addr; 204 int compls_size; 205 struct idxd_desc **descs; 206 struct sbitmap_queue sbq; 207 struct idxd_dma_chan *idxd_chan; 208 char name[WQ_NAME_SIZE + 1]; 209 u64 max_xfer_bytes; 210 u32 max_batch_size; 211 bool ats_dis; 212 }; 213 214 struct idxd_engine { 215 struct idxd_dev idxd_dev; 216 int id; 217 struct idxd_group *group; 218 struct idxd_device *idxd; 219 }; 220 221 /* shadow registers */ 222 struct idxd_hw { 223 u32 version; 224 union gen_cap_reg gen_cap; 225 union wq_cap_reg wq_cap; 226 union group_cap_reg group_cap; 227 union engine_cap_reg engine_cap; 228 struct opcap opcap; 229 u32 cmd_cap; 230 }; 231 232 enum idxd_device_state { 233 IDXD_DEV_HALTED = -1, 234 IDXD_DEV_DISABLED = 0, 235 IDXD_DEV_ENABLED, 236 }; 237 238 enum idxd_device_flag { 239 IDXD_FLAG_CONFIGURABLE = 0, 240 IDXD_FLAG_CMD_RUNNING, 241 IDXD_FLAG_PASID_ENABLED, 242 IDXD_FLAG_USER_PASID_ENABLED, 243 }; 244 245 struct idxd_dma_dev { 246 struct idxd_device *idxd; 247 struct dma_device dma; 248 }; 249 250 struct idxd_driver_data { 251 const char *name_prefix; 252 enum idxd_type type; 253 struct device_type *dev_type; 254 int compl_size; 255 int align; 256 }; 257 258 struct idxd_device { 259 struct idxd_dev idxd_dev; 260 struct idxd_driver_data *data; 261 struct list_head list; 262 struct idxd_hw hw; 263 enum idxd_device_state state; 264 unsigned long flags; 265 int id; 266 int major; 267 u32 cmd_status; 268 struct idxd_irq_entry ie; /* misc irq, msix 0 */ 269 270 struct pci_dev *pdev; 271 void __iomem *reg_base; 272 273 spinlock_t dev_lock; /* spinlock for device */ 274 spinlock_t cmd_lock; /* spinlock for device commands */ 275 struct completion *cmd_done; 276 struct idxd_group **groups; 277 struct idxd_wq **wqs; 278 struct idxd_engine **engines; 279 280 struct iommu_sva *sva; 281 unsigned int pasid; 282 283 int num_groups; 284 int irq_cnt; 285 bool request_int_handles; 286 287 u32 msix_perm_offset; 288 u32 wqcfg_offset; 289 u32 grpcfg_offset; 290 u32 perfmon_offset; 291 292 u64 max_xfer_bytes; 293 u32 max_batch_size; 294 int max_groups; 295 int max_engines; 296 int max_rdbufs; 297 int max_wqs; 298 int max_wq_size; 299 int rdbuf_limit; 300 int nr_rdbufs; /* non-reserved read buffers */ 301 unsigned int wqcfg_size; 302 303 union sw_err_reg sw_err; 304 wait_queue_head_t cmd_waitq; 305 306 struct idxd_dma_dev *idxd_dma; 307 struct workqueue_struct *wq; 308 struct work_struct work; 309 310 struct idxd_pmu *idxd_pmu; 311 }; 312 313 /* IDXD software descriptor */ 314 struct idxd_desc { 315 union { 316 struct dsa_hw_desc *hw; 317 struct iax_hw_desc *iax_hw; 318 }; 319 dma_addr_t desc_dma; 320 union { 321 struct dsa_completion_record *completion; 322 struct iax_completion_record *iax_completion; 323 }; 324 dma_addr_t compl_dma; 325 struct dma_async_tx_descriptor txd; 326 struct llist_node llnode; 327 struct list_head list; 328 int id; 329 int cpu; 330 struct idxd_wq *wq; 331 }; 332 333 /* 334 * This is software defined error for the completion status. We overload the error code 335 * that will never appear in completion status and only SWERR register. 336 */ 337 enum idxd_completion_status { 338 IDXD_COMP_DESC_ABORT = 0xff, 339 }; 340 341 #define idxd_confdev(idxd) &idxd->idxd_dev.conf_dev 342 #define wq_confdev(wq) &wq->idxd_dev.conf_dev 343 #define engine_confdev(engine) &engine->idxd_dev.conf_dev 344 #define group_confdev(group) &group->idxd_dev.conf_dev 345 #define cdev_dev(cdev) &cdev->idxd_dev.conf_dev 346 347 #define confdev_to_idxd_dev(dev) container_of(dev, struct idxd_dev, conf_dev) 348 #define idxd_dev_to_idxd(idxd_dev) container_of(idxd_dev, struct idxd_device, idxd_dev) 349 #define idxd_dev_to_wq(idxd_dev) container_of(idxd_dev, struct idxd_wq, idxd_dev) 350 351 static inline struct idxd_device *confdev_to_idxd(struct device *dev) 352 { 353 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 354 355 return idxd_dev_to_idxd(idxd_dev); 356 } 357 358 static inline struct idxd_wq *confdev_to_wq(struct device *dev) 359 { 360 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 361 362 return idxd_dev_to_wq(idxd_dev); 363 } 364 365 static inline struct idxd_engine *confdev_to_engine(struct device *dev) 366 { 367 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 368 369 return container_of(idxd_dev, struct idxd_engine, idxd_dev); 370 } 371 372 static inline struct idxd_group *confdev_to_group(struct device *dev) 373 { 374 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 375 376 return container_of(idxd_dev, struct idxd_group, idxd_dev); 377 } 378 379 static inline struct idxd_cdev *dev_to_cdev(struct device *dev) 380 { 381 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 382 383 return container_of(idxd_dev, struct idxd_cdev, idxd_dev); 384 } 385 386 static inline void idxd_dev_set_type(struct idxd_dev *idev, int type) 387 { 388 if (type >= IDXD_DEV_MAX_TYPE) { 389 idev->type = IDXD_DEV_NONE; 390 return; 391 } 392 393 idev->type = type; 394 } 395 396 static inline struct idxd_irq_entry *idxd_get_ie(struct idxd_device *idxd, int idx) 397 { 398 return (idx == 0) ? &idxd->ie : &idxd->wqs[idx - 1]->ie; 399 } 400 401 static inline struct idxd_wq *ie_to_wq(struct idxd_irq_entry *ie) 402 { 403 return container_of(ie, struct idxd_wq, ie); 404 } 405 406 static inline struct idxd_device *ie_to_idxd(struct idxd_irq_entry *ie) 407 { 408 return container_of(ie, struct idxd_device, ie); 409 } 410 411 extern struct bus_type dsa_bus_type; 412 413 extern bool support_enqcmd; 414 extern struct ida idxd_ida; 415 extern struct device_type dsa_device_type; 416 extern struct device_type iax_device_type; 417 extern struct device_type idxd_wq_device_type; 418 extern struct device_type idxd_engine_device_type; 419 extern struct device_type idxd_group_device_type; 420 421 static inline bool is_dsa_dev(struct idxd_dev *idxd_dev) 422 { 423 return idxd_dev->type == IDXD_DEV_DSA; 424 } 425 426 static inline bool is_iax_dev(struct idxd_dev *idxd_dev) 427 { 428 return idxd_dev->type == IDXD_DEV_IAX; 429 } 430 431 static inline bool is_idxd_dev(struct idxd_dev *idxd_dev) 432 { 433 return is_dsa_dev(idxd_dev) || is_iax_dev(idxd_dev); 434 } 435 436 static inline bool is_idxd_wq_dev(struct idxd_dev *idxd_dev) 437 { 438 return idxd_dev->type == IDXD_DEV_WQ; 439 } 440 441 static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq) 442 { 443 if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0) 444 return true; 445 return false; 446 } 447 448 static inline bool is_idxd_wq_user(struct idxd_wq *wq) 449 { 450 return wq->type == IDXD_WQT_USER; 451 } 452 453 static inline bool is_idxd_wq_kernel(struct idxd_wq *wq) 454 { 455 return wq->type == IDXD_WQT_KERNEL; 456 } 457 458 static inline bool wq_dedicated(struct idxd_wq *wq) 459 { 460 return test_bit(WQ_FLAG_DEDICATED, &wq->flags); 461 } 462 463 static inline bool wq_shared(struct idxd_wq *wq) 464 { 465 return !test_bit(WQ_FLAG_DEDICATED, &wq->flags); 466 } 467 468 static inline bool device_pasid_enabled(struct idxd_device *idxd) 469 { 470 return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 471 } 472 473 static inline bool device_user_pasid_enabled(struct idxd_device *idxd) 474 { 475 return test_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags); 476 } 477 478 static inline bool wq_pasid_enabled(struct idxd_wq *wq) 479 { 480 return (is_idxd_wq_kernel(wq) && device_pasid_enabled(wq->idxd)) || 481 (is_idxd_wq_user(wq) && device_user_pasid_enabled(wq->idxd)); 482 } 483 484 static inline bool wq_shared_supported(struct idxd_wq *wq) 485 { 486 return (support_enqcmd && wq_pasid_enabled(wq)); 487 } 488 489 enum idxd_portal_prot { 490 IDXD_PORTAL_UNLIMITED = 0, 491 IDXD_PORTAL_LIMITED, 492 }; 493 494 enum idxd_interrupt_type { 495 IDXD_IRQ_MSIX = 0, 496 IDXD_IRQ_IMS, 497 }; 498 499 static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot) 500 { 501 return prot * 0x1000; 502 } 503 504 static inline int idxd_get_wq_portal_full_offset(int wq_id, 505 enum idxd_portal_prot prot) 506 { 507 return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot); 508 } 509 510 #define IDXD_PORTAL_MASK (PAGE_SIZE - 1) 511 512 /* 513 * Even though this function can be accessed by multiple threads, it is safe to use. 514 * At worst the address gets used more than once before it gets incremented. We don't 515 * hit a threshold until iops becomes many million times a second. So the occasional 516 * reuse of the same address is tolerable compare to using an atomic variable. This is 517 * safe on a system that has atomic load/store for 32bit integers. Given that this is an 518 * Intel iEP device, that should not be a problem. 519 */ 520 static inline void __iomem *idxd_wq_portal_addr(struct idxd_wq *wq) 521 { 522 int ofs = wq->portal_offset; 523 524 wq->portal_offset = (ofs + sizeof(struct dsa_raw_desc)) & IDXD_PORTAL_MASK; 525 return wq->portal + ofs; 526 } 527 528 static inline void idxd_wq_get(struct idxd_wq *wq) 529 { 530 wq->client_count++; 531 } 532 533 static inline void idxd_wq_put(struct idxd_wq *wq) 534 { 535 wq->client_count--; 536 } 537 538 static inline int idxd_wq_refcount(struct idxd_wq *wq) 539 { 540 return wq->client_count; 541 }; 542 543 int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv, 544 struct module *module, const char *mod_name); 545 #define idxd_driver_register(driver) \ 546 __idxd_driver_register(driver, THIS_MODULE, KBUILD_MODNAME) 547 548 void idxd_driver_unregister(struct idxd_device_driver *idxd_drv); 549 550 #define module_idxd_driver(__idxd_driver) \ 551 module_driver(__idxd_driver, idxd_driver_register, idxd_driver_unregister) 552 553 int idxd_register_bus_type(void); 554 void idxd_unregister_bus_type(void); 555 int idxd_register_devices(struct idxd_device *idxd); 556 void idxd_unregister_devices(struct idxd_device *idxd); 557 int idxd_register_driver(void); 558 void idxd_unregister_driver(void); 559 void idxd_wqs_quiesce(struct idxd_device *idxd); 560 bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc); 561 562 /* device interrupt control */ 563 irqreturn_t idxd_misc_thread(int vec, void *data); 564 irqreturn_t idxd_wq_thread(int irq, void *data); 565 void idxd_mask_error_interrupts(struct idxd_device *idxd); 566 void idxd_unmask_error_interrupts(struct idxd_device *idxd); 567 568 /* device control */ 569 int idxd_register_idxd_drv(void); 570 void idxd_unregister_idxd_drv(void); 571 int idxd_device_drv_probe(struct idxd_dev *idxd_dev); 572 void idxd_device_drv_remove(struct idxd_dev *idxd_dev); 573 int drv_enable_wq(struct idxd_wq *wq); 574 void drv_disable_wq(struct idxd_wq *wq); 575 int idxd_device_init_reset(struct idxd_device *idxd); 576 int idxd_device_enable(struct idxd_device *idxd); 577 int idxd_device_disable(struct idxd_device *idxd); 578 void idxd_device_reset(struct idxd_device *idxd); 579 void idxd_device_clear_state(struct idxd_device *idxd); 580 int idxd_device_config(struct idxd_device *idxd); 581 void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid); 582 int idxd_device_load_config(struct idxd_device *idxd); 583 int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle, 584 enum idxd_interrupt_type irq_type); 585 int idxd_device_release_int_handle(struct idxd_device *idxd, int handle, 586 enum idxd_interrupt_type irq_type); 587 588 /* work queue control */ 589 void idxd_wqs_unmap_portal(struct idxd_device *idxd); 590 int idxd_wq_alloc_resources(struct idxd_wq *wq); 591 void idxd_wq_free_resources(struct idxd_wq *wq); 592 int idxd_wq_enable(struct idxd_wq *wq); 593 int idxd_wq_disable(struct idxd_wq *wq, bool reset_config); 594 void idxd_wq_drain(struct idxd_wq *wq); 595 void idxd_wq_reset(struct idxd_wq *wq); 596 int idxd_wq_map_portal(struct idxd_wq *wq); 597 void idxd_wq_unmap_portal(struct idxd_wq *wq); 598 int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid); 599 int idxd_wq_disable_pasid(struct idxd_wq *wq); 600 void __idxd_wq_quiesce(struct idxd_wq *wq); 601 void idxd_wq_quiesce(struct idxd_wq *wq); 602 int idxd_wq_init_percpu_ref(struct idxd_wq *wq); 603 void idxd_wq_free_irq(struct idxd_wq *wq); 604 int idxd_wq_request_irq(struct idxd_wq *wq); 605 606 /* submission */ 607 int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc); 608 struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype); 609 void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc); 610 int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc); 611 612 /* dmaengine */ 613 int idxd_register_dma_device(struct idxd_device *idxd); 614 void idxd_unregister_dma_device(struct idxd_device *idxd); 615 void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res); 616 void idxd_dma_complete_txd(struct idxd_desc *desc, 617 enum idxd_complete_type comp_type, bool free_desc); 618 619 /* cdev */ 620 int idxd_cdev_register(void); 621 void idxd_cdev_remove(void); 622 int idxd_cdev_get_major(struct idxd_device *idxd); 623 int idxd_wq_add_cdev(struct idxd_wq *wq); 624 void idxd_wq_del_cdev(struct idxd_wq *wq); 625 626 /* perfmon */ 627 #if IS_ENABLED(CONFIG_INTEL_IDXD_PERFMON) 628 int perfmon_pmu_init(struct idxd_device *idxd); 629 void perfmon_pmu_remove(struct idxd_device *idxd); 630 void perfmon_counter_overflow(struct idxd_device *idxd); 631 void perfmon_init(void); 632 void perfmon_exit(void); 633 #else 634 static inline int perfmon_pmu_init(struct idxd_device *idxd) { return 0; } 635 static inline void perfmon_pmu_remove(struct idxd_device *idxd) {} 636 static inline void perfmon_counter_overflow(struct idxd_device *idxd) {} 637 static inline void perfmon_init(void) {} 638 static inline void perfmon_exit(void) {} 639 #endif 640 641 #endif 642