1 /* SPDX-License-Identifier: MIT */ 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #ifndef __INTEL_ENGINE_TYPES__ 7 #define __INTEL_ENGINE_TYPES__ 8 9 #include <linux/average.h> 10 #include <linux/hashtable.h> 11 #include <linux/irq_work.h> 12 #include <linux/kref.h> 13 #include <linux/list.h> 14 #include <linux/llist.h> 15 #include <linux/rbtree.h> 16 #include <linux/timer.h> 17 #include <linux/types.h> 18 #include <linux/workqueue.h> 19 20 #include "i915_gem.h" 21 #include "i915_pmu.h" 22 #include "i915_priolist_types.h" 23 #include "i915_selftest.h" 24 #include "intel_sseu.h" 25 #include "intel_timeline_types.h" 26 #include "intel_uncore.h" 27 #include "intel_wakeref.h" 28 #include "intel_workarounds_types.h" 29 30 /* HW Engine class + instance */ 31 #define RENDER_CLASS 0 32 #define VIDEO_DECODE_CLASS 1 33 #define VIDEO_ENHANCEMENT_CLASS 2 34 #define COPY_ENGINE_CLASS 3 35 #define OTHER_CLASS 4 36 #define COMPUTE_CLASS 5 37 #define MAX_ENGINE_CLASS 5 38 #define MAX_ENGINE_INSTANCE 8 39 40 #define I915_MAX_SLICES 3 41 #define I915_MAX_SUBSLICES 8 42 43 #define I915_CMD_HASH_ORDER 9 44 45 struct dma_fence; 46 struct drm_i915_gem_object; 47 struct drm_i915_reg_table; 48 struct i915_gem_context; 49 struct i915_request; 50 struct i915_sched_attr; 51 struct i915_sched_engine; 52 struct intel_gt; 53 struct intel_ring; 54 struct intel_uncore; 55 struct intel_breadcrumbs; 56 struct intel_engine_cs; 57 struct i915_perf_group; 58 59 typedef u32 intel_engine_mask_t; 60 #define ALL_ENGINES ((intel_engine_mask_t)~0ul) 61 62 struct intel_hw_status_page { 63 struct list_head timelines; 64 struct i915_vma *vma; 65 u32 *addr; 66 }; 67 68 struct intel_instdone { 69 u32 instdone; 70 /* The following exist only in the RCS engine */ 71 u32 slice_common; 72 u32 slice_common_extra[2]; 73 u32 sampler[GEN_MAX_GSLICES][I915_MAX_SUBSLICES]; 74 u32 row[GEN_MAX_GSLICES][I915_MAX_SUBSLICES]; 75 76 /* Added in XeHPG */ 77 u32 geom_svg[GEN_MAX_GSLICES][I915_MAX_SUBSLICES]; 78 }; 79 80 /* 81 * we use a single page to load ctx workarounds so all of these 82 * values are referred in terms of dwords 83 * 84 * struct i915_wa_ctx_bb: 85 * offset: specifies batch starting position, also helpful in case 86 * if we want to have multiple batches at different offsets based on 87 * some criteria. It is not a requirement at the moment but provides 88 * an option for future use. 89 * size: size of the batch in DWORDS 90 */ 91 struct i915_ctx_workarounds { 92 struct i915_wa_ctx_bb { 93 u32 offset; 94 u32 size; 95 } indirect_ctx, per_ctx; 96 struct i915_vma *vma; 97 }; 98 99 #define I915_MAX_VCS 8 100 #define I915_MAX_VECS 4 101 #define I915_MAX_SFC (I915_MAX_VCS / 2) 102 #define I915_MAX_CCS 4 103 #define I915_MAX_RCS 1 104 #define I915_MAX_BCS 9 105 106 /* 107 * Engine IDs definitions. 108 * Keep instances of the same type engine together. 109 */ 110 enum intel_engine_id { 111 RCS0 = 0, 112 BCS0, 113 BCS1, 114 BCS2, 115 BCS3, 116 BCS4, 117 BCS5, 118 BCS6, 119 BCS7, 120 BCS8, 121 #define _BCS(n) (BCS0 + (n)) 122 VCS0, 123 VCS1, 124 VCS2, 125 VCS3, 126 VCS4, 127 VCS5, 128 VCS6, 129 VCS7, 130 #define _VCS(n) (VCS0 + (n)) 131 VECS0, 132 VECS1, 133 VECS2, 134 VECS3, 135 #define _VECS(n) (VECS0 + (n)) 136 CCS0, 137 CCS1, 138 CCS2, 139 CCS3, 140 #define _CCS(n) (CCS0 + (n)) 141 GSC0, 142 I915_NUM_ENGINES 143 #define INVALID_ENGINE ((enum intel_engine_id)-1) 144 }; 145 146 /* A simple estimator for the round-trip latency of an engine */ 147 DECLARE_EWMA(_engine_latency, 6, 4) 148 149 struct st_preempt_hang { 150 struct completion completion; 151 unsigned int count; 152 }; 153 154 /** 155 * struct intel_engine_execlists - execlist submission queue and port state 156 * 157 * The struct intel_engine_execlists represents the combined logical state of 158 * driver and the hardware state for execlist mode of submission. 159 */ 160 struct intel_engine_execlists { 161 /** 162 * @timer: kick the current context if its timeslice expires 163 */ 164 struct timer_list timer; 165 166 /** 167 * @preempt: reset the current context if it fails to give way 168 */ 169 struct timer_list preempt; 170 171 /** 172 * @preempt_target: active request at the time of the preemption request 173 * 174 * We force a preemption to occur if the pending contexts have not 175 * been promoted to active upon receipt of the CS ack event within 176 * the timeout. This timeout maybe chosen based on the target, 177 * using a very short timeout if the context is no longer schedulable. 178 * That short timeout may not be applicable to other contexts, so 179 * if a context switch should happen within before the preemption 180 * timeout, we may shoot early at an innocent context. To prevent this, 181 * we record which context was active at the time of the preemption 182 * request and only reset that context upon the timeout. 183 */ 184 const struct i915_request *preempt_target; 185 186 /** 187 * @ccid: identifier for contexts submitted to this engine 188 */ 189 u32 ccid; 190 191 /** 192 * @yield: CCID at the time of the last semaphore-wait interrupt. 193 * 194 * Instead of leaving a semaphore busy-spinning on an engine, we would 195 * like to switch to another ready context, i.e. yielding the semaphore 196 * timeslice. 197 */ 198 u32 yield; 199 200 /** 201 * @error_interrupt: CS Master EIR 202 * 203 * The CS generates an interrupt when it detects an error. We capture 204 * the first error interrupt, record the EIR and schedule the tasklet. 205 * In the tasklet, we process the pending CS events to ensure we have 206 * the guilty request, and then reset the engine. 207 * 208 * Low 16b are used by HW, with the upper 16b used as the enabling mask. 209 * Reserve the upper 16b for tracking internal errors. 210 */ 211 u32 error_interrupt; 212 #define ERROR_CSB BIT(31) 213 #define ERROR_PREEMPT BIT(30) 214 215 /** 216 * @reset_ccid: Active CCID [EXECLISTS_STATUS_HI] at the time of reset 217 */ 218 u32 reset_ccid; 219 220 /** 221 * @submit_reg: gen-specific execlist submission register 222 * set to the ExecList Submission Port (elsp) register pre-Gen11 and to 223 * the ExecList Submission Queue Contents register array for Gen11+ 224 */ 225 u32 __iomem *submit_reg; 226 227 /** 228 * @ctrl_reg: the enhanced execlists control register, used to load the 229 * submit queue on the HW and to request preemptions to idle 230 */ 231 u32 __iomem *ctrl_reg; 232 233 #define EXECLIST_MAX_PORTS 2 234 /** 235 * @active: the currently known context executing on HW 236 */ 237 struct i915_request * const *active; 238 /** 239 * @inflight: the set of contexts submitted and acknowleged by HW 240 * 241 * The set of inflight contexts is managed by reading CS events 242 * from the HW. On a context-switch event (not preemption), we 243 * know the HW has transitioned from port0 to port1, and we 244 * advance our inflight/active tracking accordingly. 245 */ 246 struct i915_request *inflight[EXECLIST_MAX_PORTS + 1 /* sentinel */]; 247 /** 248 * @pending: the next set of contexts submitted to ELSP 249 * 250 * We store the array of contexts that we submit to HW (via ELSP) and 251 * promote them to the inflight array once HW has signaled the 252 * preemption or idle-to-active event. 253 */ 254 struct i915_request *pending[EXECLIST_MAX_PORTS + 1]; 255 256 /** 257 * @port_mask: number of execlist ports - 1 258 */ 259 unsigned int port_mask; 260 261 /** 262 * @virtual: Queue of requets on a virtual engine, sorted by priority. 263 * Each RB entry is a struct i915_priolist containing a list of requests 264 * of the same priority. 265 */ 266 struct rb_root_cached virtual; 267 268 /** 269 * @csb_write: control register for Context Switch buffer 270 * 271 * Note this register may be either mmio or HWSP shadow. 272 */ 273 u32 *csb_write; 274 275 /** 276 * @csb_status: status array for Context Switch buffer 277 * 278 * Note these register may be either mmio or HWSP shadow. 279 */ 280 u64 *csb_status; 281 282 /** 283 * @csb_size: context status buffer FIFO size 284 */ 285 u8 csb_size; 286 287 /** 288 * @csb_head: context status buffer head 289 */ 290 u8 csb_head; 291 292 I915_SELFTEST_DECLARE(struct st_preempt_hang preempt_hang;) 293 }; 294 295 #define INTEL_ENGINE_CS_MAX_NAME 8 296 297 struct intel_engine_execlists_stats { 298 /** 299 * @active: Number of contexts currently scheduled in. 300 */ 301 unsigned int active; 302 303 /** 304 * @lock: Lock protecting the below fields. 305 */ 306 seqcount_t lock; 307 308 /** 309 * @total: Total time this engine was busy. 310 * 311 * Accumulated time not counting the most recent block in cases where 312 * engine is currently busy (active > 0). 313 */ 314 ktime_t total; 315 316 /** 317 * @start: Timestamp of the last idle to active transition. 318 * 319 * Idle is defined as active == 0, active is active > 0. 320 */ 321 ktime_t start; 322 }; 323 324 struct intel_engine_guc_stats { 325 /** 326 * @running: Active state of the engine when busyness was last sampled. 327 */ 328 bool running; 329 330 /** 331 * @prev_total: Previous value of total runtime clock cycles. 332 */ 333 u32 prev_total; 334 335 /** 336 * @total_gt_clks: Total gt clock cycles this engine was busy. 337 */ 338 u64 total_gt_clks; 339 340 /** 341 * @start_gt_clk: GT clock time of last idle to active transition. 342 */ 343 u64 start_gt_clk; 344 }; 345 346 union intel_engine_tlb_inv_reg { 347 i915_reg_t reg; 348 i915_mcr_reg_t mcr_reg; 349 }; 350 351 struct intel_engine_tlb_inv { 352 bool mcr; 353 union intel_engine_tlb_inv_reg reg; 354 u32 request; 355 u32 done; 356 }; 357 358 struct intel_engine_cs { 359 struct drm_i915_private *i915; 360 struct intel_gt *gt; 361 struct intel_uncore *uncore; 362 char name[INTEL_ENGINE_CS_MAX_NAME]; 363 364 enum intel_engine_id id; 365 enum intel_engine_id legacy_idx; 366 367 unsigned int guc_id; 368 369 intel_engine_mask_t mask; 370 u32 reset_domain; 371 /** 372 * @logical_mask: logical mask of engine, reported to user space via 373 * query IOCTL and used to communicate with the GuC in logical space. 374 * The logical instance of a physical engine can change based on product 375 * and fusing. 376 */ 377 intel_engine_mask_t logical_mask; 378 379 u8 class; 380 u8 instance; 381 382 u16 uabi_class; 383 u16 uabi_instance; 384 385 u32 uabi_capabilities; 386 u32 context_size; 387 u32 mmio_base; 388 389 struct intel_engine_tlb_inv tlb_inv; 390 391 /* 392 * Some w/a require forcewake to be held (which prevents RC6) while 393 * a particular engine is active. If so, we set fw_domain to which 394 * domains need to be held for the duration of request activity, 395 * and 0 if none. We try to limit the duration of the hold as much 396 * as possible. 397 */ 398 enum forcewake_domains fw_domain; 399 unsigned int fw_active; 400 401 unsigned long context_tag; 402 403 struct rb_node uabi_node; 404 405 struct intel_sseu sseu; 406 407 struct i915_sched_engine *sched_engine; 408 409 /* keep a request in reserve for a [pm] barrier under oom */ 410 struct i915_request *request_pool; 411 412 struct intel_context *hung_ce; 413 414 struct llist_head barrier_tasks; 415 416 struct intel_context *kernel_context; /* pinned */ 417 418 /** 419 * pinned_contexts_list: List of pinned contexts. This list is only 420 * assumed to be manipulated during driver load- or unload time and 421 * does therefore not have any additional protection. 422 */ 423 struct list_head pinned_contexts_list; 424 425 intel_engine_mask_t saturated; /* submitting semaphores too late? */ 426 427 struct { 428 struct delayed_work work; 429 struct i915_request *systole; 430 unsigned long blocked; 431 } heartbeat; 432 433 unsigned long serial; 434 435 unsigned long wakeref_serial; 436 struct intel_wakeref wakeref; 437 struct file *default_state; 438 439 struct { 440 struct intel_ring *ring; 441 struct intel_timeline *timeline; 442 } legacy; 443 444 /* 445 * We track the average duration of the idle pulse on parking the 446 * engine to keep an estimate of the how the fast the engine is 447 * under ideal conditions. 448 */ 449 struct ewma__engine_latency latency; 450 451 /* Keep track of all the seqno used, a trail of breadcrumbs */ 452 struct intel_breadcrumbs *breadcrumbs; 453 454 struct intel_engine_pmu { 455 /** 456 * @enable: Bitmask of enable sample events on this engine. 457 * 458 * Bits correspond to sample event types, for instance 459 * I915_SAMPLE_QUEUED is bit 0 etc. 460 */ 461 u32 enable; 462 /** 463 * @enable_count: Reference count for the enabled samplers. 464 * 465 * Index number corresponds to @enum drm_i915_pmu_engine_sample. 466 */ 467 unsigned int enable_count[I915_ENGINE_SAMPLE_COUNT]; 468 /** 469 * @sample: Counter values for sampling events. 470 * 471 * Our internal timer stores the current counters in this field. 472 * 473 * Index number corresponds to @enum drm_i915_pmu_engine_sample. 474 */ 475 struct i915_pmu_sample sample[I915_ENGINE_SAMPLE_COUNT]; 476 } pmu; 477 478 struct intel_hw_status_page status_page; 479 struct i915_ctx_workarounds wa_ctx; 480 struct i915_wa_list ctx_wa_list; 481 struct i915_wa_list wa_list; 482 struct i915_wa_list whitelist; 483 484 u32 irq_keep_mask; /* always keep these interrupts */ 485 u32 irq_enable_mask; /* bitmask to enable ring interrupt */ 486 void (*irq_enable)(struct intel_engine_cs *engine); 487 void (*irq_disable)(struct intel_engine_cs *engine); 488 void (*irq_handler)(struct intel_engine_cs *engine, u16 iir); 489 490 void (*sanitize)(struct intel_engine_cs *engine); 491 int (*resume)(struct intel_engine_cs *engine); 492 493 struct { 494 void (*prepare)(struct intel_engine_cs *engine); 495 496 void (*rewind)(struct intel_engine_cs *engine, bool stalled); 497 void (*cancel)(struct intel_engine_cs *engine); 498 499 void (*finish)(struct intel_engine_cs *engine); 500 } reset; 501 502 void (*park)(struct intel_engine_cs *engine); 503 void (*unpark)(struct intel_engine_cs *engine); 504 505 void (*bump_serial)(struct intel_engine_cs *engine); 506 507 void (*set_default_submission)(struct intel_engine_cs *engine); 508 509 const struct intel_context_ops *cops; 510 511 int (*request_alloc)(struct i915_request *rq); 512 513 int (*emit_flush)(struct i915_request *request, u32 mode); 514 #define EMIT_INVALIDATE BIT(0) 515 #define EMIT_FLUSH BIT(1) 516 #define EMIT_BARRIER (EMIT_INVALIDATE | EMIT_FLUSH) 517 int (*emit_bb_start)(struct i915_request *rq, 518 u64 offset, u32 length, 519 unsigned int dispatch_flags); 520 #define I915_DISPATCH_SECURE BIT(0) 521 #define I915_DISPATCH_PINNED BIT(1) 522 int (*emit_init_breadcrumb)(struct i915_request *rq); 523 u32 *(*emit_fini_breadcrumb)(struct i915_request *rq, 524 u32 *cs); 525 unsigned int emit_fini_breadcrumb_dw; 526 527 /* Pass the request to the hardware queue (e.g. directly into 528 * the legacy ringbuffer or to the end of an execlist). 529 * 530 * This is called from an atomic context with irqs disabled; must 531 * be irq safe. 532 */ 533 void (*submit_request)(struct i915_request *rq); 534 535 void (*release)(struct intel_engine_cs *engine); 536 537 /* 538 * Add / remove request from engine active tracking 539 */ 540 void (*add_active_request)(struct i915_request *rq); 541 void (*remove_active_request)(struct i915_request *rq); 542 543 /* 544 * Get engine busyness and the time at which the busyness was sampled. 545 */ 546 ktime_t (*busyness)(struct intel_engine_cs *engine, 547 ktime_t *now); 548 549 struct intel_engine_execlists execlists; 550 551 /* 552 * Keep track of completed timelines on this engine for early 553 * retirement with the goal of quickly enabling powersaving as 554 * soon as the engine is idle. 555 */ 556 struct intel_timeline *retire; 557 struct work_struct retire_work; 558 559 /* status_notifier: list of callbacks for context-switch changes */ 560 struct atomic_notifier_head context_status_notifier; 561 562 #define I915_ENGINE_USING_CMD_PARSER BIT(0) 563 #define I915_ENGINE_SUPPORTS_STATS BIT(1) 564 #define I915_ENGINE_HAS_PREEMPTION BIT(2) 565 #define I915_ENGINE_HAS_SEMAPHORES BIT(3) 566 #define I915_ENGINE_HAS_TIMESLICES BIT(4) 567 #define I915_ENGINE_IS_VIRTUAL BIT(5) 568 #define I915_ENGINE_HAS_RELATIVE_MMIO BIT(6) 569 #define I915_ENGINE_REQUIRES_CMD_PARSER BIT(7) 570 #define I915_ENGINE_WANT_FORCED_PREEMPTION BIT(8) 571 #define I915_ENGINE_HAS_RCS_REG_STATE BIT(9) 572 #define I915_ENGINE_HAS_EU_PRIORITY BIT(10) 573 #define I915_ENGINE_FIRST_RENDER_COMPUTE BIT(11) 574 #define I915_ENGINE_USES_WA_HOLD_CCS_SWITCHOUT BIT(12) 575 unsigned int flags; 576 577 /* 578 * Table of commands the command parser needs to know about 579 * for this engine. 580 */ 581 DECLARE_HASHTABLE(cmd_hash, I915_CMD_HASH_ORDER); 582 583 /* 584 * Table of registers allowed in commands that read/write registers. 585 */ 586 const struct drm_i915_reg_table *reg_tables; 587 int reg_table_count; 588 589 /* 590 * Returns the bitmask for the length field of the specified command. 591 * Return 0 for an unrecognized/invalid command. 592 * 593 * If the command parser finds an entry for a command in the engine's 594 * cmd_tables, it gets the command's length based on the table entry. 595 * If not, it calls this function to determine the per-engine length 596 * field encoding for the command (i.e. different opcode ranges use 597 * certain bits to encode the command length in the header). 598 */ 599 u32 (*get_cmd_length_mask)(u32 cmd_header); 600 601 struct { 602 union { 603 struct intel_engine_execlists_stats execlists; 604 struct intel_engine_guc_stats guc; 605 }; 606 607 /** 608 * @rps: Utilisation at last RPS sampling. 609 */ 610 ktime_t rps; 611 } stats; 612 613 struct { 614 unsigned long heartbeat_interval_ms; 615 unsigned long max_busywait_duration_ns; 616 unsigned long preempt_timeout_ms; 617 unsigned long stop_timeout_ms; 618 unsigned long timeslice_duration_ms; 619 } props, defaults; 620 621 I915_SELFTEST_DECLARE(struct fault_attr reset_timeout); 622 623 /* 624 * The perf group maps to one OA unit which controls one OA buffer. All 625 * reports corresponding to this engine will be reported to this OA 626 * buffer. An engine will map to a single OA unit, but a single OA unit 627 * can generate reports for multiple engines. 628 */ 629 struct i915_perf_group *oa_group; 630 }; 631 632 static inline bool 633 intel_engine_using_cmd_parser(const struct intel_engine_cs *engine) 634 { 635 return engine->flags & I915_ENGINE_USING_CMD_PARSER; 636 } 637 638 static inline bool 639 intel_engine_requires_cmd_parser(const struct intel_engine_cs *engine) 640 { 641 return engine->flags & I915_ENGINE_REQUIRES_CMD_PARSER; 642 } 643 644 static inline bool 645 intel_engine_supports_stats(const struct intel_engine_cs *engine) 646 { 647 return engine->flags & I915_ENGINE_SUPPORTS_STATS; 648 } 649 650 static inline bool 651 intel_engine_has_preemption(const struct intel_engine_cs *engine) 652 { 653 return engine->flags & I915_ENGINE_HAS_PREEMPTION; 654 } 655 656 static inline bool 657 intel_engine_has_semaphores(const struct intel_engine_cs *engine) 658 { 659 return engine->flags & I915_ENGINE_HAS_SEMAPHORES; 660 } 661 662 static inline bool 663 intel_engine_has_timeslices(const struct intel_engine_cs *engine) 664 { 665 if (!CONFIG_DRM_I915_TIMESLICE_DURATION) 666 return false; 667 668 return engine->flags & I915_ENGINE_HAS_TIMESLICES; 669 } 670 671 static inline bool 672 intel_engine_is_virtual(const struct intel_engine_cs *engine) 673 { 674 return engine->flags & I915_ENGINE_IS_VIRTUAL; 675 } 676 677 static inline bool 678 intel_engine_has_relative_mmio(const struct intel_engine_cs * const engine) 679 { 680 return engine->flags & I915_ENGINE_HAS_RELATIVE_MMIO; 681 } 682 683 /* Wa_14014475959:dg2 */ 684 static inline bool 685 intel_engine_uses_wa_hold_ccs_switchout(struct intel_engine_cs *engine) 686 { 687 return engine->flags & I915_ENGINE_USES_WA_HOLD_CCS_SWITCHOUT; 688 } 689 690 #endif /* __INTEL_ENGINE_TYPES_H__ */ 691