1 /* SPDX-License-Identifier: MIT */ 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #ifndef __INTEL_ENGINE_TYPES__ 7 #define __INTEL_ENGINE_TYPES__ 8 9 #include <linux/average.h> 10 #include <linux/hashtable.h> 11 #include <linux/irq_work.h> 12 #include <linux/kref.h> 13 #include <linux/list.h> 14 #include <linux/llist.h> 15 #include <linux/rbtree.h> 16 #include <linux/timer.h> 17 #include <linux/types.h> 18 #include <linux/workqueue.h> 19 20 #include "i915_gem.h" 21 #include "i915_pmu.h" 22 #include "i915_priolist_types.h" 23 #include "i915_selftest.h" 24 #include "intel_sseu.h" 25 #include "intel_timeline_types.h" 26 #include "intel_uncore.h" 27 #include "intel_wakeref.h" 28 #include "intel_workarounds_types.h" 29 30 /* HW Engine class + instance */ 31 #define RENDER_CLASS 0 32 #define VIDEO_DECODE_CLASS 1 33 #define VIDEO_ENHANCEMENT_CLASS 2 34 #define COPY_ENGINE_CLASS 3 35 #define OTHER_CLASS 4 36 #define COMPUTE_CLASS 5 37 #define MAX_ENGINE_CLASS 5 38 #define MAX_ENGINE_INSTANCE 8 39 40 #define I915_MAX_SLICES 3 41 #define I915_MAX_SUBSLICES 8 42 43 #define I915_CMD_HASH_ORDER 9 44 45 struct dma_fence; 46 struct drm_i915_gem_object; 47 struct drm_i915_reg_table; 48 struct i915_gem_context; 49 struct i915_request; 50 struct i915_sched_attr; 51 struct i915_sched_engine; 52 struct intel_gt; 53 struct intel_ring; 54 struct intel_uncore; 55 struct intel_breadcrumbs; 56 57 typedef u32 intel_engine_mask_t; 58 #define ALL_ENGINES ((intel_engine_mask_t)~0ul) 59 60 struct intel_hw_status_page { 61 struct list_head timelines; 62 struct i915_vma *vma; 63 u32 *addr; 64 }; 65 66 struct intel_instdone { 67 u32 instdone; 68 /* The following exist only in the RCS engine */ 69 u32 slice_common; 70 u32 slice_common_extra[2]; 71 u32 sampler[GEN_MAX_GSLICES][I915_MAX_SUBSLICES]; 72 u32 row[GEN_MAX_GSLICES][I915_MAX_SUBSLICES]; 73 74 /* Added in XeHPG */ 75 u32 geom_svg[GEN_MAX_GSLICES][I915_MAX_SUBSLICES]; 76 }; 77 78 /* 79 * we use a single page to load ctx workarounds so all of these 80 * values are referred in terms of dwords 81 * 82 * struct i915_wa_ctx_bb: 83 * offset: specifies batch starting position, also helpful in case 84 * if we want to have multiple batches at different offsets based on 85 * some criteria. It is not a requirement at the moment but provides 86 * an option for future use. 87 * size: size of the batch in DWORDS 88 */ 89 struct i915_ctx_workarounds { 90 struct i915_wa_ctx_bb { 91 u32 offset; 92 u32 size; 93 } indirect_ctx, per_ctx; 94 struct i915_vma *vma; 95 }; 96 97 #define I915_MAX_VCS 8 98 #define I915_MAX_VECS 4 99 #define I915_MAX_SFC (I915_MAX_VCS / 2) 100 #define I915_MAX_CCS 4 101 #define I915_MAX_RCS 1 102 #define I915_MAX_BCS 9 103 104 /* 105 * Engine IDs definitions. 106 * Keep instances of the same type engine together. 107 */ 108 enum intel_engine_id { 109 RCS0 = 0, 110 BCS0, 111 BCS1, 112 BCS2, 113 BCS3, 114 BCS4, 115 BCS5, 116 BCS6, 117 BCS7, 118 BCS8, 119 #define _BCS(n) (BCS0 + (n)) 120 VCS0, 121 VCS1, 122 VCS2, 123 VCS3, 124 VCS4, 125 VCS5, 126 VCS6, 127 VCS7, 128 #define _VCS(n) (VCS0 + (n)) 129 VECS0, 130 VECS1, 131 VECS2, 132 VECS3, 133 #define _VECS(n) (VECS0 + (n)) 134 CCS0, 135 CCS1, 136 CCS2, 137 CCS3, 138 #define _CCS(n) (CCS0 + (n)) 139 I915_NUM_ENGINES 140 #define INVALID_ENGINE ((enum intel_engine_id)-1) 141 }; 142 143 /* A simple estimator for the round-trip latency of an engine */ 144 DECLARE_EWMA(_engine_latency, 6, 4) 145 146 struct st_preempt_hang { 147 struct completion completion; 148 unsigned int count; 149 }; 150 151 /** 152 * struct intel_engine_execlists - execlist submission queue and port state 153 * 154 * The struct intel_engine_execlists represents the combined logical state of 155 * driver and the hardware state for execlist mode of submission. 156 */ 157 struct intel_engine_execlists { 158 /** 159 * @timer: kick the current context if its timeslice expires 160 */ 161 struct timer_list timer; 162 163 /** 164 * @preempt: reset the current context if it fails to give way 165 */ 166 struct timer_list preempt; 167 168 /** 169 * @ccid: identifier for contexts submitted to this engine 170 */ 171 u32 ccid; 172 173 /** 174 * @yield: CCID at the time of the last semaphore-wait interrupt. 175 * 176 * Instead of leaving a semaphore busy-spinning on an engine, we would 177 * like to switch to another ready context, i.e. yielding the semaphore 178 * timeslice. 179 */ 180 u32 yield; 181 182 /** 183 * @error_interrupt: CS Master EIR 184 * 185 * The CS generates an interrupt when it detects an error. We capture 186 * the first error interrupt, record the EIR and schedule the tasklet. 187 * In the tasklet, we process the pending CS events to ensure we have 188 * the guilty request, and then reset the engine. 189 * 190 * Low 16b are used by HW, with the upper 16b used as the enabling mask. 191 * Reserve the upper 16b for tracking internal errors. 192 */ 193 u32 error_interrupt; 194 #define ERROR_CSB BIT(31) 195 #define ERROR_PREEMPT BIT(30) 196 197 /** 198 * @reset_ccid: Active CCID [EXECLISTS_STATUS_HI] at the time of reset 199 */ 200 u32 reset_ccid; 201 202 /** 203 * @submit_reg: gen-specific execlist submission register 204 * set to the ExecList Submission Port (elsp) register pre-Gen11 and to 205 * the ExecList Submission Queue Contents register array for Gen11+ 206 */ 207 u32 __iomem *submit_reg; 208 209 /** 210 * @ctrl_reg: the enhanced execlists control register, used to load the 211 * submit queue on the HW and to request preemptions to idle 212 */ 213 u32 __iomem *ctrl_reg; 214 215 #define EXECLIST_MAX_PORTS 2 216 /** 217 * @active: the currently known context executing on HW 218 */ 219 struct i915_request * const *active; 220 /** 221 * @inflight: the set of contexts submitted and acknowleged by HW 222 * 223 * The set of inflight contexts is managed by reading CS events 224 * from the HW. On a context-switch event (not preemption), we 225 * know the HW has transitioned from port0 to port1, and we 226 * advance our inflight/active tracking accordingly. 227 */ 228 struct i915_request *inflight[EXECLIST_MAX_PORTS + 1 /* sentinel */]; 229 /** 230 * @pending: the next set of contexts submitted to ELSP 231 * 232 * We store the array of contexts that we submit to HW (via ELSP) and 233 * promote them to the inflight array once HW has signaled the 234 * preemption or idle-to-active event. 235 */ 236 struct i915_request *pending[EXECLIST_MAX_PORTS + 1]; 237 238 /** 239 * @port_mask: number of execlist ports - 1 240 */ 241 unsigned int port_mask; 242 243 /** 244 * @virtual: Queue of requets on a virtual engine, sorted by priority. 245 * Each RB entry is a struct i915_priolist containing a list of requests 246 * of the same priority. 247 */ 248 struct rb_root_cached virtual; 249 250 /** 251 * @csb_write: control register for Context Switch buffer 252 * 253 * Note this register may be either mmio or HWSP shadow. 254 */ 255 u32 *csb_write; 256 257 /** 258 * @csb_status: status array for Context Switch buffer 259 * 260 * Note these register may be either mmio or HWSP shadow. 261 */ 262 u64 *csb_status; 263 264 /** 265 * @csb_size: context status buffer FIFO size 266 */ 267 u8 csb_size; 268 269 /** 270 * @csb_head: context status buffer head 271 */ 272 u8 csb_head; 273 274 I915_SELFTEST_DECLARE(struct st_preempt_hang preempt_hang;) 275 }; 276 277 #define INTEL_ENGINE_CS_MAX_NAME 8 278 279 struct intel_engine_execlists_stats { 280 /** 281 * @active: Number of contexts currently scheduled in. 282 */ 283 unsigned int active; 284 285 /** 286 * @lock: Lock protecting the below fields. 287 */ 288 seqcount_t lock; 289 290 /** 291 * @total: Total time this engine was busy. 292 * 293 * Accumulated time not counting the most recent block in cases where 294 * engine is currently busy (active > 0). 295 */ 296 ktime_t total; 297 298 /** 299 * @start: Timestamp of the last idle to active transition. 300 * 301 * Idle is defined as active == 0, active is active > 0. 302 */ 303 ktime_t start; 304 }; 305 306 struct intel_engine_guc_stats { 307 /** 308 * @running: Active state of the engine when busyness was last sampled. 309 */ 310 bool running; 311 312 /** 313 * @prev_total: Previous value of total runtime clock cycles. 314 */ 315 u32 prev_total; 316 317 /** 318 * @total_gt_clks: Total gt clock cycles this engine was busy. 319 */ 320 u64 total_gt_clks; 321 322 /** 323 * @start_gt_clk: GT clock time of last idle to active transition. 324 */ 325 u64 start_gt_clk; 326 }; 327 328 struct intel_engine_cs { 329 struct drm_i915_private *i915; 330 struct intel_gt *gt; 331 struct intel_uncore *uncore; 332 char name[INTEL_ENGINE_CS_MAX_NAME]; 333 334 enum intel_engine_id id; 335 enum intel_engine_id legacy_idx; 336 337 unsigned int guc_id; 338 339 intel_engine_mask_t mask; 340 u32 reset_domain; 341 /** 342 * @logical_mask: logical mask of engine, reported to user space via 343 * query IOCTL and used to communicate with the GuC in logical space. 344 * The logical instance of a physical engine can change based on product 345 * and fusing. 346 */ 347 intel_engine_mask_t logical_mask; 348 349 u8 class; 350 u8 instance; 351 352 u16 uabi_class; 353 u16 uabi_instance; 354 355 u32 uabi_capabilities; 356 u32 context_size; 357 u32 mmio_base; 358 359 /* 360 * Some w/a require forcewake to be held (which prevents RC6) while 361 * a particular engine is active. If so, we set fw_domain to which 362 * domains need to be held for the duration of request activity, 363 * and 0 if none. We try to limit the duration of the hold as much 364 * as possible. 365 */ 366 enum forcewake_domains fw_domain; 367 unsigned int fw_active; 368 369 unsigned long context_tag; 370 371 struct rb_node uabi_node; 372 373 struct intel_sseu sseu; 374 375 struct i915_sched_engine *sched_engine; 376 377 /* keep a request in reserve for a [pm] barrier under oom */ 378 struct i915_request *request_pool; 379 380 struct intel_context *hung_ce; 381 382 struct llist_head barrier_tasks; 383 384 struct intel_context *kernel_context; /* pinned */ 385 386 /** 387 * pinned_contexts_list: List of pinned contexts. This list is only 388 * assumed to be manipulated during driver load- or unload time and 389 * does therefore not have any additional protection. 390 */ 391 struct list_head pinned_contexts_list; 392 393 intel_engine_mask_t saturated; /* submitting semaphores too late? */ 394 395 struct { 396 struct delayed_work work; 397 struct i915_request *systole; 398 unsigned long blocked; 399 } heartbeat; 400 401 unsigned long serial; 402 403 unsigned long wakeref_serial; 404 struct intel_wakeref wakeref; 405 struct file *default_state; 406 407 struct { 408 struct intel_ring *ring; 409 struct intel_timeline *timeline; 410 } legacy; 411 412 /* 413 * We track the average duration of the idle pulse on parking the 414 * engine to keep an estimate of the how the fast the engine is 415 * under ideal conditions. 416 */ 417 struct ewma__engine_latency latency; 418 419 /* Keep track of all the seqno used, a trail of breadcrumbs */ 420 struct intel_breadcrumbs *breadcrumbs; 421 422 struct intel_engine_pmu { 423 /** 424 * @enable: Bitmask of enable sample events on this engine. 425 * 426 * Bits correspond to sample event types, for instance 427 * I915_SAMPLE_QUEUED is bit 0 etc. 428 */ 429 u32 enable; 430 /** 431 * @enable_count: Reference count for the enabled samplers. 432 * 433 * Index number corresponds to @enum drm_i915_pmu_engine_sample. 434 */ 435 unsigned int enable_count[I915_ENGINE_SAMPLE_COUNT]; 436 /** 437 * @sample: Counter values for sampling events. 438 * 439 * Our internal timer stores the current counters in this field. 440 * 441 * Index number corresponds to @enum drm_i915_pmu_engine_sample. 442 */ 443 struct i915_pmu_sample sample[I915_ENGINE_SAMPLE_COUNT]; 444 } pmu; 445 446 struct intel_hw_status_page status_page; 447 struct i915_ctx_workarounds wa_ctx; 448 struct i915_wa_list ctx_wa_list; 449 struct i915_wa_list wa_list; 450 struct i915_wa_list whitelist; 451 452 u32 irq_keep_mask; /* always keep these interrupts */ 453 u32 irq_enable_mask; /* bitmask to enable ring interrupt */ 454 void (*irq_enable)(struct intel_engine_cs *engine); 455 void (*irq_disable)(struct intel_engine_cs *engine); 456 void (*irq_handler)(struct intel_engine_cs *engine, u16 iir); 457 458 void (*sanitize)(struct intel_engine_cs *engine); 459 int (*resume)(struct intel_engine_cs *engine); 460 461 struct { 462 void (*prepare)(struct intel_engine_cs *engine); 463 464 void (*rewind)(struct intel_engine_cs *engine, bool stalled); 465 void (*cancel)(struct intel_engine_cs *engine); 466 467 void (*finish)(struct intel_engine_cs *engine); 468 } reset; 469 470 void (*park)(struct intel_engine_cs *engine); 471 void (*unpark)(struct intel_engine_cs *engine); 472 473 void (*bump_serial)(struct intel_engine_cs *engine); 474 475 void (*set_default_submission)(struct intel_engine_cs *engine); 476 477 const struct intel_context_ops *cops; 478 479 int (*request_alloc)(struct i915_request *rq); 480 481 int (*emit_flush)(struct i915_request *request, u32 mode); 482 #define EMIT_INVALIDATE BIT(0) 483 #define EMIT_FLUSH BIT(1) 484 #define EMIT_BARRIER (EMIT_INVALIDATE | EMIT_FLUSH) 485 int (*emit_bb_start)(struct i915_request *rq, 486 u64 offset, u32 length, 487 unsigned int dispatch_flags); 488 #define I915_DISPATCH_SECURE BIT(0) 489 #define I915_DISPATCH_PINNED BIT(1) 490 int (*emit_init_breadcrumb)(struct i915_request *rq); 491 u32 *(*emit_fini_breadcrumb)(struct i915_request *rq, 492 u32 *cs); 493 unsigned int emit_fini_breadcrumb_dw; 494 495 /* Pass the request to the hardware queue (e.g. directly into 496 * the legacy ringbuffer or to the end of an execlist). 497 * 498 * This is called from an atomic context with irqs disabled; must 499 * be irq safe. 500 */ 501 void (*submit_request)(struct i915_request *rq); 502 503 void (*release)(struct intel_engine_cs *engine); 504 505 /* 506 * Add / remove request from engine active tracking 507 */ 508 void (*add_active_request)(struct i915_request *rq); 509 void (*remove_active_request)(struct i915_request *rq); 510 511 /* 512 * Get engine busyness and the time at which the busyness was sampled. 513 */ 514 ktime_t (*busyness)(struct intel_engine_cs *engine, 515 ktime_t *now); 516 517 struct intel_engine_execlists execlists; 518 519 /* 520 * Keep track of completed timelines on this engine for early 521 * retirement with the goal of quickly enabling powersaving as 522 * soon as the engine is idle. 523 */ 524 struct intel_timeline *retire; 525 struct work_struct retire_work; 526 527 /* status_notifier: list of callbacks for context-switch changes */ 528 struct atomic_notifier_head context_status_notifier; 529 530 #define I915_ENGINE_USING_CMD_PARSER BIT(0) 531 #define I915_ENGINE_SUPPORTS_STATS BIT(1) 532 #define I915_ENGINE_HAS_PREEMPTION BIT(2) 533 #define I915_ENGINE_HAS_SEMAPHORES BIT(3) 534 #define I915_ENGINE_HAS_TIMESLICES BIT(4) 535 #define I915_ENGINE_IS_VIRTUAL BIT(5) 536 #define I915_ENGINE_HAS_RELATIVE_MMIO BIT(6) 537 #define I915_ENGINE_REQUIRES_CMD_PARSER BIT(7) 538 #define I915_ENGINE_WANT_FORCED_PREEMPTION BIT(8) 539 #define I915_ENGINE_HAS_RCS_REG_STATE BIT(9) 540 #define I915_ENGINE_HAS_EU_PRIORITY BIT(10) 541 #define I915_ENGINE_FIRST_RENDER_COMPUTE BIT(11) 542 #define I915_ENGINE_USES_WA_HOLD_CCS_SWITCHOUT BIT(12) 543 unsigned int flags; 544 545 /* 546 * Table of commands the command parser needs to know about 547 * for this engine. 548 */ 549 DECLARE_HASHTABLE(cmd_hash, I915_CMD_HASH_ORDER); 550 551 /* 552 * Table of registers allowed in commands that read/write registers. 553 */ 554 const struct drm_i915_reg_table *reg_tables; 555 int reg_table_count; 556 557 /* 558 * Returns the bitmask for the length field of the specified command. 559 * Return 0 for an unrecognized/invalid command. 560 * 561 * If the command parser finds an entry for a command in the engine's 562 * cmd_tables, it gets the command's length based on the table entry. 563 * If not, it calls this function to determine the per-engine length 564 * field encoding for the command (i.e. different opcode ranges use 565 * certain bits to encode the command length in the header). 566 */ 567 u32 (*get_cmd_length_mask)(u32 cmd_header); 568 569 struct { 570 union { 571 struct intel_engine_execlists_stats execlists; 572 struct intel_engine_guc_stats guc; 573 }; 574 575 /** 576 * @rps: Utilisation at last RPS sampling. 577 */ 578 ktime_t rps; 579 } stats; 580 581 struct { 582 unsigned long heartbeat_interval_ms; 583 unsigned long max_busywait_duration_ns; 584 unsigned long preempt_timeout_ms; 585 unsigned long stop_timeout_ms; 586 unsigned long timeslice_duration_ms; 587 } props, defaults; 588 589 I915_SELFTEST_DECLARE(struct fault_attr reset_timeout); 590 }; 591 592 static inline bool 593 intel_engine_using_cmd_parser(const struct intel_engine_cs *engine) 594 { 595 return engine->flags & I915_ENGINE_USING_CMD_PARSER; 596 } 597 598 static inline bool 599 intel_engine_requires_cmd_parser(const struct intel_engine_cs *engine) 600 { 601 return engine->flags & I915_ENGINE_REQUIRES_CMD_PARSER; 602 } 603 604 static inline bool 605 intel_engine_supports_stats(const struct intel_engine_cs *engine) 606 { 607 return engine->flags & I915_ENGINE_SUPPORTS_STATS; 608 } 609 610 static inline bool 611 intel_engine_has_preemption(const struct intel_engine_cs *engine) 612 { 613 return engine->flags & I915_ENGINE_HAS_PREEMPTION; 614 } 615 616 static inline bool 617 intel_engine_has_semaphores(const struct intel_engine_cs *engine) 618 { 619 return engine->flags & I915_ENGINE_HAS_SEMAPHORES; 620 } 621 622 static inline bool 623 intel_engine_has_timeslices(const struct intel_engine_cs *engine) 624 { 625 if (!CONFIG_DRM_I915_TIMESLICE_DURATION) 626 return false; 627 628 return engine->flags & I915_ENGINE_HAS_TIMESLICES; 629 } 630 631 static inline bool 632 intel_engine_is_virtual(const struct intel_engine_cs *engine) 633 { 634 return engine->flags & I915_ENGINE_IS_VIRTUAL; 635 } 636 637 static inline bool 638 intel_engine_has_relative_mmio(const struct intel_engine_cs * const engine) 639 { 640 return engine->flags & I915_ENGINE_HAS_RELATIVE_MMIO; 641 } 642 643 /* Wa_14014475959:dg2 */ 644 static inline bool 645 intel_engine_uses_wa_hold_ccs_switchout(struct intel_engine_cs *engine) 646 { 647 return engine->flags & I915_ENGINE_USES_WA_HOLD_CCS_SWITCHOUT; 648 } 649 650 #endif /* __INTEL_ENGINE_TYPES_H__ */ 651