1 /* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <drm/drm_print.h> 26 27 #include "gem/i915_gem_context.h" 28 29 #include "i915_drv.h" 30 31 #include "intel_breadcrumbs.h" 32 #include "intel_context.h" 33 #include "intel_engine.h" 34 #include "intel_engine_pm.h" 35 #include "intel_engine_user.h" 36 #include "intel_gt.h" 37 #include "intel_gt_requests.h" 38 #include "intel_gt_pm.h" 39 #include "intel_lrc.h" 40 #include "intel_reset.h" 41 #include "intel_ring.h" 42 43 /* Haswell does have the CXT_SIZE register however it does not appear to be 44 * valid. Now, docs explain in dwords what is in the context object. The full 45 * size is 70720 bytes, however, the power context and execlist context will 46 * never be saved (power context is stored elsewhere, and execlists don't work 47 * on HSW) - so the final size, including the extra state required for the 48 * Resource Streamer, is 66944 bytes, which rounds to 17 pages. 49 */ 50 #define HSW_CXT_TOTAL_SIZE (17 * PAGE_SIZE) 51 52 #define DEFAULT_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE) 53 #define GEN8_LR_CONTEXT_RENDER_SIZE (20 * PAGE_SIZE) 54 #define GEN9_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE) 55 #define GEN10_LR_CONTEXT_RENDER_SIZE (18 * PAGE_SIZE) 56 #define GEN11_LR_CONTEXT_RENDER_SIZE (14 * PAGE_SIZE) 57 58 #define GEN8_LR_CONTEXT_OTHER_SIZE ( 2 * PAGE_SIZE) 59 60 #define MAX_MMIO_BASES 3 61 struct engine_info { 62 unsigned int hw_id; 63 u8 class; 64 u8 instance; 65 /* mmio bases table *must* be sorted in reverse gen order */ 66 struct engine_mmio_base { 67 u32 gen : 8; 68 u32 base : 24; 69 } mmio_bases[MAX_MMIO_BASES]; 70 }; 71 72 static const struct engine_info intel_engines[] = { 73 [RCS0] = { 74 .hw_id = RCS0_HW, 75 .class = RENDER_CLASS, 76 .instance = 0, 77 .mmio_bases = { 78 { .gen = 1, .base = RENDER_RING_BASE } 79 }, 80 }, 81 [BCS0] = { 82 .hw_id = BCS0_HW, 83 .class = COPY_ENGINE_CLASS, 84 .instance = 0, 85 .mmio_bases = { 86 { .gen = 6, .base = BLT_RING_BASE } 87 }, 88 }, 89 [VCS0] = { 90 .hw_id = VCS0_HW, 91 .class = VIDEO_DECODE_CLASS, 92 .instance = 0, 93 .mmio_bases = { 94 { .gen = 11, .base = GEN11_BSD_RING_BASE }, 95 { .gen = 6, .base = GEN6_BSD_RING_BASE }, 96 { .gen = 4, .base = BSD_RING_BASE } 97 }, 98 }, 99 [VCS1] = { 100 .hw_id = VCS1_HW, 101 .class = VIDEO_DECODE_CLASS, 102 .instance = 1, 103 .mmio_bases = { 104 { .gen = 11, .base = GEN11_BSD2_RING_BASE }, 105 { .gen = 8, .base = GEN8_BSD2_RING_BASE } 106 }, 107 }, 108 [VCS2] = { 109 .hw_id = VCS2_HW, 110 .class = VIDEO_DECODE_CLASS, 111 .instance = 2, 112 .mmio_bases = { 113 { .gen = 11, .base = GEN11_BSD3_RING_BASE } 114 }, 115 }, 116 [VCS3] = { 117 .hw_id = VCS3_HW, 118 .class = VIDEO_DECODE_CLASS, 119 .instance = 3, 120 .mmio_bases = { 121 { .gen = 11, .base = GEN11_BSD4_RING_BASE } 122 }, 123 }, 124 [VECS0] = { 125 .hw_id = VECS0_HW, 126 .class = VIDEO_ENHANCEMENT_CLASS, 127 .instance = 0, 128 .mmio_bases = { 129 { .gen = 11, .base = GEN11_VEBOX_RING_BASE }, 130 { .gen = 7, .base = VEBOX_RING_BASE } 131 }, 132 }, 133 [VECS1] = { 134 .hw_id = VECS1_HW, 135 .class = VIDEO_ENHANCEMENT_CLASS, 136 .instance = 1, 137 .mmio_bases = { 138 { .gen = 11, .base = GEN11_VEBOX2_RING_BASE } 139 }, 140 }, 141 }; 142 143 /** 144 * intel_engine_context_size() - return the size of the context for an engine 145 * @gt: the gt 146 * @class: engine class 147 * 148 * Each engine class may require a different amount of space for a context 149 * image. 150 * 151 * Return: size (in bytes) of an engine class specific context image 152 * 153 * Note: this size includes the HWSP, which is part of the context image 154 * in LRC mode, but does not include the "shared data page" used with 155 * GuC submission. The caller should account for this if using the GuC. 156 */ 157 u32 intel_engine_context_size(struct intel_gt *gt, u8 class) 158 { 159 struct intel_uncore *uncore = gt->uncore; 160 u32 cxt_size; 161 162 BUILD_BUG_ON(I915_GTT_PAGE_SIZE != PAGE_SIZE); 163 164 switch (class) { 165 case RENDER_CLASS: 166 switch (INTEL_GEN(gt->i915)) { 167 default: 168 MISSING_CASE(INTEL_GEN(gt->i915)); 169 return DEFAULT_LR_CONTEXT_RENDER_SIZE; 170 case 12: 171 case 11: 172 return GEN11_LR_CONTEXT_RENDER_SIZE; 173 case 10: 174 return GEN10_LR_CONTEXT_RENDER_SIZE; 175 case 9: 176 return GEN9_LR_CONTEXT_RENDER_SIZE; 177 case 8: 178 return GEN8_LR_CONTEXT_RENDER_SIZE; 179 case 7: 180 if (IS_HASWELL(gt->i915)) 181 return HSW_CXT_TOTAL_SIZE; 182 183 cxt_size = intel_uncore_read(uncore, GEN7_CXT_SIZE); 184 return round_up(GEN7_CXT_TOTAL_SIZE(cxt_size) * 64, 185 PAGE_SIZE); 186 case 6: 187 cxt_size = intel_uncore_read(uncore, CXT_SIZE); 188 return round_up(GEN6_CXT_TOTAL_SIZE(cxt_size) * 64, 189 PAGE_SIZE); 190 case 5: 191 case 4: 192 /* 193 * There is a discrepancy here between the size reported 194 * by the register and the size of the context layout 195 * in the docs. Both are described as authorative! 196 * 197 * The discrepancy is on the order of a few cachelines, 198 * but the total is under one page (4k), which is our 199 * minimum allocation anyway so it should all come 200 * out in the wash. 201 */ 202 cxt_size = intel_uncore_read(uncore, CXT_SIZE) + 1; 203 drm_dbg(>->i915->drm, 204 "gen%d CXT_SIZE = %d bytes [0x%08x]\n", 205 INTEL_GEN(gt->i915), cxt_size * 64, 206 cxt_size - 1); 207 return round_up(cxt_size * 64, PAGE_SIZE); 208 case 3: 209 case 2: 210 /* For the special day when i810 gets merged. */ 211 case 1: 212 return 0; 213 } 214 break; 215 default: 216 MISSING_CASE(class); 217 fallthrough; 218 case VIDEO_DECODE_CLASS: 219 case VIDEO_ENHANCEMENT_CLASS: 220 case COPY_ENGINE_CLASS: 221 if (INTEL_GEN(gt->i915) < 8) 222 return 0; 223 return GEN8_LR_CONTEXT_OTHER_SIZE; 224 } 225 } 226 227 static u32 __engine_mmio_base(struct drm_i915_private *i915, 228 const struct engine_mmio_base *bases) 229 { 230 int i; 231 232 for (i = 0; i < MAX_MMIO_BASES; i++) 233 if (INTEL_GEN(i915) >= bases[i].gen) 234 break; 235 236 GEM_BUG_ON(i == MAX_MMIO_BASES); 237 GEM_BUG_ON(!bases[i].base); 238 239 return bases[i].base; 240 } 241 242 static void __sprint_engine_name(struct intel_engine_cs *engine) 243 { 244 /* 245 * Before we know what the uABI name for this engine will be, 246 * we still would like to keep track of this engine in the debug logs. 247 * We throw in a ' here as a reminder that this isn't its final name. 248 */ 249 GEM_WARN_ON(snprintf(engine->name, sizeof(engine->name), "%s'%u", 250 intel_engine_class_repr(engine->class), 251 engine->instance) >= sizeof(engine->name)); 252 } 253 254 void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask) 255 { 256 /* 257 * Though they added more rings on g4x/ilk, they did not add 258 * per-engine HWSTAM until gen6. 259 */ 260 if (INTEL_GEN(engine->i915) < 6 && engine->class != RENDER_CLASS) 261 return; 262 263 if (INTEL_GEN(engine->i915) >= 3) 264 ENGINE_WRITE(engine, RING_HWSTAM, mask); 265 else 266 ENGINE_WRITE16(engine, RING_HWSTAM, mask); 267 } 268 269 static void intel_engine_sanitize_mmio(struct intel_engine_cs *engine) 270 { 271 /* Mask off all writes into the unknown HWSP */ 272 intel_engine_set_hwsp_writemask(engine, ~0u); 273 } 274 275 static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) 276 { 277 const struct engine_info *info = &intel_engines[id]; 278 struct drm_i915_private *i915 = gt->i915; 279 struct intel_engine_cs *engine; 280 281 BUILD_BUG_ON(MAX_ENGINE_CLASS >= BIT(GEN11_ENGINE_CLASS_WIDTH)); 282 BUILD_BUG_ON(MAX_ENGINE_INSTANCE >= BIT(GEN11_ENGINE_INSTANCE_WIDTH)); 283 284 if (GEM_DEBUG_WARN_ON(id >= ARRAY_SIZE(gt->engine))) 285 return -EINVAL; 286 287 if (GEM_DEBUG_WARN_ON(info->class > MAX_ENGINE_CLASS)) 288 return -EINVAL; 289 290 if (GEM_DEBUG_WARN_ON(info->instance > MAX_ENGINE_INSTANCE)) 291 return -EINVAL; 292 293 if (GEM_DEBUG_WARN_ON(gt->engine_class[info->class][info->instance])) 294 return -EINVAL; 295 296 engine = kzalloc(sizeof(*engine), GFP_KERNEL); 297 if (!engine) 298 return -ENOMEM; 299 300 BUILD_BUG_ON(BITS_PER_TYPE(engine->mask) < I915_NUM_ENGINES); 301 302 engine->id = id; 303 engine->legacy_idx = INVALID_ENGINE; 304 engine->mask = BIT(id); 305 engine->i915 = i915; 306 engine->gt = gt; 307 engine->uncore = gt->uncore; 308 engine->hw_id = engine->guc_id = info->hw_id; 309 engine->mmio_base = __engine_mmio_base(i915, info->mmio_bases); 310 311 engine->class = info->class; 312 engine->instance = info->instance; 313 __sprint_engine_name(engine); 314 315 engine->props.heartbeat_interval_ms = 316 CONFIG_DRM_I915_HEARTBEAT_INTERVAL; 317 engine->props.max_busywait_duration_ns = 318 CONFIG_DRM_I915_MAX_REQUEST_BUSYWAIT; 319 engine->props.preempt_timeout_ms = 320 CONFIG_DRM_I915_PREEMPT_TIMEOUT; 321 engine->props.stop_timeout_ms = 322 CONFIG_DRM_I915_STOP_TIMEOUT; 323 engine->props.timeslice_duration_ms = 324 CONFIG_DRM_I915_TIMESLICE_DURATION; 325 326 /* Override to uninterruptible for OpenCL workloads. */ 327 if (INTEL_GEN(i915) == 12 && engine->class == RENDER_CLASS) 328 engine->props.preempt_timeout_ms = 0; 329 330 engine->defaults = engine->props; /* never to change again */ 331 332 engine->context_size = intel_engine_context_size(gt, engine->class); 333 if (WARN_ON(engine->context_size > BIT(20))) 334 engine->context_size = 0; 335 if (engine->context_size) 336 DRIVER_CAPS(i915)->has_logical_contexts = true; 337 338 /* Nothing to do here, execute in order of dependencies */ 339 engine->schedule = NULL; 340 341 ewma__engine_latency_init(&engine->latency); 342 seqlock_init(&engine->stats.lock); 343 344 ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier); 345 346 /* Scrub mmio state on takeover */ 347 intel_engine_sanitize_mmio(engine); 348 349 gt->engine_class[info->class][info->instance] = engine; 350 gt->engine[id] = engine; 351 352 return 0; 353 } 354 355 static void __setup_engine_capabilities(struct intel_engine_cs *engine) 356 { 357 struct drm_i915_private *i915 = engine->i915; 358 359 if (engine->class == VIDEO_DECODE_CLASS) { 360 /* 361 * HEVC support is present on first engine instance 362 * before Gen11 and on all instances afterwards. 363 */ 364 if (INTEL_GEN(i915) >= 11 || 365 (INTEL_GEN(i915) >= 9 && engine->instance == 0)) 366 engine->uabi_capabilities |= 367 I915_VIDEO_CLASS_CAPABILITY_HEVC; 368 369 /* 370 * SFC block is present only on even logical engine 371 * instances. 372 */ 373 if ((INTEL_GEN(i915) >= 11 && 374 engine->gt->info.vdbox_sfc_access & engine->mask) || 375 (INTEL_GEN(i915) >= 9 && engine->instance == 0)) 376 engine->uabi_capabilities |= 377 I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC; 378 } else if (engine->class == VIDEO_ENHANCEMENT_CLASS) { 379 if (INTEL_GEN(i915) >= 9) 380 engine->uabi_capabilities |= 381 I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC; 382 } 383 } 384 385 static void intel_setup_engine_capabilities(struct intel_gt *gt) 386 { 387 struct intel_engine_cs *engine; 388 enum intel_engine_id id; 389 390 for_each_engine(engine, gt, id) 391 __setup_engine_capabilities(engine); 392 } 393 394 /** 395 * intel_engines_release() - free the resources allocated for Command Streamers 396 * @gt: pointer to struct intel_gt 397 */ 398 void intel_engines_release(struct intel_gt *gt) 399 { 400 struct intel_engine_cs *engine; 401 enum intel_engine_id id; 402 403 /* 404 * Before we release the resources held by engine, we must be certain 405 * that the HW is no longer accessing them -- having the GPU scribble 406 * to or read from a page being used for something else causes no end 407 * of fun. 408 * 409 * The GPU should be reset by this point, but assume the worst just 410 * in case we aborted before completely initialising the engines. 411 */ 412 GEM_BUG_ON(intel_gt_pm_is_awake(gt)); 413 if (!INTEL_INFO(gt->i915)->gpu_reset_clobbers_display) 414 __intel_gt_reset(gt, ALL_ENGINES); 415 416 /* Decouple the backend; but keep the layout for late GPU resets */ 417 for_each_engine(engine, gt, id) { 418 if (!engine->release) 419 continue; 420 421 intel_wakeref_wait_for_idle(&engine->wakeref); 422 GEM_BUG_ON(intel_engine_pm_is_awake(engine)); 423 424 engine->release(engine); 425 engine->release = NULL; 426 427 memset(&engine->reset, 0, sizeof(engine->reset)); 428 } 429 } 430 431 void intel_engine_free_request_pool(struct intel_engine_cs *engine) 432 { 433 if (!engine->request_pool) 434 return; 435 436 kmem_cache_free(i915_request_slab_cache(), engine->request_pool); 437 } 438 439 void intel_engines_free(struct intel_gt *gt) 440 { 441 struct intel_engine_cs *engine; 442 enum intel_engine_id id; 443 444 /* Free the requests! dma-resv keeps fences around for an eternity */ 445 rcu_barrier(); 446 447 for_each_engine(engine, gt, id) { 448 intel_engine_free_request_pool(engine); 449 kfree(engine); 450 gt->engine[id] = NULL; 451 } 452 } 453 454 /* 455 * Determine which engines are fused off in our particular hardware. 456 * Note that we have a catch-22 situation where we need to be able to access 457 * the blitter forcewake domain to read the engine fuses, but at the same time 458 * we need to know which engines are available on the system to know which 459 * forcewake domains are present. We solve this by intializing the forcewake 460 * domains based on the full engine mask in the platform capabilities before 461 * calling this function and pruning the domains for fused-off engines 462 * afterwards. 463 */ 464 static intel_engine_mask_t init_engine_mask(struct intel_gt *gt) 465 { 466 struct drm_i915_private *i915 = gt->i915; 467 struct intel_gt_info *info = >->info; 468 struct intel_uncore *uncore = gt->uncore; 469 unsigned int logical_vdbox = 0; 470 unsigned int i; 471 u32 media_fuse; 472 u16 vdbox_mask; 473 u16 vebox_mask; 474 475 info->engine_mask = INTEL_INFO(i915)->platform_engine_mask; 476 477 if (INTEL_GEN(i915) < 11) 478 return info->engine_mask; 479 480 media_fuse = ~intel_uncore_read(uncore, GEN11_GT_VEBOX_VDBOX_DISABLE); 481 482 vdbox_mask = media_fuse & GEN11_GT_VDBOX_DISABLE_MASK; 483 vebox_mask = (media_fuse & GEN11_GT_VEBOX_DISABLE_MASK) >> 484 GEN11_GT_VEBOX_DISABLE_SHIFT; 485 486 for (i = 0; i < I915_MAX_VCS; i++) { 487 if (!HAS_ENGINE(gt, _VCS(i))) { 488 vdbox_mask &= ~BIT(i); 489 continue; 490 } 491 492 if (!(BIT(i) & vdbox_mask)) { 493 info->engine_mask &= ~BIT(_VCS(i)); 494 drm_dbg(&i915->drm, "vcs%u fused off\n", i); 495 continue; 496 } 497 498 /* 499 * In Gen11, only even numbered logical VDBOXes are 500 * hooked up to an SFC (Scaler & Format Converter) unit. 501 * In TGL each VDBOX has access to an SFC. 502 */ 503 if (INTEL_GEN(i915) >= 12 || logical_vdbox++ % 2 == 0) 504 gt->info.vdbox_sfc_access |= BIT(i); 505 } 506 drm_dbg(&i915->drm, "vdbox enable: %04x, instances: %04lx\n", 507 vdbox_mask, VDBOX_MASK(gt)); 508 GEM_BUG_ON(vdbox_mask != VDBOX_MASK(gt)); 509 510 for (i = 0; i < I915_MAX_VECS; i++) { 511 if (!HAS_ENGINE(gt, _VECS(i))) { 512 vebox_mask &= ~BIT(i); 513 continue; 514 } 515 516 if (!(BIT(i) & vebox_mask)) { 517 info->engine_mask &= ~BIT(_VECS(i)); 518 drm_dbg(&i915->drm, "vecs%u fused off\n", i); 519 } 520 } 521 drm_dbg(&i915->drm, "vebox enable: %04x, instances: %04lx\n", 522 vebox_mask, VEBOX_MASK(gt)); 523 GEM_BUG_ON(vebox_mask != VEBOX_MASK(gt)); 524 525 return info->engine_mask; 526 } 527 528 /** 529 * intel_engines_init_mmio() - allocate and prepare the Engine Command Streamers 530 * @gt: pointer to struct intel_gt 531 * 532 * Return: non-zero if the initialization failed. 533 */ 534 int intel_engines_init_mmio(struct intel_gt *gt) 535 { 536 struct drm_i915_private *i915 = gt->i915; 537 const unsigned int engine_mask = init_engine_mask(gt); 538 unsigned int mask = 0; 539 unsigned int i; 540 int err; 541 542 drm_WARN_ON(&i915->drm, engine_mask == 0); 543 drm_WARN_ON(&i915->drm, engine_mask & 544 GENMASK(BITS_PER_TYPE(mask) - 1, I915_NUM_ENGINES)); 545 546 if (i915_inject_probe_failure(i915)) 547 return -ENODEV; 548 549 for (i = 0; i < ARRAY_SIZE(intel_engines); i++) { 550 if (!HAS_ENGINE(gt, i)) 551 continue; 552 553 err = intel_engine_setup(gt, i); 554 if (err) 555 goto cleanup; 556 557 mask |= BIT(i); 558 } 559 560 /* 561 * Catch failures to update intel_engines table when the new engines 562 * are added to the driver by a warning and disabling the forgotten 563 * engines. 564 */ 565 if (drm_WARN_ON(&i915->drm, mask != engine_mask)) 566 gt->info.engine_mask = mask; 567 568 gt->info.num_engines = hweight32(mask); 569 570 intel_gt_check_and_clear_faults(gt); 571 572 intel_setup_engine_capabilities(gt); 573 574 intel_uncore_prune_engine_fw_domains(gt->uncore, gt); 575 576 return 0; 577 578 cleanup: 579 intel_engines_free(gt); 580 return err; 581 } 582 583 void intel_engine_init_execlists(struct intel_engine_cs *engine) 584 { 585 struct intel_engine_execlists * const execlists = &engine->execlists; 586 587 execlists->port_mask = 1; 588 GEM_BUG_ON(!is_power_of_2(execlists_num_ports(execlists))); 589 GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS); 590 591 memset(execlists->pending, 0, sizeof(execlists->pending)); 592 execlists->active = 593 memset(execlists->inflight, 0, sizeof(execlists->inflight)); 594 595 execlists->queue_priority_hint = INT_MIN; 596 execlists->queue = RB_ROOT_CACHED; 597 } 598 599 static void cleanup_status_page(struct intel_engine_cs *engine) 600 { 601 struct i915_vma *vma; 602 603 /* Prevent writes into HWSP after returning the page to the system */ 604 intel_engine_set_hwsp_writemask(engine, ~0u); 605 606 vma = fetch_and_zero(&engine->status_page.vma); 607 if (!vma) 608 return; 609 610 if (!HWS_NEEDS_PHYSICAL(engine->i915)) 611 i915_vma_unpin(vma); 612 613 i915_gem_object_unpin_map(vma->obj); 614 i915_gem_object_put(vma->obj); 615 } 616 617 static int pin_ggtt_status_page(struct intel_engine_cs *engine, 618 struct i915_vma *vma) 619 { 620 unsigned int flags; 621 622 if (!HAS_LLC(engine->i915) && i915_ggtt_has_aperture(engine->gt->ggtt)) 623 /* 624 * On g33, we cannot place HWS above 256MiB, so 625 * restrict its pinning to the low mappable arena. 626 * Though this restriction is not documented for 627 * gen4, gen5, or byt, they also behave similarly 628 * and hang if the HWS is placed at the top of the 629 * GTT. To generalise, it appears that all !llc 630 * platforms have issues with us placing the HWS 631 * above the mappable region (even though we never 632 * actually map it). 633 */ 634 flags = PIN_MAPPABLE; 635 else 636 flags = PIN_HIGH; 637 638 return i915_ggtt_pin(vma, NULL, 0, flags); 639 } 640 641 static int init_status_page(struct intel_engine_cs *engine) 642 { 643 struct drm_i915_gem_object *obj; 644 struct i915_vma *vma; 645 void *vaddr; 646 int ret; 647 648 /* 649 * Though the HWS register does support 36bit addresses, historically 650 * we have had hangs and corruption reported due to wild writes if 651 * the HWS is placed above 4G. We only allow objects to be allocated 652 * in GFP_DMA32 for i965, and no earlier physical address users had 653 * access to more than 4G. 654 */ 655 obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE); 656 if (IS_ERR(obj)) { 657 drm_err(&engine->i915->drm, 658 "Failed to allocate status page\n"); 659 return PTR_ERR(obj); 660 } 661 662 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC); 663 664 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); 665 if (IS_ERR(vma)) { 666 ret = PTR_ERR(vma); 667 goto err; 668 } 669 670 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); 671 if (IS_ERR(vaddr)) { 672 ret = PTR_ERR(vaddr); 673 goto err; 674 } 675 676 engine->status_page.addr = memset(vaddr, 0, PAGE_SIZE); 677 engine->status_page.vma = vma; 678 679 if (!HWS_NEEDS_PHYSICAL(engine->i915)) { 680 ret = pin_ggtt_status_page(engine, vma); 681 if (ret) 682 goto err_unpin; 683 } 684 685 return 0; 686 687 err_unpin: 688 i915_gem_object_unpin_map(obj); 689 err: 690 i915_gem_object_put(obj); 691 return ret; 692 } 693 694 static int engine_setup_common(struct intel_engine_cs *engine) 695 { 696 int err; 697 698 init_llist_head(&engine->barrier_tasks); 699 700 err = init_status_page(engine); 701 if (err) 702 return err; 703 704 engine->breadcrumbs = intel_breadcrumbs_create(engine); 705 if (!engine->breadcrumbs) { 706 err = -ENOMEM; 707 goto err_status; 708 } 709 710 intel_engine_init_active(engine, ENGINE_PHYSICAL); 711 intel_engine_init_execlists(engine); 712 intel_engine_init_cmd_parser(engine); 713 intel_engine_init__pm(engine); 714 intel_engine_init_retire(engine); 715 716 /* Use the whole device by default */ 717 engine->sseu = 718 intel_sseu_from_device_info(&engine->gt->info.sseu); 719 720 intel_engine_init_workarounds(engine); 721 intel_engine_init_whitelist(engine); 722 intel_engine_init_ctx_wa(engine); 723 724 return 0; 725 726 err_status: 727 cleanup_status_page(engine); 728 return err; 729 } 730 731 struct measure_breadcrumb { 732 struct i915_request rq; 733 struct intel_ring ring; 734 u32 cs[2048]; 735 }; 736 737 static int measure_breadcrumb_dw(struct intel_context *ce) 738 { 739 struct intel_engine_cs *engine = ce->engine; 740 struct measure_breadcrumb *frame; 741 int dw; 742 743 GEM_BUG_ON(!engine->gt->scratch); 744 745 frame = kzalloc(sizeof(*frame), GFP_KERNEL); 746 if (!frame) 747 return -ENOMEM; 748 749 frame->rq.engine = engine; 750 frame->rq.context = ce; 751 rcu_assign_pointer(frame->rq.timeline, ce->timeline); 752 753 frame->ring.vaddr = frame->cs; 754 frame->ring.size = sizeof(frame->cs); 755 frame->ring.wrap = 756 BITS_PER_TYPE(frame->ring.size) - ilog2(frame->ring.size); 757 frame->ring.effective_size = frame->ring.size; 758 intel_ring_update_space(&frame->ring); 759 frame->rq.ring = &frame->ring; 760 761 mutex_lock(&ce->timeline->mutex); 762 spin_lock_irq(&engine->active.lock); 763 764 dw = engine->emit_fini_breadcrumb(&frame->rq, frame->cs) - frame->cs; 765 766 spin_unlock_irq(&engine->active.lock); 767 mutex_unlock(&ce->timeline->mutex); 768 769 GEM_BUG_ON(dw & 1); /* RING_TAIL must be qword aligned */ 770 771 kfree(frame); 772 return dw; 773 } 774 775 void 776 intel_engine_init_active(struct intel_engine_cs *engine, unsigned int subclass) 777 { 778 INIT_LIST_HEAD(&engine->active.requests); 779 INIT_LIST_HEAD(&engine->active.hold); 780 781 spin_lock_init(&engine->active.lock); 782 lockdep_set_subclass(&engine->active.lock, subclass); 783 784 /* 785 * Due to an interesting quirk in lockdep's internal debug tracking, 786 * after setting a subclass we must ensure the lock is used. Otherwise, 787 * nr_unused_locks is incremented once too often. 788 */ 789 #ifdef CONFIG_DEBUG_LOCK_ALLOC 790 local_irq_disable(); 791 lock_map_acquire(&engine->active.lock.dep_map); 792 lock_map_release(&engine->active.lock.dep_map); 793 local_irq_enable(); 794 #endif 795 } 796 797 static struct intel_context * 798 create_pinned_context(struct intel_engine_cs *engine, 799 unsigned int hwsp, 800 struct lock_class_key *key, 801 const char *name) 802 { 803 struct intel_context *ce; 804 int err; 805 806 ce = intel_context_create(engine); 807 if (IS_ERR(ce)) 808 return ce; 809 810 __set_bit(CONTEXT_BARRIER_BIT, &ce->flags); 811 ce->timeline = page_pack_bits(NULL, hwsp); 812 813 err = intel_context_pin(ce); /* perma-pin so it is always available */ 814 if (err) { 815 intel_context_put(ce); 816 return ERR_PTR(err); 817 } 818 819 /* 820 * Give our perma-pinned kernel timelines a separate lockdep class, 821 * so that we can use them from within the normal user timelines 822 * should we need to inject GPU operations during their request 823 * construction. 824 */ 825 lockdep_set_class_and_name(&ce->timeline->mutex, key, name); 826 827 return ce; 828 } 829 830 static struct intel_context * 831 create_kernel_context(struct intel_engine_cs *engine) 832 { 833 static struct lock_class_key kernel; 834 835 return create_pinned_context(engine, I915_GEM_HWS_SEQNO_ADDR, 836 &kernel, "kernel_context"); 837 } 838 839 /** 840 * intel_engines_init_common - initialize cengine state which might require hw access 841 * @engine: Engine to initialize. 842 * 843 * Initializes @engine@ structure members shared between legacy and execlists 844 * submission modes which do require hardware access. 845 * 846 * Typcally done at later stages of submission mode specific engine setup. 847 * 848 * Returns zero on success or an error code on failure. 849 */ 850 static int engine_init_common(struct intel_engine_cs *engine) 851 { 852 struct intel_context *ce; 853 int ret; 854 855 engine->set_default_submission(engine); 856 857 /* 858 * We may need to do things with the shrinker which 859 * require us to immediately switch back to the default 860 * context. This can cause a problem as pinning the 861 * default context also requires GTT space which may not 862 * be available. To avoid this we always pin the default 863 * context. 864 */ 865 ce = create_kernel_context(engine); 866 if (IS_ERR(ce)) 867 return PTR_ERR(ce); 868 869 ret = measure_breadcrumb_dw(ce); 870 if (ret < 0) 871 goto err_context; 872 873 engine->emit_fini_breadcrumb_dw = ret; 874 engine->kernel_context = ce; 875 876 return 0; 877 878 err_context: 879 intel_context_put(ce); 880 return ret; 881 } 882 883 int intel_engines_init(struct intel_gt *gt) 884 { 885 int (*setup)(struct intel_engine_cs *engine); 886 struct intel_engine_cs *engine; 887 enum intel_engine_id id; 888 int err; 889 890 if (HAS_EXECLISTS(gt->i915)) 891 setup = intel_execlists_submission_setup; 892 else 893 setup = intel_ring_submission_setup; 894 895 for_each_engine(engine, gt, id) { 896 err = engine_setup_common(engine); 897 if (err) 898 return err; 899 900 err = setup(engine); 901 if (err) 902 return err; 903 904 err = engine_init_common(engine); 905 if (err) 906 return err; 907 908 intel_engine_add_user(engine); 909 } 910 911 return 0; 912 } 913 914 /** 915 * intel_engines_cleanup_common - cleans up the engine state created by 916 * the common initiailizers. 917 * @engine: Engine to cleanup. 918 * 919 * This cleans up everything created by the common helpers. 920 */ 921 void intel_engine_cleanup_common(struct intel_engine_cs *engine) 922 { 923 GEM_BUG_ON(!list_empty(&engine->active.requests)); 924 tasklet_kill(&engine->execlists.tasklet); /* flush the callback */ 925 926 cleanup_status_page(engine); 927 intel_breadcrumbs_free(engine->breadcrumbs); 928 929 intel_engine_fini_retire(engine); 930 intel_engine_cleanup_cmd_parser(engine); 931 932 if (engine->default_state) 933 fput(engine->default_state); 934 935 if (engine->kernel_context) { 936 intel_context_unpin(engine->kernel_context); 937 intel_context_put(engine->kernel_context); 938 } 939 GEM_BUG_ON(!llist_empty(&engine->barrier_tasks)); 940 941 intel_wa_list_free(&engine->ctx_wa_list); 942 intel_wa_list_free(&engine->wa_list); 943 intel_wa_list_free(&engine->whitelist); 944 } 945 946 /** 947 * intel_engine_resume - re-initializes the HW state of the engine 948 * @engine: Engine to resume. 949 * 950 * Returns zero on success or an error code on failure. 951 */ 952 int intel_engine_resume(struct intel_engine_cs *engine) 953 { 954 intel_engine_apply_workarounds(engine); 955 intel_engine_apply_whitelist(engine); 956 957 return engine->resume(engine); 958 } 959 960 u64 intel_engine_get_active_head(const struct intel_engine_cs *engine) 961 { 962 struct drm_i915_private *i915 = engine->i915; 963 964 u64 acthd; 965 966 if (INTEL_GEN(i915) >= 8) 967 acthd = ENGINE_READ64(engine, RING_ACTHD, RING_ACTHD_UDW); 968 else if (INTEL_GEN(i915) >= 4) 969 acthd = ENGINE_READ(engine, RING_ACTHD); 970 else 971 acthd = ENGINE_READ(engine, ACTHD); 972 973 return acthd; 974 } 975 976 u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine) 977 { 978 u64 bbaddr; 979 980 if (INTEL_GEN(engine->i915) >= 8) 981 bbaddr = ENGINE_READ64(engine, RING_BBADDR, RING_BBADDR_UDW); 982 else 983 bbaddr = ENGINE_READ(engine, RING_BBADDR); 984 985 return bbaddr; 986 } 987 988 static unsigned long stop_timeout(const struct intel_engine_cs *engine) 989 { 990 if (in_atomic() || irqs_disabled()) /* inside atomic preempt-reset? */ 991 return 0; 992 993 /* 994 * If we are doing a normal GPU reset, we can take our time and allow 995 * the engine to quiesce. We've stopped submission to the engine, and 996 * if we wait long enough an innocent context should complete and 997 * leave the engine idle. So they should not be caught unaware by 998 * the forthcoming GPU reset (which usually follows the stop_cs)! 999 */ 1000 return READ_ONCE(engine->props.stop_timeout_ms); 1001 } 1002 1003 int intel_engine_stop_cs(struct intel_engine_cs *engine) 1004 { 1005 struct intel_uncore *uncore = engine->uncore; 1006 const u32 base = engine->mmio_base; 1007 const i915_reg_t mode = RING_MI_MODE(base); 1008 int err; 1009 1010 if (INTEL_GEN(engine->i915) < 3) 1011 return -ENODEV; 1012 1013 ENGINE_TRACE(engine, "\n"); 1014 1015 intel_uncore_write_fw(uncore, mode, _MASKED_BIT_ENABLE(STOP_RING)); 1016 1017 err = 0; 1018 if (__intel_wait_for_register_fw(uncore, 1019 mode, MODE_IDLE, MODE_IDLE, 1020 1000, stop_timeout(engine), 1021 NULL)) { 1022 ENGINE_TRACE(engine, "timed out on STOP_RING -> IDLE\n"); 1023 err = -ETIMEDOUT; 1024 } 1025 1026 /* A final mmio read to let GPU writes be hopefully flushed to memory */ 1027 intel_uncore_posting_read_fw(uncore, mode); 1028 1029 return err; 1030 } 1031 1032 void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine) 1033 { 1034 ENGINE_TRACE(engine, "\n"); 1035 1036 ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING)); 1037 } 1038 1039 const char *i915_cache_level_str(struct drm_i915_private *i915, int type) 1040 { 1041 switch (type) { 1042 case I915_CACHE_NONE: return " uncached"; 1043 case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped"; 1044 case I915_CACHE_L3_LLC: return " L3+LLC"; 1045 case I915_CACHE_WT: return " WT"; 1046 default: return ""; 1047 } 1048 } 1049 1050 static u32 1051 read_subslice_reg(const struct intel_engine_cs *engine, 1052 int slice, int subslice, i915_reg_t reg) 1053 { 1054 struct drm_i915_private *i915 = engine->i915; 1055 struct intel_uncore *uncore = engine->uncore; 1056 u32 mcr_mask, mcr_ss, mcr, old_mcr, val; 1057 enum forcewake_domains fw_domains; 1058 1059 if (INTEL_GEN(i915) >= 11) { 1060 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK; 1061 mcr_ss = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice); 1062 } else { 1063 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK; 1064 mcr_ss = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice); 1065 } 1066 1067 fw_domains = intel_uncore_forcewake_for_reg(uncore, reg, 1068 FW_REG_READ); 1069 fw_domains |= intel_uncore_forcewake_for_reg(uncore, 1070 GEN8_MCR_SELECTOR, 1071 FW_REG_READ | FW_REG_WRITE); 1072 1073 spin_lock_irq(&uncore->lock); 1074 intel_uncore_forcewake_get__locked(uncore, fw_domains); 1075 1076 old_mcr = mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR); 1077 1078 mcr &= ~mcr_mask; 1079 mcr |= mcr_ss; 1080 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr); 1081 1082 val = intel_uncore_read_fw(uncore, reg); 1083 1084 mcr &= ~mcr_mask; 1085 mcr |= old_mcr & mcr_mask; 1086 1087 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr); 1088 1089 intel_uncore_forcewake_put__locked(uncore, fw_domains); 1090 spin_unlock_irq(&uncore->lock); 1091 1092 return val; 1093 } 1094 1095 /* NB: please notice the memset */ 1096 void intel_engine_get_instdone(const struct intel_engine_cs *engine, 1097 struct intel_instdone *instdone) 1098 { 1099 struct drm_i915_private *i915 = engine->i915; 1100 const struct sseu_dev_info *sseu = &engine->gt->info.sseu; 1101 struct intel_uncore *uncore = engine->uncore; 1102 u32 mmio_base = engine->mmio_base; 1103 int slice; 1104 int subslice; 1105 1106 memset(instdone, 0, sizeof(*instdone)); 1107 1108 switch (INTEL_GEN(i915)) { 1109 default: 1110 instdone->instdone = 1111 intel_uncore_read(uncore, RING_INSTDONE(mmio_base)); 1112 1113 if (engine->id != RCS0) 1114 break; 1115 1116 instdone->slice_common = 1117 intel_uncore_read(uncore, GEN7_SC_INSTDONE); 1118 if (INTEL_GEN(i915) >= 12) { 1119 instdone->slice_common_extra[0] = 1120 intel_uncore_read(uncore, GEN12_SC_INSTDONE_EXTRA); 1121 instdone->slice_common_extra[1] = 1122 intel_uncore_read(uncore, GEN12_SC_INSTDONE_EXTRA2); 1123 } 1124 for_each_instdone_slice_subslice(i915, sseu, slice, subslice) { 1125 instdone->sampler[slice][subslice] = 1126 read_subslice_reg(engine, slice, subslice, 1127 GEN7_SAMPLER_INSTDONE); 1128 instdone->row[slice][subslice] = 1129 read_subslice_reg(engine, slice, subslice, 1130 GEN7_ROW_INSTDONE); 1131 } 1132 break; 1133 case 7: 1134 instdone->instdone = 1135 intel_uncore_read(uncore, RING_INSTDONE(mmio_base)); 1136 1137 if (engine->id != RCS0) 1138 break; 1139 1140 instdone->slice_common = 1141 intel_uncore_read(uncore, GEN7_SC_INSTDONE); 1142 instdone->sampler[0][0] = 1143 intel_uncore_read(uncore, GEN7_SAMPLER_INSTDONE); 1144 instdone->row[0][0] = 1145 intel_uncore_read(uncore, GEN7_ROW_INSTDONE); 1146 1147 break; 1148 case 6: 1149 case 5: 1150 case 4: 1151 instdone->instdone = 1152 intel_uncore_read(uncore, RING_INSTDONE(mmio_base)); 1153 if (engine->id == RCS0) 1154 /* HACK: Using the wrong struct member */ 1155 instdone->slice_common = 1156 intel_uncore_read(uncore, GEN4_INSTDONE1); 1157 break; 1158 case 3: 1159 case 2: 1160 instdone->instdone = intel_uncore_read(uncore, GEN2_INSTDONE); 1161 break; 1162 } 1163 } 1164 1165 static bool ring_is_idle(struct intel_engine_cs *engine) 1166 { 1167 bool idle = true; 1168 1169 if (I915_SELFTEST_ONLY(!engine->mmio_base)) 1170 return true; 1171 1172 if (!intel_engine_pm_get_if_awake(engine)) 1173 return true; 1174 1175 /* First check that no commands are left in the ring */ 1176 if ((ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR) != 1177 (ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR)) 1178 idle = false; 1179 1180 /* No bit for gen2, so assume the CS parser is idle */ 1181 if (INTEL_GEN(engine->i915) > 2 && 1182 !(ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE)) 1183 idle = false; 1184 1185 intel_engine_pm_put(engine); 1186 1187 return idle; 1188 } 1189 1190 void intel_engine_flush_submission(struct intel_engine_cs *engine) 1191 { 1192 struct tasklet_struct *t = &engine->execlists.tasklet; 1193 1194 if (!t->func) 1195 return; 1196 1197 /* Synchronise and wait for the tasklet on another CPU */ 1198 tasklet_kill(t); 1199 1200 /* Having cancelled the tasklet, ensure that is run */ 1201 local_bh_disable(); 1202 if (tasklet_trylock(t)) { 1203 /* Must wait for any GPU reset in progress. */ 1204 if (__tasklet_is_enabled(t)) 1205 t->func(t->data); 1206 tasklet_unlock(t); 1207 } 1208 local_bh_enable(); 1209 } 1210 1211 /** 1212 * intel_engine_is_idle() - Report if the engine has finished process all work 1213 * @engine: the intel_engine_cs 1214 * 1215 * Return true if there are no requests pending, nothing left to be submitted 1216 * to hardware, and that the engine is idle. 1217 */ 1218 bool intel_engine_is_idle(struct intel_engine_cs *engine) 1219 { 1220 /* More white lies, if wedged, hw state is inconsistent */ 1221 if (intel_gt_is_wedged(engine->gt)) 1222 return true; 1223 1224 if (!intel_engine_pm_is_awake(engine)) 1225 return true; 1226 1227 /* Waiting to drain ELSP? */ 1228 if (execlists_active(&engine->execlists)) { 1229 synchronize_hardirq(engine->i915->drm.pdev->irq); 1230 1231 intel_engine_flush_submission(engine); 1232 1233 if (execlists_active(&engine->execlists)) 1234 return false; 1235 } 1236 1237 /* ELSP is empty, but there are ready requests? E.g. after reset */ 1238 if (!RB_EMPTY_ROOT(&engine->execlists.queue.rb_root)) 1239 return false; 1240 1241 /* Ring stopped? */ 1242 return ring_is_idle(engine); 1243 } 1244 1245 bool intel_engines_are_idle(struct intel_gt *gt) 1246 { 1247 struct intel_engine_cs *engine; 1248 enum intel_engine_id id; 1249 1250 /* 1251 * If the driver is wedged, HW state may be very inconsistent and 1252 * report that it is still busy, even though we have stopped using it. 1253 */ 1254 if (intel_gt_is_wedged(gt)) 1255 return true; 1256 1257 /* Already parked (and passed an idleness test); must still be idle */ 1258 if (!READ_ONCE(gt->awake)) 1259 return true; 1260 1261 for_each_engine(engine, gt, id) { 1262 if (!intel_engine_is_idle(engine)) 1263 return false; 1264 } 1265 1266 return true; 1267 } 1268 1269 void intel_engines_reset_default_submission(struct intel_gt *gt) 1270 { 1271 struct intel_engine_cs *engine; 1272 enum intel_engine_id id; 1273 1274 for_each_engine(engine, gt, id) 1275 engine->set_default_submission(engine); 1276 } 1277 1278 bool intel_engine_can_store_dword(struct intel_engine_cs *engine) 1279 { 1280 switch (INTEL_GEN(engine->i915)) { 1281 case 2: 1282 return false; /* uses physical not virtual addresses */ 1283 case 3: 1284 /* maybe only uses physical not virtual addresses */ 1285 return !(IS_I915G(engine->i915) || IS_I915GM(engine->i915)); 1286 case 4: 1287 return !IS_I965G(engine->i915); /* who knows! */ 1288 case 6: 1289 return engine->class != VIDEO_DECODE_CLASS; /* b0rked */ 1290 default: 1291 return true; 1292 } 1293 } 1294 1295 static int print_sched_attr(const struct i915_sched_attr *attr, 1296 char *buf, int x, int len) 1297 { 1298 if (attr->priority == I915_PRIORITY_INVALID) 1299 return x; 1300 1301 x += snprintf(buf + x, len - x, 1302 " prio=%d", attr->priority); 1303 1304 return x; 1305 } 1306 1307 static void print_request(struct drm_printer *m, 1308 struct i915_request *rq, 1309 const char *prefix) 1310 { 1311 const char *name = rq->fence.ops->get_timeline_name(&rq->fence); 1312 char buf[80] = ""; 1313 int x = 0; 1314 1315 x = print_sched_attr(&rq->sched.attr, buf, x, sizeof(buf)); 1316 1317 drm_printf(m, "%s %llx:%llx%s%s %s @ %dms: %s\n", 1318 prefix, 1319 rq->fence.context, rq->fence.seqno, 1320 i915_request_completed(rq) ? "!" : 1321 i915_request_started(rq) ? "*" : 1322 "", 1323 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, 1324 &rq->fence.flags) ? "+" : 1325 test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, 1326 &rq->fence.flags) ? "-" : 1327 "", 1328 buf, 1329 jiffies_to_msecs(jiffies - rq->emitted_jiffies), 1330 name); 1331 } 1332 1333 static struct intel_timeline *get_timeline(struct i915_request *rq) 1334 { 1335 struct intel_timeline *tl; 1336 1337 /* 1338 * Even though we are holding the engine->active.lock here, there 1339 * is no control over the submission queue per-se and we are 1340 * inspecting the active state at a random point in time, with an 1341 * unknown queue. Play safe and make sure the timeline remains valid. 1342 * (Only being used for pretty printing, one extra kref shouldn't 1343 * cause a camel stampede!) 1344 */ 1345 rcu_read_lock(); 1346 tl = rcu_dereference(rq->timeline); 1347 if (!kref_get_unless_zero(&tl->kref)) 1348 tl = NULL; 1349 rcu_read_unlock(); 1350 1351 return tl; 1352 } 1353 1354 static int print_ring(char *buf, int sz, struct i915_request *rq) 1355 { 1356 int len = 0; 1357 1358 if (!i915_request_signaled(rq)) { 1359 struct intel_timeline *tl = get_timeline(rq); 1360 1361 len = scnprintf(buf, sz, 1362 "ring:{start:%08x, hwsp:%08x, seqno:%08x, runtime:%llums}, ", 1363 i915_ggtt_offset(rq->ring->vma), 1364 tl ? tl->hwsp_offset : 0, 1365 hwsp_seqno(rq), 1366 DIV_ROUND_CLOSEST_ULL(intel_context_get_total_runtime_ns(rq->context), 1367 1000 * 1000)); 1368 1369 if (tl) 1370 intel_timeline_put(tl); 1371 } 1372 1373 return len; 1374 } 1375 1376 static void hexdump(struct drm_printer *m, const void *buf, size_t len) 1377 { 1378 const size_t rowsize = 8 * sizeof(u32); 1379 const void *prev = NULL; 1380 bool skip = false; 1381 size_t pos; 1382 1383 for (pos = 0; pos < len; pos += rowsize) { 1384 char line[128]; 1385 1386 if (prev && !memcmp(prev, buf + pos, rowsize)) { 1387 if (!skip) { 1388 drm_printf(m, "*\n"); 1389 skip = true; 1390 } 1391 continue; 1392 } 1393 1394 WARN_ON_ONCE(hex_dump_to_buffer(buf + pos, len - pos, 1395 rowsize, sizeof(u32), 1396 line, sizeof(line), 1397 false) >= sizeof(line)); 1398 drm_printf(m, "[%04zx] %s\n", pos, line); 1399 1400 prev = buf + pos; 1401 skip = false; 1402 } 1403 } 1404 1405 static const char *repr_timer(const struct timer_list *t) 1406 { 1407 if (!READ_ONCE(t->expires)) 1408 return "inactive"; 1409 1410 if (timer_pending(t)) 1411 return "active"; 1412 1413 return "expired"; 1414 } 1415 1416 static void intel_engine_print_registers(struct intel_engine_cs *engine, 1417 struct drm_printer *m) 1418 { 1419 struct drm_i915_private *dev_priv = engine->i915; 1420 struct intel_engine_execlists * const execlists = &engine->execlists; 1421 u64 addr; 1422 1423 if (engine->id == RENDER_CLASS && IS_GEN_RANGE(dev_priv, 4, 7)) 1424 drm_printf(m, "\tCCID: 0x%08x\n", ENGINE_READ(engine, CCID)); 1425 if (HAS_EXECLISTS(dev_priv)) { 1426 drm_printf(m, "\tEL_STAT_HI: 0x%08x\n", 1427 ENGINE_READ(engine, RING_EXECLIST_STATUS_HI)); 1428 drm_printf(m, "\tEL_STAT_LO: 0x%08x\n", 1429 ENGINE_READ(engine, RING_EXECLIST_STATUS_LO)); 1430 } 1431 drm_printf(m, "\tRING_START: 0x%08x\n", 1432 ENGINE_READ(engine, RING_START)); 1433 drm_printf(m, "\tRING_HEAD: 0x%08x\n", 1434 ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR); 1435 drm_printf(m, "\tRING_TAIL: 0x%08x\n", 1436 ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR); 1437 drm_printf(m, "\tRING_CTL: 0x%08x%s\n", 1438 ENGINE_READ(engine, RING_CTL), 1439 ENGINE_READ(engine, RING_CTL) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? " [waiting]" : ""); 1440 if (INTEL_GEN(engine->i915) > 2) { 1441 drm_printf(m, "\tRING_MODE: 0x%08x%s\n", 1442 ENGINE_READ(engine, RING_MI_MODE), 1443 ENGINE_READ(engine, RING_MI_MODE) & (MODE_IDLE) ? " [idle]" : ""); 1444 } 1445 1446 if (INTEL_GEN(dev_priv) >= 6) { 1447 drm_printf(m, "\tRING_IMR: 0x%08x\n", 1448 ENGINE_READ(engine, RING_IMR)); 1449 drm_printf(m, "\tRING_ESR: 0x%08x\n", 1450 ENGINE_READ(engine, RING_ESR)); 1451 drm_printf(m, "\tRING_EMR: 0x%08x\n", 1452 ENGINE_READ(engine, RING_EMR)); 1453 drm_printf(m, "\tRING_EIR: 0x%08x\n", 1454 ENGINE_READ(engine, RING_EIR)); 1455 } 1456 1457 addr = intel_engine_get_active_head(engine); 1458 drm_printf(m, "\tACTHD: 0x%08x_%08x\n", 1459 upper_32_bits(addr), lower_32_bits(addr)); 1460 addr = intel_engine_get_last_batch_head(engine); 1461 drm_printf(m, "\tBBADDR: 0x%08x_%08x\n", 1462 upper_32_bits(addr), lower_32_bits(addr)); 1463 if (INTEL_GEN(dev_priv) >= 8) 1464 addr = ENGINE_READ64(engine, RING_DMA_FADD, RING_DMA_FADD_UDW); 1465 else if (INTEL_GEN(dev_priv) >= 4) 1466 addr = ENGINE_READ(engine, RING_DMA_FADD); 1467 else 1468 addr = ENGINE_READ(engine, DMA_FADD_I8XX); 1469 drm_printf(m, "\tDMA_FADDR: 0x%08x_%08x\n", 1470 upper_32_bits(addr), lower_32_bits(addr)); 1471 if (INTEL_GEN(dev_priv) >= 4) { 1472 drm_printf(m, "\tIPEIR: 0x%08x\n", 1473 ENGINE_READ(engine, RING_IPEIR)); 1474 drm_printf(m, "\tIPEHR: 0x%08x\n", 1475 ENGINE_READ(engine, RING_IPEHR)); 1476 } else { 1477 drm_printf(m, "\tIPEIR: 0x%08x\n", ENGINE_READ(engine, IPEIR)); 1478 drm_printf(m, "\tIPEHR: 0x%08x\n", ENGINE_READ(engine, IPEHR)); 1479 } 1480 1481 if (HAS_EXECLISTS(dev_priv)) { 1482 struct i915_request * const *port, *rq; 1483 const u32 *hws = 1484 &engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX]; 1485 const u8 num_entries = execlists->csb_size; 1486 unsigned int idx; 1487 u8 read, write; 1488 1489 drm_printf(m, "\tExeclist tasklet queued? %s (%s), preempt? %s, timeslice? %s\n", 1490 yesno(test_bit(TASKLET_STATE_SCHED, 1491 &engine->execlists.tasklet.state)), 1492 enableddisabled(!atomic_read(&engine->execlists.tasklet.count)), 1493 repr_timer(&engine->execlists.preempt), 1494 repr_timer(&engine->execlists.timer)); 1495 1496 read = execlists->csb_head; 1497 write = READ_ONCE(*execlists->csb_write); 1498 1499 drm_printf(m, "\tExeclist status: 0x%08x %08x; CSB read:%d, write:%d, entries:%d\n", 1500 ENGINE_READ(engine, RING_EXECLIST_STATUS_LO), 1501 ENGINE_READ(engine, RING_EXECLIST_STATUS_HI), 1502 read, write, num_entries); 1503 1504 if (read >= num_entries) 1505 read = 0; 1506 if (write >= num_entries) 1507 write = 0; 1508 if (read > write) 1509 write += num_entries; 1510 while (read < write) { 1511 idx = ++read % num_entries; 1512 drm_printf(m, "\tExeclist CSB[%d]: 0x%08x, context: %d\n", 1513 idx, hws[idx * 2], hws[idx * 2 + 1]); 1514 } 1515 1516 execlists_active_lock_bh(execlists); 1517 rcu_read_lock(); 1518 for (port = execlists->active; (rq = *port); port++) { 1519 char hdr[160]; 1520 int len; 1521 1522 len = scnprintf(hdr, sizeof(hdr), 1523 "\t\tActive[%d]: ccid:%08x%s%s, ", 1524 (int)(port - execlists->active), 1525 rq->context->lrc.ccid, 1526 intel_context_is_closed(rq->context) ? "!" : "", 1527 intel_context_is_banned(rq->context) ? "*" : ""); 1528 len += print_ring(hdr + len, sizeof(hdr) - len, rq); 1529 scnprintf(hdr + len, sizeof(hdr) - len, "rq: "); 1530 print_request(m, rq, hdr); 1531 } 1532 for (port = execlists->pending; (rq = *port); port++) { 1533 char hdr[160]; 1534 int len; 1535 1536 len = scnprintf(hdr, sizeof(hdr), 1537 "\t\tPending[%d]: ccid:%08x%s%s, ", 1538 (int)(port - execlists->pending), 1539 rq->context->lrc.ccid, 1540 intel_context_is_closed(rq->context) ? "!" : "", 1541 intel_context_is_banned(rq->context) ? "*" : ""); 1542 len += print_ring(hdr + len, sizeof(hdr) - len, rq); 1543 scnprintf(hdr + len, sizeof(hdr) - len, "rq: "); 1544 print_request(m, rq, hdr); 1545 } 1546 rcu_read_unlock(); 1547 execlists_active_unlock_bh(execlists); 1548 } else if (INTEL_GEN(dev_priv) > 6) { 1549 drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n", 1550 ENGINE_READ(engine, RING_PP_DIR_BASE)); 1551 drm_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n", 1552 ENGINE_READ(engine, RING_PP_DIR_BASE_READ)); 1553 drm_printf(m, "\tPP_DIR_DCLV: 0x%08x\n", 1554 ENGINE_READ(engine, RING_PP_DIR_DCLV)); 1555 } 1556 } 1557 1558 static void print_request_ring(struct drm_printer *m, struct i915_request *rq) 1559 { 1560 void *ring; 1561 int size; 1562 1563 drm_printf(m, 1564 "[head %04x, postfix %04x, tail %04x, batch 0x%08x_%08x]:\n", 1565 rq->head, rq->postfix, rq->tail, 1566 rq->batch ? upper_32_bits(rq->batch->node.start) : ~0u, 1567 rq->batch ? lower_32_bits(rq->batch->node.start) : ~0u); 1568 1569 size = rq->tail - rq->head; 1570 if (rq->tail < rq->head) 1571 size += rq->ring->size; 1572 1573 ring = kmalloc(size, GFP_ATOMIC); 1574 if (ring) { 1575 const void *vaddr = rq->ring->vaddr; 1576 unsigned int head = rq->head; 1577 unsigned int len = 0; 1578 1579 if (rq->tail < head) { 1580 len = rq->ring->size - head; 1581 memcpy(ring, vaddr + head, len); 1582 head = 0; 1583 } 1584 memcpy(ring + len, vaddr + head, size - len); 1585 1586 hexdump(m, ring, size); 1587 kfree(ring); 1588 } 1589 } 1590 1591 static unsigned long list_count(struct list_head *list) 1592 { 1593 struct list_head *pos; 1594 unsigned long count = 0; 1595 1596 list_for_each(pos, list) 1597 count++; 1598 1599 return count; 1600 } 1601 1602 void intel_engine_dump(struct intel_engine_cs *engine, 1603 struct drm_printer *m, 1604 const char *header, ...) 1605 { 1606 struct i915_gpu_error * const error = &engine->i915->gpu_error; 1607 struct i915_request *rq; 1608 intel_wakeref_t wakeref; 1609 unsigned long flags; 1610 ktime_t dummy; 1611 1612 if (header) { 1613 va_list ap; 1614 1615 va_start(ap, header); 1616 drm_vprintf(m, header, &ap); 1617 va_end(ap); 1618 } 1619 1620 if (intel_gt_is_wedged(engine->gt)) 1621 drm_printf(m, "*** WEDGED ***\n"); 1622 1623 drm_printf(m, "\tAwake? %d\n", atomic_read(&engine->wakeref.count)); 1624 drm_printf(m, "\tBarriers?: %s\n", 1625 yesno(!llist_empty(&engine->barrier_tasks))); 1626 drm_printf(m, "\tLatency: %luus\n", 1627 ewma__engine_latency_read(&engine->latency)); 1628 if (intel_engine_supports_stats(engine)) 1629 drm_printf(m, "\tRuntime: %llums\n", 1630 ktime_to_ms(intel_engine_get_busy_time(engine, 1631 &dummy))); 1632 drm_printf(m, "\tForcewake: %x domains, %d active\n", 1633 engine->fw_domain, atomic_read(&engine->fw_active)); 1634 1635 rcu_read_lock(); 1636 rq = READ_ONCE(engine->heartbeat.systole); 1637 if (rq) 1638 drm_printf(m, "\tHeartbeat: %d ms ago\n", 1639 jiffies_to_msecs(jiffies - rq->emitted_jiffies)); 1640 rcu_read_unlock(); 1641 drm_printf(m, "\tReset count: %d (global %d)\n", 1642 i915_reset_engine_count(error, engine), 1643 i915_reset_count(error)); 1644 1645 drm_printf(m, "\tRequests:\n"); 1646 1647 spin_lock_irqsave(&engine->active.lock, flags); 1648 rq = intel_engine_find_active_request(engine); 1649 if (rq) { 1650 struct intel_timeline *tl = get_timeline(rq); 1651 1652 print_request(m, rq, "\t\tactive "); 1653 1654 drm_printf(m, "\t\tring->start: 0x%08x\n", 1655 i915_ggtt_offset(rq->ring->vma)); 1656 drm_printf(m, "\t\tring->head: 0x%08x\n", 1657 rq->ring->head); 1658 drm_printf(m, "\t\tring->tail: 0x%08x\n", 1659 rq->ring->tail); 1660 drm_printf(m, "\t\tring->emit: 0x%08x\n", 1661 rq->ring->emit); 1662 drm_printf(m, "\t\tring->space: 0x%08x\n", 1663 rq->ring->space); 1664 1665 if (tl) { 1666 drm_printf(m, "\t\tring->hwsp: 0x%08x\n", 1667 tl->hwsp_offset); 1668 intel_timeline_put(tl); 1669 } 1670 1671 print_request_ring(m, rq); 1672 1673 if (rq->context->lrc_reg_state) { 1674 drm_printf(m, "Logical Ring Context:\n"); 1675 hexdump(m, rq->context->lrc_reg_state, PAGE_SIZE); 1676 } 1677 } 1678 drm_printf(m, "\tOn hold?: %lu\n", list_count(&engine->active.hold)); 1679 spin_unlock_irqrestore(&engine->active.lock, flags); 1680 1681 drm_printf(m, "\tMMIO base: 0x%08x\n", engine->mmio_base); 1682 wakeref = intel_runtime_pm_get_if_in_use(engine->uncore->rpm); 1683 if (wakeref) { 1684 intel_engine_print_registers(engine, m); 1685 intel_runtime_pm_put(engine->uncore->rpm, wakeref); 1686 } else { 1687 drm_printf(m, "\tDevice is asleep; skipping register dump\n"); 1688 } 1689 1690 intel_execlists_show_requests(engine, m, print_request, 8); 1691 1692 drm_printf(m, "HWSP:\n"); 1693 hexdump(m, engine->status_page.addr, PAGE_SIZE); 1694 1695 drm_printf(m, "Idle? %s\n", yesno(intel_engine_is_idle(engine))); 1696 1697 intel_engine_print_breadcrumbs(engine, m); 1698 } 1699 1700 static ktime_t __intel_engine_get_busy_time(struct intel_engine_cs *engine, 1701 ktime_t *now) 1702 { 1703 ktime_t total = engine->stats.total; 1704 1705 /* 1706 * If the engine is executing something at the moment 1707 * add it to the total. 1708 */ 1709 *now = ktime_get(); 1710 if (atomic_read(&engine->stats.active)) 1711 total = ktime_add(total, ktime_sub(*now, engine->stats.start)); 1712 1713 return total; 1714 } 1715 1716 /** 1717 * intel_engine_get_busy_time() - Return current accumulated engine busyness 1718 * @engine: engine to report on 1719 * @now: monotonic timestamp of sampling 1720 * 1721 * Returns accumulated time @engine was busy since engine stats were enabled. 1722 */ 1723 ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine, ktime_t *now) 1724 { 1725 unsigned int seq; 1726 ktime_t total; 1727 1728 do { 1729 seq = read_seqbegin(&engine->stats.lock); 1730 total = __intel_engine_get_busy_time(engine, now); 1731 } while (read_seqretry(&engine->stats.lock, seq)); 1732 1733 return total; 1734 } 1735 1736 static bool match_ring(struct i915_request *rq) 1737 { 1738 u32 ring = ENGINE_READ(rq->engine, RING_START); 1739 1740 return ring == i915_ggtt_offset(rq->ring->vma); 1741 } 1742 1743 struct i915_request * 1744 intel_engine_find_active_request(struct intel_engine_cs *engine) 1745 { 1746 struct i915_request *request, *active = NULL; 1747 1748 /* 1749 * We are called by the error capture, reset and to dump engine 1750 * state at random points in time. In particular, note that neither is 1751 * crucially ordered with an interrupt. After a hang, the GPU is dead 1752 * and we assume that no more writes can happen (we waited long enough 1753 * for all writes that were in transaction to be flushed) - adding an 1754 * extra delay for a recent interrupt is pointless. Hence, we do 1755 * not need an engine->irq_seqno_barrier() before the seqno reads. 1756 * At all other times, we must assume the GPU is still running, but 1757 * we only care about the snapshot of this moment. 1758 */ 1759 lockdep_assert_held(&engine->active.lock); 1760 1761 rcu_read_lock(); 1762 request = execlists_active(&engine->execlists); 1763 if (request) { 1764 struct intel_timeline *tl = request->context->timeline; 1765 1766 list_for_each_entry_from_reverse(request, &tl->requests, link) { 1767 if (i915_request_completed(request)) 1768 break; 1769 1770 active = request; 1771 } 1772 } 1773 rcu_read_unlock(); 1774 if (active) 1775 return active; 1776 1777 list_for_each_entry(request, &engine->active.requests, sched.link) { 1778 if (i915_request_completed(request)) 1779 continue; 1780 1781 if (!i915_request_started(request)) 1782 continue; 1783 1784 /* More than one preemptible request may match! */ 1785 if (!match_ring(request)) 1786 continue; 1787 1788 active = request; 1789 break; 1790 } 1791 1792 return active; 1793 } 1794 1795 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 1796 #include "mock_engine.c" 1797 #include "selftest_engine.c" 1798 #include "selftest_engine_cs.c" 1799 #endif 1800