1 /* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <drm/drm_print.h> 26 27 #include "gem/i915_gem_context.h" 28 29 #include "i915_drv.h" 30 31 #include "intel_context.h" 32 #include "intel_engine.h" 33 #include "intel_engine_pm.h" 34 #include "intel_engine_pool.h" 35 #include "intel_engine_user.h" 36 #include "intel_gt.h" 37 #include "intel_gt_requests.h" 38 #include "intel_lrc.h" 39 #include "intel_reset.h" 40 #include "intel_ring.h" 41 42 /* Haswell does have the CXT_SIZE register however it does not appear to be 43 * valid. Now, docs explain in dwords what is in the context object. The full 44 * size is 70720 bytes, however, the power context and execlist context will 45 * never be saved (power context is stored elsewhere, and execlists don't work 46 * on HSW) - so the final size, including the extra state required for the 47 * Resource Streamer, is 66944 bytes, which rounds to 17 pages. 48 */ 49 #define HSW_CXT_TOTAL_SIZE (17 * PAGE_SIZE) 50 51 #define DEFAULT_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE) 52 #define GEN8_LR_CONTEXT_RENDER_SIZE (20 * PAGE_SIZE) 53 #define GEN9_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE) 54 #define GEN10_LR_CONTEXT_RENDER_SIZE (18 * PAGE_SIZE) 55 #define GEN11_LR_CONTEXT_RENDER_SIZE (14 * PAGE_SIZE) 56 57 #define GEN8_LR_CONTEXT_OTHER_SIZE ( 2 * PAGE_SIZE) 58 59 #define MAX_MMIO_BASES 3 60 struct engine_info { 61 unsigned int hw_id; 62 u8 class; 63 u8 instance; 64 /* mmio bases table *must* be sorted in reverse gen order */ 65 struct engine_mmio_base { 66 u32 gen : 8; 67 u32 base : 24; 68 } mmio_bases[MAX_MMIO_BASES]; 69 }; 70 71 static const struct engine_info intel_engines[] = { 72 [RCS0] = { 73 .hw_id = RCS0_HW, 74 .class = RENDER_CLASS, 75 .instance = 0, 76 .mmio_bases = { 77 { .gen = 1, .base = RENDER_RING_BASE } 78 }, 79 }, 80 [BCS0] = { 81 .hw_id = BCS0_HW, 82 .class = COPY_ENGINE_CLASS, 83 .instance = 0, 84 .mmio_bases = { 85 { .gen = 6, .base = BLT_RING_BASE } 86 }, 87 }, 88 [VCS0] = { 89 .hw_id = VCS0_HW, 90 .class = VIDEO_DECODE_CLASS, 91 .instance = 0, 92 .mmio_bases = { 93 { .gen = 11, .base = GEN11_BSD_RING_BASE }, 94 { .gen = 6, .base = GEN6_BSD_RING_BASE }, 95 { .gen = 4, .base = BSD_RING_BASE } 96 }, 97 }, 98 [VCS1] = { 99 .hw_id = VCS1_HW, 100 .class = VIDEO_DECODE_CLASS, 101 .instance = 1, 102 .mmio_bases = { 103 { .gen = 11, .base = GEN11_BSD2_RING_BASE }, 104 { .gen = 8, .base = GEN8_BSD2_RING_BASE } 105 }, 106 }, 107 [VCS2] = { 108 .hw_id = VCS2_HW, 109 .class = VIDEO_DECODE_CLASS, 110 .instance = 2, 111 .mmio_bases = { 112 { .gen = 11, .base = GEN11_BSD3_RING_BASE } 113 }, 114 }, 115 [VCS3] = { 116 .hw_id = VCS3_HW, 117 .class = VIDEO_DECODE_CLASS, 118 .instance = 3, 119 .mmio_bases = { 120 { .gen = 11, .base = GEN11_BSD4_RING_BASE } 121 }, 122 }, 123 [VECS0] = { 124 .hw_id = VECS0_HW, 125 .class = VIDEO_ENHANCEMENT_CLASS, 126 .instance = 0, 127 .mmio_bases = { 128 { .gen = 11, .base = GEN11_VEBOX_RING_BASE }, 129 { .gen = 7, .base = VEBOX_RING_BASE } 130 }, 131 }, 132 [VECS1] = { 133 .hw_id = VECS1_HW, 134 .class = VIDEO_ENHANCEMENT_CLASS, 135 .instance = 1, 136 .mmio_bases = { 137 { .gen = 11, .base = GEN11_VEBOX2_RING_BASE } 138 }, 139 }, 140 }; 141 142 /** 143 * intel_engine_context_size() - return the size of the context for an engine 144 * @gt: the gt 145 * @class: engine class 146 * 147 * Each engine class may require a different amount of space for a context 148 * image. 149 * 150 * Return: size (in bytes) of an engine class specific context image 151 * 152 * Note: this size includes the HWSP, which is part of the context image 153 * in LRC mode, but does not include the "shared data page" used with 154 * GuC submission. The caller should account for this if using the GuC. 155 */ 156 u32 intel_engine_context_size(struct intel_gt *gt, u8 class) 157 { 158 struct intel_uncore *uncore = gt->uncore; 159 u32 cxt_size; 160 161 BUILD_BUG_ON(I915_GTT_PAGE_SIZE != PAGE_SIZE); 162 163 switch (class) { 164 case RENDER_CLASS: 165 switch (INTEL_GEN(gt->i915)) { 166 default: 167 MISSING_CASE(INTEL_GEN(gt->i915)); 168 return DEFAULT_LR_CONTEXT_RENDER_SIZE; 169 case 12: 170 case 11: 171 return GEN11_LR_CONTEXT_RENDER_SIZE; 172 case 10: 173 return GEN10_LR_CONTEXT_RENDER_SIZE; 174 case 9: 175 return GEN9_LR_CONTEXT_RENDER_SIZE; 176 case 8: 177 return GEN8_LR_CONTEXT_RENDER_SIZE; 178 case 7: 179 if (IS_HASWELL(gt->i915)) 180 return HSW_CXT_TOTAL_SIZE; 181 182 cxt_size = intel_uncore_read(uncore, GEN7_CXT_SIZE); 183 return round_up(GEN7_CXT_TOTAL_SIZE(cxt_size) * 64, 184 PAGE_SIZE); 185 case 6: 186 cxt_size = intel_uncore_read(uncore, CXT_SIZE); 187 return round_up(GEN6_CXT_TOTAL_SIZE(cxt_size) * 64, 188 PAGE_SIZE); 189 case 5: 190 case 4: 191 /* 192 * There is a discrepancy here between the size reported 193 * by the register and the size of the context layout 194 * in the docs. Both are described as authorative! 195 * 196 * The discrepancy is on the order of a few cachelines, 197 * but the total is under one page (4k), which is our 198 * minimum allocation anyway so it should all come 199 * out in the wash. 200 */ 201 cxt_size = intel_uncore_read(uncore, CXT_SIZE) + 1; 202 DRM_DEBUG_DRIVER("gen%d CXT_SIZE = %d bytes [0x%08x]\n", 203 INTEL_GEN(gt->i915), 204 cxt_size * 64, 205 cxt_size - 1); 206 return round_up(cxt_size * 64, PAGE_SIZE); 207 case 3: 208 case 2: 209 /* For the special day when i810 gets merged. */ 210 case 1: 211 return 0; 212 } 213 break; 214 default: 215 MISSING_CASE(class); 216 /* fall through */ 217 case VIDEO_DECODE_CLASS: 218 case VIDEO_ENHANCEMENT_CLASS: 219 case COPY_ENGINE_CLASS: 220 if (INTEL_GEN(gt->i915) < 8) 221 return 0; 222 return GEN8_LR_CONTEXT_OTHER_SIZE; 223 } 224 } 225 226 static u32 __engine_mmio_base(struct drm_i915_private *i915, 227 const struct engine_mmio_base *bases) 228 { 229 int i; 230 231 for (i = 0; i < MAX_MMIO_BASES; i++) 232 if (INTEL_GEN(i915) >= bases[i].gen) 233 break; 234 235 GEM_BUG_ON(i == MAX_MMIO_BASES); 236 GEM_BUG_ON(!bases[i].base); 237 238 return bases[i].base; 239 } 240 241 static void __sprint_engine_name(struct intel_engine_cs *engine) 242 { 243 /* 244 * Before we know what the uABI name for this engine will be, 245 * we still would like to keep track of this engine in the debug logs. 246 * We throw in a ' here as a reminder that this isn't its final name. 247 */ 248 GEM_WARN_ON(snprintf(engine->name, sizeof(engine->name), "%s'%u", 249 intel_engine_class_repr(engine->class), 250 engine->instance) >= sizeof(engine->name)); 251 } 252 253 void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask) 254 { 255 /* 256 * Though they added more rings on g4x/ilk, they did not add 257 * per-engine HWSTAM until gen6. 258 */ 259 if (INTEL_GEN(engine->i915) < 6 && engine->class != RENDER_CLASS) 260 return; 261 262 if (INTEL_GEN(engine->i915) >= 3) 263 ENGINE_WRITE(engine, RING_HWSTAM, mask); 264 else 265 ENGINE_WRITE16(engine, RING_HWSTAM, mask); 266 } 267 268 static void intel_engine_sanitize_mmio(struct intel_engine_cs *engine) 269 { 270 /* Mask off all writes into the unknown HWSP */ 271 intel_engine_set_hwsp_writemask(engine, ~0u); 272 } 273 274 static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) 275 { 276 const struct engine_info *info = &intel_engines[id]; 277 struct intel_engine_cs *engine; 278 279 BUILD_BUG_ON(MAX_ENGINE_CLASS >= BIT(GEN11_ENGINE_CLASS_WIDTH)); 280 BUILD_BUG_ON(MAX_ENGINE_INSTANCE >= BIT(GEN11_ENGINE_INSTANCE_WIDTH)); 281 282 if (GEM_DEBUG_WARN_ON(id >= ARRAY_SIZE(gt->engine))) 283 return -EINVAL; 284 285 if (GEM_DEBUG_WARN_ON(info->class > MAX_ENGINE_CLASS)) 286 return -EINVAL; 287 288 if (GEM_DEBUG_WARN_ON(info->instance > MAX_ENGINE_INSTANCE)) 289 return -EINVAL; 290 291 if (GEM_DEBUG_WARN_ON(gt->engine_class[info->class][info->instance])) 292 return -EINVAL; 293 294 engine = kzalloc(sizeof(*engine), GFP_KERNEL); 295 if (!engine) 296 return -ENOMEM; 297 298 BUILD_BUG_ON(BITS_PER_TYPE(engine->mask) < I915_NUM_ENGINES); 299 300 engine->id = id; 301 engine->legacy_idx = INVALID_ENGINE; 302 engine->mask = BIT(id); 303 engine->i915 = gt->i915; 304 engine->gt = gt; 305 engine->uncore = gt->uncore; 306 engine->hw_id = engine->guc_id = info->hw_id; 307 engine->mmio_base = __engine_mmio_base(gt->i915, info->mmio_bases); 308 309 engine->class = info->class; 310 engine->instance = info->instance; 311 __sprint_engine_name(engine); 312 313 engine->props.heartbeat_interval_ms = 314 CONFIG_DRM_I915_HEARTBEAT_INTERVAL; 315 engine->props.preempt_timeout_ms = 316 CONFIG_DRM_I915_PREEMPT_TIMEOUT; 317 engine->props.stop_timeout_ms = 318 CONFIG_DRM_I915_STOP_TIMEOUT; 319 engine->props.timeslice_duration_ms = 320 CONFIG_DRM_I915_TIMESLICE_DURATION; 321 322 engine->context_size = intel_engine_context_size(gt, engine->class); 323 if (WARN_ON(engine->context_size > BIT(20))) 324 engine->context_size = 0; 325 if (engine->context_size) 326 DRIVER_CAPS(gt->i915)->has_logical_contexts = true; 327 328 /* Nothing to do here, execute in order of dependencies */ 329 engine->schedule = NULL; 330 331 ewma__engine_latency_init(&engine->latency); 332 seqlock_init(&engine->stats.lock); 333 334 ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier); 335 336 /* Scrub mmio state on takeover */ 337 intel_engine_sanitize_mmio(engine); 338 339 gt->engine_class[info->class][info->instance] = engine; 340 gt->engine[id] = engine; 341 342 gt->i915->engine[id] = engine; 343 344 return 0; 345 } 346 347 static void __setup_engine_capabilities(struct intel_engine_cs *engine) 348 { 349 struct drm_i915_private *i915 = engine->i915; 350 351 if (engine->class == VIDEO_DECODE_CLASS) { 352 /* 353 * HEVC support is present on first engine instance 354 * before Gen11 and on all instances afterwards. 355 */ 356 if (INTEL_GEN(i915) >= 11 || 357 (INTEL_GEN(i915) >= 9 && engine->instance == 0)) 358 engine->uabi_capabilities |= 359 I915_VIDEO_CLASS_CAPABILITY_HEVC; 360 361 /* 362 * SFC block is present only on even logical engine 363 * instances. 364 */ 365 if ((INTEL_GEN(i915) >= 11 && 366 RUNTIME_INFO(i915)->vdbox_sfc_access & engine->mask) || 367 (INTEL_GEN(i915) >= 9 && engine->instance == 0)) 368 engine->uabi_capabilities |= 369 I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC; 370 } else if (engine->class == VIDEO_ENHANCEMENT_CLASS) { 371 if (INTEL_GEN(i915) >= 9) 372 engine->uabi_capabilities |= 373 I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC; 374 } 375 } 376 377 static void intel_setup_engine_capabilities(struct intel_gt *gt) 378 { 379 struct intel_engine_cs *engine; 380 enum intel_engine_id id; 381 382 for_each_engine(engine, gt, id) 383 __setup_engine_capabilities(engine); 384 } 385 386 /** 387 * intel_engines_release() - free the resources allocated for Command Streamers 388 * @gt: pointer to struct intel_gt 389 */ 390 void intel_engines_release(struct intel_gt *gt) 391 { 392 struct intel_engine_cs *engine; 393 enum intel_engine_id id; 394 395 /* Decouple the backend; but keep the layout for late GPU resets */ 396 for_each_engine(engine, gt, id) { 397 if (!engine->release) 398 continue; 399 400 engine->release(engine); 401 engine->release = NULL; 402 403 memset(&engine->reset, 0, sizeof(engine->reset)); 404 405 gt->i915->engine[id] = NULL; 406 } 407 } 408 409 void intel_engines_free(struct intel_gt *gt) 410 { 411 struct intel_engine_cs *engine; 412 enum intel_engine_id id; 413 414 for_each_engine(engine, gt, id) { 415 kfree(engine); 416 gt->engine[id] = NULL; 417 } 418 } 419 420 /** 421 * intel_engines_init_mmio() - allocate and prepare the Engine Command Streamers 422 * @gt: pointer to struct intel_gt 423 * 424 * Return: non-zero if the initialization failed. 425 */ 426 int intel_engines_init_mmio(struct intel_gt *gt) 427 { 428 struct drm_i915_private *i915 = gt->i915; 429 struct intel_device_info *device_info = mkwrite_device_info(i915); 430 const unsigned int engine_mask = INTEL_INFO(i915)->engine_mask; 431 unsigned int mask = 0; 432 unsigned int i; 433 int err; 434 435 WARN_ON(engine_mask == 0); 436 WARN_ON(engine_mask & 437 GENMASK(BITS_PER_TYPE(mask) - 1, I915_NUM_ENGINES)); 438 439 if (i915_inject_probe_failure(i915)) 440 return -ENODEV; 441 442 for (i = 0; i < ARRAY_SIZE(intel_engines); i++) { 443 if (!HAS_ENGINE(i915, i)) 444 continue; 445 446 err = intel_engine_setup(gt, i); 447 if (err) 448 goto cleanup; 449 450 mask |= BIT(i); 451 } 452 453 /* 454 * Catch failures to update intel_engines table when the new engines 455 * are added to the driver by a warning and disabling the forgotten 456 * engines. 457 */ 458 if (WARN_ON(mask != engine_mask)) 459 device_info->engine_mask = mask; 460 461 RUNTIME_INFO(i915)->num_engines = hweight32(mask); 462 463 intel_gt_check_and_clear_faults(gt); 464 465 intel_setup_engine_capabilities(gt); 466 467 return 0; 468 469 cleanup: 470 intel_engines_free(gt); 471 return err; 472 } 473 474 void intel_engine_init_execlists(struct intel_engine_cs *engine) 475 { 476 struct intel_engine_execlists * const execlists = &engine->execlists; 477 478 execlists->port_mask = 1; 479 GEM_BUG_ON(!is_power_of_2(execlists_num_ports(execlists))); 480 GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS); 481 482 memset(execlists->pending, 0, sizeof(execlists->pending)); 483 execlists->active = 484 memset(execlists->inflight, 0, sizeof(execlists->inflight)); 485 486 execlists->queue_priority_hint = INT_MIN; 487 execlists->queue = RB_ROOT_CACHED; 488 } 489 490 static void cleanup_status_page(struct intel_engine_cs *engine) 491 { 492 struct i915_vma *vma; 493 494 /* Prevent writes into HWSP after returning the page to the system */ 495 intel_engine_set_hwsp_writemask(engine, ~0u); 496 497 vma = fetch_and_zero(&engine->status_page.vma); 498 if (!vma) 499 return; 500 501 if (!HWS_NEEDS_PHYSICAL(engine->i915)) 502 i915_vma_unpin(vma); 503 504 i915_gem_object_unpin_map(vma->obj); 505 i915_gem_object_put(vma->obj); 506 } 507 508 static int pin_ggtt_status_page(struct intel_engine_cs *engine, 509 struct i915_vma *vma) 510 { 511 unsigned int flags; 512 513 flags = PIN_GLOBAL; 514 if (!HAS_LLC(engine->i915) && i915_ggtt_has_aperture(engine->gt->ggtt)) 515 /* 516 * On g33, we cannot place HWS above 256MiB, so 517 * restrict its pinning to the low mappable arena. 518 * Though this restriction is not documented for 519 * gen4, gen5, or byt, they also behave similarly 520 * and hang if the HWS is placed at the top of the 521 * GTT. To generalise, it appears that all !llc 522 * platforms have issues with us placing the HWS 523 * above the mappable region (even though we never 524 * actually map it). 525 */ 526 flags |= PIN_MAPPABLE; 527 else 528 flags |= PIN_HIGH; 529 530 return i915_vma_pin(vma, 0, 0, flags); 531 } 532 533 static int init_status_page(struct intel_engine_cs *engine) 534 { 535 struct drm_i915_gem_object *obj; 536 struct i915_vma *vma; 537 void *vaddr; 538 int ret; 539 540 /* 541 * Though the HWS register does support 36bit addresses, historically 542 * we have had hangs and corruption reported due to wild writes if 543 * the HWS is placed above 4G. We only allow objects to be allocated 544 * in GFP_DMA32 for i965, and no earlier physical address users had 545 * access to more than 4G. 546 */ 547 obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE); 548 if (IS_ERR(obj)) { 549 DRM_ERROR("Failed to allocate status page\n"); 550 return PTR_ERR(obj); 551 } 552 553 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC); 554 555 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); 556 if (IS_ERR(vma)) { 557 ret = PTR_ERR(vma); 558 goto err; 559 } 560 561 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); 562 if (IS_ERR(vaddr)) { 563 ret = PTR_ERR(vaddr); 564 goto err; 565 } 566 567 engine->status_page.addr = memset(vaddr, 0, PAGE_SIZE); 568 engine->status_page.vma = vma; 569 570 if (!HWS_NEEDS_PHYSICAL(engine->i915)) { 571 ret = pin_ggtt_status_page(engine, vma); 572 if (ret) 573 goto err_unpin; 574 } 575 576 return 0; 577 578 err_unpin: 579 i915_gem_object_unpin_map(obj); 580 err: 581 i915_gem_object_put(obj); 582 return ret; 583 } 584 585 static int engine_setup_common(struct intel_engine_cs *engine) 586 { 587 int err; 588 589 init_llist_head(&engine->barrier_tasks); 590 591 err = init_status_page(engine); 592 if (err) 593 return err; 594 595 intel_engine_init_active(engine, ENGINE_PHYSICAL); 596 intel_engine_init_breadcrumbs(engine); 597 intel_engine_init_execlists(engine); 598 intel_engine_init_cmd_parser(engine); 599 intel_engine_init__pm(engine); 600 intel_engine_init_retire(engine); 601 602 intel_engine_pool_init(&engine->pool); 603 604 /* Use the whole device by default */ 605 engine->sseu = 606 intel_sseu_from_device_info(&RUNTIME_INFO(engine->i915)->sseu); 607 608 intel_engine_init_workarounds(engine); 609 intel_engine_init_whitelist(engine); 610 intel_engine_init_ctx_wa(engine); 611 612 return 0; 613 } 614 615 struct measure_breadcrumb { 616 struct i915_request rq; 617 struct intel_timeline timeline; 618 struct intel_ring ring; 619 u32 cs[1024]; 620 }; 621 622 static int measure_breadcrumb_dw(struct intel_engine_cs *engine) 623 { 624 struct measure_breadcrumb *frame; 625 int dw = -ENOMEM; 626 627 GEM_BUG_ON(!engine->gt->scratch); 628 629 frame = kzalloc(sizeof(*frame), GFP_KERNEL); 630 if (!frame) 631 return -ENOMEM; 632 633 if (intel_timeline_init(&frame->timeline, 634 engine->gt, 635 engine->status_page.vma)) 636 goto out_frame; 637 638 mutex_lock(&frame->timeline.mutex); 639 640 frame->ring.vaddr = frame->cs; 641 frame->ring.size = sizeof(frame->cs); 642 frame->ring.effective_size = frame->ring.size; 643 intel_ring_update_space(&frame->ring); 644 645 frame->rq.i915 = engine->i915; 646 frame->rq.engine = engine; 647 frame->rq.ring = &frame->ring; 648 rcu_assign_pointer(frame->rq.timeline, &frame->timeline); 649 650 dw = intel_timeline_pin(&frame->timeline); 651 if (dw < 0) 652 goto out_timeline; 653 654 spin_lock_irq(&engine->active.lock); 655 dw = engine->emit_fini_breadcrumb(&frame->rq, frame->cs) - frame->cs; 656 spin_unlock_irq(&engine->active.lock); 657 658 GEM_BUG_ON(dw & 1); /* RING_TAIL must be qword aligned */ 659 660 intel_timeline_unpin(&frame->timeline); 661 662 out_timeline: 663 mutex_unlock(&frame->timeline.mutex); 664 intel_timeline_fini(&frame->timeline); 665 out_frame: 666 kfree(frame); 667 return dw; 668 } 669 670 void 671 intel_engine_init_active(struct intel_engine_cs *engine, unsigned int subclass) 672 { 673 INIT_LIST_HEAD(&engine->active.requests); 674 675 spin_lock_init(&engine->active.lock); 676 lockdep_set_subclass(&engine->active.lock, subclass); 677 678 /* 679 * Due to an interesting quirk in lockdep's internal debug tracking, 680 * after setting a subclass we must ensure the lock is used. Otherwise, 681 * nr_unused_locks is incremented once too often. 682 */ 683 #ifdef CONFIG_DEBUG_LOCK_ALLOC 684 local_irq_disable(); 685 lock_map_acquire(&engine->active.lock.dep_map); 686 lock_map_release(&engine->active.lock.dep_map); 687 local_irq_enable(); 688 #endif 689 } 690 691 static struct intel_context * 692 create_kernel_context(struct intel_engine_cs *engine) 693 { 694 static struct lock_class_key kernel; 695 struct intel_context *ce; 696 int err; 697 698 ce = intel_context_create(engine); 699 if (IS_ERR(ce)) 700 return ce; 701 702 __set_bit(CONTEXT_BARRIER_BIT, &ce->flags); 703 704 err = intel_context_pin(ce); /* perma-pin so it is always available */ 705 if (err) { 706 intel_context_put(ce); 707 return ERR_PTR(err); 708 } 709 710 /* 711 * Give our perma-pinned kernel timelines a separate lockdep class, 712 * so that we can use them from within the normal user timelines 713 * should we need to inject GPU operations during their request 714 * construction. 715 */ 716 lockdep_set_class(&ce->timeline->mutex, &kernel); 717 718 return ce; 719 } 720 721 /** 722 * intel_engines_init_common - initialize cengine state which might require hw access 723 * @engine: Engine to initialize. 724 * 725 * Initializes @engine@ structure members shared between legacy and execlists 726 * submission modes which do require hardware access. 727 * 728 * Typcally done at later stages of submission mode specific engine setup. 729 * 730 * Returns zero on success or an error code on failure. 731 */ 732 static int engine_init_common(struct intel_engine_cs *engine) 733 { 734 struct intel_context *ce; 735 int ret; 736 737 engine->set_default_submission(engine); 738 739 ret = measure_breadcrumb_dw(engine); 740 if (ret < 0) 741 return ret; 742 743 engine->emit_fini_breadcrumb_dw = ret; 744 745 /* 746 * We may need to do things with the shrinker which 747 * require us to immediately switch back to the default 748 * context. This can cause a problem as pinning the 749 * default context also requires GTT space which may not 750 * be available. To avoid this we always pin the default 751 * context. 752 */ 753 ce = create_kernel_context(engine); 754 if (IS_ERR(ce)) 755 return PTR_ERR(ce); 756 757 engine->kernel_context = ce; 758 759 return 0; 760 } 761 762 int intel_engines_init(struct intel_gt *gt) 763 { 764 int (*setup)(struct intel_engine_cs *engine); 765 struct intel_engine_cs *engine; 766 enum intel_engine_id id; 767 int err; 768 769 if (HAS_EXECLISTS(gt->i915)) 770 setup = intel_execlists_submission_setup; 771 else 772 setup = intel_ring_submission_setup; 773 774 for_each_engine(engine, gt, id) { 775 err = engine_setup_common(engine); 776 if (err) 777 return err; 778 779 err = setup(engine); 780 if (err) 781 return err; 782 783 err = engine_init_common(engine); 784 if (err) 785 return err; 786 787 intel_engine_add_user(engine); 788 } 789 790 return 0; 791 } 792 793 /** 794 * intel_engines_cleanup_common - cleans up the engine state created by 795 * the common initiailizers. 796 * @engine: Engine to cleanup. 797 * 798 * This cleans up everything created by the common helpers. 799 */ 800 void intel_engine_cleanup_common(struct intel_engine_cs *engine) 801 { 802 GEM_BUG_ON(!list_empty(&engine->active.requests)); 803 tasklet_kill(&engine->execlists.tasklet); /* flush the callback */ 804 805 cleanup_status_page(engine); 806 807 intel_engine_fini_retire(engine); 808 intel_engine_pool_fini(&engine->pool); 809 intel_engine_fini_breadcrumbs(engine); 810 intel_engine_cleanup_cmd_parser(engine); 811 812 if (engine->default_state) 813 i915_gem_object_put(engine->default_state); 814 815 if (engine->kernel_context) { 816 intel_context_unpin(engine->kernel_context); 817 intel_context_put(engine->kernel_context); 818 } 819 GEM_BUG_ON(!llist_empty(&engine->barrier_tasks)); 820 821 intel_wa_list_free(&engine->ctx_wa_list); 822 intel_wa_list_free(&engine->wa_list); 823 intel_wa_list_free(&engine->whitelist); 824 } 825 826 u64 intel_engine_get_active_head(const struct intel_engine_cs *engine) 827 { 828 struct drm_i915_private *i915 = engine->i915; 829 830 u64 acthd; 831 832 if (INTEL_GEN(i915) >= 8) 833 acthd = ENGINE_READ64(engine, RING_ACTHD, RING_ACTHD_UDW); 834 else if (INTEL_GEN(i915) >= 4) 835 acthd = ENGINE_READ(engine, RING_ACTHD); 836 else 837 acthd = ENGINE_READ(engine, ACTHD); 838 839 return acthd; 840 } 841 842 u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine) 843 { 844 u64 bbaddr; 845 846 if (INTEL_GEN(engine->i915) >= 8) 847 bbaddr = ENGINE_READ64(engine, RING_BBADDR, RING_BBADDR_UDW); 848 else 849 bbaddr = ENGINE_READ(engine, RING_BBADDR); 850 851 return bbaddr; 852 } 853 854 static unsigned long stop_timeout(const struct intel_engine_cs *engine) 855 { 856 if (in_atomic() || irqs_disabled()) /* inside atomic preempt-reset? */ 857 return 0; 858 859 /* 860 * If we are doing a normal GPU reset, we can take our time and allow 861 * the engine to quiesce. We've stopped submission to the engine, and 862 * if we wait long enough an innocent context should complete and 863 * leave the engine idle. So they should not be caught unaware by 864 * the forthcoming GPU reset (which usually follows the stop_cs)! 865 */ 866 return READ_ONCE(engine->props.stop_timeout_ms); 867 } 868 869 int intel_engine_stop_cs(struct intel_engine_cs *engine) 870 { 871 struct intel_uncore *uncore = engine->uncore; 872 const u32 base = engine->mmio_base; 873 const i915_reg_t mode = RING_MI_MODE(base); 874 int err; 875 876 if (INTEL_GEN(engine->i915) < 3) 877 return -ENODEV; 878 879 ENGINE_TRACE(engine, "\n"); 880 881 intel_uncore_write_fw(uncore, mode, _MASKED_BIT_ENABLE(STOP_RING)); 882 883 err = 0; 884 if (__intel_wait_for_register_fw(uncore, 885 mode, MODE_IDLE, MODE_IDLE, 886 1000, stop_timeout(engine), 887 NULL)) { 888 ENGINE_TRACE(engine, "timed out on STOP_RING -> IDLE\n"); 889 err = -ETIMEDOUT; 890 } 891 892 /* A final mmio read to let GPU writes be hopefully flushed to memory */ 893 intel_uncore_posting_read_fw(uncore, mode); 894 895 return err; 896 } 897 898 void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine) 899 { 900 ENGINE_TRACE(engine, "\n"); 901 902 ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING)); 903 } 904 905 const char *i915_cache_level_str(struct drm_i915_private *i915, int type) 906 { 907 switch (type) { 908 case I915_CACHE_NONE: return " uncached"; 909 case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped"; 910 case I915_CACHE_L3_LLC: return " L3+LLC"; 911 case I915_CACHE_WT: return " WT"; 912 default: return ""; 913 } 914 } 915 916 static u32 917 read_subslice_reg(const struct intel_engine_cs *engine, 918 int slice, int subslice, i915_reg_t reg) 919 { 920 struct drm_i915_private *i915 = engine->i915; 921 struct intel_uncore *uncore = engine->uncore; 922 u32 mcr_mask, mcr_ss, mcr, old_mcr, val; 923 enum forcewake_domains fw_domains; 924 925 if (INTEL_GEN(i915) >= 11) { 926 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK; 927 mcr_ss = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice); 928 } else { 929 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK; 930 mcr_ss = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice); 931 } 932 933 fw_domains = intel_uncore_forcewake_for_reg(uncore, reg, 934 FW_REG_READ); 935 fw_domains |= intel_uncore_forcewake_for_reg(uncore, 936 GEN8_MCR_SELECTOR, 937 FW_REG_READ | FW_REG_WRITE); 938 939 spin_lock_irq(&uncore->lock); 940 intel_uncore_forcewake_get__locked(uncore, fw_domains); 941 942 old_mcr = mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR); 943 944 mcr &= ~mcr_mask; 945 mcr |= mcr_ss; 946 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr); 947 948 val = intel_uncore_read_fw(uncore, reg); 949 950 mcr &= ~mcr_mask; 951 mcr |= old_mcr & mcr_mask; 952 953 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr); 954 955 intel_uncore_forcewake_put__locked(uncore, fw_domains); 956 spin_unlock_irq(&uncore->lock); 957 958 return val; 959 } 960 961 /* NB: please notice the memset */ 962 void intel_engine_get_instdone(const struct intel_engine_cs *engine, 963 struct intel_instdone *instdone) 964 { 965 struct drm_i915_private *i915 = engine->i915; 966 const struct sseu_dev_info *sseu = &RUNTIME_INFO(i915)->sseu; 967 struct intel_uncore *uncore = engine->uncore; 968 u32 mmio_base = engine->mmio_base; 969 int slice; 970 int subslice; 971 972 memset(instdone, 0, sizeof(*instdone)); 973 974 switch (INTEL_GEN(i915)) { 975 default: 976 instdone->instdone = 977 intel_uncore_read(uncore, RING_INSTDONE(mmio_base)); 978 979 if (engine->id != RCS0) 980 break; 981 982 instdone->slice_common = 983 intel_uncore_read(uncore, GEN7_SC_INSTDONE); 984 for_each_instdone_slice_subslice(i915, sseu, slice, subslice) { 985 instdone->sampler[slice][subslice] = 986 read_subslice_reg(engine, slice, subslice, 987 GEN7_SAMPLER_INSTDONE); 988 instdone->row[slice][subslice] = 989 read_subslice_reg(engine, slice, subslice, 990 GEN7_ROW_INSTDONE); 991 } 992 break; 993 case 7: 994 instdone->instdone = 995 intel_uncore_read(uncore, RING_INSTDONE(mmio_base)); 996 997 if (engine->id != RCS0) 998 break; 999 1000 instdone->slice_common = 1001 intel_uncore_read(uncore, GEN7_SC_INSTDONE); 1002 instdone->sampler[0][0] = 1003 intel_uncore_read(uncore, GEN7_SAMPLER_INSTDONE); 1004 instdone->row[0][0] = 1005 intel_uncore_read(uncore, GEN7_ROW_INSTDONE); 1006 1007 break; 1008 case 6: 1009 case 5: 1010 case 4: 1011 instdone->instdone = 1012 intel_uncore_read(uncore, RING_INSTDONE(mmio_base)); 1013 if (engine->id == RCS0) 1014 /* HACK: Using the wrong struct member */ 1015 instdone->slice_common = 1016 intel_uncore_read(uncore, GEN4_INSTDONE1); 1017 break; 1018 case 3: 1019 case 2: 1020 instdone->instdone = intel_uncore_read(uncore, GEN2_INSTDONE); 1021 break; 1022 } 1023 } 1024 1025 static bool ring_is_idle(struct intel_engine_cs *engine) 1026 { 1027 bool idle = true; 1028 1029 if (I915_SELFTEST_ONLY(!engine->mmio_base)) 1030 return true; 1031 1032 if (!intel_engine_pm_get_if_awake(engine)) 1033 return true; 1034 1035 /* First check that no commands are left in the ring */ 1036 if ((ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR) != 1037 (ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR)) 1038 idle = false; 1039 1040 /* No bit for gen2, so assume the CS parser is idle */ 1041 if (INTEL_GEN(engine->i915) > 2 && 1042 !(ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE)) 1043 idle = false; 1044 1045 intel_engine_pm_put(engine); 1046 1047 return idle; 1048 } 1049 1050 void intel_engine_flush_submission(struct intel_engine_cs *engine) 1051 { 1052 struct tasklet_struct *t = &engine->execlists.tasklet; 1053 1054 if (__tasklet_is_scheduled(t)) { 1055 local_bh_disable(); 1056 if (tasklet_trylock(t)) { 1057 /* Must wait for any GPU reset in progress. */ 1058 if (__tasklet_is_enabled(t)) 1059 t->func(t->data); 1060 tasklet_unlock(t); 1061 } 1062 local_bh_enable(); 1063 } 1064 1065 /* Otherwise flush the tasklet if it was running on another cpu */ 1066 tasklet_unlock_wait(t); 1067 } 1068 1069 /** 1070 * intel_engine_is_idle() - Report if the engine has finished process all work 1071 * @engine: the intel_engine_cs 1072 * 1073 * Return true if there are no requests pending, nothing left to be submitted 1074 * to hardware, and that the engine is idle. 1075 */ 1076 bool intel_engine_is_idle(struct intel_engine_cs *engine) 1077 { 1078 /* More white lies, if wedged, hw state is inconsistent */ 1079 if (intel_gt_is_wedged(engine->gt)) 1080 return true; 1081 1082 if (!intel_engine_pm_is_awake(engine)) 1083 return true; 1084 1085 /* Waiting to drain ELSP? */ 1086 if (execlists_active(&engine->execlists)) { 1087 synchronize_hardirq(engine->i915->drm.pdev->irq); 1088 1089 intel_engine_flush_submission(engine); 1090 1091 if (execlists_active(&engine->execlists)) 1092 return false; 1093 } 1094 1095 /* ELSP is empty, but there are ready requests? E.g. after reset */ 1096 if (!RB_EMPTY_ROOT(&engine->execlists.queue.rb_root)) 1097 return false; 1098 1099 /* Ring stopped? */ 1100 return ring_is_idle(engine); 1101 } 1102 1103 bool intel_engines_are_idle(struct intel_gt *gt) 1104 { 1105 struct intel_engine_cs *engine; 1106 enum intel_engine_id id; 1107 1108 /* 1109 * If the driver is wedged, HW state may be very inconsistent and 1110 * report that it is still busy, even though we have stopped using it. 1111 */ 1112 if (intel_gt_is_wedged(gt)) 1113 return true; 1114 1115 /* Already parked (and passed an idleness test); must still be idle */ 1116 if (!READ_ONCE(gt->awake)) 1117 return true; 1118 1119 for_each_engine(engine, gt, id) { 1120 if (!intel_engine_is_idle(engine)) 1121 return false; 1122 } 1123 1124 return true; 1125 } 1126 1127 void intel_engines_reset_default_submission(struct intel_gt *gt) 1128 { 1129 struct intel_engine_cs *engine; 1130 enum intel_engine_id id; 1131 1132 for_each_engine(engine, gt, id) 1133 engine->set_default_submission(engine); 1134 } 1135 1136 bool intel_engine_can_store_dword(struct intel_engine_cs *engine) 1137 { 1138 switch (INTEL_GEN(engine->i915)) { 1139 case 2: 1140 return false; /* uses physical not virtual addresses */ 1141 case 3: 1142 /* maybe only uses physical not virtual addresses */ 1143 return !(IS_I915G(engine->i915) || IS_I915GM(engine->i915)); 1144 case 4: 1145 return !IS_I965G(engine->i915); /* who knows! */ 1146 case 6: 1147 return engine->class != VIDEO_DECODE_CLASS; /* b0rked */ 1148 default: 1149 return true; 1150 } 1151 } 1152 1153 static int print_sched_attr(struct drm_i915_private *i915, 1154 const struct i915_sched_attr *attr, 1155 char *buf, int x, int len) 1156 { 1157 if (attr->priority == I915_PRIORITY_INVALID) 1158 return x; 1159 1160 x += snprintf(buf + x, len - x, 1161 " prio=%d", attr->priority); 1162 1163 return x; 1164 } 1165 1166 static void print_request(struct drm_printer *m, 1167 struct i915_request *rq, 1168 const char *prefix) 1169 { 1170 const char *name = rq->fence.ops->get_timeline_name(&rq->fence); 1171 char buf[80] = ""; 1172 int x = 0; 1173 1174 x = print_sched_attr(rq->i915, &rq->sched.attr, buf, x, sizeof(buf)); 1175 1176 drm_printf(m, "%s %llx:%llx%s%s %s @ %dms: %s\n", 1177 prefix, 1178 rq->fence.context, rq->fence.seqno, 1179 i915_request_completed(rq) ? "!" : 1180 i915_request_started(rq) ? "*" : 1181 "", 1182 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, 1183 &rq->fence.flags) ? "+" : 1184 test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, 1185 &rq->fence.flags) ? "-" : 1186 "", 1187 buf, 1188 jiffies_to_msecs(jiffies - rq->emitted_jiffies), 1189 name); 1190 } 1191 1192 static void hexdump(struct drm_printer *m, const void *buf, size_t len) 1193 { 1194 const size_t rowsize = 8 * sizeof(u32); 1195 const void *prev = NULL; 1196 bool skip = false; 1197 size_t pos; 1198 1199 for (pos = 0; pos < len; pos += rowsize) { 1200 char line[128]; 1201 1202 if (prev && !memcmp(prev, buf + pos, rowsize)) { 1203 if (!skip) { 1204 drm_printf(m, "*\n"); 1205 skip = true; 1206 } 1207 continue; 1208 } 1209 1210 WARN_ON_ONCE(hex_dump_to_buffer(buf + pos, len - pos, 1211 rowsize, sizeof(u32), 1212 line, sizeof(line), 1213 false) >= sizeof(line)); 1214 drm_printf(m, "[%04zx] %s\n", pos, line); 1215 1216 prev = buf + pos; 1217 skip = false; 1218 } 1219 } 1220 1221 static struct intel_timeline *get_timeline(struct i915_request *rq) 1222 { 1223 struct intel_timeline *tl; 1224 1225 /* 1226 * Even though we are holding the engine->active.lock here, there 1227 * is no control over the submission queue per-se and we are 1228 * inspecting the active state at a random point in time, with an 1229 * unknown queue. Play safe and make sure the timeline remains valid. 1230 * (Only being used for pretty printing, one extra kref shouldn't 1231 * cause a camel stampede!) 1232 */ 1233 rcu_read_lock(); 1234 tl = rcu_dereference(rq->timeline); 1235 if (!kref_get_unless_zero(&tl->kref)) 1236 tl = NULL; 1237 rcu_read_unlock(); 1238 1239 return tl; 1240 } 1241 1242 static const char *repr_timer(const struct timer_list *t) 1243 { 1244 if (!READ_ONCE(t->expires)) 1245 return "inactive"; 1246 1247 if (timer_pending(t)) 1248 return "active"; 1249 1250 return "expired"; 1251 } 1252 1253 static void intel_engine_print_registers(struct intel_engine_cs *engine, 1254 struct drm_printer *m) 1255 { 1256 struct drm_i915_private *dev_priv = engine->i915; 1257 struct intel_engine_execlists * const execlists = &engine->execlists; 1258 u64 addr; 1259 1260 if (engine->id == RENDER_CLASS && IS_GEN_RANGE(dev_priv, 4, 7)) 1261 drm_printf(m, "\tCCID: 0x%08x\n", ENGINE_READ(engine, CCID)); 1262 drm_printf(m, "\tRING_START: 0x%08x\n", 1263 ENGINE_READ(engine, RING_START)); 1264 drm_printf(m, "\tRING_HEAD: 0x%08x\n", 1265 ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR); 1266 drm_printf(m, "\tRING_TAIL: 0x%08x\n", 1267 ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR); 1268 drm_printf(m, "\tRING_CTL: 0x%08x%s\n", 1269 ENGINE_READ(engine, RING_CTL), 1270 ENGINE_READ(engine, RING_CTL) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? " [waiting]" : ""); 1271 if (INTEL_GEN(engine->i915) > 2) { 1272 drm_printf(m, "\tRING_MODE: 0x%08x%s\n", 1273 ENGINE_READ(engine, RING_MI_MODE), 1274 ENGINE_READ(engine, RING_MI_MODE) & (MODE_IDLE) ? " [idle]" : ""); 1275 } 1276 1277 if (INTEL_GEN(dev_priv) >= 6) { 1278 drm_printf(m, "\tRING_IMR: %08x\n", 1279 ENGINE_READ(engine, RING_IMR)); 1280 } 1281 1282 addr = intel_engine_get_active_head(engine); 1283 drm_printf(m, "\tACTHD: 0x%08x_%08x\n", 1284 upper_32_bits(addr), lower_32_bits(addr)); 1285 addr = intel_engine_get_last_batch_head(engine); 1286 drm_printf(m, "\tBBADDR: 0x%08x_%08x\n", 1287 upper_32_bits(addr), lower_32_bits(addr)); 1288 if (INTEL_GEN(dev_priv) >= 8) 1289 addr = ENGINE_READ64(engine, RING_DMA_FADD, RING_DMA_FADD_UDW); 1290 else if (INTEL_GEN(dev_priv) >= 4) 1291 addr = ENGINE_READ(engine, RING_DMA_FADD); 1292 else 1293 addr = ENGINE_READ(engine, DMA_FADD_I8XX); 1294 drm_printf(m, "\tDMA_FADDR: 0x%08x_%08x\n", 1295 upper_32_bits(addr), lower_32_bits(addr)); 1296 if (INTEL_GEN(dev_priv) >= 4) { 1297 drm_printf(m, "\tIPEIR: 0x%08x\n", 1298 ENGINE_READ(engine, RING_IPEIR)); 1299 drm_printf(m, "\tIPEHR: 0x%08x\n", 1300 ENGINE_READ(engine, RING_IPEHR)); 1301 } else { 1302 drm_printf(m, "\tIPEIR: 0x%08x\n", ENGINE_READ(engine, IPEIR)); 1303 drm_printf(m, "\tIPEHR: 0x%08x\n", ENGINE_READ(engine, IPEHR)); 1304 } 1305 1306 if (HAS_EXECLISTS(dev_priv)) { 1307 struct i915_request * const *port, *rq; 1308 const u32 *hws = 1309 &engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX]; 1310 const u8 num_entries = execlists->csb_size; 1311 unsigned int idx; 1312 u8 read, write; 1313 1314 drm_printf(m, "\tExeclist tasklet queued? %s (%s), preempt? %s, timeslice? %s\n", 1315 yesno(test_bit(TASKLET_STATE_SCHED, 1316 &engine->execlists.tasklet.state)), 1317 enableddisabled(!atomic_read(&engine->execlists.tasklet.count)), 1318 repr_timer(&engine->execlists.preempt), 1319 repr_timer(&engine->execlists.timer)); 1320 1321 read = execlists->csb_head; 1322 write = READ_ONCE(*execlists->csb_write); 1323 1324 drm_printf(m, "\tExeclist status: 0x%08x %08x; CSB read:%d, write:%d, entries:%d\n", 1325 ENGINE_READ(engine, RING_EXECLIST_STATUS_LO), 1326 ENGINE_READ(engine, RING_EXECLIST_STATUS_HI), 1327 read, write, num_entries); 1328 1329 if (read >= num_entries) 1330 read = 0; 1331 if (write >= num_entries) 1332 write = 0; 1333 if (read > write) 1334 write += num_entries; 1335 while (read < write) { 1336 idx = ++read % num_entries; 1337 drm_printf(m, "\tExeclist CSB[%d]: 0x%08x, context: %d\n", 1338 idx, hws[idx * 2], hws[idx * 2 + 1]); 1339 } 1340 1341 execlists_active_lock_bh(execlists); 1342 rcu_read_lock(); 1343 for (port = execlists->active; (rq = *port); port++) { 1344 char hdr[80]; 1345 int len; 1346 1347 len = snprintf(hdr, sizeof(hdr), 1348 "\t\tActive[%d]: ", 1349 (int)(port - execlists->active)); 1350 if (!i915_request_signaled(rq)) { 1351 struct intel_timeline *tl = get_timeline(rq); 1352 1353 len += snprintf(hdr + len, sizeof(hdr) - len, 1354 "ring:{start:%08x, hwsp:%08x, seqno:%08x}, ", 1355 i915_ggtt_offset(rq->ring->vma), 1356 tl ? tl->hwsp_offset : 0, 1357 hwsp_seqno(rq)); 1358 1359 if (tl) 1360 intel_timeline_put(tl); 1361 } 1362 snprintf(hdr + len, sizeof(hdr) - len, "rq: "); 1363 print_request(m, rq, hdr); 1364 } 1365 for (port = execlists->pending; (rq = *port); port++) { 1366 struct intel_timeline *tl = get_timeline(rq); 1367 char hdr[80]; 1368 1369 snprintf(hdr, sizeof(hdr), 1370 "\t\tPending[%d] ring:{start:%08x, hwsp:%08x, seqno:%08x}, rq: ", 1371 (int)(port - execlists->pending), 1372 i915_ggtt_offset(rq->ring->vma), 1373 tl ? tl->hwsp_offset : 0, 1374 hwsp_seqno(rq)); 1375 print_request(m, rq, hdr); 1376 1377 if (tl) 1378 intel_timeline_put(tl); 1379 } 1380 rcu_read_unlock(); 1381 execlists_active_unlock_bh(execlists); 1382 } else if (INTEL_GEN(dev_priv) > 6) { 1383 drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n", 1384 ENGINE_READ(engine, RING_PP_DIR_BASE)); 1385 drm_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n", 1386 ENGINE_READ(engine, RING_PP_DIR_BASE_READ)); 1387 drm_printf(m, "\tPP_DIR_DCLV: 0x%08x\n", 1388 ENGINE_READ(engine, RING_PP_DIR_DCLV)); 1389 } 1390 } 1391 1392 static void print_request_ring(struct drm_printer *m, struct i915_request *rq) 1393 { 1394 void *ring; 1395 int size; 1396 1397 drm_printf(m, 1398 "[head %04x, postfix %04x, tail %04x, batch 0x%08x_%08x]:\n", 1399 rq->head, rq->postfix, rq->tail, 1400 rq->batch ? upper_32_bits(rq->batch->node.start) : ~0u, 1401 rq->batch ? lower_32_bits(rq->batch->node.start) : ~0u); 1402 1403 size = rq->tail - rq->head; 1404 if (rq->tail < rq->head) 1405 size += rq->ring->size; 1406 1407 ring = kmalloc(size, GFP_ATOMIC); 1408 if (ring) { 1409 const void *vaddr = rq->ring->vaddr; 1410 unsigned int head = rq->head; 1411 unsigned int len = 0; 1412 1413 if (rq->tail < head) { 1414 len = rq->ring->size - head; 1415 memcpy(ring, vaddr + head, len); 1416 head = 0; 1417 } 1418 memcpy(ring + len, vaddr + head, size - len); 1419 1420 hexdump(m, ring, size); 1421 kfree(ring); 1422 } 1423 } 1424 1425 void intel_engine_dump(struct intel_engine_cs *engine, 1426 struct drm_printer *m, 1427 const char *header, ...) 1428 { 1429 struct i915_gpu_error * const error = &engine->i915->gpu_error; 1430 struct i915_request *rq; 1431 intel_wakeref_t wakeref; 1432 unsigned long flags; 1433 1434 if (header) { 1435 va_list ap; 1436 1437 va_start(ap, header); 1438 drm_vprintf(m, header, &ap); 1439 va_end(ap); 1440 } 1441 1442 if (intel_gt_is_wedged(engine->gt)) 1443 drm_printf(m, "*** WEDGED ***\n"); 1444 1445 drm_printf(m, "\tAwake? %d\n", atomic_read(&engine->wakeref.count)); 1446 drm_printf(m, "\tBarriers?: %s\n", 1447 yesno(!llist_empty(&engine->barrier_tasks))); 1448 drm_printf(m, "\tLatency: %luus\n", 1449 ewma__engine_latency_read(&engine->latency)); 1450 1451 rcu_read_lock(); 1452 rq = READ_ONCE(engine->heartbeat.systole); 1453 if (rq) 1454 drm_printf(m, "\tHeartbeat: %d ms ago\n", 1455 jiffies_to_msecs(jiffies - rq->emitted_jiffies)); 1456 rcu_read_unlock(); 1457 drm_printf(m, "\tReset count: %d (global %d)\n", 1458 i915_reset_engine_count(error, engine), 1459 i915_reset_count(error)); 1460 1461 drm_printf(m, "\tRequests:\n"); 1462 1463 spin_lock_irqsave(&engine->active.lock, flags); 1464 rq = intel_engine_find_active_request(engine); 1465 if (rq) { 1466 struct intel_timeline *tl = get_timeline(rq); 1467 1468 print_request(m, rq, "\t\tactive "); 1469 1470 drm_printf(m, "\t\tring->start: 0x%08x\n", 1471 i915_ggtt_offset(rq->ring->vma)); 1472 drm_printf(m, "\t\tring->head: 0x%08x\n", 1473 rq->ring->head); 1474 drm_printf(m, "\t\tring->tail: 0x%08x\n", 1475 rq->ring->tail); 1476 drm_printf(m, "\t\tring->emit: 0x%08x\n", 1477 rq->ring->emit); 1478 drm_printf(m, "\t\tring->space: 0x%08x\n", 1479 rq->ring->space); 1480 1481 if (tl) { 1482 drm_printf(m, "\t\tring->hwsp: 0x%08x\n", 1483 tl->hwsp_offset); 1484 intel_timeline_put(tl); 1485 } 1486 1487 print_request_ring(m, rq); 1488 1489 if (rq->context->lrc_reg_state) { 1490 drm_printf(m, "Logical Ring Context:\n"); 1491 hexdump(m, rq->context->lrc_reg_state, PAGE_SIZE); 1492 } 1493 } 1494 spin_unlock_irqrestore(&engine->active.lock, flags); 1495 1496 drm_printf(m, "\tMMIO base: 0x%08x\n", engine->mmio_base); 1497 wakeref = intel_runtime_pm_get_if_in_use(engine->uncore->rpm); 1498 if (wakeref) { 1499 intel_engine_print_registers(engine, m); 1500 intel_runtime_pm_put(engine->uncore->rpm, wakeref); 1501 } else { 1502 drm_printf(m, "\tDevice is asleep; skipping register dump\n"); 1503 } 1504 1505 intel_execlists_show_requests(engine, m, print_request, 8); 1506 1507 drm_printf(m, "HWSP:\n"); 1508 hexdump(m, engine->status_page.addr, PAGE_SIZE); 1509 1510 drm_printf(m, "Idle? %s\n", yesno(intel_engine_is_idle(engine))); 1511 1512 intel_engine_print_breadcrumbs(engine, m); 1513 } 1514 1515 /** 1516 * intel_enable_engine_stats() - Enable engine busy tracking on engine 1517 * @engine: engine to enable stats collection 1518 * 1519 * Start collecting the engine busyness data for @engine. 1520 * 1521 * Returns 0 on success or a negative error code. 1522 */ 1523 int intel_enable_engine_stats(struct intel_engine_cs *engine) 1524 { 1525 struct intel_engine_execlists *execlists = &engine->execlists; 1526 unsigned long flags; 1527 int err = 0; 1528 1529 if (!intel_engine_supports_stats(engine)) 1530 return -ENODEV; 1531 1532 execlists_active_lock_bh(execlists); 1533 write_seqlock_irqsave(&engine->stats.lock, flags); 1534 1535 if (unlikely(engine->stats.enabled == ~0)) { 1536 err = -EBUSY; 1537 goto unlock; 1538 } 1539 1540 if (engine->stats.enabled++ == 0) { 1541 struct i915_request * const *port; 1542 struct i915_request *rq; 1543 1544 engine->stats.enabled_at = ktime_get(); 1545 1546 /* XXX submission method oblivious? */ 1547 for (port = execlists->active; (rq = *port); port++) 1548 engine->stats.active++; 1549 1550 for (port = execlists->pending; (rq = *port); port++) { 1551 /* Exclude any contexts already counted in active */ 1552 if (!intel_context_inflight_count(rq->context)) 1553 engine->stats.active++; 1554 } 1555 1556 if (engine->stats.active) 1557 engine->stats.start = engine->stats.enabled_at; 1558 } 1559 1560 unlock: 1561 write_sequnlock_irqrestore(&engine->stats.lock, flags); 1562 execlists_active_unlock_bh(execlists); 1563 1564 return err; 1565 } 1566 1567 static ktime_t __intel_engine_get_busy_time(struct intel_engine_cs *engine) 1568 { 1569 ktime_t total = engine->stats.total; 1570 1571 /* 1572 * If the engine is executing something at the moment 1573 * add it to the total. 1574 */ 1575 if (engine->stats.active) 1576 total = ktime_add(total, 1577 ktime_sub(ktime_get(), engine->stats.start)); 1578 1579 return total; 1580 } 1581 1582 /** 1583 * intel_engine_get_busy_time() - Return current accumulated engine busyness 1584 * @engine: engine to report on 1585 * 1586 * Returns accumulated time @engine was busy since engine stats were enabled. 1587 */ 1588 ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine) 1589 { 1590 unsigned int seq; 1591 ktime_t total; 1592 1593 do { 1594 seq = read_seqbegin(&engine->stats.lock); 1595 total = __intel_engine_get_busy_time(engine); 1596 } while (read_seqretry(&engine->stats.lock, seq)); 1597 1598 return total; 1599 } 1600 1601 /** 1602 * intel_disable_engine_stats() - Disable engine busy tracking on engine 1603 * @engine: engine to disable stats collection 1604 * 1605 * Stops collecting the engine busyness data for @engine. 1606 */ 1607 void intel_disable_engine_stats(struct intel_engine_cs *engine) 1608 { 1609 unsigned long flags; 1610 1611 if (!intel_engine_supports_stats(engine)) 1612 return; 1613 1614 write_seqlock_irqsave(&engine->stats.lock, flags); 1615 WARN_ON_ONCE(engine->stats.enabled == 0); 1616 if (--engine->stats.enabled == 0) { 1617 engine->stats.total = __intel_engine_get_busy_time(engine); 1618 engine->stats.active = 0; 1619 } 1620 write_sequnlock_irqrestore(&engine->stats.lock, flags); 1621 } 1622 1623 static bool match_ring(struct i915_request *rq) 1624 { 1625 u32 ring = ENGINE_READ(rq->engine, RING_START); 1626 1627 return ring == i915_ggtt_offset(rq->ring->vma); 1628 } 1629 1630 struct i915_request * 1631 intel_engine_find_active_request(struct intel_engine_cs *engine) 1632 { 1633 struct i915_request *request, *active = NULL; 1634 1635 /* 1636 * We are called by the error capture, reset and to dump engine 1637 * state at random points in time. In particular, note that neither is 1638 * crucially ordered with an interrupt. After a hang, the GPU is dead 1639 * and we assume that no more writes can happen (we waited long enough 1640 * for all writes that were in transaction to be flushed) - adding an 1641 * extra delay for a recent interrupt is pointless. Hence, we do 1642 * not need an engine->irq_seqno_barrier() before the seqno reads. 1643 * At all other times, we must assume the GPU is still running, but 1644 * we only care about the snapshot of this moment. 1645 */ 1646 lockdep_assert_held(&engine->active.lock); 1647 list_for_each_entry(request, &engine->active.requests, sched.link) { 1648 if (i915_request_completed(request)) 1649 continue; 1650 1651 if (!i915_request_started(request)) 1652 continue; 1653 1654 /* More than one preemptible request may match! */ 1655 if (!match_ring(request)) 1656 continue; 1657 1658 active = request; 1659 break; 1660 } 1661 1662 return active; 1663 } 1664 1665 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 1666 #include "mock_engine.c" 1667 #include "selftest_engine.c" 1668 #include "selftest_engine_cs.c" 1669 #endif 1670