1 /* 2 * Copyright © 2013 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 */ 23 24 #include "i915_drv.h" 25 #include "intel_drv.h" 26 #include "i915_vgpu.h" 27 28 #include <asm/iosf_mbi.h> 29 #include <linux/pm_runtime.h> 30 31 #define FORCEWAKE_ACK_TIMEOUT_MS 50 32 #define GT_FIFO_TIMEOUT_MS 10 33 34 #define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32((dev_priv__), (reg__)) 35 36 static const char * const forcewake_domain_names[] = { 37 "render", 38 "blitter", 39 "media", 40 "vdbox0", 41 "vdbox1", 42 "vdbox2", 43 "vdbox3", 44 "vebox0", 45 "vebox1", 46 }; 47 48 const char * 49 intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id) 50 { 51 BUILD_BUG_ON(ARRAY_SIZE(forcewake_domain_names) != FW_DOMAIN_ID_COUNT); 52 53 if (id >= 0 && id < FW_DOMAIN_ID_COUNT) 54 return forcewake_domain_names[id]; 55 56 WARN_ON(id); 57 58 return "unknown"; 59 } 60 61 static inline void 62 fw_domain_reset(struct drm_i915_private *i915, 63 const struct intel_uncore_forcewake_domain *d) 64 { 65 /* 66 * We don't really know if the powerwell for the forcewake domain we are 67 * trying to reset here does exist at this point (engines could be fused 68 * off in ICL+), so no waiting for acks 69 */ 70 __raw_i915_write32(i915, d->reg_set, i915->uncore.fw_reset); 71 } 72 73 static inline void 74 fw_domain_arm_timer(struct intel_uncore_forcewake_domain *d) 75 { 76 d->wake_count++; 77 hrtimer_start_range_ns(&d->timer, 78 NSEC_PER_MSEC, 79 NSEC_PER_MSEC, 80 HRTIMER_MODE_REL); 81 } 82 83 static inline int 84 __wait_for_ack(const struct drm_i915_private *i915, 85 const struct intel_uncore_forcewake_domain *d, 86 const u32 ack, 87 const u32 value) 88 { 89 return wait_for_atomic((__raw_i915_read32(i915, d->reg_ack) & ack) == value, 90 FORCEWAKE_ACK_TIMEOUT_MS); 91 } 92 93 static inline int 94 wait_ack_clear(const struct drm_i915_private *i915, 95 const struct intel_uncore_forcewake_domain *d, 96 const u32 ack) 97 { 98 return __wait_for_ack(i915, d, ack, 0); 99 } 100 101 static inline int 102 wait_ack_set(const struct drm_i915_private *i915, 103 const struct intel_uncore_forcewake_domain *d, 104 const u32 ack) 105 { 106 return __wait_for_ack(i915, d, ack, ack); 107 } 108 109 static inline void 110 fw_domain_wait_ack_clear(const struct drm_i915_private *i915, 111 const struct intel_uncore_forcewake_domain *d) 112 { 113 if (wait_ack_clear(i915, d, FORCEWAKE_KERNEL)) 114 DRM_ERROR("%s: timed out waiting for forcewake ack to clear.\n", 115 intel_uncore_forcewake_domain_to_str(d->id)); 116 } 117 118 enum ack_type { 119 ACK_CLEAR = 0, 120 ACK_SET 121 }; 122 123 static int 124 fw_domain_wait_ack_with_fallback(const struct drm_i915_private *i915, 125 const struct intel_uncore_forcewake_domain *d, 126 const enum ack_type type) 127 { 128 const u32 ack_bit = FORCEWAKE_KERNEL; 129 const u32 value = type == ACK_SET ? ack_bit : 0; 130 unsigned int pass; 131 bool ack_detected; 132 133 /* 134 * There is a possibility of driver's wake request colliding 135 * with hardware's own wake requests and that can cause 136 * hardware to not deliver the driver's ack message. 137 * 138 * Use a fallback bit toggle to kick the gpu state machine 139 * in the hope that the original ack will be delivered along with 140 * the fallback ack. 141 * 142 * This workaround is described in HSDES #1604254524 and it's known as: 143 * WaRsForcewakeAddDelayForAck:skl,bxt,kbl,glk,cfl,cnl,icl 144 * although the name is a bit misleading. 145 */ 146 147 pass = 1; 148 do { 149 wait_ack_clear(i915, d, FORCEWAKE_KERNEL_FALLBACK); 150 151 __raw_i915_write32(i915, d->reg_set, 152 _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL_FALLBACK)); 153 /* Give gt some time to relax before the polling frenzy */ 154 udelay(10 * pass); 155 wait_ack_set(i915, d, FORCEWAKE_KERNEL_FALLBACK); 156 157 ack_detected = (__raw_i915_read32(i915, d->reg_ack) & ack_bit) == value; 158 159 __raw_i915_write32(i915, d->reg_set, 160 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL_FALLBACK)); 161 } while (!ack_detected && pass++ < 10); 162 163 DRM_DEBUG_DRIVER("%s had to use fallback to %s ack, 0x%x (passes %u)\n", 164 intel_uncore_forcewake_domain_to_str(d->id), 165 type == ACK_SET ? "set" : "clear", 166 __raw_i915_read32(i915, d->reg_ack), 167 pass); 168 169 return ack_detected ? 0 : -ETIMEDOUT; 170 } 171 172 static inline void 173 fw_domain_wait_ack_clear_fallback(const struct drm_i915_private *i915, 174 const struct intel_uncore_forcewake_domain *d) 175 { 176 if (likely(!wait_ack_clear(i915, d, FORCEWAKE_KERNEL))) 177 return; 178 179 if (fw_domain_wait_ack_with_fallback(i915, d, ACK_CLEAR)) 180 fw_domain_wait_ack_clear(i915, d); 181 } 182 183 static inline void 184 fw_domain_get(struct drm_i915_private *i915, 185 const struct intel_uncore_forcewake_domain *d) 186 { 187 __raw_i915_write32(i915, d->reg_set, i915->uncore.fw_set); 188 } 189 190 static inline void 191 fw_domain_wait_ack_set(const struct drm_i915_private *i915, 192 const struct intel_uncore_forcewake_domain *d) 193 { 194 if (wait_ack_set(i915, d, FORCEWAKE_KERNEL)) 195 DRM_ERROR("%s: timed out waiting for forcewake ack request.\n", 196 intel_uncore_forcewake_domain_to_str(d->id)); 197 } 198 199 static inline void 200 fw_domain_wait_ack_set_fallback(const struct drm_i915_private *i915, 201 const struct intel_uncore_forcewake_domain *d) 202 { 203 if (likely(!wait_ack_set(i915, d, FORCEWAKE_KERNEL))) 204 return; 205 206 if (fw_domain_wait_ack_with_fallback(i915, d, ACK_SET)) 207 fw_domain_wait_ack_set(i915, d); 208 } 209 210 static inline void 211 fw_domain_put(const struct drm_i915_private *i915, 212 const struct intel_uncore_forcewake_domain *d) 213 { 214 __raw_i915_write32(i915, d->reg_set, i915->uncore.fw_clear); 215 } 216 217 static void 218 fw_domains_get(struct drm_i915_private *i915, enum forcewake_domains fw_domains) 219 { 220 struct intel_uncore_forcewake_domain *d; 221 unsigned int tmp; 222 223 GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains); 224 225 for_each_fw_domain_masked(d, fw_domains, i915, tmp) { 226 fw_domain_wait_ack_clear(i915, d); 227 fw_domain_get(i915, d); 228 } 229 230 for_each_fw_domain_masked(d, fw_domains, i915, tmp) 231 fw_domain_wait_ack_set(i915, d); 232 233 i915->uncore.fw_domains_active |= fw_domains; 234 } 235 236 static void 237 fw_domains_get_with_fallback(struct drm_i915_private *i915, 238 enum forcewake_domains fw_domains) 239 { 240 struct intel_uncore_forcewake_domain *d; 241 unsigned int tmp; 242 243 GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains); 244 245 for_each_fw_domain_masked(d, fw_domains, i915, tmp) { 246 fw_domain_wait_ack_clear_fallback(i915, d); 247 fw_domain_get(i915, d); 248 } 249 250 for_each_fw_domain_masked(d, fw_domains, i915, tmp) 251 fw_domain_wait_ack_set_fallback(i915, d); 252 253 i915->uncore.fw_domains_active |= fw_domains; 254 } 255 256 static void 257 fw_domains_put(struct drm_i915_private *i915, enum forcewake_domains fw_domains) 258 { 259 struct intel_uncore_forcewake_domain *d; 260 unsigned int tmp; 261 262 GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains); 263 264 for_each_fw_domain_masked(d, fw_domains, i915, tmp) 265 fw_domain_put(i915, d); 266 267 i915->uncore.fw_domains_active &= ~fw_domains; 268 } 269 270 static void 271 fw_domains_reset(struct drm_i915_private *i915, 272 enum forcewake_domains fw_domains) 273 { 274 struct intel_uncore_forcewake_domain *d; 275 unsigned int tmp; 276 277 if (!fw_domains) 278 return; 279 280 GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains); 281 282 for_each_fw_domain_masked(d, fw_domains, i915, tmp) 283 fw_domain_reset(i915, d); 284 } 285 286 static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv) 287 { 288 /* w/a for a sporadic read returning 0 by waiting for the GT 289 * thread to wake up. 290 */ 291 if (wait_for_atomic_us((__raw_i915_read32(dev_priv, GEN6_GT_THREAD_STATUS_REG) & 292 GEN6_GT_THREAD_STATUS_CORE_MASK) == 0, 500)) 293 DRM_ERROR("GT thread status wait timed out\n"); 294 } 295 296 static void fw_domains_get_with_thread_status(struct drm_i915_private *dev_priv, 297 enum forcewake_domains fw_domains) 298 { 299 fw_domains_get(dev_priv, fw_domains); 300 301 /* WaRsForcewakeWaitTC0:snb,ivb,hsw,bdw,vlv */ 302 __gen6_gt_wait_for_thread_c0(dev_priv); 303 } 304 305 static inline u32 fifo_free_entries(struct drm_i915_private *dev_priv) 306 { 307 u32 count = __raw_i915_read32(dev_priv, GTFIFOCTL); 308 309 return count & GT_FIFO_FREE_ENTRIES_MASK; 310 } 311 312 static void __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv) 313 { 314 u32 n; 315 316 /* On VLV, FIFO will be shared by both SW and HW. 317 * So, we need to read the FREE_ENTRIES everytime */ 318 if (IS_VALLEYVIEW(dev_priv)) 319 n = fifo_free_entries(dev_priv); 320 else 321 n = dev_priv->uncore.fifo_count; 322 323 if (n <= GT_FIFO_NUM_RESERVED_ENTRIES) { 324 if (wait_for_atomic((n = fifo_free_entries(dev_priv)) > 325 GT_FIFO_NUM_RESERVED_ENTRIES, 326 GT_FIFO_TIMEOUT_MS)) { 327 DRM_DEBUG("GT_FIFO timeout, entries: %u\n", n); 328 return; 329 } 330 } 331 332 dev_priv->uncore.fifo_count = n - 1; 333 } 334 335 static enum hrtimer_restart 336 intel_uncore_fw_release_timer(struct hrtimer *timer) 337 { 338 struct intel_uncore_forcewake_domain *domain = 339 container_of(timer, struct intel_uncore_forcewake_domain, timer); 340 struct drm_i915_private *dev_priv = 341 container_of(domain, struct drm_i915_private, uncore.fw_domain[domain->id]); 342 unsigned long irqflags; 343 344 assert_rpm_device_not_suspended(dev_priv); 345 346 if (xchg(&domain->active, false)) 347 return HRTIMER_RESTART; 348 349 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 350 if (WARN_ON(domain->wake_count == 0)) 351 domain->wake_count++; 352 353 if (--domain->wake_count == 0) 354 dev_priv->uncore.funcs.force_wake_put(dev_priv, domain->mask); 355 356 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 357 358 return HRTIMER_NORESTART; 359 } 360 361 /* Note callers must have acquired the PUNIT->PMIC bus, before calling this. */ 362 static unsigned int 363 intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv) 364 { 365 unsigned long irqflags; 366 struct intel_uncore_forcewake_domain *domain; 367 int retry_count = 100; 368 enum forcewake_domains fw, active_domains; 369 370 iosf_mbi_assert_punit_acquired(); 371 372 /* Hold uncore.lock across reset to prevent any register access 373 * with forcewake not set correctly. Wait until all pending 374 * timers are run before holding. 375 */ 376 while (1) { 377 unsigned int tmp; 378 379 active_domains = 0; 380 381 for_each_fw_domain(domain, dev_priv, tmp) { 382 smp_store_mb(domain->active, false); 383 if (hrtimer_cancel(&domain->timer) == 0) 384 continue; 385 386 intel_uncore_fw_release_timer(&domain->timer); 387 } 388 389 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 390 391 for_each_fw_domain(domain, dev_priv, tmp) { 392 if (hrtimer_active(&domain->timer)) 393 active_domains |= domain->mask; 394 } 395 396 if (active_domains == 0) 397 break; 398 399 if (--retry_count == 0) { 400 DRM_ERROR("Timed out waiting for forcewake timers to finish\n"); 401 break; 402 } 403 404 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 405 cond_resched(); 406 } 407 408 WARN_ON(active_domains); 409 410 fw = dev_priv->uncore.fw_domains_active; 411 if (fw) 412 dev_priv->uncore.funcs.force_wake_put(dev_priv, fw); 413 414 fw_domains_reset(dev_priv, dev_priv->uncore.fw_domains); 415 assert_forcewakes_inactive(dev_priv); 416 417 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 418 419 return fw; /* track the lost user forcewake domains */ 420 } 421 422 static u64 gen9_edram_size(struct drm_i915_private *dev_priv) 423 { 424 const unsigned int ways[8] = { 4, 8, 12, 16, 16, 16, 16, 16 }; 425 const unsigned int sets[4] = { 1, 1, 2, 2 }; 426 const u32 cap = dev_priv->edram_cap; 427 428 return EDRAM_NUM_BANKS(cap) * 429 ways[EDRAM_WAYS_IDX(cap)] * 430 sets[EDRAM_SETS_IDX(cap)] * 431 1024 * 1024; 432 } 433 434 u64 intel_uncore_edram_size(struct drm_i915_private *dev_priv) 435 { 436 if (!HAS_EDRAM(dev_priv)) 437 return 0; 438 439 /* The needed capability bits for size calculation 440 * are not there with pre gen9 so return 128MB always. 441 */ 442 if (INTEL_GEN(dev_priv) < 9) 443 return 128 * 1024 * 1024; 444 445 return gen9_edram_size(dev_priv); 446 } 447 448 static void intel_uncore_edram_detect(struct drm_i915_private *dev_priv) 449 { 450 if (IS_HASWELL(dev_priv) || 451 IS_BROADWELL(dev_priv) || 452 INTEL_GEN(dev_priv) >= 9) { 453 dev_priv->edram_cap = __raw_i915_read32(dev_priv, 454 HSW_EDRAM_CAP); 455 456 /* NB: We can't write IDICR yet because we do not have gt funcs 457 * set up */ 458 } else { 459 dev_priv->edram_cap = 0; 460 } 461 462 if (HAS_EDRAM(dev_priv)) 463 DRM_INFO("Found %lluMB of eDRAM\n", 464 intel_uncore_edram_size(dev_priv) / (1024 * 1024)); 465 } 466 467 static bool 468 fpga_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv) 469 { 470 u32 dbg; 471 472 dbg = __raw_i915_read32(dev_priv, FPGA_DBG); 473 if (likely(!(dbg & FPGA_DBG_RM_NOCLAIM))) 474 return false; 475 476 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); 477 478 return true; 479 } 480 481 static bool 482 vlv_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv) 483 { 484 u32 cer; 485 486 cer = __raw_i915_read32(dev_priv, CLAIM_ER); 487 if (likely(!(cer & (CLAIM_ER_OVERFLOW | CLAIM_ER_CTR_MASK)))) 488 return false; 489 490 __raw_i915_write32(dev_priv, CLAIM_ER, CLAIM_ER_CLR); 491 492 return true; 493 } 494 495 static bool 496 gen6_check_for_fifo_debug(struct drm_i915_private *dev_priv) 497 { 498 u32 fifodbg; 499 500 fifodbg = __raw_i915_read32(dev_priv, GTFIFODBG); 501 502 if (unlikely(fifodbg)) { 503 DRM_DEBUG_DRIVER("GTFIFODBG = 0x08%x\n", fifodbg); 504 __raw_i915_write32(dev_priv, GTFIFODBG, fifodbg); 505 } 506 507 return fifodbg; 508 } 509 510 static bool 511 check_for_unclaimed_mmio(struct drm_i915_private *dev_priv) 512 { 513 bool ret = false; 514 515 if (HAS_FPGA_DBG_UNCLAIMED(dev_priv)) 516 ret |= fpga_check_for_unclaimed_mmio(dev_priv); 517 518 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) 519 ret |= vlv_check_for_unclaimed_mmio(dev_priv); 520 521 if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) 522 ret |= gen6_check_for_fifo_debug(dev_priv); 523 524 return ret; 525 } 526 527 static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv, 528 unsigned int restore_forcewake) 529 { 530 /* clear out unclaimed reg detection bit */ 531 if (check_for_unclaimed_mmio(dev_priv)) 532 DRM_DEBUG("unclaimed mmio detected on uncore init, clearing\n"); 533 534 /* WaDisableShadowRegForCpd:chv */ 535 if (IS_CHERRYVIEW(dev_priv)) { 536 __raw_i915_write32(dev_priv, GTFIFOCTL, 537 __raw_i915_read32(dev_priv, GTFIFOCTL) | 538 GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL | 539 GT_FIFO_CTL_RC6_POLICY_STALL); 540 } 541 542 iosf_mbi_punit_acquire(); 543 intel_uncore_forcewake_reset(dev_priv); 544 if (restore_forcewake) { 545 spin_lock_irq(&dev_priv->uncore.lock); 546 dev_priv->uncore.funcs.force_wake_get(dev_priv, 547 restore_forcewake); 548 549 if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) 550 dev_priv->uncore.fifo_count = 551 fifo_free_entries(dev_priv); 552 spin_unlock_irq(&dev_priv->uncore.lock); 553 } 554 iosf_mbi_punit_release(); 555 } 556 557 void intel_uncore_suspend(struct drm_i915_private *dev_priv) 558 { 559 iosf_mbi_punit_acquire(); 560 iosf_mbi_unregister_pmic_bus_access_notifier_unlocked( 561 &dev_priv->uncore.pmic_bus_access_nb); 562 dev_priv->uncore.fw_domains_saved = 563 intel_uncore_forcewake_reset(dev_priv); 564 iosf_mbi_punit_release(); 565 } 566 567 void intel_uncore_resume_early(struct drm_i915_private *dev_priv) 568 { 569 unsigned int restore_forcewake; 570 571 restore_forcewake = fetch_and_zero(&dev_priv->uncore.fw_domains_saved); 572 __intel_uncore_early_sanitize(dev_priv, restore_forcewake); 573 574 iosf_mbi_register_pmic_bus_access_notifier( 575 &dev_priv->uncore.pmic_bus_access_nb); 576 i915_check_and_clear_faults(dev_priv); 577 } 578 579 void intel_uncore_runtime_resume(struct drm_i915_private *dev_priv) 580 { 581 iosf_mbi_register_pmic_bus_access_notifier( 582 &dev_priv->uncore.pmic_bus_access_nb); 583 } 584 585 void intel_uncore_sanitize(struct drm_i915_private *dev_priv) 586 { 587 /* BIOS often leaves RC6 enabled, but disable it for hw init */ 588 intel_sanitize_gt_powersave(dev_priv); 589 } 590 591 static void __intel_uncore_forcewake_get(struct drm_i915_private *dev_priv, 592 enum forcewake_domains fw_domains) 593 { 594 struct intel_uncore_forcewake_domain *domain; 595 unsigned int tmp; 596 597 fw_domains &= dev_priv->uncore.fw_domains; 598 599 for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp) { 600 if (domain->wake_count++) { 601 fw_domains &= ~domain->mask; 602 domain->active = true; 603 } 604 } 605 606 if (fw_domains) 607 dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains); 608 } 609 610 /** 611 * intel_uncore_forcewake_get - grab forcewake domain references 612 * @dev_priv: i915 device instance 613 * @fw_domains: forcewake domains to get reference on 614 * 615 * This function can be used get GT's forcewake domain references. 616 * Normal register access will handle the forcewake domains automatically. 617 * However if some sequence requires the GT to not power down a particular 618 * forcewake domains this function should be called at the beginning of the 619 * sequence. And subsequently the reference should be dropped by symmetric 620 * call to intel_unforce_forcewake_put(). Usually caller wants all the domains 621 * to be kept awake so the @fw_domains would be then FORCEWAKE_ALL. 622 */ 623 void intel_uncore_forcewake_get(struct drm_i915_private *dev_priv, 624 enum forcewake_domains fw_domains) 625 { 626 unsigned long irqflags; 627 628 if (!dev_priv->uncore.funcs.force_wake_get) 629 return; 630 631 assert_rpm_wakelock_held(dev_priv); 632 633 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 634 __intel_uncore_forcewake_get(dev_priv, fw_domains); 635 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 636 } 637 638 /** 639 * intel_uncore_forcewake_user_get - claim forcewake on behalf of userspace 640 * @dev_priv: i915 device instance 641 * 642 * This function is a wrapper around intel_uncore_forcewake_get() to acquire 643 * the GT powerwell and in the process disable our debugging for the 644 * duration of userspace's bypass. 645 */ 646 void intel_uncore_forcewake_user_get(struct drm_i915_private *dev_priv) 647 { 648 spin_lock_irq(&dev_priv->uncore.lock); 649 if (!dev_priv->uncore.user_forcewake.count++) { 650 intel_uncore_forcewake_get__locked(dev_priv, FORCEWAKE_ALL); 651 652 /* Save and disable mmio debugging for the user bypass */ 653 dev_priv->uncore.user_forcewake.saved_mmio_check = 654 dev_priv->uncore.unclaimed_mmio_check; 655 dev_priv->uncore.user_forcewake.saved_mmio_debug = 656 i915_modparams.mmio_debug; 657 658 dev_priv->uncore.unclaimed_mmio_check = 0; 659 i915_modparams.mmio_debug = 0; 660 } 661 spin_unlock_irq(&dev_priv->uncore.lock); 662 } 663 664 /** 665 * intel_uncore_forcewake_user_put - release forcewake on behalf of userspace 666 * @dev_priv: i915 device instance 667 * 668 * This function complements intel_uncore_forcewake_user_get() and releases 669 * the GT powerwell taken on behalf of the userspace bypass. 670 */ 671 void intel_uncore_forcewake_user_put(struct drm_i915_private *dev_priv) 672 { 673 spin_lock_irq(&dev_priv->uncore.lock); 674 if (!--dev_priv->uncore.user_forcewake.count) { 675 if (intel_uncore_unclaimed_mmio(dev_priv)) 676 dev_info(dev_priv->drm.dev, 677 "Invalid mmio detected during user access\n"); 678 679 dev_priv->uncore.unclaimed_mmio_check = 680 dev_priv->uncore.user_forcewake.saved_mmio_check; 681 i915_modparams.mmio_debug = 682 dev_priv->uncore.user_forcewake.saved_mmio_debug; 683 684 intel_uncore_forcewake_put__locked(dev_priv, FORCEWAKE_ALL); 685 } 686 spin_unlock_irq(&dev_priv->uncore.lock); 687 } 688 689 /** 690 * intel_uncore_forcewake_get__locked - grab forcewake domain references 691 * @dev_priv: i915 device instance 692 * @fw_domains: forcewake domains to get reference on 693 * 694 * See intel_uncore_forcewake_get(). This variant places the onus 695 * on the caller to explicitly handle the dev_priv->uncore.lock spinlock. 696 */ 697 void intel_uncore_forcewake_get__locked(struct drm_i915_private *dev_priv, 698 enum forcewake_domains fw_domains) 699 { 700 lockdep_assert_held(&dev_priv->uncore.lock); 701 702 if (!dev_priv->uncore.funcs.force_wake_get) 703 return; 704 705 __intel_uncore_forcewake_get(dev_priv, fw_domains); 706 } 707 708 static void __intel_uncore_forcewake_put(struct drm_i915_private *dev_priv, 709 enum forcewake_domains fw_domains) 710 { 711 struct intel_uncore_forcewake_domain *domain; 712 unsigned int tmp; 713 714 fw_domains &= dev_priv->uncore.fw_domains; 715 716 for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp) { 717 if (WARN_ON(domain->wake_count == 0)) 718 continue; 719 720 if (--domain->wake_count) { 721 domain->active = true; 722 continue; 723 } 724 725 fw_domain_arm_timer(domain); 726 } 727 } 728 729 /** 730 * intel_uncore_forcewake_put - release a forcewake domain reference 731 * @dev_priv: i915 device instance 732 * @fw_domains: forcewake domains to put references 733 * 734 * This function drops the device-level forcewakes for specified 735 * domains obtained by intel_uncore_forcewake_get(). 736 */ 737 void intel_uncore_forcewake_put(struct drm_i915_private *dev_priv, 738 enum forcewake_domains fw_domains) 739 { 740 unsigned long irqflags; 741 742 if (!dev_priv->uncore.funcs.force_wake_put) 743 return; 744 745 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 746 __intel_uncore_forcewake_put(dev_priv, fw_domains); 747 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 748 } 749 750 /** 751 * intel_uncore_forcewake_put__locked - grab forcewake domain references 752 * @dev_priv: i915 device instance 753 * @fw_domains: forcewake domains to get reference on 754 * 755 * See intel_uncore_forcewake_put(). This variant places the onus 756 * on the caller to explicitly handle the dev_priv->uncore.lock spinlock. 757 */ 758 void intel_uncore_forcewake_put__locked(struct drm_i915_private *dev_priv, 759 enum forcewake_domains fw_domains) 760 { 761 lockdep_assert_held(&dev_priv->uncore.lock); 762 763 if (!dev_priv->uncore.funcs.force_wake_put) 764 return; 765 766 __intel_uncore_forcewake_put(dev_priv, fw_domains); 767 } 768 769 void assert_forcewakes_inactive(struct drm_i915_private *dev_priv) 770 { 771 if (!dev_priv->uncore.funcs.force_wake_get) 772 return; 773 774 WARN(dev_priv->uncore.fw_domains_active, 775 "Expected all fw_domains to be inactive, but %08x are still on\n", 776 dev_priv->uncore.fw_domains_active); 777 } 778 779 void assert_forcewakes_active(struct drm_i915_private *dev_priv, 780 enum forcewake_domains fw_domains) 781 { 782 if (!dev_priv->uncore.funcs.force_wake_get) 783 return; 784 785 assert_rpm_wakelock_held(dev_priv); 786 787 fw_domains &= dev_priv->uncore.fw_domains; 788 WARN(fw_domains & ~dev_priv->uncore.fw_domains_active, 789 "Expected %08x fw_domains to be active, but %08x are off\n", 790 fw_domains, fw_domains & ~dev_priv->uncore.fw_domains_active); 791 } 792 793 /* We give fast paths for the really cool registers */ 794 #define NEEDS_FORCE_WAKE(reg) ((reg) < 0x40000) 795 796 #define GEN11_NEEDS_FORCE_WAKE(reg) \ 797 ((reg) < 0x40000 || ((reg) >= 0x1c0000 && (reg) < 0x1dc000)) 798 799 #define __gen6_reg_read_fw_domains(offset) \ 800 ({ \ 801 enum forcewake_domains __fwd; \ 802 if (NEEDS_FORCE_WAKE(offset)) \ 803 __fwd = FORCEWAKE_RENDER; \ 804 else \ 805 __fwd = 0; \ 806 __fwd; \ 807 }) 808 809 static int fw_range_cmp(u32 offset, const struct intel_forcewake_range *entry) 810 { 811 if (offset < entry->start) 812 return -1; 813 else if (offset > entry->end) 814 return 1; 815 else 816 return 0; 817 } 818 819 /* Copied and "macroized" from lib/bsearch.c */ 820 #define BSEARCH(key, base, num, cmp) ({ \ 821 unsigned int start__ = 0, end__ = (num); \ 822 typeof(base) result__ = NULL; \ 823 while (start__ < end__) { \ 824 unsigned int mid__ = start__ + (end__ - start__) / 2; \ 825 int ret__ = (cmp)((key), (base) + mid__); \ 826 if (ret__ < 0) { \ 827 end__ = mid__; \ 828 } else if (ret__ > 0) { \ 829 start__ = mid__ + 1; \ 830 } else { \ 831 result__ = (base) + mid__; \ 832 break; \ 833 } \ 834 } \ 835 result__; \ 836 }) 837 838 static enum forcewake_domains 839 find_fw_domain(struct drm_i915_private *dev_priv, u32 offset) 840 { 841 const struct intel_forcewake_range *entry; 842 843 entry = BSEARCH(offset, 844 dev_priv->uncore.fw_domains_table, 845 dev_priv->uncore.fw_domains_table_entries, 846 fw_range_cmp); 847 848 if (!entry) 849 return 0; 850 851 /* 852 * The list of FW domains depends on the SKU in gen11+ so we 853 * can't determine it statically. We use FORCEWAKE_ALL and 854 * translate it here to the list of available domains. 855 */ 856 if (entry->domains == FORCEWAKE_ALL) 857 return dev_priv->uncore.fw_domains; 858 859 WARN(entry->domains & ~dev_priv->uncore.fw_domains, 860 "Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n", 861 entry->domains & ~dev_priv->uncore.fw_domains, offset); 862 863 return entry->domains; 864 } 865 866 #define GEN_FW_RANGE(s, e, d) \ 867 { .start = (s), .end = (e), .domains = (d) } 868 869 #define HAS_FWTABLE(dev_priv) \ 870 (INTEL_GEN(dev_priv) >= 9 || \ 871 IS_CHERRYVIEW(dev_priv) || \ 872 IS_VALLEYVIEW(dev_priv)) 873 874 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ 875 static const struct intel_forcewake_range __vlv_fw_ranges[] = { 876 GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER), 877 GEN_FW_RANGE(0x5000, 0x7fff, FORCEWAKE_RENDER), 878 GEN_FW_RANGE(0xb000, 0x11fff, FORCEWAKE_RENDER), 879 GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA), 880 GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_MEDIA), 881 GEN_FW_RANGE(0x2e000, 0x2ffff, FORCEWAKE_RENDER), 882 GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA), 883 }; 884 885 #define __fwtable_reg_read_fw_domains(offset) \ 886 ({ \ 887 enum forcewake_domains __fwd = 0; \ 888 if (NEEDS_FORCE_WAKE((offset))) \ 889 __fwd = find_fw_domain(dev_priv, offset); \ 890 __fwd; \ 891 }) 892 893 #define __gen11_fwtable_reg_read_fw_domains(offset) \ 894 ({ \ 895 enum forcewake_domains __fwd = 0; \ 896 if (GEN11_NEEDS_FORCE_WAKE((offset))) \ 897 __fwd = find_fw_domain(dev_priv, offset); \ 898 __fwd; \ 899 }) 900 901 /* *Must* be sorted by offset! See intel_shadow_table_check(). */ 902 static const i915_reg_t gen8_shadowed_regs[] = { 903 RING_TAIL(RENDER_RING_BASE), /* 0x2000 (base) */ 904 GEN6_RPNSWREQ, /* 0xA008 */ 905 GEN6_RC_VIDEO_FREQ, /* 0xA00C */ 906 RING_TAIL(GEN6_BSD_RING_BASE), /* 0x12000 (base) */ 907 RING_TAIL(VEBOX_RING_BASE), /* 0x1a000 (base) */ 908 RING_TAIL(BLT_RING_BASE), /* 0x22000 (base) */ 909 /* TODO: Other registers are not yet used */ 910 }; 911 912 static const i915_reg_t gen11_shadowed_regs[] = { 913 RING_TAIL(RENDER_RING_BASE), /* 0x2000 (base) */ 914 GEN6_RPNSWREQ, /* 0xA008 */ 915 GEN6_RC_VIDEO_FREQ, /* 0xA00C */ 916 RING_TAIL(BLT_RING_BASE), /* 0x22000 (base) */ 917 RING_TAIL(GEN11_BSD_RING_BASE), /* 0x1C0000 (base) */ 918 RING_TAIL(GEN11_BSD2_RING_BASE), /* 0x1C4000 (base) */ 919 RING_TAIL(GEN11_VEBOX_RING_BASE), /* 0x1C8000 (base) */ 920 RING_TAIL(GEN11_BSD3_RING_BASE), /* 0x1D0000 (base) */ 921 RING_TAIL(GEN11_BSD4_RING_BASE), /* 0x1D4000 (base) */ 922 RING_TAIL(GEN11_VEBOX2_RING_BASE), /* 0x1D8000 (base) */ 923 /* TODO: Other registers are not yet used */ 924 }; 925 926 static int mmio_reg_cmp(u32 key, const i915_reg_t *reg) 927 { 928 u32 offset = i915_mmio_reg_offset(*reg); 929 930 if (key < offset) 931 return -1; 932 else if (key > offset) 933 return 1; 934 else 935 return 0; 936 } 937 938 #define __is_genX_shadowed(x) \ 939 static bool is_gen##x##_shadowed(u32 offset) \ 940 { \ 941 const i915_reg_t *regs = gen##x##_shadowed_regs; \ 942 return BSEARCH(offset, regs, ARRAY_SIZE(gen##x##_shadowed_regs), \ 943 mmio_reg_cmp); \ 944 } 945 946 __is_genX_shadowed(8) 947 __is_genX_shadowed(11) 948 949 #define __gen8_reg_write_fw_domains(offset) \ 950 ({ \ 951 enum forcewake_domains __fwd; \ 952 if (NEEDS_FORCE_WAKE(offset) && !is_gen8_shadowed(offset)) \ 953 __fwd = FORCEWAKE_RENDER; \ 954 else \ 955 __fwd = 0; \ 956 __fwd; \ 957 }) 958 959 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ 960 static const struct intel_forcewake_range __chv_fw_ranges[] = { 961 GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER), 962 GEN_FW_RANGE(0x4000, 0x4fff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 963 GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), 964 GEN_FW_RANGE(0x8000, 0x82ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 965 GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER), 966 GEN_FW_RANGE(0x8500, 0x85ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 967 GEN_FW_RANGE(0x8800, 0x88ff, FORCEWAKE_MEDIA), 968 GEN_FW_RANGE(0x9000, 0xafff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 969 GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER), 970 GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA), 971 GEN_FW_RANGE(0xe000, 0xe7ff, FORCEWAKE_RENDER), 972 GEN_FW_RANGE(0xf000, 0xffff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 973 GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA), 974 GEN_FW_RANGE(0x1a000, 0x1bfff, FORCEWAKE_MEDIA), 975 GEN_FW_RANGE(0x1e800, 0x1e9ff, FORCEWAKE_MEDIA), 976 GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA), 977 }; 978 979 #define __fwtable_reg_write_fw_domains(offset) \ 980 ({ \ 981 enum forcewake_domains __fwd = 0; \ 982 if (NEEDS_FORCE_WAKE((offset)) && !is_gen8_shadowed(offset)) \ 983 __fwd = find_fw_domain(dev_priv, offset); \ 984 __fwd; \ 985 }) 986 987 #define __gen11_fwtable_reg_write_fw_domains(offset) \ 988 ({ \ 989 enum forcewake_domains __fwd = 0; \ 990 if (GEN11_NEEDS_FORCE_WAKE((offset)) && !is_gen11_shadowed(offset)) \ 991 __fwd = find_fw_domain(dev_priv, offset); \ 992 __fwd; \ 993 }) 994 995 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ 996 static const struct intel_forcewake_range __gen9_fw_ranges[] = { 997 GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER), 998 GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */ 999 GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER), 1000 GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER), 1001 GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER), 1002 GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER), 1003 GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), 1004 GEN_FW_RANGE(0x8000, 0x812f, FORCEWAKE_BLITTER), 1005 GEN_FW_RANGE(0x8130, 0x813f, FORCEWAKE_MEDIA), 1006 GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER), 1007 GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER), 1008 GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER), 1009 GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_BLITTER), 1010 GEN_FW_RANGE(0x8800, 0x89ff, FORCEWAKE_MEDIA), 1011 GEN_FW_RANGE(0x8a00, 0x8bff, FORCEWAKE_BLITTER), 1012 GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER), 1013 GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER), 1014 GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 1015 GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER), 1016 GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER), 1017 GEN_FW_RANGE(0xb480, 0xcfff, FORCEWAKE_BLITTER), 1018 GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA), 1019 GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_BLITTER), 1020 GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER), 1021 GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_BLITTER), 1022 GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA), 1023 GEN_FW_RANGE(0x14000, 0x19fff, FORCEWAKE_BLITTER), 1024 GEN_FW_RANGE(0x1a000, 0x1e9ff, FORCEWAKE_MEDIA), 1025 GEN_FW_RANGE(0x1ea00, 0x243ff, FORCEWAKE_BLITTER), 1026 GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER), 1027 GEN_FW_RANGE(0x24800, 0x2ffff, FORCEWAKE_BLITTER), 1028 GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA), 1029 }; 1030 1031 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ 1032 static const struct intel_forcewake_range __gen11_fw_ranges[] = { 1033 GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER), 1034 GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */ 1035 GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER), 1036 GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER), 1037 GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER), 1038 GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER), 1039 GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), 1040 GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_BLITTER), 1041 GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER), 1042 GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER), 1043 GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER), 1044 GEN_FW_RANGE(0x8500, 0x8bff, FORCEWAKE_BLITTER), 1045 GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER), 1046 GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER), 1047 GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_ALL), 1048 GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER), 1049 GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER), 1050 GEN_FW_RANGE(0xb480, 0xdfff, FORCEWAKE_BLITTER), 1051 GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER), 1052 GEN_FW_RANGE(0xe900, 0x243ff, FORCEWAKE_BLITTER), 1053 GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER), 1054 GEN_FW_RANGE(0x24800, 0x3ffff, FORCEWAKE_BLITTER), 1055 GEN_FW_RANGE(0x40000, 0x1bffff, 0), 1056 GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), 1057 GEN_FW_RANGE(0x1c4000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX1), 1058 GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), 1059 GEN_FW_RANGE(0x1cc000, 0x1cffff, FORCEWAKE_BLITTER), 1060 GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2), 1061 GEN_FW_RANGE(0x1d4000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX3), 1062 GEN_FW_RANGE(0x1d8000, 0x1dbfff, FORCEWAKE_MEDIA_VEBOX1) 1063 }; 1064 1065 static void 1066 ilk_dummy_write(struct drm_i915_private *dev_priv) 1067 { 1068 /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up 1069 * the chip from rc6 before touching it for real. MI_MODE is masked, 1070 * hence harmless to write 0 into. */ 1071 __raw_i915_write32(dev_priv, MI_MODE, 0); 1072 } 1073 1074 static void 1075 __unclaimed_reg_debug(struct drm_i915_private *dev_priv, 1076 const i915_reg_t reg, 1077 const bool read, 1078 const bool before) 1079 { 1080 if (WARN(check_for_unclaimed_mmio(dev_priv) && !before, 1081 "Unclaimed %s register 0x%x\n", 1082 read ? "read from" : "write to", 1083 i915_mmio_reg_offset(reg))) 1084 /* Only report the first N failures */ 1085 i915_modparams.mmio_debug--; 1086 } 1087 1088 static inline void 1089 unclaimed_reg_debug(struct drm_i915_private *dev_priv, 1090 const i915_reg_t reg, 1091 const bool read, 1092 const bool before) 1093 { 1094 if (likely(!i915_modparams.mmio_debug)) 1095 return; 1096 1097 __unclaimed_reg_debug(dev_priv, reg, read, before); 1098 } 1099 1100 #define GEN2_READ_HEADER(x) \ 1101 u##x val = 0; \ 1102 assert_rpm_wakelock_held(dev_priv); 1103 1104 #define GEN2_READ_FOOTER \ 1105 trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \ 1106 return val 1107 1108 #define __gen2_read(x) \ 1109 static u##x \ 1110 gen2_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \ 1111 GEN2_READ_HEADER(x); \ 1112 val = __raw_i915_read##x(dev_priv, reg); \ 1113 GEN2_READ_FOOTER; \ 1114 } 1115 1116 #define __gen5_read(x) \ 1117 static u##x \ 1118 gen5_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \ 1119 GEN2_READ_HEADER(x); \ 1120 ilk_dummy_write(dev_priv); \ 1121 val = __raw_i915_read##x(dev_priv, reg); \ 1122 GEN2_READ_FOOTER; \ 1123 } 1124 1125 __gen5_read(8) 1126 __gen5_read(16) 1127 __gen5_read(32) 1128 __gen5_read(64) 1129 __gen2_read(8) 1130 __gen2_read(16) 1131 __gen2_read(32) 1132 __gen2_read(64) 1133 1134 #undef __gen5_read 1135 #undef __gen2_read 1136 1137 #undef GEN2_READ_FOOTER 1138 #undef GEN2_READ_HEADER 1139 1140 #define GEN6_READ_HEADER(x) \ 1141 u32 offset = i915_mmio_reg_offset(reg); \ 1142 unsigned long irqflags; \ 1143 u##x val = 0; \ 1144 assert_rpm_wakelock_held(dev_priv); \ 1145 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \ 1146 unclaimed_reg_debug(dev_priv, reg, true, true) 1147 1148 #define GEN6_READ_FOOTER \ 1149 unclaimed_reg_debug(dev_priv, reg, true, false); \ 1150 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \ 1151 trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \ 1152 return val 1153 1154 static noinline void ___force_wake_auto(struct drm_i915_private *dev_priv, 1155 enum forcewake_domains fw_domains) 1156 { 1157 struct intel_uncore_forcewake_domain *domain; 1158 unsigned int tmp; 1159 1160 GEM_BUG_ON(fw_domains & ~dev_priv->uncore.fw_domains); 1161 1162 for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp) 1163 fw_domain_arm_timer(domain); 1164 1165 dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains); 1166 } 1167 1168 static inline void __force_wake_auto(struct drm_i915_private *dev_priv, 1169 enum forcewake_domains fw_domains) 1170 { 1171 if (WARN_ON(!fw_domains)) 1172 return; 1173 1174 /* Turn on all requested but inactive supported forcewake domains. */ 1175 fw_domains &= dev_priv->uncore.fw_domains; 1176 fw_domains &= ~dev_priv->uncore.fw_domains_active; 1177 1178 if (fw_domains) 1179 ___force_wake_auto(dev_priv, fw_domains); 1180 } 1181 1182 #define __gen_read(func, x) \ 1183 static u##x \ 1184 func##_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \ 1185 enum forcewake_domains fw_engine; \ 1186 GEN6_READ_HEADER(x); \ 1187 fw_engine = __##func##_reg_read_fw_domains(offset); \ 1188 if (fw_engine) \ 1189 __force_wake_auto(dev_priv, fw_engine); \ 1190 val = __raw_i915_read##x(dev_priv, reg); \ 1191 GEN6_READ_FOOTER; \ 1192 } 1193 #define __gen6_read(x) __gen_read(gen6, x) 1194 #define __fwtable_read(x) __gen_read(fwtable, x) 1195 #define __gen11_fwtable_read(x) __gen_read(gen11_fwtable, x) 1196 1197 __gen11_fwtable_read(8) 1198 __gen11_fwtable_read(16) 1199 __gen11_fwtable_read(32) 1200 __gen11_fwtable_read(64) 1201 __fwtable_read(8) 1202 __fwtable_read(16) 1203 __fwtable_read(32) 1204 __fwtable_read(64) 1205 __gen6_read(8) 1206 __gen6_read(16) 1207 __gen6_read(32) 1208 __gen6_read(64) 1209 1210 #undef __gen11_fwtable_read 1211 #undef __fwtable_read 1212 #undef __gen6_read 1213 #undef GEN6_READ_FOOTER 1214 #undef GEN6_READ_HEADER 1215 1216 #define GEN2_WRITE_HEADER \ 1217 trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \ 1218 assert_rpm_wakelock_held(dev_priv); \ 1219 1220 #define GEN2_WRITE_FOOTER 1221 1222 #define __gen2_write(x) \ 1223 static void \ 1224 gen2_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \ 1225 GEN2_WRITE_HEADER; \ 1226 __raw_i915_write##x(dev_priv, reg, val); \ 1227 GEN2_WRITE_FOOTER; \ 1228 } 1229 1230 #define __gen5_write(x) \ 1231 static void \ 1232 gen5_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \ 1233 GEN2_WRITE_HEADER; \ 1234 ilk_dummy_write(dev_priv); \ 1235 __raw_i915_write##x(dev_priv, reg, val); \ 1236 GEN2_WRITE_FOOTER; \ 1237 } 1238 1239 __gen5_write(8) 1240 __gen5_write(16) 1241 __gen5_write(32) 1242 __gen2_write(8) 1243 __gen2_write(16) 1244 __gen2_write(32) 1245 1246 #undef __gen5_write 1247 #undef __gen2_write 1248 1249 #undef GEN2_WRITE_FOOTER 1250 #undef GEN2_WRITE_HEADER 1251 1252 #define GEN6_WRITE_HEADER \ 1253 u32 offset = i915_mmio_reg_offset(reg); \ 1254 unsigned long irqflags; \ 1255 trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \ 1256 assert_rpm_wakelock_held(dev_priv); \ 1257 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \ 1258 unclaimed_reg_debug(dev_priv, reg, false, true) 1259 1260 #define GEN6_WRITE_FOOTER \ 1261 unclaimed_reg_debug(dev_priv, reg, false, false); \ 1262 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags) 1263 1264 #define __gen6_write(x) \ 1265 static void \ 1266 gen6_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \ 1267 GEN6_WRITE_HEADER; \ 1268 if (NEEDS_FORCE_WAKE(offset)) \ 1269 __gen6_gt_wait_for_fifo(dev_priv); \ 1270 __raw_i915_write##x(dev_priv, reg, val); \ 1271 GEN6_WRITE_FOOTER; \ 1272 } 1273 1274 #define __gen_write(func, x) \ 1275 static void \ 1276 func##_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \ 1277 enum forcewake_domains fw_engine; \ 1278 GEN6_WRITE_HEADER; \ 1279 fw_engine = __##func##_reg_write_fw_domains(offset); \ 1280 if (fw_engine) \ 1281 __force_wake_auto(dev_priv, fw_engine); \ 1282 __raw_i915_write##x(dev_priv, reg, val); \ 1283 GEN6_WRITE_FOOTER; \ 1284 } 1285 #define __gen8_write(x) __gen_write(gen8, x) 1286 #define __fwtable_write(x) __gen_write(fwtable, x) 1287 #define __gen11_fwtable_write(x) __gen_write(gen11_fwtable, x) 1288 1289 __gen11_fwtable_write(8) 1290 __gen11_fwtable_write(16) 1291 __gen11_fwtable_write(32) 1292 __fwtable_write(8) 1293 __fwtable_write(16) 1294 __fwtable_write(32) 1295 __gen8_write(8) 1296 __gen8_write(16) 1297 __gen8_write(32) 1298 __gen6_write(8) 1299 __gen6_write(16) 1300 __gen6_write(32) 1301 1302 #undef __gen11_fwtable_write 1303 #undef __fwtable_write 1304 #undef __gen8_write 1305 #undef __gen6_write 1306 #undef GEN6_WRITE_FOOTER 1307 #undef GEN6_WRITE_HEADER 1308 1309 #define ASSIGN_WRITE_MMIO_VFUNCS(i915, x) \ 1310 do { \ 1311 (i915)->uncore.funcs.mmio_writeb = x##_write8; \ 1312 (i915)->uncore.funcs.mmio_writew = x##_write16; \ 1313 (i915)->uncore.funcs.mmio_writel = x##_write32; \ 1314 } while (0) 1315 1316 #define ASSIGN_READ_MMIO_VFUNCS(i915, x) \ 1317 do { \ 1318 (i915)->uncore.funcs.mmio_readb = x##_read8; \ 1319 (i915)->uncore.funcs.mmio_readw = x##_read16; \ 1320 (i915)->uncore.funcs.mmio_readl = x##_read32; \ 1321 (i915)->uncore.funcs.mmio_readq = x##_read64; \ 1322 } while (0) 1323 1324 1325 static void fw_domain_init(struct drm_i915_private *dev_priv, 1326 enum forcewake_domain_id domain_id, 1327 i915_reg_t reg_set, 1328 i915_reg_t reg_ack) 1329 { 1330 struct intel_uncore_forcewake_domain *d; 1331 1332 if (WARN_ON(domain_id >= FW_DOMAIN_ID_COUNT)) 1333 return; 1334 1335 d = &dev_priv->uncore.fw_domain[domain_id]; 1336 1337 WARN_ON(d->wake_count); 1338 1339 WARN_ON(!i915_mmio_reg_valid(reg_set)); 1340 WARN_ON(!i915_mmio_reg_valid(reg_ack)); 1341 1342 d->wake_count = 0; 1343 d->reg_set = reg_set; 1344 d->reg_ack = reg_ack; 1345 1346 d->id = domain_id; 1347 1348 BUILD_BUG_ON(FORCEWAKE_RENDER != (1 << FW_DOMAIN_ID_RENDER)); 1349 BUILD_BUG_ON(FORCEWAKE_BLITTER != (1 << FW_DOMAIN_ID_BLITTER)); 1350 BUILD_BUG_ON(FORCEWAKE_MEDIA != (1 << FW_DOMAIN_ID_MEDIA)); 1351 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX0)); 1352 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX1)); 1353 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX2 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX2)); 1354 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX3 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX3)); 1355 BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX0)); 1356 BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX1)); 1357 1358 1359 d->mask = BIT(domain_id); 1360 1361 hrtimer_init(&d->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1362 d->timer.function = intel_uncore_fw_release_timer; 1363 1364 dev_priv->uncore.fw_domains |= BIT(domain_id); 1365 1366 fw_domain_reset(dev_priv, d); 1367 } 1368 1369 static void fw_domain_fini(struct drm_i915_private *dev_priv, 1370 enum forcewake_domain_id domain_id) 1371 { 1372 struct intel_uncore_forcewake_domain *d; 1373 1374 if (WARN_ON(domain_id >= FW_DOMAIN_ID_COUNT)) 1375 return; 1376 1377 d = &dev_priv->uncore.fw_domain[domain_id]; 1378 1379 WARN_ON(d->wake_count); 1380 WARN_ON(hrtimer_cancel(&d->timer)); 1381 memset(d, 0, sizeof(*d)); 1382 1383 dev_priv->uncore.fw_domains &= ~BIT(domain_id); 1384 } 1385 1386 static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv) 1387 { 1388 if (INTEL_GEN(dev_priv) <= 5 || intel_vgpu_active(dev_priv)) 1389 return; 1390 1391 if (IS_GEN6(dev_priv)) { 1392 dev_priv->uncore.fw_reset = 0; 1393 dev_priv->uncore.fw_set = FORCEWAKE_KERNEL; 1394 dev_priv->uncore.fw_clear = 0; 1395 } else { 1396 /* WaRsClearFWBitsAtReset:bdw,skl */ 1397 dev_priv->uncore.fw_reset = _MASKED_BIT_DISABLE(0xffff); 1398 dev_priv->uncore.fw_set = _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL); 1399 dev_priv->uncore.fw_clear = _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL); 1400 } 1401 1402 if (INTEL_GEN(dev_priv) >= 11) { 1403 int i; 1404 1405 dev_priv->uncore.funcs.force_wake_get = 1406 fw_domains_get_with_fallback; 1407 dev_priv->uncore.funcs.force_wake_put = fw_domains_put; 1408 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, 1409 FORCEWAKE_RENDER_GEN9, 1410 FORCEWAKE_ACK_RENDER_GEN9); 1411 fw_domain_init(dev_priv, FW_DOMAIN_ID_BLITTER, 1412 FORCEWAKE_BLITTER_GEN9, 1413 FORCEWAKE_ACK_BLITTER_GEN9); 1414 for (i = 0; i < I915_MAX_VCS; i++) { 1415 if (!HAS_ENGINE(dev_priv, _VCS(i))) 1416 continue; 1417 1418 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA_VDBOX0 + i, 1419 FORCEWAKE_MEDIA_VDBOX_GEN11(i), 1420 FORCEWAKE_ACK_MEDIA_VDBOX_GEN11(i)); 1421 } 1422 for (i = 0; i < I915_MAX_VECS; i++) { 1423 if (!HAS_ENGINE(dev_priv, _VECS(i))) 1424 continue; 1425 1426 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA_VEBOX0 + i, 1427 FORCEWAKE_MEDIA_VEBOX_GEN11(i), 1428 FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(i)); 1429 } 1430 } else if (IS_GEN9(dev_priv) || IS_GEN10(dev_priv)) { 1431 dev_priv->uncore.funcs.force_wake_get = 1432 fw_domains_get_with_fallback; 1433 dev_priv->uncore.funcs.force_wake_put = fw_domains_put; 1434 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, 1435 FORCEWAKE_RENDER_GEN9, 1436 FORCEWAKE_ACK_RENDER_GEN9); 1437 fw_domain_init(dev_priv, FW_DOMAIN_ID_BLITTER, 1438 FORCEWAKE_BLITTER_GEN9, 1439 FORCEWAKE_ACK_BLITTER_GEN9); 1440 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA, 1441 FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9); 1442 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { 1443 dev_priv->uncore.funcs.force_wake_get = fw_domains_get; 1444 dev_priv->uncore.funcs.force_wake_put = fw_domains_put; 1445 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, 1446 FORCEWAKE_VLV, FORCEWAKE_ACK_VLV); 1447 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA, 1448 FORCEWAKE_MEDIA_VLV, FORCEWAKE_ACK_MEDIA_VLV); 1449 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { 1450 dev_priv->uncore.funcs.force_wake_get = 1451 fw_domains_get_with_thread_status; 1452 dev_priv->uncore.funcs.force_wake_put = fw_domains_put; 1453 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, 1454 FORCEWAKE_MT, FORCEWAKE_ACK_HSW); 1455 } else if (IS_IVYBRIDGE(dev_priv)) { 1456 u32 ecobus; 1457 1458 /* IVB configs may use multi-threaded forcewake */ 1459 1460 /* A small trick here - if the bios hasn't configured 1461 * MT forcewake, and if the device is in RC6, then 1462 * force_wake_mt_get will not wake the device and the 1463 * ECOBUS read will return zero. Which will be 1464 * (correctly) interpreted by the test below as MT 1465 * forcewake being disabled. 1466 */ 1467 dev_priv->uncore.funcs.force_wake_get = 1468 fw_domains_get_with_thread_status; 1469 dev_priv->uncore.funcs.force_wake_put = fw_domains_put; 1470 1471 /* We need to init first for ECOBUS access and then 1472 * determine later if we want to reinit, in case of MT access is 1473 * not working. In this stage we don't know which flavour this 1474 * ivb is, so it is better to reset also the gen6 fw registers 1475 * before the ecobus check. 1476 */ 1477 1478 __raw_i915_write32(dev_priv, FORCEWAKE, 0); 1479 __raw_posting_read(dev_priv, ECOBUS); 1480 1481 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, 1482 FORCEWAKE_MT, FORCEWAKE_MT_ACK); 1483 1484 spin_lock_irq(&dev_priv->uncore.lock); 1485 fw_domains_get_with_thread_status(dev_priv, FORCEWAKE_RENDER); 1486 ecobus = __raw_i915_read32(dev_priv, ECOBUS); 1487 fw_domains_put(dev_priv, FORCEWAKE_RENDER); 1488 spin_unlock_irq(&dev_priv->uncore.lock); 1489 1490 if (!(ecobus & FORCEWAKE_MT_ENABLE)) { 1491 DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n"); 1492 DRM_INFO("when using vblank-synced partial screen updates.\n"); 1493 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, 1494 FORCEWAKE, FORCEWAKE_ACK); 1495 } 1496 } else if (IS_GEN6(dev_priv)) { 1497 dev_priv->uncore.funcs.force_wake_get = 1498 fw_domains_get_with_thread_status; 1499 dev_priv->uncore.funcs.force_wake_put = fw_domains_put; 1500 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, 1501 FORCEWAKE, FORCEWAKE_ACK); 1502 } 1503 1504 /* All future platforms are expected to require complex power gating */ 1505 WARN_ON(dev_priv->uncore.fw_domains == 0); 1506 } 1507 1508 #define ASSIGN_FW_DOMAINS_TABLE(d) \ 1509 { \ 1510 dev_priv->uncore.fw_domains_table = \ 1511 (struct intel_forcewake_range *)(d); \ 1512 dev_priv->uncore.fw_domains_table_entries = ARRAY_SIZE((d)); \ 1513 } 1514 1515 static int i915_pmic_bus_access_notifier(struct notifier_block *nb, 1516 unsigned long action, void *data) 1517 { 1518 struct drm_i915_private *dev_priv = container_of(nb, 1519 struct drm_i915_private, uncore.pmic_bus_access_nb); 1520 1521 switch (action) { 1522 case MBI_PMIC_BUS_ACCESS_BEGIN: 1523 /* 1524 * forcewake all now to make sure that we don't need to do a 1525 * forcewake later which on systems where this notifier gets 1526 * called requires the punit to access to the shared pmic i2c 1527 * bus, which will be busy after this notification, leading to: 1528 * "render: timed out waiting for forcewake ack request." 1529 * errors. 1530 * 1531 * The notifier is unregistered during intel_runtime_suspend(), 1532 * so it's ok to access the HW here without holding a RPM 1533 * wake reference -> disable wakeref asserts for the time of 1534 * the access. 1535 */ 1536 disable_rpm_wakeref_asserts(dev_priv); 1537 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 1538 enable_rpm_wakeref_asserts(dev_priv); 1539 break; 1540 case MBI_PMIC_BUS_ACCESS_END: 1541 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 1542 break; 1543 } 1544 1545 return NOTIFY_OK; 1546 } 1547 1548 void intel_uncore_init(struct drm_i915_private *dev_priv) 1549 { 1550 i915_check_vgpu(dev_priv); 1551 1552 intel_uncore_edram_detect(dev_priv); 1553 intel_uncore_fw_domains_init(dev_priv); 1554 __intel_uncore_early_sanitize(dev_priv, 0); 1555 1556 dev_priv->uncore.unclaimed_mmio_check = 1; 1557 dev_priv->uncore.pmic_bus_access_nb.notifier_call = 1558 i915_pmic_bus_access_notifier; 1559 1560 if (IS_GEN(dev_priv, 2, 4) || intel_vgpu_active(dev_priv)) { 1561 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen2); 1562 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen2); 1563 } else if (IS_GEN5(dev_priv)) { 1564 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen5); 1565 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen5); 1566 } else if (IS_GEN(dev_priv, 6, 7)) { 1567 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen6); 1568 1569 if (IS_VALLEYVIEW(dev_priv)) { 1570 ASSIGN_FW_DOMAINS_TABLE(__vlv_fw_ranges); 1571 ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable); 1572 } else { 1573 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6); 1574 } 1575 } else if (IS_GEN8(dev_priv)) { 1576 if (IS_CHERRYVIEW(dev_priv)) { 1577 ASSIGN_FW_DOMAINS_TABLE(__chv_fw_ranges); 1578 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable); 1579 ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable); 1580 1581 } else { 1582 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen8); 1583 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6); 1584 } 1585 } else if (IS_GEN(dev_priv, 9, 10)) { 1586 ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges); 1587 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable); 1588 ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable); 1589 } else { 1590 ASSIGN_FW_DOMAINS_TABLE(__gen11_fw_ranges); 1591 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen11_fwtable); 1592 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen11_fwtable); 1593 } 1594 1595 iosf_mbi_register_pmic_bus_access_notifier( 1596 &dev_priv->uncore.pmic_bus_access_nb); 1597 } 1598 1599 /* 1600 * We might have detected that some engines are fused off after we initialized 1601 * the forcewake domains. Prune them, to make sure they only reference existing 1602 * engines. 1603 */ 1604 void intel_uncore_prune(struct drm_i915_private *dev_priv) 1605 { 1606 if (INTEL_GEN(dev_priv) >= 11) { 1607 enum forcewake_domains fw_domains = dev_priv->uncore.fw_domains; 1608 enum forcewake_domain_id domain_id; 1609 int i; 1610 1611 for (i = 0; i < I915_MAX_VCS; i++) { 1612 domain_id = FW_DOMAIN_ID_MEDIA_VDBOX0 + i; 1613 1614 if (HAS_ENGINE(dev_priv, _VCS(i))) 1615 continue; 1616 1617 if (fw_domains & BIT(domain_id)) 1618 fw_domain_fini(dev_priv, domain_id); 1619 } 1620 1621 for (i = 0; i < I915_MAX_VECS; i++) { 1622 domain_id = FW_DOMAIN_ID_MEDIA_VEBOX0 + i; 1623 1624 if (HAS_ENGINE(dev_priv, _VECS(i))) 1625 continue; 1626 1627 if (fw_domains & BIT(domain_id)) 1628 fw_domain_fini(dev_priv, domain_id); 1629 } 1630 } 1631 } 1632 1633 void intel_uncore_fini(struct drm_i915_private *dev_priv) 1634 { 1635 /* Paranoia: make sure we have disabled everything before we exit. */ 1636 intel_uncore_sanitize(dev_priv); 1637 1638 iosf_mbi_punit_acquire(); 1639 iosf_mbi_unregister_pmic_bus_access_notifier_unlocked( 1640 &dev_priv->uncore.pmic_bus_access_nb); 1641 intel_uncore_forcewake_reset(dev_priv); 1642 iosf_mbi_punit_release(); 1643 } 1644 1645 static const struct reg_whitelist { 1646 i915_reg_t offset_ldw; 1647 i915_reg_t offset_udw; 1648 u16 gen_mask; 1649 u8 size; 1650 } reg_read_whitelist[] = { { 1651 .offset_ldw = RING_TIMESTAMP(RENDER_RING_BASE), 1652 .offset_udw = RING_TIMESTAMP_UDW(RENDER_RING_BASE), 1653 .gen_mask = INTEL_GEN_MASK(4, 11), 1654 .size = 8 1655 } }; 1656 1657 int i915_reg_read_ioctl(struct drm_device *dev, 1658 void *data, struct drm_file *file) 1659 { 1660 struct drm_i915_private *dev_priv = to_i915(dev); 1661 struct drm_i915_reg_read *reg = data; 1662 struct reg_whitelist const *entry; 1663 unsigned int flags; 1664 int remain; 1665 int ret = 0; 1666 1667 entry = reg_read_whitelist; 1668 remain = ARRAY_SIZE(reg_read_whitelist); 1669 while (remain) { 1670 u32 entry_offset = i915_mmio_reg_offset(entry->offset_ldw); 1671 1672 GEM_BUG_ON(!is_power_of_2(entry->size)); 1673 GEM_BUG_ON(entry->size > 8); 1674 GEM_BUG_ON(entry_offset & (entry->size - 1)); 1675 1676 if (INTEL_INFO(dev_priv)->gen_mask & entry->gen_mask && 1677 entry_offset == (reg->offset & -entry->size)) 1678 break; 1679 entry++; 1680 remain--; 1681 } 1682 1683 if (!remain) 1684 return -EINVAL; 1685 1686 flags = reg->offset & (entry->size - 1); 1687 1688 intel_runtime_pm_get(dev_priv); 1689 if (entry->size == 8 && flags == I915_REG_READ_8B_WA) 1690 reg->val = I915_READ64_2x32(entry->offset_ldw, 1691 entry->offset_udw); 1692 else if (entry->size == 8 && flags == 0) 1693 reg->val = I915_READ64(entry->offset_ldw); 1694 else if (entry->size == 4 && flags == 0) 1695 reg->val = I915_READ(entry->offset_ldw); 1696 else if (entry->size == 2 && flags == 0) 1697 reg->val = I915_READ16(entry->offset_ldw); 1698 else if (entry->size == 1 && flags == 0) 1699 reg->val = I915_READ8(entry->offset_ldw); 1700 else 1701 ret = -EINVAL; 1702 intel_runtime_pm_put(dev_priv); 1703 1704 return ret; 1705 } 1706 1707 static void gen3_stop_engine(struct intel_engine_cs *engine) 1708 { 1709 struct drm_i915_private *dev_priv = engine->i915; 1710 const u32 base = engine->mmio_base; 1711 1712 if (intel_engine_stop_cs(engine)) 1713 DRM_DEBUG_DRIVER("%s: timed out on STOP_RING\n", engine->name); 1714 1715 I915_WRITE_FW(RING_HEAD(base), I915_READ_FW(RING_TAIL(base))); 1716 POSTING_READ_FW(RING_HEAD(base)); /* paranoia */ 1717 1718 I915_WRITE_FW(RING_HEAD(base), 0); 1719 I915_WRITE_FW(RING_TAIL(base), 0); 1720 POSTING_READ_FW(RING_TAIL(base)); 1721 1722 /* The ring must be empty before it is disabled */ 1723 I915_WRITE_FW(RING_CTL(base), 0); 1724 1725 /* Check acts as a post */ 1726 if (I915_READ_FW(RING_HEAD(base)) != 0) 1727 DRM_DEBUG_DRIVER("%s: ring head not parked\n", 1728 engine->name); 1729 } 1730 1731 static void i915_stop_engines(struct drm_i915_private *dev_priv, 1732 unsigned engine_mask) 1733 { 1734 struct intel_engine_cs *engine; 1735 enum intel_engine_id id; 1736 1737 if (INTEL_GEN(dev_priv) < 3) 1738 return; 1739 1740 for_each_engine_masked(engine, dev_priv, engine_mask, id) 1741 gen3_stop_engine(engine); 1742 } 1743 1744 static bool i915_in_reset(struct pci_dev *pdev) 1745 { 1746 u8 gdrst; 1747 1748 pci_read_config_byte(pdev, I915_GDRST, &gdrst); 1749 return gdrst & GRDOM_RESET_STATUS; 1750 } 1751 1752 static int i915_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask) 1753 { 1754 struct pci_dev *pdev = dev_priv->drm.pdev; 1755 int err; 1756 1757 /* Assert reset for at least 20 usec, and wait for acknowledgement. */ 1758 pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE); 1759 usleep_range(50, 200); 1760 err = wait_for(i915_in_reset(pdev), 500); 1761 1762 /* Clear the reset request. */ 1763 pci_write_config_byte(pdev, I915_GDRST, 0); 1764 usleep_range(50, 200); 1765 if (!err) 1766 err = wait_for(!i915_in_reset(pdev), 500); 1767 1768 return err; 1769 } 1770 1771 static bool g4x_reset_complete(struct pci_dev *pdev) 1772 { 1773 u8 gdrst; 1774 1775 pci_read_config_byte(pdev, I915_GDRST, &gdrst); 1776 return (gdrst & GRDOM_RESET_ENABLE) == 0; 1777 } 1778 1779 static int g33_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask) 1780 { 1781 struct pci_dev *pdev = dev_priv->drm.pdev; 1782 1783 pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE); 1784 return wait_for(g4x_reset_complete(pdev), 500); 1785 } 1786 1787 static int g4x_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask) 1788 { 1789 struct pci_dev *pdev = dev_priv->drm.pdev; 1790 int ret; 1791 1792 /* WaVcpClkGateDisableForMediaReset:ctg,elk */ 1793 I915_WRITE(VDECCLK_GATE_D, 1794 I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE); 1795 POSTING_READ(VDECCLK_GATE_D); 1796 1797 pci_write_config_byte(pdev, I915_GDRST, 1798 GRDOM_MEDIA | GRDOM_RESET_ENABLE); 1799 ret = wait_for(g4x_reset_complete(pdev), 500); 1800 if (ret) { 1801 DRM_DEBUG_DRIVER("Wait for media reset failed\n"); 1802 goto out; 1803 } 1804 1805 pci_write_config_byte(pdev, I915_GDRST, 1806 GRDOM_RENDER | GRDOM_RESET_ENABLE); 1807 ret = wait_for(g4x_reset_complete(pdev), 500); 1808 if (ret) { 1809 DRM_DEBUG_DRIVER("Wait for render reset failed\n"); 1810 goto out; 1811 } 1812 1813 out: 1814 pci_write_config_byte(pdev, I915_GDRST, 0); 1815 1816 I915_WRITE(VDECCLK_GATE_D, 1817 I915_READ(VDECCLK_GATE_D) & ~VCP_UNIT_CLOCK_GATE_DISABLE); 1818 POSTING_READ(VDECCLK_GATE_D); 1819 1820 return ret; 1821 } 1822 1823 static int ironlake_do_reset(struct drm_i915_private *dev_priv, 1824 unsigned engine_mask) 1825 { 1826 int ret; 1827 1828 I915_WRITE(ILK_GDSR, ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE); 1829 ret = intel_wait_for_register(dev_priv, 1830 ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0, 1831 500); 1832 if (ret) { 1833 DRM_DEBUG_DRIVER("Wait for render reset failed\n"); 1834 goto out; 1835 } 1836 1837 I915_WRITE(ILK_GDSR, ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE); 1838 ret = intel_wait_for_register(dev_priv, 1839 ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0, 1840 500); 1841 if (ret) { 1842 DRM_DEBUG_DRIVER("Wait for media reset failed\n"); 1843 goto out; 1844 } 1845 1846 out: 1847 I915_WRITE(ILK_GDSR, 0); 1848 POSTING_READ(ILK_GDSR); 1849 return ret; 1850 } 1851 1852 /* Reset the hardware domains (GENX_GRDOM_*) specified by mask */ 1853 static int gen6_hw_domain_reset(struct drm_i915_private *dev_priv, 1854 u32 hw_domain_mask) 1855 { 1856 int err; 1857 1858 /* GEN6_GDRST is not in the gt power well, no need to check 1859 * for fifo space for the write or forcewake the chip for 1860 * the read 1861 */ 1862 __raw_i915_write32(dev_priv, GEN6_GDRST, hw_domain_mask); 1863 1864 /* Wait for the device to ack the reset requests */ 1865 err = __intel_wait_for_register_fw(dev_priv, 1866 GEN6_GDRST, hw_domain_mask, 0, 1867 500, 0, 1868 NULL); 1869 if (err) 1870 DRM_DEBUG_DRIVER("Wait for 0x%08x engines reset failed\n", 1871 hw_domain_mask); 1872 1873 return err; 1874 } 1875 1876 /** 1877 * gen6_reset_engines - reset individual engines 1878 * @dev_priv: i915 device 1879 * @engine_mask: mask of intel_ring_flag() engines or ALL_ENGINES for full reset 1880 * 1881 * This function will reset the individual engines that are set in engine_mask. 1882 * If you provide ALL_ENGINES as mask, full global domain reset will be issued. 1883 * 1884 * Note: It is responsibility of the caller to handle the difference between 1885 * asking full domain reset versus reset for all available individual engines. 1886 * 1887 * Returns 0 on success, nonzero on error. 1888 */ 1889 static int gen6_reset_engines(struct drm_i915_private *dev_priv, 1890 unsigned engine_mask) 1891 { 1892 struct intel_engine_cs *engine; 1893 const u32 hw_engine_mask[I915_NUM_ENGINES] = { 1894 [RCS] = GEN6_GRDOM_RENDER, 1895 [BCS] = GEN6_GRDOM_BLT, 1896 [VCS] = GEN6_GRDOM_MEDIA, 1897 [VCS2] = GEN8_GRDOM_MEDIA2, 1898 [VECS] = GEN6_GRDOM_VECS, 1899 }; 1900 u32 hw_mask; 1901 1902 if (engine_mask == ALL_ENGINES) { 1903 hw_mask = GEN6_GRDOM_FULL; 1904 } else { 1905 unsigned int tmp; 1906 1907 hw_mask = 0; 1908 for_each_engine_masked(engine, dev_priv, engine_mask, tmp) 1909 hw_mask |= hw_engine_mask[engine->id]; 1910 } 1911 1912 return gen6_hw_domain_reset(dev_priv, hw_mask); 1913 } 1914 1915 /** 1916 * gen11_reset_engines - reset individual engines 1917 * @dev_priv: i915 device 1918 * @engine_mask: mask of intel_ring_flag() engines or ALL_ENGINES for full reset 1919 * 1920 * This function will reset the individual engines that are set in engine_mask. 1921 * If you provide ALL_ENGINES as mask, full global domain reset will be issued. 1922 * 1923 * Note: It is responsibility of the caller to handle the difference between 1924 * asking full domain reset versus reset for all available individual engines. 1925 * 1926 * Returns 0 on success, nonzero on error. 1927 */ 1928 static int gen11_reset_engines(struct drm_i915_private *dev_priv, 1929 unsigned engine_mask) 1930 { 1931 struct intel_engine_cs *engine; 1932 const u32 hw_engine_mask[I915_NUM_ENGINES] = { 1933 [RCS] = GEN11_GRDOM_RENDER, 1934 [BCS] = GEN11_GRDOM_BLT, 1935 [VCS] = GEN11_GRDOM_MEDIA, 1936 [VCS2] = GEN11_GRDOM_MEDIA2, 1937 [VCS3] = GEN11_GRDOM_MEDIA3, 1938 [VCS4] = GEN11_GRDOM_MEDIA4, 1939 [VECS] = GEN11_GRDOM_VECS, 1940 [VECS2] = GEN11_GRDOM_VECS2, 1941 }; 1942 u32 hw_mask; 1943 1944 BUILD_BUG_ON(VECS2 + 1 != I915_NUM_ENGINES); 1945 1946 if (engine_mask == ALL_ENGINES) { 1947 hw_mask = GEN11_GRDOM_FULL; 1948 } else { 1949 unsigned int tmp; 1950 1951 hw_mask = 0; 1952 for_each_engine_masked(engine, dev_priv, engine_mask, tmp) 1953 hw_mask |= hw_engine_mask[engine->id]; 1954 } 1955 1956 return gen6_hw_domain_reset(dev_priv, hw_mask); 1957 } 1958 1959 /** 1960 * __intel_wait_for_register_fw - wait until register matches expected state 1961 * @dev_priv: the i915 device 1962 * @reg: the register to read 1963 * @mask: mask to apply to register value 1964 * @value: expected value 1965 * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait 1966 * @slow_timeout_ms: slow timeout in millisecond 1967 * @out_value: optional placeholder to hold registry value 1968 * 1969 * This routine waits until the target register @reg contains the expected 1970 * @value after applying the @mask, i.e. it waits until :: 1971 * 1972 * (I915_READ_FW(reg) & mask) == value 1973 * 1974 * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds. 1975 * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us 1976 * must be not larger than 20,0000 microseconds. 1977 * 1978 * Note that this routine assumes the caller holds forcewake asserted, it is 1979 * not suitable for very long waits. See intel_wait_for_register() if you 1980 * wish to wait without holding forcewake for the duration (i.e. you expect 1981 * the wait to be slow). 1982 * 1983 * Returns 0 if the register matches the desired condition, or -ETIMEOUT. 1984 */ 1985 int __intel_wait_for_register_fw(struct drm_i915_private *dev_priv, 1986 i915_reg_t reg, 1987 u32 mask, 1988 u32 value, 1989 unsigned int fast_timeout_us, 1990 unsigned int slow_timeout_ms, 1991 u32 *out_value) 1992 { 1993 u32 uninitialized_var(reg_value); 1994 #define done (((reg_value = I915_READ_FW(reg)) & mask) == value) 1995 int ret; 1996 1997 /* Catch any overuse of this function */ 1998 might_sleep_if(slow_timeout_ms); 1999 GEM_BUG_ON(fast_timeout_us > 20000); 2000 2001 ret = -ETIMEDOUT; 2002 if (fast_timeout_us && fast_timeout_us <= 20000) 2003 ret = _wait_for_atomic(done, fast_timeout_us, 0); 2004 if (ret && slow_timeout_ms) 2005 ret = wait_for(done, slow_timeout_ms); 2006 2007 if (out_value) 2008 *out_value = reg_value; 2009 2010 return ret; 2011 #undef done 2012 } 2013 2014 /** 2015 * __intel_wait_for_register - wait until register matches expected state 2016 * @dev_priv: the i915 device 2017 * @reg: the register to read 2018 * @mask: mask to apply to register value 2019 * @value: expected value 2020 * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait 2021 * @slow_timeout_ms: slow timeout in millisecond 2022 * @out_value: optional placeholder to hold registry value 2023 * 2024 * This routine waits until the target register @reg contains the expected 2025 * @value after applying the @mask, i.e. it waits until :: 2026 * 2027 * (I915_READ(reg) & mask) == value 2028 * 2029 * Otherwise, the wait will timeout after @timeout_ms milliseconds. 2030 * 2031 * Returns 0 if the register matches the desired condition, or -ETIMEOUT. 2032 */ 2033 int __intel_wait_for_register(struct drm_i915_private *dev_priv, 2034 i915_reg_t reg, 2035 u32 mask, 2036 u32 value, 2037 unsigned int fast_timeout_us, 2038 unsigned int slow_timeout_ms, 2039 u32 *out_value) 2040 { 2041 unsigned fw = 2042 intel_uncore_forcewake_for_reg(dev_priv, reg, FW_REG_READ); 2043 u32 reg_value; 2044 int ret; 2045 2046 might_sleep_if(slow_timeout_ms); 2047 2048 spin_lock_irq(&dev_priv->uncore.lock); 2049 intel_uncore_forcewake_get__locked(dev_priv, fw); 2050 2051 ret = __intel_wait_for_register_fw(dev_priv, 2052 reg, mask, value, 2053 fast_timeout_us, 0, ®_value); 2054 2055 intel_uncore_forcewake_put__locked(dev_priv, fw); 2056 spin_unlock_irq(&dev_priv->uncore.lock); 2057 2058 if (ret && slow_timeout_ms) 2059 ret = __wait_for(reg_value = I915_READ_NOTRACE(reg), 2060 (reg_value & mask) == value, 2061 slow_timeout_ms * 1000, 10, 1000); 2062 2063 if (out_value) 2064 *out_value = reg_value; 2065 2066 return ret; 2067 } 2068 2069 static int gen8_reset_engine_start(struct intel_engine_cs *engine) 2070 { 2071 struct drm_i915_private *dev_priv = engine->i915; 2072 int ret; 2073 2074 I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base), 2075 _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET)); 2076 2077 ret = __intel_wait_for_register_fw(dev_priv, 2078 RING_RESET_CTL(engine->mmio_base), 2079 RESET_CTL_READY_TO_RESET, 2080 RESET_CTL_READY_TO_RESET, 2081 700, 0, 2082 NULL); 2083 if (ret) 2084 DRM_ERROR("%s: reset request timeout\n", engine->name); 2085 2086 return ret; 2087 } 2088 2089 static void gen8_reset_engine_cancel(struct intel_engine_cs *engine) 2090 { 2091 struct drm_i915_private *dev_priv = engine->i915; 2092 2093 I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base), 2094 _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET)); 2095 } 2096 2097 static int gen8_reset_engines(struct drm_i915_private *dev_priv, 2098 unsigned engine_mask) 2099 { 2100 struct intel_engine_cs *engine; 2101 unsigned int tmp; 2102 int ret; 2103 2104 for_each_engine_masked(engine, dev_priv, engine_mask, tmp) { 2105 if (gen8_reset_engine_start(engine)) { 2106 ret = -EIO; 2107 goto not_ready; 2108 } 2109 } 2110 2111 if (INTEL_GEN(dev_priv) >= 11) 2112 ret = gen11_reset_engines(dev_priv, engine_mask); 2113 else 2114 ret = gen6_reset_engines(dev_priv, engine_mask); 2115 2116 not_ready: 2117 for_each_engine_masked(engine, dev_priv, engine_mask, tmp) 2118 gen8_reset_engine_cancel(engine); 2119 2120 return ret; 2121 } 2122 2123 typedef int (*reset_func)(struct drm_i915_private *, unsigned engine_mask); 2124 2125 static reset_func intel_get_gpu_reset(struct drm_i915_private *dev_priv) 2126 { 2127 if (!i915_modparams.reset) 2128 return NULL; 2129 2130 if (INTEL_GEN(dev_priv) >= 8) 2131 return gen8_reset_engines; 2132 else if (INTEL_GEN(dev_priv) >= 6) 2133 return gen6_reset_engines; 2134 else if (IS_GEN5(dev_priv)) 2135 return ironlake_do_reset; 2136 else if (IS_G4X(dev_priv)) 2137 return g4x_do_reset; 2138 else if (IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) 2139 return g33_do_reset; 2140 else if (INTEL_GEN(dev_priv) >= 3) 2141 return i915_do_reset; 2142 else 2143 return NULL; 2144 } 2145 2146 int intel_gpu_reset(struct drm_i915_private *dev_priv, unsigned engine_mask) 2147 { 2148 reset_func reset = intel_get_gpu_reset(dev_priv); 2149 int retry; 2150 int ret; 2151 2152 /* 2153 * We want to perform per-engine reset from atomic context (e.g. 2154 * softirq), which imposes the constraint that we cannot sleep. 2155 * However, experience suggests that spending a bit of time waiting 2156 * for a reset helps in various cases, so for a full-device reset 2157 * we apply the opposite rule and wait if we want to. As we should 2158 * always follow up a failed per-engine reset with a full device reset, 2159 * being a little faster, stricter and more error prone for the 2160 * atomic case seems an acceptable compromise. 2161 * 2162 * Unfortunately this leads to a bimodal routine, when the goal was 2163 * to have a single reset function that worked for resetting any 2164 * number of engines simultaneously. 2165 */ 2166 might_sleep_if(engine_mask == ALL_ENGINES); 2167 2168 /* 2169 * If the power well sleeps during the reset, the reset 2170 * request may be dropped and never completes (causing -EIO). 2171 */ 2172 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 2173 for (retry = 0; retry < 3; retry++) { 2174 2175 /* 2176 * We stop engines, otherwise we might get failed reset and a 2177 * dead gpu (on elk). Also as modern gpu as kbl can suffer 2178 * from system hang if batchbuffer is progressing when 2179 * the reset is issued, regardless of READY_TO_RESET ack. 2180 * Thus assume it is best to stop engines on all gens 2181 * where we have a gpu reset. 2182 * 2183 * WaKBLVECSSemaphoreWaitPoll:kbl (on ALL_ENGINES) 2184 * 2185 * WaMediaResetMainRingCleanup:ctg,elk (presumably) 2186 * 2187 * FIXME: Wa for more modern gens needs to be validated 2188 */ 2189 i915_stop_engines(dev_priv, engine_mask); 2190 2191 ret = -ENODEV; 2192 if (reset) { 2193 GEM_TRACE("engine_mask=%x\n", engine_mask); 2194 ret = reset(dev_priv, engine_mask); 2195 } 2196 if (ret != -ETIMEDOUT || engine_mask != ALL_ENGINES) 2197 break; 2198 2199 cond_resched(); 2200 } 2201 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 2202 2203 return ret; 2204 } 2205 2206 bool intel_has_gpu_reset(struct drm_i915_private *dev_priv) 2207 { 2208 return intel_get_gpu_reset(dev_priv) != NULL; 2209 } 2210 2211 bool intel_has_reset_engine(struct drm_i915_private *dev_priv) 2212 { 2213 return (dev_priv->info.has_reset_engine && 2214 i915_modparams.reset >= 2); 2215 } 2216 2217 int intel_reset_guc(struct drm_i915_private *dev_priv) 2218 { 2219 u32 guc_domain = INTEL_GEN(dev_priv) >= 11 ? GEN11_GRDOM_GUC : 2220 GEN9_GRDOM_GUC; 2221 int ret; 2222 2223 GEM_BUG_ON(!HAS_GUC(dev_priv)); 2224 2225 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 2226 ret = gen6_hw_domain_reset(dev_priv, guc_domain); 2227 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 2228 2229 return ret; 2230 } 2231 2232 bool intel_uncore_unclaimed_mmio(struct drm_i915_private *dev_priv) 2233 { 2234 return check_for_unclaimed_mmio(dev_priv); 2235 } 2236 2237 bool 2238 intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private *dev_priv) 2239 { 2240 if (unlikely(i915_modparams.mmio_debug || 2241 dev_priv->uncore.unclaimed_mmio_check <= 0)) 2242 return false; 2243 2244 if (unlikely(intel_uncore_unclaimed_mmio(dev_priv))) { 2245 DRM_DEBUG("Unclaimed register detected, " 2246 "enabling oneshot unclaimed register reporting. " 2247 "Please use i915.mmio_debug=N for more information.\n"); 2248 i915_modparams.mmio_debug++; 2249 dev_priv->uncore.unclaimed_mmio_check--; 2250 return true; 2251 } 2252 2253 return false; 2254 } 2255 2256 static enum forcewake_domains 2257 intel_uncore_forcewake_for_read(struct drm_i915_private *dev_priv, 2258 i915_reg_t reg) 2259 { 2260 u32 offset = i915_mmio_reg_offset(reg); 2261 enum forcewake_domains fw_domains; 2262 2263 if (INTEL_GEN(dev_priv) >= 11) { 2264 fw_domains = __gen11_fwtable_reg_read_fw_domains(offset); 2265 } else if (HAS_FWTABLE(dev_priv)) { 2266 fw_domains = __fwtable_reg_read_fw_domains(offset); 2267 } else if (INTEL_GEN(dev_priv) >= 6) { 2268 fw_domains = __gen6_reg_read_fw_domains(offset); 2269 } else { 2270 WARN_ON(!IS_GEN(dev_priv, 2, 5)); 2271 fw_domains = 0; 2272 } 2273 2274 WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains); 2275 2276 return fw_domains; 2277 } 2278 2279 static enum forcewake_domains 2280 intel_uncore_forcewake_for_write(struct drm_i915_private *dev_priv, 2281 i915_reg_t reg) 2282 { 2283 u32 offset = i915_mmio_reg_offset(reg); 2284 enum forcewake_domains fw_domains; 2285 2286 if (INTEL_GEN(dev_priv) >= 11) { 2287 fw_domains = __gen11_fwtable_reg_write_fw_domains(offset); 2288 } else if (HAS_FWTABLE(dev_priv) && !IS_VALLEYVIEW(dev_priv)) { 2289 fw_domains = __fwtable_reg_write_fw_domains(offset); 2290 } else if (IS_GEN8(dev_priv)) { 2291 fw_domains = __gen8_reg_write_fw_domains(offset); 2292 } else if (IS_GEN(dev_priv, 6, 7)) { 2293 fw_domains = FORCEWAKE_RENDER; 2294 } else { 2295 WARN_ON(!IS_GEN(dev_priv, 2, 5)); 2296 fw_domains = 0; 2297 } 2298 2299 WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains); 2300 2301 return fw_domains; 2302 } 2303 2304 /** 2305 * intel_uncore_forcewake_for_reg - which forcewake domains are needed to access 2306 * a register 2307 * @dev_priv: pointer to struct drm_i915_private 2308 * @reg: register in question 2309 * @op: operation bitmask of FW_REG_READ and/or FW_REG_WRITE 2310 * 2311 * Returns a set of forcewake domains required to be taken with for example 2312 * intel_uncore_forcewake_get for the specified register to be accessible in the 2313 * specified mode (read, write or read/write) with raw mmio accessors. 2314 * 2315 * NOTE: On Gen6 and Gen7 write forcewake domain (FORCEWAKE_RENDER) requires the 2316 * callers to do FIFO management on their own or risk losing writes. 2317 */ 2318 enum forcewake_domains 2319 intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv, 2320 i915_reg_t reg, unsigned int op) 2321 { 2322 enum forcewake_domains fw_domains = 0; 2323 2324 WARN_ON(!op); 2325 2326 if (intel_vgpu_active(dev_priv)) 2327 return 0; 2328 2329 if (op & FW_REG_READ) 2330 fw_domains = intel_uncore_forcewake_for_read(dev_priv, reg); 2331 2332 if (op & FW_REG_WRITE) 2333 fw_domains |= intel_uncore_forcewake_for_write(dev_priv, reg); 2334 2335 return fw_domains; 2336 } 2337 2338 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2339 #include "selftests/mock_uncore.c" 2340 #include "selftests/intel_uncore.c" 2341 #endif 2342