1 /* 2 * Copyright © 2013 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 */ 23 24 #include "i915_drv.h" 25 #include "intel_drv.h" 26 #include "i915_vgpu.h" 27 28 #include <asm/iosf_mbi.h> 29 #include <linux/pm_runtime.h> 30 31 #define FORCEWAKE_ACK_TIMEOUT_MS 50 32 #define GT_FIFO_TIMEOUT_MS 10 33 34 #define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32((dev_priv__), (reg__)) 35 36 static const char * const forcewake_domain_names[] = { 37 "render", 38 "blitter", 39 "media", 40 "vdbox0", 41 "vdbox1", 42 "vdbox2", 43 "vdbox3", 44 "vebox0", 45 "vebox1", 46 }; 47 48 const char * 49 intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id) 50 { 51 BUILD_BUG_ON(ARRAY_SIZE(forcewake_domain_names) != FW_DOMAIN_ID_COUNT); 52 53 if (id >= 0 && id < FW_DOMAIN_ID_COUNT) 54 return forcewake_domain_names[id]; 55 56 WARN_ON(id); 57 58 return "unknown"; 59 } 60 61 static inline void 62 fw_domain_reset(struct drm_i915_private *i915, 63 const struct intel_uncore_forcewake_domain *d) 64 { 65 /* 66 * We don't really know if the powerwell for the forcewake domain we are 67 * trying to reset here does exist at this point (engines could be fused 68 * off in ICL+), so no waiting for acks 69 */ 70 __raw_i915_write32(i915, d->reg_set, i915->uncore.fw_reset); 71 } 72 73 static inline void 74 fw_domain_arm_timer(struct intel_uncore_forcewake_domain *d) 75 { 76 d->wake_count++; 77 hrtimer_start_range_ns(&d->timer, 78 NSEC_PER_MSEC, 79 NSEC_PER_MSEC, 80 HRTIMER_MODE_REL); 81 } 82 83 static inline int 84 __wait_for_ack(const struct drm_i915_private *i915, 85 const struct intel_uncore_forcewake_domain *d, 86 const u32 ack, 87 const u32 value) 88 { 89 return wait_for_atomic((__raw_i915_read32(i915, d->reg_ack) & ack) == value, 90 FORCEWAKE_ACK_TIMEOUT_MS); 91 } 92 93 static inline int 94 wait_ack_clear(const struct drm_i915_private *i915, 95 const struct intel_uncore_forcewake_domain *d, 96 const u32 ack) 97 { 98 return __wait_for_ack(i915, d, ack, 0); 99 } 100 101 static inline int 102 wait_ack_set(const struct drm_i915_private *i915, 103 const struct intel_uncore_forcewake_domain *d, 104 const u32 ack) 105 { 106 return __wait_for_ack(i915, d, ack, ack); 107 } 108 109 static inline void 110 fw_domain_wait_ack_clear(const struct drm_i915_private *i915, 111 const struct intel_uncore_forcewake_domain *d) 112 { 113 if (wait_ack_clear(i915, d, FORCEWAKE_KERNEL)) 114 DRM_ERROR("%s: timed out waiting for forcewake ack to clear.\n", 115 intel_uncore_forcewake_domain_to_str(d->id)); 116 } 117 118 enum ack_type { 119 ACK_CLEAR = 0, 120 ACK_SET 121 }; 122 123 static int 124 fw_domain_wait_ack_with_fallback(const struct drm_i915_private *i915, 125 const struct intel_uncore_forcewake_domain *d, 126 const enum ack_type type) 127 { 128 const u32 ack_bit = FORCEWAKE_KERNEL; 129 const u32 value = type == ACK_SET ? ack_bit : 0; 130 unsigned int pass; 131 bool ack_detected; 132 133 /* 134 * There is a possibility of driver's wake request colliding 135 * with hardware's own wake requests and that can cause 136 * hardware to not deliver the driver's ack message. 137 * 138 * Use a fallback bit toggle to kick the gpu state machine 139 * in the hope that the original ack will be delivered along with 140 * the fallback ack. 141 * 142 * This workaround is described in HSDES #1604254524 and it's known as: 143 * WaRsForcewakeAddDelayForAck:skl,bxt,kbl,glk,cfl,cnl,icl 144 * although the name is a bit misleading. 145 */ 146 147 pass = 1; 148 do { 149 wait_ack_clear(i915, d, FORCEWAKE_KERNEL_FALLBACK); 150 151 __raw_i915_write32(i915, d->reg_set, 152 _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL_FALLBACK)); 153 /* Give gt some time to relax before the polling frenzy */ 154 udelay(10 * pass); 155 wait_ack_set(i915, d, FORCEWAKE_KERNEL_FALLBACK); 156 157 ack_detected = (__raw_i915_read32(i915, d->reg_ack) & ack_bit) == value; 158 159 __raw_i915_write32(i915, d->reg_set, 160 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL_FALLBACK)); 161 } while (!ack_detected && pass++ < 10); 162 163 DRM_DEBUG_DRIVER("%s had to use fallback to %s ack, 0x%x (passes %u)\n", 164 intel_uncore_forcewake_domain_to_str(d->id), 165 type == ACK_SET ? "set" : "clear", 166 __raw_i915_read32(i915, d->reg_ack), 167 pass); 168 169 return ack_detected ? 0 : -ETIMEDOUT; 170 } 171 172 static inline void 173 fw_domain_wait_ack_clear_fallback(const struct drm_i915_private *i915, 174 const struct intel_uncore_forcewake_domain *d) 175 { 176 if (likely(!wait_ack_clear(i915, d, FORCEWAKE_KERNEL))) 177 return; 178 179 if (fw_domain_wait_ack_with_fallback(i915, d, ACK_CLEAR)) 180 fw_domain_wait_ack_clear(i915, d); 181 } 182 183 static inline void 184 fw_domain_get(struct drm_i915_private *i915, 185 const struct intel_uncore_forcewake_domain *d) 186 { 187 __raw_i915_write32(i915, d->reg_set, i915->uncore.fw_set); 188 } 189 190 static inline void 191 fw_domain_wait_ack_set(const struct drm_i915_private *i915, 192 const struct intel_uncore_forcewake_domain *d) 193 { 194 if (wait_ack_set(i915, d, FORCEWAKE_KERNEL)) 195 DRM_ERROR("%s: timed out waiting for forcewake ack request.\n", 196 intel_uncore_forcewake_domain_to_str(d->id)); 197 } 198 199 static inline void 200 fw_domain_wait_ack_set_fallback(const struct drm_i915_private *i915, 201 const struct intel_uncore_forcewake_domain *d) 202 { 203 if (likely(!wait_ack_set(i915, d, FORCEWAKE_KERNEL))) 204 return; 205 206 if (fw_domain_wait_ack_with_fallback(i915, d, ACK_SET)) 207 fw_domain_wait_ack_set(i915, d); 208 } 209 210 static inline void 211 fw_domain_put(const struct drm_i915_private *i915, 212 const struct intel_uncore_forcewake_domain *d) 213 { 214 __raw_i915_write32(i915, d->reg_set, i915->uncore.fw_clear); 215 } 216 217 static void 218 fw_domains_get(struct drm_i915_private *i915, enum forcewake_domains fw_domains) 219 { 220 struct intel_uncore_forcewake_domain *d; 221 unsigned int tmp; 222 223 GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains); 224 225 for_each_fw_domain_masked(d, fw_domains, i915, tmp) { 226 fw_domain_wait_ack_clear(i915, d); 227 fw_domain_get(i915, d); 228 } 229 230 for_each_fw_domain_masked(d, fw_domains, i915, tmp) 231 fw_domain_wait_ack_set(i915, d); 232 233 i915->uncore.fw_domains_active |= fw_domains; 234 } 235 236 static void 237 fw_domains_get_with_fallback(struct drm_i915_private *i915, 238 enum forcewake_domains fw_domains) 239 { 240 struct intel_uncore_forcewake_domain *d; 241 unsigned int tmp; 242 243 GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains); 244 245 for_each_fw_domain_masked(d, fw_domains, i915, tmp) { 246 fw_domain_wait_ack_clear_fallback(i915, d); 247 fw_domain_get(i915, d); 248 } 249 250 for_each_fw_domain_masked(d, fw_domains, i915, tmp) 251 fw_domain_wait_ack_set_fallback(i915, d); 252 253 i915->uncore.fw_domains_active |= fw_domains; 254 } 255 256 static void 257 fw_domains_put(struct drm_i915_private *i915, enum forcewake_domains fw_domains) 258 { 259 struct intel_uncore_forcewake_domain *d; 260 unsigned int tmp; 261 262 GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains); 263 264 for_each_fw_domain_masked(d, fw_domains, i915, tmp) 265 fw_domain_put(i915, d); 266 267 i915->uncore.fw_domains_active &= ~fw_domains; 268 } 269 270 static void 271 fw_domains_reset(struct drm_i915_private *i915, 272 enum forcewake_domains fw_domains) 273 { 274 struct intel_uncore_forcewake_domain *d; 275 unsigned int tmp; 276 277 if (!fw_domains) 278 return; 279 280 GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains); 281 282 for_each_fw_domain_masked(d, fw_domains, i915, tmp) 283 fw_domain_reset(i915, d); 284 } 285 286 static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv) 287 { 288 /* w/a for a sporadic read returning 0 by waiting for the GT 289 * thread to wake up. 290 */ 291 if (wait_for_atomic_us((__raw_i915_read32(dev_priv, GEN6_GT_THREAD_STATUS_REG) & 292 GEN6_GT_THREAD_STATUS_CORE_MASK) == 0, 500)) 293 DRM_ERROR("GT thread status wait timed out\n"); 294 } 295 296 static void fw_domains_get_with_thread_status(struct drm_i915_private *dev_priv, 297 enum forcewake_domains fw_domains) 298 { 299 fw_domains_get(dev_priv, fw_domains); 300 301 /* WaRsForcewakeWaitTC0:snb,ivb,hsw,bdw,vlv */ 302 __gen6_gt_wait_for_thread_c0(dev_priv); 303 } 304 305 static inline u32 fifo_free_entries(struct drm_i915_private *dev_priv) 306 { 307 u32 count = __raw_i915_read32(dev_priv, GTFIFOCTL); 308 309 return count & GT_FIFO_FREE_ENTRIES_MASK; 310 } 311 312 static void __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv) 313 { 314 u32 n; 315 316 /* On VLV, FIFO will be shared by both SW and HW. 317 * So, we need to read the FREE_ENTRIES everytime */ 318 if (IS_VALLEYVIEW(dev_priv)) 319 n = fifo_free_entries(dev_priv); 320 else 321 n = dev_priv->uncore.fifo_count; 322 323 if (n <= GT_FIFO_NUM_RESERVED_ENTRIES) { 324 if (wait_for_atomic((n = fifo_free_entries(dev_priv)) > 325 GT_FIFO_NUM_RESERVED_ENTRIES, 326 GT_FIFO_TIMEOUT_MS)) { 327 DRM_DEBUG("GT_FIFO timeout, entries: %u\n", n); 328 return; 329 } 330 } 331 332 dev_priv->uncore.fifo_count = n - 1; 333 } 334 335 static enum hrtimer_restart 336 intel_uncore_fw_release_timer(struct hrtimer *timer) 337 { 338 struct intel_uncore_forcewake_domain *domain = 339 container_of(timer, struct intel_uncore_forcewake_domain, timer); 340 struct drm_i915_private *dev_priv = 341 container_of(domain, struct drm_i915_private, uncore.fw_domain[domain->id]); 342 unsigned long irqflags; 343 344 assert_rpm_device_not_suspended(dev_priv); 345 346 if (xchg(&domain->active, false)) 347 return HRTIMER_RESTART; 348 349 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 350 if (WARN_ON(domain->wake_count == 0)) 351 domain->wake_count++; 352 353 if (--domain->wake_count == 0) 354 dev_priv->uncore.funcs.force_wake_put(dev_priv, domain->mask); 355 356 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 357 358 return HRTIMER_NORESTART; 359 } 360 361 /* Note callers must have acquired the PUNIT->PMIC bus, before calling this. */ 362 static void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv, 363 bool restore) 364 { 365 unsigned long irqflags; 366 struct intel_uncore_forcewake_domain *domain; 367 int retry_count = 100; 368 enum forcewake_domains fw, active_domains; 369 370 iosf_mbi_assert_punit_acquired(); 371 372 /* Hold uncore.lock across reset to prevent any register access 373 * with forcewake not set correctly. Wait until all pending 374 * timers are run before holding. 375 */ 376 while (1) { 377 unsigned int tmp; 378 379 active_domains = 0; 380 381 for_each_fw_domain(domain, dev_priv, tmp) { 382 smp_store_mb(domain->active, false); 383 if (hrtimer_cancel(&domain->timer) == 0) 384 continue; 385 386 intel_uncore_fw_release_timer(&domain->timer); 387 } 388 389 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 390 391 for_each_fw_domain(domain, dev_priv, tmp) { 392 if (hrtimer_active(&domain->timer)) 393 active_domains |= domain->mask; 394 } 395 396 if (active_domains == 0) 397 break; 398 399 if (--retry_count == 0) { 400 DRM_ERROR("Timed out waiting for forcewake timers to finish\n"); 401 break; 402 } 403 404 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 405 cond_resched(); 406 } 407 408 WARN_ON(active_domains); 409 410 fw = dev_priv->uncore.fw_domains_active; 411 if (fw) 412 dev_priv->uncore.funcs.force_wake_put(dev_priv, fw); 413 414 fw_domains_reset(dev_priv, dev_priv->uncore.fw_domains); 415 416 if (restore) { /* If reset with a user forcewake, try to restore */ 417 if (fw) 418 dev_priv->uncore.funcs.force_wake_get(dev_priv, fw); 419 420 if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) 421 dev_priv->uncore.fifo_count = 422 fifo_free_entries(dev_priv); 423 } 424 425 if (!restore) 426 assert_forcewakes_inactive(dev_priv); 427 428 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 429 } 430 431 static u64 gen9_edram_size(struct drm_i915_private *dev_priv) 432 { 433 const unsigned int ways[8] = { 4, 8, 12, 16, 16, 16, 16, 16 }; 434 const unsigned int sets[4] = { 1, 1, 2, 2 }; 435 const u32 cap = dev_priv->edram_cap; 436 437 return EDRAM_NUM_BANKS(cap) * 438 ways[EDRAM_WAYS_IDX(cap)] * 439 sets[EDRAM_SETS_IDX(cap)] * 440 1024 * 1024; 441 } 442 443 u64 intel_uncore_edram_size(struct drm_i915_private *dev_priv) 444 { 445 if (!HAS_EDRAM(dev_priv)) 446 return 0; 447 448 /* The needed capability bits for size calculation 449 * are not there with pre gen9 so return 128MB always. 450 */ 451 if (INTEL_GEN(dev_priv) < 9) 452 return 128 * 1024 * 1024; 453 454 return gen9_edram_size(dev_priv); 455 } 456 457 static void intel_uncore_edram_detect(struct drm_i915_private *dev_priv) 458 { 459 if (IS_HASWELL(dev_priv) || 460 IS_BROADWELL(dev_priv) || 461 INTEL_GEN(dev_priv) >= 9) { 462 dev_priv->edram_cap = __raw_i915_read32(dev_priv, 463 HSW_EDRAM_CAP); 464 465 /* NB: We can't write IDICR yet because we do not have gt funcs 466 * set up */ 467 } else { 468 dev_priv->edram_cap = 0; 469 } 470 471 if (HAS_EDRAM(dev_priv)) 472 DRM_INFO("Found %lluMB of eDRAM\n", 473 intel_uncore_edram_size(dev_priv) / (1024 * 1024)); 474 } 475 476 static bool 477 fpga_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv) 478 { 479 u32 dbg; 480 481 dbg = __raw_i915_read32(dev_priv, FPGA_DBG); 482 if (likely(!(dbg & FPGA_DBG_RM_NOCLAIM))) 483 return false; 484 485 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); 486 487 return true; 488 } 489 490 static bool 491 vlv_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv) 492 { 493 u32 cer; 494 495 cer = __raw_i915_read32(dev_priv, CLAIM_ER); 496 if (likely(!(cer & (CLAIM_ER_OVERFLOW | CLAIM_ER_CTR_MASK)))) 497 return false; 498 499 __raw_i915_write32(dev_priv, CLAIM_ER, CLAIM_ER_CLR); 500 501 return true; 502 } 503 504 static bool 505 gen6_check_for_fifo_debug(struct drm_i915_private *dev_priv) 506 { 507 u32 fifodbg; 508 509 fifodbg = __raw_i915_read32(dev_priv, GTFIFODBG); 510 511 if (unlikely(fifodbg)) { 512 DRM_DEBUG_DRIVER("GTFIFODBG = 0x08%x\n", fifodbg); 513 __raw_i915_write32(dev_priv, GTFIFODBG, fifodbg); 514 } 515 516 return fifodbg; 517 } 518 519 static bool 520 check_for_unclaimed_mmio(struct drm_i915_private *dev_priv) 521 { 522 bool ret = false; 523 524 if (HAS_FPGA_DBG_UNCLAIMED(dev_priv)) 525 ret |= fpga_check_for_unclaimed_mmio(dev_priv); 526 527 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) 528 ret |= vlv_check_for_unclaimed_mmio(dev_priv); 529 530 if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) 531 ret |= gen6_check_for_fifo_debug(dev_priv); 532 533 return ret; 534 } 535 536 static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv, 537 bool restore_forcewake) 538 { 539 /* clear out unclaimed reg detection bit */ 540 if (check_for_unclaimed_mmio(dev_priv)) 541 DRM_DEBUG("unclaimed mmio detected on uncore init, clearing\n"); 542 543 /* WaDisableShadowRegForCpd:chv */ 544 if (IS_CHERRYVIEW(dev_priv)) { 545 __raw_i915_write32(dev_priv, GTFIFOCTL, 546 __raw_i915_read32(dev_priv, GTFIFOCTL) | 547 GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL | 548 GT_FIFO_CTL_RC6_POLICY_STALL); 549 } 550 551 iosf_mbi_punit_acquire(); 552 intel_uncore_forcewake_reset(dev_priv, restore_forcewake); 553 iosf_mbi_punit_release(); 554 } 555 556 void intel_uncore_suspend(struct drm_i915_private *dev_priv) 557 { 558 iosf_mbi_punit_acquire(); 559 iosf_mbi_unregister_pmic_bus_access_notifier_unlocked( 560 &dev_priv->uncore.pmic_bus_access_nb); 561 intel_uncore_forcewake_reset(dev_priv, false); 562 iosf_mbi_punit_release(); 563 } 564 565 void intel_uncore_resume_early(struct drm_i915_private *dev_priv) 566 { 567 __intel_uncore_early_sanitize(dev_priv, true); 568 iosf_mbi_register_pmic_bus_access_notifier( 569 &dev_priv->uncore.pmic_bus_access_nb); 570 i915_check_and_clear_faults(dev_priv); 571 } 572 573 void intel_uncore_runtime_resume(struct drm_i915_private *dev_priv) 574 { 575 iosf_mbi_register_pmic_bus_access_notifier( 576 &dev_priv->uncore.pmic_bus_access_nb); 577 } 578 579 void intel_uncore_sanitize(struct drm_i915_private *dev_priv) 580 { 581 /* BIOS often leaves RC6 enabled, but disable it for hw init */ 582 intel_sanitize_gt_powersave(dev_priv); 583 } 584 585 static void __intel_uncore_forcewake_get(struct drm_i915_private *dev_priv, 586 enum forcewake_domains fw_domains) 587 { 588 struct intel_uncore_forcewake_domain *domain; 589 unsigned int tmp; 590 591 fw_domains &= dev_priv->uncore.fw_domains; 592 593 for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp) { 594 if (domain->wake_count++) { 595 fw_domains &= ~domain->mask; 596 domain->active = true; 597 } 598 } 599 600 if (fw_domains) 601 dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains); 602 } 603 604 /** 605 * intel_uncore_forcewake_get - grab forcewake domain references 606 * @dev_priv: i915 device instance 607 * @fw_domains: forcewake domains to get reference on 608 * 609 * This function can be used get GT's forcewake domain references. 610 * Normal register access will handle the forcewake domains automatically. 611 * However if some sequence requires the GT to not power down a particular 612 * forcewake domains this function should be called at the beginning of the 613 * sequence. And subsequently the reference should be dropped by symmetric 614 * call to intel_unforce_forcewake_put(). Usually caller wants all the domains 615 * to be kept awake so the @fw_domains would be then FORCEWAKE_ALL. 616 */ 617 void intel_uncore_forcewake_get(struct drm_i915_private *dev_priv, 618 enum forcewake_domains fw_domains) 619 { 620 unsigned long irqflags; 621 622 if (!dev_priv->uncore.funcs.force_wake_get) 623 return; 624 625 assert_rpm_wakelock_held(dev_priv); 626 627 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 628 __intel_uncore_forcewake_get(dev_priv, fw_domains); 629 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 630 } 631 632 /** 633 * intel_uncore_forcewake_user_get - claim forcewake on behalf of userspace 634 * @dev_priv: i915 device instance 635 * 636 * This function is a wrapper around intel_uncore_forcewake_get() to acquire 637 * the GT powerwell and in the process disable our debugging for the 638 * duration of userspace's bypass. 639 */ 640 void intel_uncore_forcewake_user_get(struct drm_i915_private *dev_priv) 641 { 642 spin_lock_irq(&dev_priv->uncore.lock); 643 if (!dev_priv->uncore.user_forcewake.count++) { 644 intel_uncore_forcewake_get__locked(dev_priv, FORCEWAKE_ALL); 645 646 /* Save and disable mmio debugging for the user bypass */ 647 dev_priv->uncore.user_forcewake.saved_mmio_check = 648 dev_priv->uncore.unclaimed_mmio_check; 649 dev_priv->uncore.user_forcewake.saved_mmio_debug = 650 i915_modparams.mmio_debug; 651 652 dev_priv->uncore.unclaimed_mmio_check = 0; 653 i915_modparams.mmio_debug = 0; 654 } 655 spin_unlock_irq(&dev_priv->uncore.lock); 656 } 657 658 /** 659 * intel_uncore_forcewake_user_put - release forcewake on behalf of userspace 660 * @dev_priv: i915 device instance 661 * 662 * This function complements intel_uncore_forcewake_user_get() and releases 663 * the GT powerwell taken on behalf of the userspace bypass. 664 */ 665 void intel_uncore_forcewake_user_put(struct drm_i915_private *dev_priv) 666 { 667 spin_lock_irq(&dev_priv->uncore.lock); 668 if (!--dev_priv->uncore.user_forcewake.count) { 669 if (intel_uncore_unclaimed_mmio(dev_priv)) 670 dev_info(dev_priv->drm.dev, 671 "Invalid mmio detected during user access\n"); 672 673 dev_priv->uncore.unclaimed_mmio_check = 674 dev_priv->uncore.user_forcewake.saved_mmio_check; 675 i915_modparams.mmio_debug = 676 dev_priv->uncore.user_forcewake.saved_mmio_debug; 677 678 intel_uncore_forcewake_put__locked(dev_priv, FORCEWAKE_ALL); 679 } 680 spin_unlock_irq(&dev_priv->uncore.lock); 681 } 682 683 /** 684 * intel_uncore_forcewake_get__locked - grab forcewake domain references 685 * @dev_priv: i915 device instance 686 * @fw_domains: forcewake domains to get reference on 687 * 688 * See intel_uncore_forcewake_get(). This variant places the onus 689 * on the caller to explicitly handle the dev_priv->uncore.lock spinlock. 690 */ 691 void intel_uncore_forcewake_get__locked(struct drm_i915_private *dev_priv, 692 enum forcewake_domains fw_domains) 693 { 694 lockdep_assert_held(&dev_priv->uncore.lock); 695 696 if (!dev_priv->uncore.funcs.force_wake_get) 697 return; 698 699 __intel_uncore_forcewake_get(dev_priv, fw_domains); 700 } 701 702 static void __intel_uncore_forcewake_put(struct drm_i915_private *dev_priv, 703 enum forcewake_domains fw_domains) 704 { 705 struct intel_uncore_forcewake_domain *domain; 706 unsigned int tmp; 707 708 fw_domains &= dev_priv->uncore.fw_domains; 709 710 for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp) { 711 if (WARN_ON(domain->wake_count == 0)) 712 continue; 713 714 if (--domain->wake_count) { 715 domain->active = true; 716 continue; 717 } 718 719 fw_domain_arm_timer(domain); 720 } 721 } 722 723 /** 724 * intel_uncore_forcewake_put - release a forcewake domain reference 725 * @dev_priv: i915 device instance 726 * @fw_domains: forcewake domains to put references 727 * 728 * This function drops the device-level forcewakes for specified 729 * domains obtained by intel_uncore_forcewake_get(). 730 */ 731 void intel_uncore_forcewake_put(struct drm_i915_private *dev_priv, 732 enum forcewake_domains fw_domains) 733 { 734 unsigned long irqflags; 735 736 if (!dev_priv->uncore.funcs.force_wake_put) 737 return; 738 739 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 740 __intel_uncore_forcewake_put(dev_priv, fw_domains); 741 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 742 } 743 744 /** 745 * intel_uncore_forcewake_put__locked - grab forcewake domain references 746 * @dev_priv: i915 device instance 747 * @fw_domains: forcewake domains to get reference on 748 * 749 * See intel_uncore_forcewake_put(). This variant places the onus 750 * on the caller to explicitly handle the dev_priv->uncore.lock spinlock. 751 */ 752 void intel_uncore_forcewake_put__locked(struct drm_i915_private *dev_priv, 753 enum forcewake_domains fw_domains) 754 { 755 lockdep_assert_held(&dev_priv->uncore.lock); 756 757 if (!dev_priv->uncore.funcs.force_wake_put) 758 return; 759 760 __intel_uncore_forcewake_put(dev_priv, fw_domains); 761 } 762 763 void assert_forcewakes_inactive(struct drm_i915_private *dev_priv) 764 { 765 if (!dev_priv->uncore.funcs.force_wake_get) 766 return; 767 768 WARN(dev_priv->uncore.fw_domains_active, 769 "Expected all fw_domains to be inactive, but %08x are still on\n", 770 dev_priv->uncore.fw_domains_active); 771 } 772 773 void assert_forcewakes_active(struct drm_i915_private *dev_priv, 774 enum forcewake_domains fw_domains) 775 { 776 if (!dev_priv->uncore.funcs.force_wake_get) 777 return; 778 779 assert_rpm_wakelock_held(dev_priv); 780 781 fw_domains &= dev_priv->uncore.fw_domains; 782 WARN(fw_domains & ~dev_priv->uncore.fw_domains_active, 783 "Expected %08x fw_domains to be active, but %08x are off\n", 784 fw_domains, fw_domains & ~dev_priv->uncore.fw_domains_active); 785 } 786 787 /* We give fast paths for the really cool registers */ 788 #define NEEDS_FORCE_WAKE(reg) ((reg) < 0x40000) 789 790 #define GEN11_NEEDS_FORCE_WAKE(reg) \ 791 ((reg) < 0x40000 || ((reg) >= 0x1c0000 && (reg) < 0x1dc000)) 792 793 #define __gen6_reg_read_fw_domains(offset) \ 794 ({ \ 795 enum forcewake_domains __fwd; \ 796 if (NEEDS_FORCE_WAKE(offset)) \ 797 __fwd = FORCEWAKE_RENDER; \ 798 else \ 799 __fwd = 0; \ 800 __fwd; \ 801 }) 802 803 static int fw_range_cmp(u32 offset, const struct intel_forcewake_range *entry) 804 { 805 if (offset < entry->start) 806 return -1; 807 else if (offset > entry->end) 808 return 1; 809 else 810 return 0; 811 } 812 813 /* Copied and "macroized" from lib/bsearch.c */ 814 #define BSEARCH(key, base, num, cmp) ({ \ 815 unsigned int start__ = 0, end__ = (num); \ 816 typeof(base) result__ = NULL; \ 817 while (start__ < end__) { \ 818 unsigned int mid__ = start__ + (end__ - start__) / 2; \ 819 int ret__ = (cmp)((key), (base) + mid__); \ 820 if (ret__ < 0) { \ 821 end__ = mid__; \ 822 } else if (ret__ > 0) { \ 823 start__ = mid__ + 1; \ 824 } else { \ 825 result__ = (base) + mid__; \ 826 break; \ 827 } \ 828 } \ 829 result__; \ 830 }) 831 832 static enum forcewake_domains 833 find_fw_domain(struct drm_i915_private *dev_priv, u32 offset) 834 { 835 const struct intel_forcewake_range *entry; 836 837 entry = BSEARCH(offset, 838 dev_priv->uncore.fw_domains_table, 839 dev_priv->uncore.fw_domains_table_entries, 840 fw_range_cmp); 841 842 if (!entry) 843 return 0; 844 845 /* 846 * The list of FW domains depends on the SKU in gen11+ so we 847 * can't determine it statically. We use FORCEWAKE_ALL and 848 * translate it here to the list of available domains. 849 */ 850 if (entry->domains == FORCEWAKE_ALL) 851 return dev_priv->uncore.fw_domains; 852 853 WARN(entry->domains & ~dev_priv->uncore.fw_domains, 854 "Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n", 855 entry->domains & ~dev_priv->uncore.fw_domains, offset); 856 857 return entry->domains; 858 } 859 860 #define GEN_FW_RANGE(s, e, d) \ 861 { .start = (s), .end = (e), .domains = (d) } 862 863 #define HAS_FWTABLE(dev_priv) \ 864 (INTEL_GEN(dev_priv) >= 9 || \ 865 IS_CHERRYVIEW(dev_priv) || \ 866 IS_VALLEYVIEW(dev_priv)) 867 868 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ 869 static const struct intel_forcewake_range __vlv_fw_ranges[] = { 870 GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER), 871 GEN_FW_RANGE(0x5000, 0x7fff, FORCEWAKE_RENDER), 872 GEN_FW_RANGE(0xb000, 0x11fff, FORCEWAKE_RENDER), 873 GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA), 874 GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_MEDIA), 875 GEN_FW_RANGE(0x2e000, 0x2ffff, FORCEWAKE_RENDER), 876 GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA), 877 }; 878 879 #define __fwtable_reg_read_fw_domains(offset) \ 880 ({ \ 881 enum forcewake_domains __fwd = 0; \ 882 if (NEEDS_FORCE_WAKE((offset))) \ 883 __fwd = find_fw_domain(dev_priv, offset); \ 884 __fwd; \ 885 }) 886 887 #define __gen11_fwtable_reg_read_fw_domains(offset) \ 888 ({ \ 889 enum forcewake_domains __fwd = 0; \ 890 if (GEN11_NEEDS_FORCE_WAKE((offset))) \ 891 __fwd = find_fw_domain(dev_priv, offset); \ 892 __fwd; \ 893 }) 894 895 /* *Must* be sorted by offset! See intel_shadow_table_check(). */ 896 static const i915_reg_t gen8_shadowed_regs[] = { 897 RING_TAIL(RENDER_RING_BASE), /* 0x2000 (base) */ 898 GEN6_RPNSWREQ, /* 0xA008 */ 899 GEN6_RC_VIDEO_FREQ, /* 0xA00C */ 900 RING_TAIL(GEN6_BSD_RING_BASE), /* 0x12000 (base) */ 901 RING_TAIL(VEBOX_RING_BASE), /* 0x1a000 (base) */ 902 RING_TAIL(BLT_RING_BASE), /* 0x22000 (base) */ 903 /* TODO: Other registers are not yet used */ 904 }; 905 906 static const i915_reg_t gen11_shadowed_regs[] = { 907 RING_TAIL(RENDER_RING_BASE), /* 0x2000 (base) */ 908 GEN6_RPNSWREQ, /* 0xA008 */ 909 GEN6_RC_VIDEO_FREQ, /* 0xA00C */ 910 RING_TAIL(BLT_RING_BASE), /* 0x22000 (base) */ 911 RING_TAIL(GEN11_BSD_RING_BASE), /* 0x1C0000 (base) */ 912 RING_TAIL(GEN11_BSD2_RING_BASE), /* 0x1C4000 (base) */ 913 RING_TAIL(GEN11_VEBOX_RING_BASE), /* 0x1C8000 (base) */ 914 RING_TAIL(GEN11_BSD3_RING_BASE), /* 0x1D0000 (base) */ 915 RING_TAIL(GEN11_BSD4_RING_BASE), /* 0x1D4000 (base) */ 916 RING_TAIL(GEN11_VEBOX2_RING_BASE), /* 0x1D8000 (base) */ 917 /* TODO: Other registers are not yet used */ 918 }; 919 920 static int mmio_reg_cmp(u32 key, const i915_reg_t *reg) 921 { 922 u32 offset = i915_mmio_reg_offset(*reg); 923 924 if (key < offset) 925 return -1; 926 else if (key > offset) 927 return 1; 928 else 929 return 0; 930 } 931 932 #define __is_genX_shadowed(x) \ 933 static bool is_gen##x##_shadowed(u32 offset) \ 934 { \ 935 const i915_reg_t *regs = gen##x##_shadowed_regs; \ 936 return BSEARCH(offset, regs, ARRAY_SIZE(gen##x##_shadowed_regs), \ 937 mmio_reg_cmp); \ 938 } 939 940 __is_genX_shadowed(8) 941 __is_genX_shadowed(11) 942 943 #define __gen8_reg_write_fw_domains(offset) \ 944 ({ \ 945 enum forcewake_domains __fwd; \ 946 if (NEEDS_FORCE_WAKE(offset) && !is_gen8_shadowed(offset)) \ 947 __fwd = FORCEWAKE_RENDER; \ 948 else \ 949 __fwd = 0; \ 950 __fwd; \ 951 }) 952 953 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ 954 static const struct intel_forcewake_range __chv_fw_ranges[] = { 955 GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER), 956 GEN_FW_RANGE(0x4000, 0x4fff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 957 GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), 958 GEN_FW_RANGE(0x8000, 0x82ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 959 GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER), 960 GEN_FW_RANGE(0x8500, 0x85ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 961 GEN_FW_RANGE(0x8800, 0x88ff, FORCEWAKE_MEDIA), 962 GEN_FW_RANGE(0x9000, 0xafff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 963 GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER), 964 GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA), 965 GEN_FW_RANGE(0xe000, 0xe7ff, FORCEWAKE_RENDER), 966 GEN_FW_RANGE(0xf000, 0xffff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 967 GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA), 968 GEN_FW_RANGE(0x1a000, 0x1bfff, FORCEWAKE_MEDIA), 969 GEN_FW_RANGE(0x1e800, 0x1e9ff, FORCEWAKE_MEDIA), 970 GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA), 971 }; 972 973 #define __fwtable_reg_write_fw_domains(offset) \ 974 ({ \ 975 enum forcewake_domains __fwd = 0; \ 976 if (NEEDS_FORCE_WAKE((offset)) && !is_gen8_shadowed(offset)) \ 977 __fwd = find_fw_domain(dev_priv, offset); \ 978 __fwd; \ 979 }) 980 981 #define __gen11_fwtable_reg_write_fw_domains(offset) \ 982 ({ \ 983 enum forcewake_domains __fwd = 0; \ 984 if (GEN11_NEEDS_FORCE_WAKE((offset)) && !is_gen11_shadowed(offset)) \ 985 __fwd = find_fw_domain(dev_priv, offset); \ 986 __fwd; \ 987 }) 988 989 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ 990 static const struct intel_forcewake_range __gen9_fw_ranges[] = { 991 GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER), 992 GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */ 993 GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER), 994 GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER), 995 GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER), 996 GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER), 997 GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), 998 GEN_FW_RANGE(0x8000, 0x812f, FORCEWAKE_BLITTER), 999 GEN_FW_RANGE(0x8130, 0x813f, FORCEWAKE_MEDIA), 1000 GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER), 1001 GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER), 1002 GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER), 1003 GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_BLITTER), 1004 GEN_FW_RANGE(0x8800, 0x89ff, FORCEWAKE_MEDIA), 1005 GEN_FW_RANGE(0x8a00, 0x8bff, FORCEWAKE_BLITTER), 1006 GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER), 1007 GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER), 1008 GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 1009 GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER), 1010 GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER), 1011 GEN_FW_RANGE(0xb480, 0xcfff, FORCEWAKE_BLITTER), 1012 GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA), 1013 GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_BLITTER), 1014 GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER), 1015 GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_BLITTER), 1016 GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA), 1017 GEN_FW_RANGE(0x14000, 0x19fff, FORCEWAKE_BLITTER), 1018 GEN_FW_RANGE(0x1a000, 0x1e9ff, FORCEWAKE_MEDIA), 1019 GEN_FW_RANGE(0x1ea00, 0x243ff, FORCEWAKE_BLITTER), 1020 GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER), 1021 GEN_FW_RANGE(0x24800, 0x2ffff, FORCEWAKE_BLITTER), 1022 GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA), 1023 }; 1024 1025 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ 1026 static const struct intel_forcewake_range __gen11_fw_ranges[] = { 1027 GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER), 1028 GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */ 1029 GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER), 1030 GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER), 1031 GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER), 1032 GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER), 1033 GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), 1034 GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_BLITTER), 1035 GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER), 1036 GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER), 1037 GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER), 1038 GEN_FW_RANGE(0x8500, 0x8bff, FORCEWAKE_BLITTER), 1039 GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER), 1040 GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER), 1041 GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_ALL), 1042 GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER), 1043 GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER), 1044 GEN_FW_RANGE(0xb480, 0xdfff, FORCEWAKE_BLITTER), 1045 GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER), 1046 GEN_FW_RANGE(0xe900, 0x243ff, FORCEWAKE_BLITTER), 1047 GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER), 1048 GEN_FW_RANGE(0x24800, 0x3ffff, FORCEWAKE_BLITTER), 1049 GEN_FW_RANGE(0x40000, 0x1bffff, 0), 1050 GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), 1051 GEN_FW_RANGE(0x1c4000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX1), 1052 GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), 1053 GEN_FW_RANGE(0x1cc000, 0x1cffff, FORCEWAKE_BLITTER), 1054 GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2), 1055 GEN_FW_RANGE(0x1d4000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX3), 1056 GEN_FW_RANGE(0x1d8000, 0x1dbfff, FORCEWAKE_MEDIA_VEBOX1) 1057 }; 1058 1059 static void 1060 ilk_dummy_write(struct drm_i915_private *dev_priv) 1061 { 1062 /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up 1063 * the chip from rc6 before touching it for real. MI_MODE is masked, 1064 * hence harmless to write 0 into. */ 1065 __raw_i915_write32(dev_priv, MI_MODE, 0); 1066 } 1067 1068 static void 1069 __unclaimed_reg_debug(struct drm_i915_private *dev_priv, 1070 const i915_reg_t reg, 1071 const bool read, 1072 const bool before) 1073 { 1074 if (WARN(check_for_unclaimed_mmio(dev_priv) && !before, 1075 "Unclaimed %s register 0x%x\n", 1076 read ? "read from" : "write to", 1077 i915_mmio_reg_offset(reg))) 1078 /* Only report the first N failures */ 1079 i915_modparams.mmio_debug--; 1080 } 1081 1082 static inline void 1083 unclaimed_reg_debug(struct drm_i915_private *dev_priv, 1084 const i915_reg_t reg, 1085 const bool read, 1086 const bool before) 1087 { 1088 if (likely(!i915_modparams.mmio_debug)) 1089 return; 1090 1091 __unclaimed_reg_debug(dev_priv, reg, read, before); 1092 } 1093 1094 #define GEN2_READ_HEADER(x) \ 1095 u##x val = 0; \ 1096 assert_rpm_wakelock_held(dev_priv); 1097 1098 #define GEN2_READ_FOOTER \ 1099 trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \ 1100 return val 1101 1102 #define __gen2_read(x) \ 1103 static u##x \ 1104 gen2_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \ 1105 GEN2_READ_HEADER(x); \ 1106 val = __raw_i915_read##x(dev_priv, reg); \ 1107 GEN2_READ_FOOTER; \ 1108 } 1109 1110 #define __gen5_read(x) \ 1111 static u##x \ 1112 gen5_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \ 1113 GEN2_READ_HEADER(x); \ 1114 ilk_dummy_write(dev_priv); \ 1115 val = __raw_i915_read##x(dev_priv, reg); \ 1116 GEN2_READ_FOOTER; \ 1117 } 1118 1119 __gen5_read(8) 1120 __gen5_read(16) 1121 __gen5_read(32) 1122 __gen5_read(64) 1123 __gen2_read(8) 1124 __gen2_read(16) 1125 __gen2_read(32) 1126 __gen2_read(64) 1127 1128 #undef __gen5_read 1129 #undef __gen2_read 1130 1131 #undef GEN2_READ_FOOTER 1132 #undef GEN2_READ_HEADER 1133 1134 #define GEN6_READ_HEADER(x) \ 1135 u32 offset = i915_mmio_reg_offset(reg); \ 1136 unsigned long irqflags; \ 1137 u##x val = 0; \ 1138 assert_rpm_wakelock_held(dev_priv); \ 1139 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \ 1140 unclaimed_reg_debug(dev_priv, reg, true, true) 1141 1142 #define GEN6_READ_FOOTER \ 1143 unclaimed_reg_debug(dev_priv, reg, true, false); \ 1144 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \ 1145 trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \ 1146 return val 1147 1148 static noinline void ___force_wake_auto(struct drm_i915_private *dev_priv, 1149 enum forcewake_domains fw_domains) 1150 { 1151 struct intel_uncore_forcewake_domain *domain; 1152 unsigned int tmp; 1153 1154 GEM_BUG_ON(fw_domains & ~dev_priv->uncore.fw_domains); 1155 1156 for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp) 1157 fw_domain_arm_timer(domain); 1158 1159 dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains); 1160 } 1161 1162 static inline void __force_wake_auto(struct drm_i915_private *dev_priv, 1163 enum forcewake_domains fw_domains) 1164 { 1165 if (WARN_ON(!fw_domains)) 1166 return; 1167 1168 /* Turn on all requested but inactive supported forcewake domains. */ 1169 fw_domains &= dev_priv->uncore.fw_domains; 1170 fw_domains &= ~dev_priv->uncore.fw_domains_active; 1171 1172 if (fw_domains) 1173 ___force_wake_auto(dev_priv, fw_domains); 1174 } 1175 1176 #define __gen_read(func, x) \ 1177 static u##x \ 1178 func##_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \ 1179 enum forcewake_domains fw_engine; \ 1180 GEN6_READ_HEADER(x); \ 1181 fw_engine = __##func##_reg_read_fw_domains(offset); \ 1182 if (fw_engine) \ 1183 __force_wake_auto(dev_priv, fw_engine); \ 1184 val = __raw_i915_read##x(dev_priv, reg); \ 1185 GEN6_READ_FOOTER; \ 1186 } 1187 #define __gen6_read(x) __gen_read(gen6, x) 1188 #define __fwtable_read(x) __gen_read(fwtable, x) 1189 #define __gen11_fwtable_read(x) __gen_read(gen11_fwtable, x) 1190 1191 __gen11_fwtable_read(8) 1192 __gen11_fwtable_read(16) 1193 __gen11_fwtable_read(32) 1194 __gen11_fwtable_read(64) 1195 __fwtable_read(8) 1196 __fwtable_read(16) 1197 __fwtable_read(32) 1198 __fwtable_read(64) 1199 __gen6_read(8) 1200 __gen6_read(16) 1201 __gen6_read(32) 1202 __gen6_read(64) 1203 1204 #undef __gen11_fwtable_read 1205 #undef __fwtable_read 1206 #undef __gen6_read 1207 #undef GEN6_READ_FOOTER 1208 #undef GEN6_READ_HEADER 1209 1210 #define GEN2_WRITE_HEADER \ 1211 trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \ 1212 assert_rpm_wakelock_held(dev_priv); \ 1213 1214 #define GEN2_WRITE_FOOTER 1215 1216 #define __gen2_write(x) \ 1217 static void \ 1218 gen2_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \ 1219 GEN2_WRITE_HEADER; \ 1220 __raw_i915_write##x(dev_priv, reg, val); \ 1221 GEN2_WRITE_FOOTER; \ 1222 } 1223 1224 #define __gen5_write(x) \ 1225 static void \ 1226 gen5_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \ 1227 GEN2_WRITE_HEADER; \ 1228 ilk_dummy_write(dev_priv); \ 1229 __raw_i915_write##x(dev_priv, reg, val); \ 1230 GEN2_WRITE_FOOTER; \ 1231 } 1232 1233 __gen5_write(8) 1234 __gen5_write(16) 1235 __gen5_write(32) 1236 __gen2_write(8) 1237 __gen2_write(16) 1238 __gen2_write(32) 1239 1240 #undef __gen5_write 1241 #undef __gen2_write 1242 1243 #undef GEN2_WRITE_FOOTER 1244 #undef GEN2_WRITE_HEADER 1245 1246 #define GEN6_WRITE_HEADER \ 1247 u32 offset = i915_mmio_reg_offset(reg); \ 1248 unsigned long irqflags; \ 1249 trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \ 1250 assert_rpm_wakelock_held(dev_priv); \ 1251 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \ 1252 unclaimed_reg_debug(dev_priv, reg, false, true) 1253 1254 #define GEN6_WRITE_FOOTER \ 1255 unclaimed_reg_debug(dev_priv, reg, false, false); \ 1256 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags) 1257 1258 #define __gen6_write(x) \ 1259 static void \ 1260 gen6_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \ 1261 GEN6_WRITE_HEADER; \ 1262 if (NEEDS_FORCE_WAKE(offset)) \ 1263 __gen6_gt_wait_for_fifo(dev_priv); \ 1264 __raw_i915_write##x(dev_priv, reg, val); \ 1265 GEN6_WRITE_FOOTER; \ 1266 } 1267 1268 #define __gen_write(func, x) \ 1269 static void \ 1270 func##_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \ 1271 enum forcewake_domains fw_engine; \ 1272 GEN6_WRITE_HEADER; \ 1273 fw_engine = __##func##_reg_write_fw_domains(offset); \ 1274 if (fw_engine) \ 1275 __force_wake_auto(dev_priv, fw_engine); \ 1276 __raw_i915_write##x(dev_priv, reg, val); \ 1277 GEN6_WRITE_FOOTER; \ 1278 } 1279 #define __gen8_write(x) __gen_write(gen8, x) 1280 #define __fwtable_write(x) __gen_write(fwtable, x) 1281 #define __gen11_fwtable_write(x) __gen_write(gen11_fwtable, x) 1282 1283 __gen11_fwtable_write(8) 1284 __gen11_fwtable_write(16) 1285 __gen11_fwtable_write(32) 1286 __fwtable_write(8) 1287 __fwtable_write(16) 1288 __fwtable_write(32) 1289 __gen8_write(8) 1290 __gen8_write(16) 1291 __gen8_write(32) 1292 __gen6_write(8) 1293 __gen6_write(16) 1294 __gen6_write(32) 1295 1296 #undef __gen11_fwtable_write 1297 #undef __fwtable_write 1298 #undef __gen8_write 1299 #undef __gen6_write 1300 #undef GEN6_WRITE_FOOTER 1301 #undef GEN6_WRITE_HEADER 1302 1303 #define ASSIGN_WRITE_MMIO_VFUNCS(i915, x) \ 1304 do { \ 1305 (i915)->uncore.funcs.mmio_writeb = x##_write8; \ 1306 (i915)->uncore.funcs.mmio_writew = x##_write16; \ 1307 (i915)->uncore.funcs.mmio_writel = x##_write32; \ 1308 } while (0) 1309 1310 #define ASSIGN_READ_MMIO_VFUNCS(i915, x) \ 1311 do { \ 1312 (i915)->uncore.funcs.mmio_readb = x##_read8; \ 1313 (i915)->uncore.funcs.mmio_readw = x##_read16; \ 1314 (i915)->uncore.funcs.mmio_readl = x##_read32; \ 1315 (i915)->uncore.funcs.mmio_readq = x##_read64; \ 1316 } while (0) 1317 1318 1319 static void fw_domain_init(struct drm_i915_private *dev_priv, 1320 enum forcewake_domain_id domain_id, 1321 i915_reg_t reg_set, 1322 i915_reg_t reg_ack) 1323 { 1324 struct intel_uncore_forcewake_domain *d; 1325 1326 if (WARN_ON(domain_id >= FW_DOMAIN_ID_COUNT)) 1327 return; 1328 1329 d = &dev_priv->uncore.fw_domain[domain_id]; 1330 1331 WARN_ON(d->wake_count); 1332 1333 WARN_ON(!i915_mmio_reg_valid(reg_set)); 1334 WARN_ON(!i915_mmio_reg_valid(reg_ack)); 1335 1336 d->wake_count = 0; 1337 d->reg_set = reg_set; 1338 d->reg_ack = reg_ack; 1339 1340 d->id = domain_id; 1341 1342 BUILD_BUG_ON(FORCEWAKE_RENDER != (1 << FW_DOMAIN_ID_RENDER)); 1343 BUILD_BUG_ON(FORCEWAKE_BLITTER != (1 << FW_DOMAIN_ID_BLITTER)); 1344 BUILD_BUG_ON(FORCEWAKE_MEDIA != (1 << FW_DOMAIN_ID_MEDIA)); 1345 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX0)); 1346 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX1)); 1347 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX2 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX2)); 1348 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX3 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX3)); 1349 BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX0)); 1350 BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX1)); 1351 1352 1353 d->mask = BIT(domain_id); 1354 1355 hrtimer_init(&d->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1356 d->timer.function = intel_uncore_fw_release_timer; 1357 1358 dev_priv->uncore.fw_domains |= BIT(domain_id); 1359 1360 fw_domain_reset(dev_priv, d); 1361 } 1362 1363 static void fw_domain_fini(struct drm_i915_private *dev_priv, 1364 enum forcewake_domain_id domain_id) 1365 { 1366 struct intel_uncore_forcewake_domain *d; 1367 1368 if (WARN_ON(domain_id >= FW_DOMAIN_ID_COUNT)) 1369 return; 1370 1371 d = &dev_priv->uncore.fw_domain[domain_id]; 1372 1373 WARN_ON(d->wake_count); 1374 WARN_ON(hrtimer_cancel(&d->timer)); 1375 memset(d, 0, sizeof(*d)); 1376 1377 dev_priv->uncore.fw_domains &= ~BIT(domain_id); 1378 } 1379 1380 static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv) 1381 { 1382 if (INTEL_GEN(dev_priv) <= 5 || intel_vgpu_active(dev_priv)) 1383 return; 1384 1385 if (IS_GEN6(dev_priv)) { 1386 dev_priv->uncore.fw_reset = 0; 1387 dev_priv->uncore.fw_set = FORCEWAKE_KERNEL; 1388 dev_priv->uncore.fw_clear = 0; 1389 } else { 1390 /* WaRsClearFWBitsAtReset:bdw,skl */ 1391 dev_priv->uncore.fw_reset = _MASKED_BIT_DISABLE(0xffff); 1392 dev_priv->uncore.fw_set = _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL); 1393 dev_priv->uncore.fw_clear = _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL); 1394 } 1395 1396 if (INTEL_GEN(dev_priv) >= 11) { 1397 int i; 1398 1399 dev_priv->uncore.funcs.force_wake_get = 1400 fw_domains_get_with_fallback; 1401 dev_priv->uncore.funcs.force_wake_put = fw_domains_put; 1402 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, 1403 FORCEWAKE_RENDER_GEN9, 1404 FORCEWAKE_ACK_RENDER_GEN9); 1405 fw_domain_init(dev_priv, FW_DOMAIN_ID_BLITTER, 1406 FORCEWAKE_BLITTER_GEN9, 1407 FORCEWAKE_ACK_BLITTER_GEN9); 1408 for (i = 0; i < I915_MAX_VCS; i++) { 1409 if (!HAS_ENGINE(dev_priv, _VCS(i))) 1410 continue; 1411 1412 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA_VDBOX0 + i, 1413 FORCEWAKE_MEDIA_VDBOX_GEN11(i), 1414 FORCEWAKE_ACK_MEDIA_VDBOX_GEN11(i)); 1415 } 1416 for (i = 0; i < I915_MAX_VECS; i++) { 1417 if (!HAS_ENGINE(dev_priv, _VECS(i))) 1418 continue; 1419 1420 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA_VEBOX0 + i, 1421 FORCEWAKE_MEDIA_VEBOX_GEN11(i), 1422 FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(i)); 1423 } 1424 } else if (IS_GEN9(dev_priv) || IS_GEN10(dev_priv)) { 1425 dev_priv->uncore.funcs.force_wake_get = 1426 fw_domains_get_with_fallback; 1427 dev_priv->uncore.funcs.force_wake_put = fw_domains_put; 1428 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, 1429 FORCEWAKE_RENDER_GEN9, 1430 FORCEWAKE_ACK_RENDER_GEN9); 1431 fw_domain_init(dev_priv, FW_DOMAIN_ID_BLITTER, 1432 FORCEWAKE_BLITTER_GEN9, 1433 FORCEWAKE_ACK_BLITTER_GEN9); 1434 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA, 1435 FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9); 1436 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { 1437 dev_priv->uncore.funcs.force_wake_get = fw_domains_get; 1438 dev_priv->uncore.funcs.force_wake_put = fw_domains_put; 1439 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, 1440 FORCEWAKE_VLV, FORCEWAKE_ACK_VLV); 1441 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA, 1442 FORCEWAKE_MEDIA_VLV, FORCEWAKE_ACK_MEDIA_VLV); 1443 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { 1444 dev_priv->uncore.funcs.force_wake_get = 1445 fw_domains_get_with_thread_status; 1446 dev_priv->uncore.funcs.force_wake_put = fw_domains_put; 1447 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, 1448 FORCEWAKE_MT, FORCEWAKE_ACK_HSW); 1449 } else if (IS_IVYBRIDGE(dev_priv)) { 1450 u32 ecobus; 1451 1452 /* IVB configs may use multi-threaded forcewake */ 1453 1454 /* A small trick here - if the bios hasn't configured 1455 * MT forcewake, and if the device is in RC6, then 1456 * force_wake_mt_get will not wake the device and the 1457 * ECOBUS read will return zero. Which will be 1458 * (correctly) interpreted by the test below as MT 1459 * forcewake being disabled. 1460 */ 1461 dev_priv->uncore.funcs.force_wake_get = 1462 fw_domains_get_with_thread_status; 1463 dev_priv->uncore.funcs.force_wake_put = fw_domains_put; 1464 1465 /* We need to init first for ECOBUS access and then 1466 * determine later if we want to reinit, in case of MT access is 1467 * not working. In this stage we don't know which flavour this 1468 * ivb is, so it is better to reset also the gen6 fw registers 1469 * before the ecobus check. 1470 */ 1471 1472 __raw_i915_write32(dev_priv, FORCEWAKE, 0); 1473 __raw_posting_read(dev_priv, ECOBUS); 1474 1475 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, 1476 FORCEWAKE_MT, FORCEWAKE_MT_ACK); 1477 1478 spin_lock_irq(&dev_priv->uncore.lock); 1479 fw_domains_get_with_thread_status(dev_priv, FORCEWAKE_RENDER); 1480 ecobus = __raw_i915_read32(dev_priv, ECOBUS); 1481 fw_domains_put(dev_priv, FORCEWAKE_RENDER); 1482 spin_unlock_irq(&dev_priv->uncore.lock); 1483 1484 if (!(ecobus & FORCEWAKE_MT_ENABLE)) { 1485 DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n"); 1486 DRM_INFO("when using vblank-synced partial screen updates.\n"); 1487 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, 1488 FORCEWAKE, FORCEWAKE_ACK); 1489 } 1490 } else if (IS_GEN6(dev_priv)) { 1491 dev_priv->uncore.funcs.force_wake_get = 1492 fw_domains_get_with_thread_status; 1493 dev_priv->uncore.funcs.force_wake_put = fw_domains_put; 1494 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, 1495 FORCEWAKE, FORCEWAKE_ACK); 1496 } 1497 1498 /* All future platforms are expected to require complex power gating */ 1499 WARN_ON(dev_priv->uncore.fw_domains == 0); 1500 } 1501 1502 #define ASSIGN_FW_DOMAINS_TABLE(d) \ 1503 { \ 1504 dev_priv->uncore.fw_domains_table = \ 1505 (struct intel_forcewake_range *)(d); \ 1506 dev_priv->uncore.fw_domains_table_entries = ARRAY_SIZE((d)); \ 1507 } 1508 1509 static int i915_pmic_bus_access_notifier(struct notifier_block *nb, 1510 unsigned long action, void *data) 1511 { 1512 struct drm_i915_private *dev_priv = container_of(nb, 1513 struct drm_i915_private, uncore.pmic_bus_access_nb); 1514 1515 switch (action) { 1516 case MBI_PMIC_BUS_ACCESS_BEGIN: 1517 /* 1518 * forcewake all now to make sure that we don't need to do a 1519 * forcewake later which on systems where this notifier gets 1520 * called requires the punit to access to the shared pmic i2c 1521 * bus, which will be busy after this notification, leading to: 1522 * "render: timed out waiting for forcewake ack request." 1523 * errors. 1524 * 1525 * The notifier is unregistered during intel_runtime_suspend(), 1526 * so it's ok to access the HW here without holding a RPM 1527 * wake reference -> disable wakeref asserts for the time of 1528 * the access. 1529 */ 1530 disable_rpm_wakeref_asserts(dev_priv); 1531 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 1532 enable_rpm_wakeref_asserts(dev_priv); 1533 break; 1534 case MBI_PMIC_BUS_ACCESS_END: 1535 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 1536 break; 1537 } 1538 1539 return NOTIFY_OK; 1540 } 1541 1542 void intel_uncore_init(struct drm_i915_private *dev_priv) 1543 { 1544 i915_check_vgpu(dev_priv); 1545 1546 intel_uncore_edram_detect(dev_priv); 1547 intel_uncore_fw_domains_init(dev_priv); 1548 __intel_uncore_early_sanitize(dev_priv, false); 1549 1550 dev_priv->uncore.unclaimed_mmio_check = 1; 1551 dev_priv->uncore.pmic_bus_access_nb.notifier_call = 1552 i915_pmic_bus_access_notifier; 1553 1554 if (IS_GEN(dev_priv, 2, 4) || intel_vgpu_active(dev_priv)) { 1555 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen2); 1556 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen2); 1557 } else if (IS_GEN5(dev_priv)) { 1558 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen5); 1559 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen5); 1560 } else if (IS_GEN(dev_priv, 6, 7)) { 1561 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen6); 1562 1563 if (IS_VALLEYVIEW(dev_priv)) { 1564 ASSIGN_FW_DOMAINS_TABLE(__vlv_fw_ranges); 1565 ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable); 1566 } else { 1567 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6); 1568 } 1569 } else if (IS_GEN8(dev_priv)) { 1570 if (IS_CHERRYVIEW(dev_priv)) { 1571 ASSIGN_FW_DOMAINS_TABLE(__chv_fw_ranges); 1572 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable); 1573 ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable); 1574 1575 } else { 1576 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen8); 1577 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6); 1578 } 1579 } else if (IS_GEN(dev_priv, 9, 10)) { 1580 ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges); 1581 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable); 1582 ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable); 1583 } else { 1584 ASSIGN_FW_DOMAINS_TABLE(__gen11_fw_ranges); 1585 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen11_fwtable); 1586 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen11_fwtable); 1587 } 1588 1589 iosf_mbi_register_pmic_bus_access_notifier( 1590 &dev_priv->uncore.pmic_bus_access_nb); 1591 } 1592 1593 /* 1594 * We might have detected that some engines are fused off after we initialized 1595 * the forcewake domains. Prune them, to make sure they only reference existing 1596 * engines. 1597 */ 1598 void intel_uncore_prune(struct drm_i915_private *dev_priv) 1599 { 1600 if (INTEL_GEN(dev_priv) >= 11) { 1601 enum forcewake_domains fw_domains = dev_priv->uncore.fw_domains; 1602 enum forcewake_domain_id domain_id; 1603 int i; 1604 1605 for (i = 0; i < I915_MAX_VCS; i++) { 1606 domain_id = FW_DOMAIN_ID_MEDIA_VDBOX0 + i; 1607 1608 if (HAS_ENGINE(dev_priv, _VCS(i))) 1609 continue; 1610 1611 if (fw_domains & BIT(domain_id)) 1612 fw_domain_fini(dev_priv, domain_id); 1613 } 1614 1615 for (i = 0; i < I915_MAX_VECS; i++) { 1616 domain_id = FW_DOMAIN_ID_MEDIA_VEBOX0 + i; 1617 1618 if (HAS_ENGINE(dev_priv, _VECS(i))) 1619 continue; 1620 1621 if (fw_domains & BIT(domain_id)) 1622 fw_domain_fini(dev_priv, domain_id); 1623 } 1624 } 1625 } 1626 1627 void intel_uncore_fini(struct drm_i915_private *dev_priv) 1628 { 1629 /* Paranoia: make sure we have disabled everything before we exit. */ 1630 intel_uncore_sanitize(dev_priv); 1631 1632 iosf_mbi_punit_acquire(); 1633 iosf_mbi_unregister_pmic_bus_access_notifier_unlocked( 1634 &dev_priv->uncore.pmic_bus_access_nb); 1635 intel_uncore_forcewake_reset(dev_priv, false); 1636 iosf_mbi_punit_release(); 1637 } 1638 1639 static const struct reg_whitelist { 1640 i915_reg_t offset_ldw; 1641 i915_reg_t offset_udw; 1642 u16 gen_mask; 1643 u8 size; 1644 } reg_read_whitelist[] = { { 1645 .offset_ldw = RING_TIMESTAMP(RENDER_RING_BASE), 1646 .offset_udw = RING_TIMESTAMP_UDW(RENDER_RING_BASE), 1647 .gen_mask = INTEL_GEN_MASK(4, 11), 1648 .size = 8 1649 } }; 1650 1651 int i915_reg_read_ioctl(struct drm_device *dev, 1652 void *data, struct drm_file *file) 1653 { 1654 struct drm_i915_private *dev_priv = to_i915(dev); 1655 struct drm_i915_reg_read *reg = data; 1656 struct reg_whitelist const *entry; 1657 unsigned int flags; 1658 int remain; 1659 int ret = 0; 1660 1661 entry = reg_read_whitelist; 1662 remain = ARRAY_SIZE(reg_read_whitelist); 1663 while (remain) { 1664 u32 entry_offset = i915_mmio_reg_offset(entry->offset_ldw); 1665 1666 GEM_BUG_ON(!is_power_of_2(entry->size)); 1667 GEM_BUG_ON(entry->size > 8); 1668 GEM_BUG_ON(entry_offset & (entry->size - 1)); 1669 1670 if (INTEL_INFO(dev_priv)->gen_mask & entry->gen_mask && 1671 entry_offset == (reg->offset & -entry->size)) 1672 break; 1673 entry++; 1674 remain--; 1675 } 1676 1677 if (!remain) 1678 return -EINVAL; 1679 1680 flags = reg->offset & (entry->size - 1); 1681 1682 intel_runtime_pm_get(dev_priv); 1683 if (entry->size == 8 && flags == I915_REG_READ_8B_WA) 1684 reg->val = I915_READ64_2x32(entry->offset_ldw, 1685 entry->offset_udw); 1686 else if (entry->size == 8 && flags == 0) 1687 reg->val = I915_READ64(entry->offset_ldw); 1688 else if (entry->size == 4 && flags == 0) 1689 reg->val = I915_READ(entry->offset_ldw); 1690 else if (entry->size == 2 && flags == 0) 1691 reg->val = I915_READ16(entry->offset_ldw); 1692 else if (entry->size == 1 && flags == 0) 1693 reg->val = I915_READ8(entry->offset_ldw); 1694 else 1695 ret = -EINVAL; 1696 intel_runtime_pm_put(dev_priv); 1697 1698 return ret; 1699 } 1700 1701 static void gen3_stop_engine(struct intel_engine_cs *engine) 1702 { 1703 struct drm_i915_private *dev_priv = engine->i915; 1704 const u32 base = engine->mmio_base; 1705 const i915_reg_t mode = RING_MI_MODE(base); 1706 1707 I915_WRITE_FW(mode, _MASKED_BIT_ENABLE(STOP_RING)); 1708 if (__intel_wait_for_register_fw(dev_priv, 1709 mode, MODE_IDLE, MODE_IDLE, 1710 500, 0, 1711 NULL)) 1712 DRM_DEBUG_DRIVER("%s: timed out on STOP_RING\n", 1713 engine->name); 1714 1715 I915_WRITE_FW(RING_HEAD(base), I915_READ_FW(RING_TAIL(base))); 1716 POSTING_READ_FW(RING_HEAD(base)); /* paranoia */ 1717 1718 I915_WRITE_FW(RING_HEAD(base), 0); 1719 I915_WRITE_FW(RING_TAIL(base), 0); 1720 POSTING_READ_FW(RING_TAIL(base)); 1721 1722 /* The ring must be empty before it is disabled */ 1723 I915_WRITE_FW(RING_CTL(base), 0); 1724 1725 /* Check acts as a post */ 1726 if (I915_READ_FW(RING_HEAD(base)) != 0) 1727 DRM_DEBUG_DRIVER("%s: ring head not parked\n", 1728 engine->name); 1729 } 1730 1731 static void i915_stop_engines(struct drm_i915_private *dev_priv, 1732 unsigned engine_mask) 1733 { 1734 struct intel_engine_cs *engine; 1735 enum intel_engine_id id; 1736 1737 if (INTEL_GEN(dev_priv) < 3) 1738 return; 1739 1740 for_each_engine_masked(engine, dev_priv, engine_mask, id) 1741 gen3_stop_engine(engine); 1742 } 1743 1744 static bool i915_in_reset(struct pci_dev *pdev) 1745 { 1746 u8 gdrst; 1747 1748 pci_read_config_byte(pdev, I915_GDRST, &gdrst); 1749 return gdrst & GRDOM_RESET_STATUS; 1750 } 1751 1752 static int i915_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask) 1753 { 1754 struct pci_dev *pdev = dev_priv->drm.pdev; 1755 int err; 1756 1757 /* Assert reset for at least 20 usec, and wait for acknowledgement. */ 1758 pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE); 1759 usleep_range(50, 200); 1760 err = wait_for(i915_in_reset(pdev), 500); 1761 1762 /* Clear the reset request. */ 1763 pci_write_config_byte(pdev, I915_GDRST, 0); 1764 usleep_range(50, 200); 1765 if (!err) 1766 err = wait_for(!i915_in_reset(pdev), 500); 1767 1768 return err; 1769 } 1770 1771 static bool g4x_reset_complete(struct pci_dev *pdev) 1772 { 1773 u8 gdrst; 1774 1775 pci_read_config_byte(pdev, I915_GDRST, &gdrst); 1776 return (gdrst & GRDOM_RESET_ENABLE) == 0; 1777 } 1778 1779 static int g33_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask) 1780 { 1781 struct pci_dev *pdev = dev_priv->drm.pdev; 1782 1783 pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE); 1784 return wait_for(g4x_reset_complete(pdev), 500); 1785 } 1786 1787 static int g4x_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask) 1788 { 1789 struct pci_dev *pdev = dev_priv->drm.pdev; 1790 int ret; 1791 1792 /* WaVcpClkGateDisableForMediaReset:ctg,elk */ 1793 I915_WRITE(VDECCLK_GATE_D, 1794 I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE); 1795 POSTING_READ(VDECCLK_GATE_D); 1796 1797 pci_write_config_byte(pdev, I915_GDRST, 1798 GRDOM_MEDIA | GRDOM_RESET_ENABLE); 1799 ret = wait_for(g4x_reset_complete(pdev), 500); 1800 if (ret) { 1801 DRM_DEBUG_DRIVER("Wait for media reset failed\n"); 1802 goto out; 1803 } 1804 1805 pci_write_config_byte(pdev, I915_GDRST, 1806 GRDOM_RENDER | GRDOM_RESET_ENABLE); 1807 ret = wait_for(g4x_reset_complete(pdev), 500); 1808 if (ret) { 1809 DRM_DEBUG_DRIVER("Wait for render reset failed\n"); 1810 goto out; 1811 } 1812 1813 out: 1814 pci_write_config_byte(pdev, I915_GDRST, 0); 1815 1816 I915_WRITE(VDECCLK_GATE_D, 1817 I915_READ(VDECCLK_GATE_D) & ~VCP_UNIT_CLOCK_GATE_DISABLE); 1818 POSTING_READ(VDECCLK_GATE_D); 1819 1820 return ret; 1821 } 1822 1823 static int ironlake_do_reset(struct drm_i915_private *dev_priv, 1824 unsigned engine_mask) 1825 { 1826 int ret; 1827 1828 I915_WRITE(ILK_GDSR, ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE); 1829 ret = intel_wait_for_register(dev_priv, 1830 ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0, 1831 500); 1832 if (ret) { 1833 DRM_DEBUG_DRIVER("Wait for render reset failed\n"); 1834 goto out; 1835 } 1836 1837 I915_WRITE(ILK_GDSR, ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE); 1838 ret = intel_wait_for_register(dev_priv, 1839 ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0, 1840 500); 1841 if (ret) { 1842 DRM_DEBUG_DRIVER("Wait for media reset failed\n"); 1843 goto out; 1844 } 1845 1846 out: 1847 I915_WRITE(ILK_GDSR, 0); 1848 POSTING_READ(ILK_GDSR); 1849 return ret; 1850 } 1851 1852 /* Reset the hardware domains (GENX_GRDOM_*) specified by mask */ 1853 static int gen6_hw_domain_reset(struct drm_i915_private *dev_priv, 1854 u32 hw_domain_mask) 1855 { 1856 int err; 1857 1858 /* GEN6_GDRST is not in the gt power well, no need to check 1859 * for fifo space for the write or forcewake the chip for 1860 * the read 1861 */ 1862 __raw_i915_write32(dev_priv, GEN6_GDRST, hw_domain_mask); 1863 1864 /* Wait for the device to ack the reset requests */ 1865 err = __intel_wait_for_register_fw(dev_priv, 1866 GEN6_GDRST, hw_domain_mask, 0, 1867 500, 0, 1868 NULL); 1869 if (err) 1870 DRM_DEBUG_DRIVER("Wait for 0x%08x engines reset failed\n", 1871 hw_domain_mask); 1872 1873 return err; 1874 } 1875 1876 /** 1877 * gen6_reset_engines - reset individual engines 1878 * @dev_priv: i915 device 1879 * @engine_mask: mask of intel_ring_flag() engines or ALL_ENGINES for full reset 1880 * 1881 * This function will reset the individual engines that are set in engine_mask. 1882 * If you provide ALL_ENGINES as mask, full global domain reset will be issued. 1883 * 1884 * Note: It is responsibility of the caller to handle the difference between 1885 * asking full domain reset versus reset for all available individual engines. 1886 * 1887 * Returns 0 on success, nonzero on error. 1888 */ 1889 static int gen6_reset_engines(struct drm_i915_private *dev_priv, 1890 unsigned engine_mask) 1891 { 1892 struct intel_engine_cs *engine; 1893 const u32 hw_engine_mask[I915_NUM_ENGINES] = { 1894 [RCS] = GEN6_GRDOM_RENDER, 1895 [BCS] = GEN6_GRDOM_BLT, 1896 [VCS] = GEN6_GRDOM_MEDIA, 1897 [VCS2] = GEN8_GRDOM_MEDIA2, 1898 [VECS] = GEN6_GRDOM_VECS, 1899 }; 1900 u32 hw_mask; 1901 1902 if (engine_mask == ALL_ENGINES) { 1903 hw_mask = GEN6_GRDOM_FULL; 1904 } else { 1905 unsigned int tmp; 1906 1907 hw_mask = 0; 1908 for_each_engine_masked(engine, dev_priv, engine_mask, tmp) 1909 hw_mask |= hw_engine_mask[engine->id]; 1910 } 1911 1912 return gen6_hw_domain_reset(dev_priv, hw_mask); 1913 } 1914 1915 /** 1916 * gen11_reset_engines - reset individual engines 1917 * @dev_priv: i915 device 1918 * @engine_mask: mask of intel_ring_flag() engines or ALL_ENGINES for full reset 1919 * 1920 * This function will reset the individual engines that are set in engine_mask. 1921 * If you provide ALL_ENGINES as mask, full global domain reset will be issued. 1922 * 1923 * Note: It is responsibility of the caller to handle the difference between 1924 * asking full domain reset versus reset for all available individual engines. 1925 * 1926 * Returns 0 on success, nonzero on error. 1927 */ 1928 static int gen11_reset_engines(struct drm_i915_private *dev_priv, 1929 unsigned engine_mask) 1930 { 1931 struct intel_engine_cs *engine; 1932 const u32 hw_engine_mask[I915_NUM_ENGINES] = { 1933 [RCS] = GEN11_GRDOM_RENDER, 1934 [BCS] = GEN11_GRDOM_BLT, 1935 [VCS] = GEN11_GRDOM_MEDIA, 1936 [VCS2] = GEN11_GRDOM_MEDIA2, 1937 [VCS3] = GEN11_GRDOM_MEDIA3, 1938 [VCS4] = GEN11_GRDOM_MEDIA4, 1939 [VECS] = GEN11_GRDOM_VECS, 1940 [VECS2] = GEN11_GRDOM_VECS2, 1941 }; 1942 u32 hw_mask; 1943 1944 BUILD_BUG_ON(VECS2 + 1 != I915_NUM_ENGINES); 1945 1946 if (engine_mask == ALL_ENGINES) { 1947 hw_mask = GEN11_GRDOM_FULL; 1948 } else { 1949 unsigned int tmp; 1950 1951 hw_mask = 0; 1952 for_each_engine_masked(engine, dev_priv, engine_mask, tmp) 1953 hw_mask |= hw_engine_mask[engine->id]; 1954 } 1955 1956 return gen6_hw_domain_reset(dev_priv, hw_mask); 1957 } 1958 1959 /** 1960 * __intel_wait_for_register_fw - wait until register matches expected state 1961 * @dev_priv: the i915 device 1962 * @reg: the register to read 1963 * @mask: mask to apply to register value 1964 * @value: expected value 1965 * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait 1966 * @slow_timeout_ms: slow timeout in millisecond 1967 * @out_value: optional placeholder to hold registry value 1968 * 1969 * This routine waits until the target register @reg contains the expected 1970 * @value after applying the @mask, i.e. it waits until :: 1971 * 1972 * (I915_READ_FW(reg) & mask) == value 1973 * 1974 * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds. 1975 * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us 1976 * must be not larger than 20,0000 microseconds. 1977 * 1978 * Note that this routine assumes the caller holds forcewake asserted, it is 1979 * not suitable for very long waits. See intel_wait_for_register() if you 1980 * wish to wait without holding forcewake for the duration (i.e. you expect 1981 * the wait to be slow). 1982 * 1983 * Returns 0 if the register matches the desired condition, or -ETIMEOUT. 1984 */ 1985 int __intel_wait_for_register_fw(struct drm_i915_private *dev_priv, 1986 i915_reg_t reg, 1987 u32 mask, 1988 u32 value, 1989 unsigned int fast_timeout_us, 1990 unsigned int slow_timeout_ms, 1991 u32 *out_value) 1992 { 1993 u32 uninitialized_var(reg_value); 1994 #define done (((reg_value = I915_READ_FW(reg)) & mask) == value) 1995 int ret; 1996 1997 /* Catch any overuse of this function */ 1998 might_sleep_if(slow_timeout_ms); 1999 GEM_BUG_ON(fast_timeout_us > 20000); 2000 2001 ret = -ETIMEDOUT; 2002 if (fast_timeout_us && fast_timeout_us <= 20000) 2003 ret = _wait_for_atomic(done, fast_timeout_us, 0); 2004 if (ret && slow_timeout_ms) 2005 ret = wait_for(done, slow_timeout_ms); 2006 2007 if (out_value) 2008 *out_value = reg_value; 2009 2010 return ret; 2011 #undef done 2012 } 2013 2014 /** 2015 * __intel_wait_for_register - wait until register matches expected state 2016 * @dev_priv: the i915 device 2017 * @reg: the register to read 2018 * @mask: mask to apply to register value 2019 * @value: expected value 2020 * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait 2021 * @slow_timeout_ms: slow timeout in millisecond 2022 * @out_value: optional placeholder to hold registry value 2023 * 2024 * This routine waits until the target register @reg contains the expected 2025 * @value after applying the @mask, i.e. it waits until :: 2026 * 2027 * (I915_READ(reg) & mask) == value 2028 * 2029 * Otherwise, the wait will timeout after @timeout_ms milliseconds. 2030 * 2031 * Returns 0 if the register matches the desired condition, or -ETIMEOUT. 2032 */ 2033 int __intel_wait_for_register(struct drm_i915_private *dev_priv, 2034 i915_reg_t reg, 2035 u32 mask, 2036 u32 value, 2037 unsigned int fast_timeout_us, 2038 unsigned int slow_timeout_ms, 2039 u32 *out_value) 2040 { 2041 unsigned fw = 2042 intel_uncore_forcewake_for_reg(dev_priv, reg, FW_REG_READ); 2043 u32 reg_value; 2044 int ret; 2045 2046 might_sleep_if(slow_timeout_ms); 2047 2048 spin_lock_irq(&dev_priv->uncore.lock); 2049 intel_uncore_forcewake_get__locked(dev_priv, fw); 2050 2051 ret = __intel_wait_for_register_fw(dev_priv, 2052 reg, mask, value, 2053 fast_timeout_us, 0, ®_value); 2054 2055 intel_uncore_forcewake_put__locked(dev_priv, fw); 2056 spin_unlock_irq(&dev_priv->uncore.lock); 2057 2058 if (ret && slow_timeout_ms) 2059 ret = __wait_for(reg_value = I915_READ_NOTRACE(reg), 2060 (reg_value & mask) == value, 2061 slow_timeout_ms * 1000, 10, 1000); 2062 2063 if (out_value) 2064 *out_value = reg_value; 2065 2066 return ret; 2067 } 2068 2069 static int gen8_reset_engine_start(struct intel_engine_cs *engine) 2070 { 2071 struct drm_i915_private *dev_priv = engine->i915; 2072 int ret; 2073 2074 I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base), 2075 _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET)); 2076 2077 ret = __intel_wait_for_register_fw(dev_priv, 2078 RING_RESET_CTL(engine->mmio_base), 2079 RESET_CTL_READY_TO_RESET, 2080 RESET_CTL_READY_TO_RESET, 2081 700, 0, 2082 NULL); 2083 if (ret) 2084 DRM_ERROR("%s: reset request timeout\n", engine->name); 2085 2086 return ret; 2087 } 2088 2089 static void gen8_reset_engine_cancel(struct intel_engine_cs *engine) 2090 { 2091 struct drm_i915_private *dev_priv = engine->i915; 2092 2093 I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base), 2094 _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET)); 2095 } 2096 2097 static int gen8_reset_engines(struct drm_i915_private *dev_priv, 2098 unsigned engine_mask) 2099 { 2100 struct intel_engine_cs *engine; 2101 unsigned int tmp; 2102 2103 for_each_engine_masked(engine, dev_priv, engine_mask, tmp) 2104 if (gen8_reset_engine_start(engine)) 2105 goto not_ready; 2106 2107 if (INTEL_GEN(dev_priv) >= 11) 2108 return gen11_reset_engines(dev_priv, engine_mask); 2109 else 2110 return gen6_reset_engines(dev_priv, engine_mask); 2111 2112 not_ready: 2113 for_each_engine_masked(engine, dev_priv, engine_mask, tmp) 2114 gen8_reset_engine_cancel(engine); 2115 2116 return -EIO; 2117 } 2118 2119 typedef int (*reset_func)(struct drm_i915_private *, unsigned engine_mask); 2120 2121 static reset_func intel_get_gpu_reset(struct drm_i915_private *dev_priv) 2122 { 2123 if (!i915_modparams.reset) 2124 return NULL; 2125 2126 if (INTEL_GEN(dev_priv) >= 8) 2127 return gen8_reset_engines; 2128 else if (INTEL_GEN(dev_priv) >= 6) 2129 return gen6_reset_engines; 2130 else if (IS_GEN5(dev_priv)) 2131 return ironlake_do_reset; 2132 else if (IS_G4X(dev_priv)) 2133 return g4x_do_reset; 2134 else if (IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) 2135 return g33_do_reset; 2136 else if (INTEL_GEN(dev_priv) >= 3) 2137 return i915_do_reset; 2138 else 2139 return NULL; 2140 } 2141 2142 int intel_gpu_reset(struct drm_i915_private *dev_priv, unsigned engine_mask) 2143 { 2144 reset_func reset = intel_get_gpu_reset(dev_priv); 2145 int retry; 2146 int ret; 2147 2148 /* 2149 * We want to perform per-engine reset from atomic context (e.g. 2150 * softirq), which imposes the constraint that we cannot sleep. 2151 * However, experience suggests that spending a bit of time waiting 2152 * for a reset helps in various cases, so for a full-device reset 2153 * we apply the opposite rule and wait if we want to. As we should 2154 * always follow up a failed per-engine reset with a full device reset, 2155 * being a little faster, stricter and more error prone for the 2156 * atomic case seems an acceptable compromise. 2157 * 2158 * Unfortunately this leads to a bimodal routine, when the goal was 2159 * to have a single reset function that worked for resetting any 2160 * number of engines simultaneously. 2161 */ 2162 might_sleep_if(engine_mask == ALL_ENGINES); 2163 2164 /* 2165 * If the power well sleeps during the reset, the reset 2166 * request may be dropped and never completes (causing -EIO). 2167 */ 2168 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 2169 for (retry = 0; retry < 3; retry++) { 2170 2171 /* 2172 * We stop engines, otherwise we might get failed reset and a 2173 * dead gpu (on elk). Also as modern gpu as kbl can suffer 2174 * from system hang if batchbuffer is progressing when 2175 * the reset is issued, regardless of READY_TO_RESET ack. 2176 * Thus assume it is best to stop engines on all gens 2177 * where we have a gpu reset. 2178 * 2179 * WaMediaResetMainRingCleanup:ctg,elk (presumably) 2180 * 2181 * FIXME: Wa for more modern gens needs to be validated 2182 */ 2183 i915_stop_engines(dev_priv, engine_mask); 2184 2185 ret = -ENODEV; 2186 if (reset) { 2187 GEM_TRACE("engine_mask=%x\n", engine_mask); 2188 ret = reset(dev_priv, engine_mask); 2189 } 2190 if (ret != -ETIMEDOUT || engine_mask != ALL_ENGINES) 2191 break; 2192 2193 cond_resched(); 2194 } 2195 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 2196 2197 return ret; 2198 } 2199 2200 bool intel_has_gpu_reset(struct drm_i915_private *dev_priv) 2201 { 2202 return intel_get_gpu_reset(dev_priv) != NULL; 2203 } 2204 2205 bool intel_has_reset_engine(struct drm_i915_private *dev_priv) 2206 { 2207 return (dev_priv->info.has_reset_engine && 2208 i915_modparams.reset >= 2); 2209 } 2210 2211 int intel_reset_guc(struct drm_i915_private *dev_priv) 2212 { 2213 u32 guc_domain = INTEL_GEN(dev_priv) >= 11 ? GEN11_GRDOM_GUC : 2214 GEN9_GRDOM_GUC; 2215 int ret; 2216 2217 GEM_BUG_ON(!HAS_GUC(dev_priv)); 2218 2219 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 2220 ret = gen6_hw_domain_reset(dev_priv, guc_domain); 2221 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 2222 2223 return ret; 2224 } 2225 2226 bool intel_uncore_unclaimed_mmio(struct drm_i915_private *dev_priv) 2227 { 2228 return check_for_unclaimed_mmio(dev_priv); 2229 } 2230 2231 bool 2232 intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private *dev_priv) 2233 { 2234 if (unlikely(i915_modparams.mmio_debug || 2235 dev_priv->uncore.unclaimed_mmio_check <= 0)) 2236 return false; 2237 2238 if (unlikely(intel_uncore_unclaimed_mmio(dev_priv))) { 2239 DRM_DEBUG("Unclaimed register detected, " 2240 "enabling oneshot unclaimed register reporting. " 2241 "Please use i915.mmio_debug=N for more information.\n"); 2242 i915_modparams.mmio_debug++; 2243 dev_priv->uncore.unclaimed_mmio_check--; 2244 return true; 2245 } 2246 2247 return false; 2248 } 2249 2250 static enum forcewake_domains 2251 intel_uncore_forcewake_for_read(struct drm_i915_private *dev_priv, 2252 i915_reg_t reg) 2253 { 2254 u32 offset = i915_mmio_reg_offset(reg); 2255 enum forcewake_domains fw_domains; 2256 2257 if (INTEL_GEN(dev_priv) >= 11) { 2258 fw_domains = __gen11_fwtable_reg_read_fw_domains(offset); 2259 } else if (HAS_FWTABLE(dev_priv)) { 2260 fw_domains = __fwtable_reg_read_fw_domains(offset); 2261 } else if (INTEL_GEN(dev_priv) >= 6) { 2262 fw_domains = __gen6_reg_read_fw_domains(offset); 2263 } else { 2264 WARN_ON(!IS_GEN(dev_priv, 2, 5)); 2265 fw_domains = 0; 2266 } 2267 2268 WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains); 2269 2270 return fw_domains; 2271 } 2272 2273 static enum forcewake_domains 2274 intel_uncore_forcewake_for_write(struct drm_i915_private *dev_priv, 2275 i915_reg_t reg) 2276 { 2277 u32 offset = i915_mmio_reg_offset(reg); 2278 enum forcewake_domains fw_domains; 2279 2280 if (INTEL_GEN(dev_priv) >= 11) { 2281 fw_domains = __gen11_fwtable_reg_write_fw_domains(offset); 2282 } else if (HAS_FWTABLE(dev_priv) && !IS_VALLEYVIEW(dev_priv)) { 2283 fw_domains = __fwtable_reg_write_fw_domains(offset); 2284 } else if (IS_GEN8(dev_priv)) { 2285 fw_domains = __gen8_reg_write_fw_domains(offset); 2286 } else if (IS_GEN(dev_priv, 6, 7)) { 2287 fw_domains = FORCEWAKE_RENDER; 2288 } else { 2289 WARN_ON(!IS_GEN(dev_priv, 2, 5)); 2290 fw_domains = 0; 2291 } 2292 2293 WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains); 2294 2295 return fw_domains; 2296 } 2297 2298 /** 2299 * intel_uncore_forcewake_for_reg - which forcewake domains are needed to access 2300 * a register 2301 * @dev_priv: pointer to struct drm_i915_private 2302 * @reg: register in question 2303 * @op: operation bitmask of FW_REG_READ and/or FW_REG_WRITE 2304 * 2305 * Returns a set of forcewake domains required to be taken with for example 2306 * intel_uncore_forcewake_get for the specified register to be accessible in the 2307 * specified mode (read, write or read/write) with raw mmio accessors. 2308 * 2309 * NOTE: On Gen6 and Gen7 write forcewake domain (FORCEWAKE_RENDER) requires the 2310 * callers to do FIFO management on their own or risk losing writes. 2311 */ 2312 enum forcewake_domains 2313 intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv, 2314 i915_reg_t reg, unsigned int op) 2315 { 2316 enum forcewake_domains fw_domains = 0; 2317 2318 WARN_ON(!op); 2319 2320 if (intel_vgpu_active(dev_priv)) 2321 return 0; 2322 2323 if (op & FW_REG_READ) 2324 fw_domains = intel_uncore_forcewake_for_read(dev_priv, reg); 2325 2326 if (op & FW_REG_WRITE) 2327 fw_domains |= intel_uncore_forcewake_for_write(dev_priv, reg); 2328 2329 return fw_domains; 2330 } 2331 2332 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2333 #include "selftests/mock_uncore.c" 2334 #include "selftests/intel_uncore.c" 2335 #endif 2336