1 /* 2 * Copyright © 2013 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 */ 23 24 #include "i915_drv.h" 25 #include "intel_drv.h" 26 #include "i915_vgpu.h" 27 28 #include <asm/iosf_mbi.h> 29 #include <linux/pm_runtime.h> 30 31 #define FORCEWAKE_ACK_TIMEOUT_MS 50 32 #define GT_FIFO_TIMEOUT_MS 10 33 34 #define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32((dev_priv__), (reg__)) 35 36 static const char * const forcewake_domain_names[] = { 37 "render", 38 "blitter", 39 "media", 40 "vdbox0", 41 "vdbox1", 42 "vdbox2", 43 "vdbox3", 44 "vebox0", 45 "vebox1", 46 }; 47 48 const char * 49 intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id) 50 { 51 BUILD_BUG_ON(ARRAY_SIZE(forcewake_domain_names) != FW_DOMAIN_ID_COUNT); 52 53 if (id >= 0 && id < FW_DOMAIN_ID_COUNT) 54 return forcewake_domain_names[id]; 55 56 WARN_ON(id); 57 58 return "unknown"; 59 } 60 61 static inline void 62 fw_domain_reset(struct drm_i915_private *i915, 63 const struct intel_uncore_forcewake_domain *d) 64 { 65 __raw_i915_write32(i915, d->reg_set, i915->uncore.fw_reset); 66 } 67 68 static inline void 69 fw_domain_arm_timer(struct intel_uncore_forcewake_domain *d) 70 { 71 d->wake_count++; 72 hrtimer_start_range_ns(&d->timer, 73 NSEC_PER_MSEC, 74 NSEC_PER_MSEC, 75 HRTIMER_MODE_REL); 76 } 77 78 static inline int 79 __wait_for_ack(const struct drm_i915_private *i915, 80 const struct intel_uncore_forcewake_domain *d, 81 const u32 ack, 82 const u32 value) 83 { 84 return wait_for_atomic((__raw_i915_read32(i915, d->reg_ack) & ack) == value, 85 FORCEWAKE_ACK_TIMEOUT_MS); 86 } 87 88 static inline int 89 wait_ack_clear(const struct drm_i915_private *i915, 90 const struct intel_uncore_forcewake_domain *d, 91 const u32 ack) 92 { 93 return __wait_for_ack(i915, d, ack, 0); 94 } 95 96 static inline int 97 wait_ack_set(const struct drm_i915_private *i915, 98 const struct intel_uncore_forcewake_domain *d, 99 const u32 ack) 100 { 101 return __wait_for_ack(i915, d, ack, ack); 102 } 103 104 static inline void 105 fw_domain_wait_ack_clear(const struct drm_i915_private *i915, 106 const struct intel_uncore_forcewake_domain *d) 107 { 108 if (wait_ack_clear(i915, d, FORCEWAKE_KERNEL)) 109 DRM_ERROR("%s: timed out waiting for forcewake ack to clear.\n", 110 intel_uncore_forcewake_domain_to_str(d->id)); 111 } 112 113 enum ack_type { 114 ACK_CLEAR = 0, 115 ACK_SET 116 }; 117 118 static int 119 fw_domain_wait_ack_with_fallback(const struct drm_i915_private *i915, 120 const struct intel_uncore_forcewake_domain *d, 121 const enum ack_type type) 122 { 123 const u32 ack_bit = FORCEWAKE_KERNEL; 124 const u32 value = type == ACK_SET ? ack_bit : 0; 125 unsigned int pass; 126 bool ack_detected; 127 128 /* 129 * There is a possibility of driver's wake request colliding 130 * with hardware's own wake requests and that can cause 131 * hardware to not deliver the driver's ack message. 132 * 133 * Use a fallback bit toggle to kick the gpu state machine 134 * in the hope that the original ack will be delivered along with 135 * the fallback ack. 136 * 137 * This workaround is described in HSDES #1604254524 138 */ 139 140 pass = 1; 141 do { 142 wait_ack_clear(i915, d, FORCEWAKE_KERNEL_FALLBACK); 143 144 __raw_i915_write32(i915, d->reg_set, 145 _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL_FALLBACK)); 146 /* Give gt some time to relax before the polling frenzy */ 147 udelay(10 * pass); 148 wait_ack_set(i915, d, FORCEWAKE_KERNEL_FALLBACK); 149 150 ack_detected = (__raw_i915_read32(i915, d->reg_ack) & ack_bit) == value; 151 152 __raw_i915_write32(i915, d->reg_set, 153 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL_FALLBACK)); 154 } while (!ack_detected && pass++ < 10); 155 156 DRM_DEBUG_DRIVER("%s had to use fallback to %s ack, 0x%x (passes %u)\n", 157 intel_uncore_forcewake_domain_to_str(d->id), 158 type == ACK_SET ? "set" : "clear", 159 __raw_i915_read32(i915, d->reg_ack), 160 pass); 161 162 return ack_detected ? 0 : -ETIMEDOUT; 163 } 164 165 static inline void 166 fw_domain_wait_ack_clear_fallback(const struct drm_i915_private *i915, 167 const struct intel_uncore_forcewake_domain *d) 168 { 169 if (likely(!wait_ack_clear(i915, d, FORCEWAKE_KERNEL))) 170 return; 171 172 if (fw_domain_wait_ack_with_fallback(i915, d, ACK_CLEAR)) 173 fw_domain_wait_ack_clear(i915, d); 174 } 175 176 static inline void 177 fw_domain_get(struct drm_i915_private *i915, 178 const struct intel_uncore_forcewake_domain *d) 179 { 180 __raw_i915_write32(i915, d->reg_set, i915->uncore.fw_set); 181 } 182 183 static inline void 184 fw_domain_wait_ack_set(const struct drm_i915_private *i915, 185 const struct intel_uncore_forcewake_domain *d) 186 { 187 if (wait_ack_set(i915, d, FORCEWAKE_KERNEL)) 188 DRM_ERROR("%s: timed out waiting for forcewake ack request.\n", 189 intel_uncore_forcewake_domain_to_str(d->id)); 190 } 191 192 static inline void 193 fw_domain_wait_ack_set_fallback(const struct drm_i915_private *i915, 194 const struct intel_uncore_forcewake_domain *d) 195 { 196 if (likely(!wait_ack_set(i915, d, FORCEWAKE_KERNEL))) 197 return; 198 199 if (fw_domain_wait_ack_with_fallback(i915, d, ACK_SET)) 200 fw_domain_wait_ack_set(i915, d); 201 } 202 203 static inline void 204 fw_domain_put(const struct drm_i915_private *i915, 205 const struct intel_uncore_forcewake_domain *d) 206 { 207 __raw_i915_write32(i915, d->reg_set, i915->uncore.fw_clear); 208 } 209 210 static void 211 fw_domains_get(struct drm_i915_private *i915, enum forcewake_domains fw_domains) 212 { 213 struct intel_uncore_forcewake_domain *d; 214 unsigned int tmp; 215 216 GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains); 217 218 for_each_fw_domain_masked(d, fw_domains, i915, tmp) { 219 fw_domain_wait_ack_clear(i915, d); 220 fw_domain_get(i915, d); 221 } 222 223 for_each_fw_domain_masked(d, fw_domains, i915, tmp) 224 fw_domain_wait_ack_set(i915, d); 225 226 i915->uncore.fw_domains_active |= fw_domains; 227 } 228 229 static void 230 fw_domains_get_with_fallback(struct drm_i915_private *i915, 231 enum forcewake_domains fw_domains) 232 { 233 struct intel_uncore_forcewake_domain *d; 234 unsigned int tmp; 235 236 GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains); 237 238 for_each_fw_domain_masked(d, fw_domains, i915, tmp) { 239 fw_domain_wait_ack_clear_fallback(i915, d); 240 fw_domain_get(i915, d); 241 } 242 243 for_each_fw_domain_masked(d, fw_domains, i915, tmp) 244 fw_domain_wait_ack_set_fallback(i915, d); 245 246 i915->uncore.fw_domains_active |= fw_domains; 247 } 248 249 static void 250 fw_domains_put(struct drm_i915_private *i915, enum forcewake_domains fw_domains) 251 { 252 struct intel_uncore_forcewake_domain *d; 253 unsigned int tmp; 254 255 GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains); 256 257 for_each_fw_domain_masked(d, fw_domains, i915, tmp) 258 fw_domain_put(i915, d); 259 260 i915->uncore.fw_domains_active &= ~fw_domains; 261 } 262 263 static void 264 fw_domains_reset(struct drm_i915_private *i915, 265 enum forcewake_domains fw_domains) 266 { 267 struct intel_uncore_forcewake_domain *d; 268 unsigned int tmp; 269 270 if (!fw_domains) 271 return; 272 273 GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains); 274 275 for_each_fw_domain_masked(d, fw_domains, i915, tmp) 276 fw_domain_reset(i915, d); 277 } 278 279 static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv) 280 { 281 /* w/a for a sporadic read returning 0 by waiting for the GT 282 * thread to wake up. 283 */ 284 if (wait_for_atomic_us((__raw_i915_read32(dev_priv, GEN6_GT_THREAD_STATUS_REG) & 285 GEN6_GT_THREAD_STATUS_CORE_MASK) == 0, 500)) 286 DRM_ERROR("GT thread status wait timed out\n"); 287 } 288 289 static void fw_domains_get_with_thread_status(struct drm_i915_private *dev_priv, 290 enum forcewake_domains fw_domains) 291 { 292 fw_domains_get(dev_priv, fw_domains); 293 294 /* WaRsForcewakeWaitTC0:snb,ivb,hsw,bdw,vlv */ 295 __gen6_gt_wait_for_thread_c0(dev_priv); 296 } 297 298 static inline u32 fifo_free_entries(struct drm_i915_private *dev_priv) 299 { 300 u32 count = __raw_i915_read32(dev_priv, GTFIFOCTL); 301 302 return count & GT_FIFO_FREE_ENTRIES_MASK; 303 } 304 305 static void __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv) 306 { 307 u32 n; 308 309 /* On VLV, FIFO will be shared by both SW and HW. 310 * So, we need to read the FREE_ENTRIES everytime */ 311 if (IS_VALLEYVIEW(dev_priv)) 312 n = fifo_free_entries(dev_priv); 313 else 314 n = dev_priv->uncore.fifo_count; 315 316 if (n <= GT_FIFO_NUM_RESERVED_ENTRIES) { 317 if (wait_for_atomic((n = fifo_free_entries(dev_priv)) > 318 GT_FIFO_NUM_RESERVED_ENTRIES, 319 GT_FIFO_TIMEOUT_MS)) { 320 DRM_DEBUG("GT_FIFO timeout, entries: %u\n", n); 321 return; 322 } 323 } 324 325 dev_priv->uncore.fifo_count = n - 1; 326 } 327 328 static enum hrtimer_restart 329 intel_uncore_fw_release_timer(struct hrtimer *timer) 330 { 331 struct intel_uncore_forcewake_domain *domain = 332 container_of(timer, struct intel_uncore_forcewake_domain, timer); 333 struct drm_i915_private *dev_priv = 334 container_of(domain, struct drm_i915_private, uncore.fw_domain[domain->id]); 335 unsigned long irqflags; 336 337 assert_rpm_device_not_suspended(dev_priv); 338 339 if (xchg(&domain->active, false)) 340 return HRTIMER_RESTART; 341 342 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 343 if (WARN_ON(domain->wake_count == 0)) 344 domain->wake_count++; 345 346 if (--domain->wake_count == 0) 347 dev_priv->uncore.funcs.force_wake_put(dev_priv, domain->mask); 348 349 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 350 351 return HRTIMER_NORESTART; 352 } 353 354 /* Note callers must have acquired the PUNIT->PMIC bus, before calling this. */ 355 static void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv, 356 bool restore) 357 { 358 unsigned long irqflags; 359 struct intel_uncore_forcewake_domain *domain; 360 int retry_count = 100; 361 enum forcewake_domains fw, active_domains; 362 363 iosf_mbi_assert_punit_acquired(); 364 365 /* Hold uncore.lock across reset to prevent any register access 366 * with forcewake not set correctly. Wait until all pending 367 * timers are run before holding. 368 */ 369 while (1) { 370 unsigned int tmp; 371 372 active_domains = 0; 373 374 for_each_fw_domain(domain, dev_priv, tmp) { 375 smp_store_mb(domain->active, false); 376 if (hrtimer_cancel(&domain->timer) == 0) 377 continue; 378 379 intel_uncore_fw_release_timer(&domain->timer); 380 } 381 382 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 383 384 for_each_fw_domain(domain, dev_priv, tmp) { 385 if (hrtimer_active(&domain->timer)) 386 active_domains |= domain->mask; 387 } 388 389 if (active_domains == 0) 390 break; 391 392 if (--retry_count == 0) { 393 DRM_ERROR("Timed out waiting for forcewake timers to finish\n"); 394 break; 395 } 396 397 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 398 cond_resched(); 399 } 400 401 WARN_ON(active_domains); 402 403 fw = dev_priv->uncore.fw_domains_active; 404 if (fw) 405 dev_priv->uncore.funcs.force_wake_put(dev_priv, fw); 406 407 fw_domains_reset(dev_priv, dev_priv->uncore.fw_domains); 408 409 if (restore) { /* If reset with a user forcewake, try to restore */ 410 if (fw) 411 dev_priv->uncore.funcs.force_wake_get(dev_priv, fw); 412 413 if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) 414 dev_priv->uncore.fifo_count = 415 fifo_free_entries(dev_priv); 416 } 417 418 if (!restore) 419 assert_forcewakes_inactive(dev_priv); 420 421 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 422 } 423 424 static u64 gen9_edram_size(struct drm_i915_private *dev_priv) 425 { 426 const unsigned int ways[8] = { 4, 8, 12, 16, 16, 16, 16, 16 }; 427 const unsigned int sets[4] = { 1, 1, 2, 2 }; 428 const u32 cap = dev_priv->edram_cap; 429 430 return EDRAM_NUM_BANKS(cap) * 431 ways[EDRAM_WAYS_IDX(cap)] * 432 sets[EDRAM_SETS_IDX(cap)] * 433 1024 * 1024; 434 } 435 436 u64 intel_uncore_edram_size(struct drm_i915_private *dev_priv) 437 { 438 if (!HAS_EDRAM(dev_priv)) 439 return 0; 440 441 /* The needed capability bits for size calculation 442 * are not there with pre gen9 so return 128MB always. 443 */ 444 if (INTEL_GEN(dev_priv) < 9) 445 return 128 * 1024 * 1024; 446 447 return gen9_edram_size(dev_priv); 448 } 449 450 static void intel_uncore_edram_detect(struct drm_i915_private *dev_priv) 451 { 452 if (IS_HASWELL(dev_priv) || 453 IS_BROADWELL(dev_priv) || 454 INTEL_GEN(dev_priv) >= 9) { 455 dev_priv->edram_cap = __raw_i915_read32(dev_priv, 456 HSW_EDRAM_CAP); 457 458 /* NB: We can't write IDICR yet because we do not have gt funcs 459 * set up */ 460 } else { 461 dev_priv->edram_cap = 0; 462 } 463 464 if (HAS_EDRAM(dev_priv)) 465 DRM_INFO("Found %lluMB of eDRAM\n", 466 intel_uncore_edram_size(dev_priv) / (1024 * 1024)); 467 } 468 469 static bool 470 fpga_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv) 471 { 472 u32 dbg; 473 474 dbg = __raw_i915_read32(dev_priv, FPGA_DBG); 475 if (likely(!(dbg & FPGA_DBG_RM_NOCLAIM))) 476 return false; 477 478 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); 479 480 return true; 481 } 482 483 static bool 484 vlv_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv) 485 { 486 u32 cer; 487 488 cer = __raw_i915_read32(dev_priv, CLAIM_ER); 489 if (likely(!(cer & (CLAIM_ER_OVERFLOW | CLAIM_ER_CTR_MASK)))) 490 return false; 491 492 __raw_i915_write32(dev_priv, CLAIM_ER, CLAIM_ER_CLR); 493 494 return true; 495 } 496 497 static bool 498 gen6_check_for_fifo_debug(struct drm_i915_private *dev_priv) 499 { 500 u32 fifodbg; 501 502 fifodbg = __raw_i915_read32(dev_priv, GTFIFODBG); 503 504 if (unlikely(fifodbg)) { 505 DRM_DEBUG_DRIVER("GTFIFODBG = 0x08%x\n", fifodbg); 506 __raw_i915_write32(dev_priv, GTFIFODBG, fifodbg); 507 } 508 509 return fifodbg; 510 } 511 512 static bool 513 check_for_unclaimed_mmio(struct drm_i915_private *dev_priv) 514 { 515 bool ret = false; 516 517 if (HAS_FPGA_DBG_UNCLAIMED(dev_priv)) 518 ret |= fpga_check_for_unclaimed_mmio(dev_priv); 519 520 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) 521 ret |= vlv_check_for_unclaimed_mmio(dev_priv); 522 523 if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) 524 ret |= gen6_check_for_fifo_debug(dev_priv); 525 526 return ret; 527 } 528 529 static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv, 530 bool restore_forcewake) 531 { 532 /* clear out unclaimed reg detection bit */ 533 if (check_for_unclaimed_mmio(dev_priv)) 534 DRM_DEBUG("unclaimed mmio detected on uncore init, clearing\n"); 535 536 /* WaDisableShadowRegForCpd:chv */ 537 if (IS_CHERRYVIEW(dev_priv)) { 538 __raw_i915_write32(dev_priv, GTFIFOCTL, 539 __raw_i915_read32(dev_priv, GTFIFOCTL) | 540 GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL | 541 GT_FIFO_CTL_RC6_POLICY_STALL); 542 } 543 544 iosf_mbi_punit_acquire(); 545 intel_uncore_forcewake_reset(dev_priv, restore_forcewake); 546 iosf_mbi_punit_release(); 547 } 548 549 void intel_uncore_suspend(struct drm_i915_private *dev_priv) 550 { 551 iosf_mbi_punit_acquire(); 552 iosf_mbi_unregister_pmic_bus_access_notifier_unlocked( 553 &dev_priv->uncore.pmic_bus_access_nb); 554 intel_uncore_forcewake_reset(dev_priv, false); 555 iosf_mbi_punit_release(); 556 } 557 558 void intel_uncore_resume_early(struct drm_i915_private *dev_priv) 559 { 560 __intel_uncore_early_sanitize(dev_priv, true); 561 iosf_mbi_register_pmic_bus_access_notifier( 562 &dev_priv->uncore.pmic_bus_access_nb); 563 i915_check_and_clear_faults(dev_priv); 564 } 565 566 void intel_uncore_runtime_resume(struct drm_i915_private *dev_priv) 567 { 568 iosf_mbi_register_pmic_bus_access_notifier( 569 &dev_priv->uncore.pmic_bus_access_nb); 570 } 571 572 void intel_uncore_sanitize(struct drm_i915_private *dev_priv) 573 { 574 /* BIOS often leaves RC6 enabled, but disable it for hw init */ 575 intel_sanitize_gt_powersave(dev_priv); 576 } 577 578 static void __intel_uncore_forcewake_get(struct drm_i915_private *dev_priv, 579 enum forcewake_domains fw_domains) 580 { 581 struct intel_uncore_forcewake_domain *domain; 582 unsigned int tmp; 583 584 fw_domains &= dev_priv->uncore.fw_domains; 585 586 for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp) { 587 if (domain->wake_count++) { 588 fw_domains &= ~domain->mask; 589 domain->active = true; 590 } 591 } 592 593 if (fw_domains) 594 dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains); 595 } 596 597 /** 598 * intel_uncore_forcewake_get - grab forcewake domain references 599 * @dev_priv: i915 device instance 600 * @fw_domains: forcewake domains to get reference on 601 * 602 * This function can be used get GT's forcewake domain references. 603 * Normal register access will handle the forcewake domains automatically. 604 * However if some sequence requires the GT to not power down a particular 605 * forcewake domains this function should be called at the beginning of the 606 * sequence. And subsequently the reference should be dropped by symmetric 607 * call to intel_unforce_forcewake_put(). Usually caller wants all the domains 608 * to be kept awake so the @fw_domains would be then FORCEWAKE_ALL. 609 */ 610 void intel_uncore_forcewake_get(struct drm_i915_private *dev_priv, 611 enum forcewake_domains fw_domains) 612 { 613 unsigned long irqflags; 614 615 if (!dev_priv->uncore.funcs.force_wake_get) 616 return; 617 618 assert_rpm_wakelock_held(dev_priv); 619 620 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 621 __intel_uncore_forcewake_get(dev_priv, fw_domains); 622 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 623 } 624 625 /** 626 * intel_uncore_forcewake_user_get - claim forcewake on behalf of userspace 627 * @dev_priv: i915 device instance 628 * 629 * This function is a wrapper around intel_uncore_forcewake_get() to acquire 630 * the GT powerwell and in the process disable our debugging for the 631 * duration of userspace's bypass. 632 */ 633 void intel_uncore_forcewake_user_get(struct drm_i915_private *dev_priv) 634 { 635 spin_lock_irq(&dev_priv->uncore.lock); 636 if (!dev_priv->uncore.user_forcewake.count++) { 637 intel_uncore_forcewake_get__locked(dev_priv, FORCEWAKE_ALL); 638 639 /* Save and disable mmio debugging for the user bypass */ 640 dev_priv->uncore.user_forcewake.saved_mmio_check = 641 dev_priv->uncore.unclaimed_mmio_check; 642 dev_priv->uncore.user_forcewake.saved_mmio_debug = 643 i915_modparams.mmio_debug; 644 645 dev_priv->uncore.unclaimed_mmio_check = 0; 646 i915_modparams.mmio_debug = 0; 647 } 648 spin_unlock_irq(&dev_priv->uncore.lock); 649 } 650 651 /** 652 * intel_uncore_forcewake_user_put - release forcewake on behalf of userspace 653 * @dev_priv: i915 device instance 654 * 655 * This function complements intel_uncore_forcewake_user_get() and releases 656 * the GT powerwell taken on behalf of the userspace bypass. 657 */ 658 void intel_uncore_forcewake_user_put(struct drm_i915_private *dev_priv) 659 { 660 spin_lock_irq(&dev_priv->uncore.lock); 661 if (!--dev_priv->uncore.user_forcewake.count) { 662 if (intel_uncore_unclaimed_mmio(dev_priv)) 663 dev_info(dev_priv->drm.dev, 664 "Invalid mmio detected during user access\n"); 665 666 dev_priv->uncore.unclaimed_mmio_check = 667 dev_priv->uncore.user_forcewake.saved_mmio_check; 668 i915_modparams.mmio_debug = 669 dev_priv->uncore.user_forcewake.saved_mmio_debug; 670 671 intel_uncore_forcewake_put__locked(dev_priv, FORCEWAKE_ALL); 672 } 673 spin_unlock_irq(&dev_priv->uncore.lock); 674 } 675 676 /** 677 * intel_uncore_forcewake_get__locked - grab forcewake domain references 678 * @dev_priv: i915 device instance 679 * @fw_domains: forcewake domains to get reference on 680 * 681 * See intel_uncore_forcewake_get(). This variant places the onus 682 * on the caller to explicitly handle the dev_priv->uncore.lock spinlock. 683 */ 684 void intel_uncore_forcewake_get__locked(struct drm_i915_private *dev_priv, 685 enum forcewake_domains fw_domains) 686 { 687 lockdep_assert_held(&dev_priv->uncore.lock); 688 689 if (!dev_priv->uncore.funcs.force_wake_get) 690 return; 691 692 __intel_uncore_forcewake_get(dev_priv, fw_domains); 693 } 694 695 static void __intel_uncore_forcewake_put(struct drm_i915_private *dev_priv, 696 enum forcewake_domains fw_domains) 697 { 698 struct intel_uncore_forcewake_domain *domain; 699 unsigned int tmp; 700 701 fw_domains &= dev_priv->uncore.fw_domains; 702 703 for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp) { 704 if (WARN_ON(domain->wake_count == 0)) 705 continue; 706 707 if (--domain->wake_count) { 708 domain->active = true; 709 continue; 710 } 711 712 fw_domain_arm_timer(domain); 713 } 714 } 715 716 /** 717 * intel_uncore_forcewake_put - release a forcewake domain reference 718 * @dev_priv: i915 device instance 719 * @fw_domains: forcewake domains to put references 720 * 721 * This function drops the device-level forcewakes for specified 722 * domains obtained by intel_uncore_forcewake_get(). 723 */ 724 void intel_uncore_forcewake_put(struct drm_i915_private *dev_priv, 725 enum forcewake_domains fw_domains) 726 { 727 unsigned long irqflags; 728 729 if (!dev_priv->uncore.funcs.force_wake_put) 730 return; 731 732 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 733 __intel_uncore_forcewake_put(dev_priv, fw_domains); 734 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 735 } 736 737 /** 738 * intel_uncore_forcewake_put__locked - grab forcewake domain references 739 * @dev_priv: i915 device instance 740 * @fw_domains: forcewake domains to get reference on 741 * 742 * See intel_uncore_forcewake_put(). This variant places the onus 743 * on the caller to explicitly handle the dev_priv->uncore.lock spinlock. 744 */ 745 void intel_uncore_forcewake_put__locked(struct drm_i915_private *dev_priv, 746 enum forcewake_domains fw_domains) 747 { 748 lockdep_assert_held(&dev_priv->uncore.lock); 749 750 if (!dev_priv->uncore.funcs.force_wake_put) 751 return; 752 753 __intel_uncore_forcewake_put(dev_priv, fw_domains); 754 } 755 756 void assert_forcewakes_inactive(struct drm_i915_private *dev_priv) 757 { 758 if (!dev_priv->uncore.funcs.force_wake_get) 759 return; 760 761 WARN(dev_priv->uncore.fw_domains_active, 762 "Expected all fw_domains to be inactive, but %08x are still on\n", 763 dev_priv->uncore.fw_domains_active); 764 } 765 766 void assert_forcewakes_active(struct drm_i915_private *dev_priv, 767 enum forcewake_domains fw_domains) 768 { 769 if (!dev_priv->uncore.funcs.force_wake_get) 770 return; 771 772 assert_rpm_wakelock_held(dev_priv); 773 774 fw_domains &= dev_priv->uncore.fw_domains; 775 WARN(fw_domains & ~dev_priv->uncore.fw_domains_active, 776 "Expected %08x fw_domains to be active, but %08x are off\n", 777 fw_domains, fw_domains & ~dev_priv->uncore.fw_domains_active); 778 } 779 780 /* We give fast paths for the really cool registers */ 781 #define NEEDS_FORCE_WAKE(reg) ((reg) < 0x40000) 782 783 #define GEN11_NEEDS_FORCE_WAKE(reg) \ 784 ((reg) < 0x40000 || ((reg) >= 0x1c0000 && (reg) < 0x1dc000)) 785 786 #define __gen6_reg_read_fw_domains(offset) \ 787 ({ \ 788 enum forcewake_domains __fwd; \ 789 if (NEEDS_FORCE_WAKE(offset)) \ 790 __fwd = FORCEWAKE_RENDER; \ 791 else \ 792 __fwd = 0; \ 793 __fwd; \ 794 }) 795 796 static int fw_range_cmp(u32 offset, const struct intel_forcewake_range *entry) 797 { 798 if (offset < entry->start) 799 return -1; 800 else if (offset > entry->end) 801 return 1; 802 else 803 return 0; 804 } 805 806 /* Copied and "macroized" from lib/bsearch.c */ 807 #define BSEARCH(key, base, num, cmp) ({ \ 808 unsigned int start__ = 0, end__ = (num); \ 809 typeof(base) result__ = NULL; \ 810 while (start__ < end__) { \ 811 unsigned int mid__ = start__ + (end__ - start__) / 2; \ 812 int ret__ = (cmp)((key), (base) + mid__); \ 813 if (ret__ < 0) { \ 814 end__ = mid__; \ 815 } else if (ret__ > 0) { \ 816 start__ = mid__ + 1; \ 817 } else { \ 818 result__ = (base) + mid__; \ 819 break; \ 820 } \ 821 } \ 822 result__; \ 823 }) 824 825 static enum forcewake_domains 826 find_fw_domain(struct drm_i915_private *dev_priv, u32 offset) 827 { 828 const struct intel_forcewake_range *entry; 829 830 entry = BSEARCH(offset, 831 dev_priv->uncore.fw_domains_table, 832 dev_priv->uncore.fw_domains_table_entries, 833 fw_range_cmp); 834 835 if (!entry) 836 return 0; 837 838 /* 839 * The list of FW domains depends on the SKU in gen11+ so we 840 * can't determine it statically. We use FORCEWAKE_ALL and 841 * translate it here to the list of available domains. 842 */ 843 if (entry->domains == FORCEWAKE_ALL) 844 return dev_priv->uncore.fw_domains; 845 846 WARN(entry->domains & ~dev_priv->uncore.fw_domains, 847 "Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n", 848 entry->domains & ~dev_priv->uncore.fw_domains, offset); 849 850 return entry->domains; 851 } 852 853 #define GEN_FW_RANGE(s, e, d) \ 854 { .start = (s), .end = (e), .domains = (d) } 855 856 #define HAS_FWTABLE(dev_priv) \ 857 (INTEL_GEN(dev_priv) >= 9 || \ 858 IS_CHERRYVIEW(dev_priv) || \ 859 IS_VALLEYVIEW(dev_priv)) 860 861 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ 862 static const struct intel_forcewake_range __vlv_fw_ranges[] = { 863 GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER), 864 GEN_FW_RANGE(0x5000, 0x7fff, FORCEWAKE_RENDER), 865 GEN_FW_RANGE(0xb000, 0x11fff, FORCEWAKE_RENDER), 866 GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA), 867 GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_MEDIA), 868 GEN_FW_RANGE(0x2e000, 0x2ffff, FORCEWAKE_RENDER), 869 GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA), 870 }; 871 872 #define __fwtable_reg_read_fw_domains(offset) \ 873 ({ \ 874 enum forcewake_domains __fwd = 0; \ 875 if (NEEDS_FORCE_WAKE((offset))) \ 876 __fwd = find_fw_domain(dev_priv, offset); \ 877 __fwd; \ 878 }) 879 880 #define __gen11_fwtable_reg_read_fw_domains(offset) \ 881 ({ \ 882 enum forcewake_domains __fwd = 0; \ 883 if (GEN11_NEEDS_FORCE_WAKE((offset))) \ 884 __fwd = find_fw_domain(dev_priv, offset); \ 885 __fwd; \ 886 }) 887 888 /* *Must* be sorted by offset! See intel_shadow_table_check(). */ 889 static const i915_reg_t gen8_shadowed_regs[] = { 890 RING_TAIL(RENDER_RING_BASE), /* 0x2000 (base) */ 891 GEN6_RPNSWREQ, /* 0xA008 */ 892 GEN6_RC_VIDEO_FREQ, /* 0xA00C */ 893 RING_TAIL(GEN6_BSD_RING_BASE), /* 0x12000 (base) */ 894 RING_TAIL(VEBOX_RING_BASE), /* 0x1a000 (base) */ 895 RING_TAIL(BLT_RING_BASE), /* 0x22000 (base) */ 896 /* TODO: Other registers are not yet used */ 897 }; 898 899 static const i915_reg_t gen11_shadowed_regs[] = { 900 RING_TAIL(RENDER_RING_BASE), /* 0x2000 (base) */ 901 GEN6_RPNSWREQ, /* 0xA008 */ 902 GEN6_RC_VIDEO_FREQ, /* 0xA00C */ 903 RING_TAIL(BLT_RING_BASE), /* 0x22000 (base) */ 904 RING_TAIL(GEN11_BSD_RING_BASE), /* 0x1C0000 (base) */ 905 RING_TAIL(GEN11_BSD2_RING_BASE), /* 0x1C4000 (base) */ 906 RING_TAIL(GEN11_VEBOX_RING_BASE), /* 0x1C8000 (base) */ 907 RING_TAIL(GEN11_BSD3_RING_BASE), /* 0x1D0000 (base) */ 908 RING_TAIL(GEN11_BSD4_RING_BASE), /* 0x1D4000 (base) */ 909 RING_TAIL(GEN11_VEBOX2_RING_BASE), /* 0x1D8000 (base) */ 910 /* TODO: Other registers are not yet used */ 911 }; 912 913 static int mmio_reg_cmp(u32 key, const i915_reg_t *reg) 914 { 915 u32 offset = i915_mmio_reg_offset(*reg); 916 917 if (key < offset) 918 return -1; 919 else if (key > offset) 920 return 1; 921 else 922 return 0; 923 } 924 925 #define __is_genX_shadowed(x) \ 926 static bool is_gen##x##_shadowed(u32 offset) \ 927 { \ 928 const i915_reg_t *regs = gen##x##_shadowed_regs; \ 929 return BSEARCH(offset, regs, ARRAY_SIZE(gen##x##_shadowed_regs), \ 930 mmio_reg_cmp); \ 931 } 932 933 __is_genX_shadowed(8) 934 __is_genX_shadowed(11) 935 936 #define __gen8_reg_write_fw_domains(offset) \ 937 ({ \ 938 enum forcewake_domains __fwd; \ 939 if (NEEDS_FORCE_WAKE(offset) && !is_gen8_shadowed(offset)) \ 940 __fwd = FORCEWAKE_RENDER; \ 941 else \ 942 __fwd = 0; \ 943 __fwd; \ 944 }) 945 946 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ 947 static const struct intel_forcewake_range __chv_fw_ranges[] = { 948 GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER), 949 GEN_FW_RANGE(0x4000, 0x4fff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 950 GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), 951 GEN_FW_RANGE(0x8000, 0x82ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 952 GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER), 953 GEN_FW_RANGE(0x8500, 0x85ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 954 GEN_FW_RANGE(0x8800, 0x88ff, FORCEWAKE_MEDIA), 955 GEN_FW_RANGE(0x9000, 0xafff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 956 GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER), 957 GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA), 958 GEN_FW_RANGE(0xe000, 0xe7ff, FORCEWAKE_RENDER), 959 GEN_FW_RANGE(0xf000, 0xffff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 960 GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA), 961 GEN_FW_RANGE(0x1a000, 0x1bfff, FORCEWAKE_MEDIA), 962 GEN_FW_RANGE(0x1e800, 0x1e9ff, FORCEWAKE_MEDIA), 963 GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA), 964 }; 965 966 #define __fwtable_reg_write_fw_domains(offset) \ 967 ({ \ 968 enum forcewake_domains __fwd = 0; \ 969 if (NEEDS_FORCE_WAKE((offset)) && !is_gen8_shadowed(offset)) \ 970 __fwd = find_fw_domain(dev_priv, offset); \ 971 __fwd; \ 972 }) 973 974 #define __gen11_fwtable_reg_write_fw_domains(offset) \ 975 ({ \ 976 enum forcewake_domains __fwd = 0; \ 977 if (GEN11_NEEDS_FORCE_WAKE((offset)) && !is_gen11_shadowed(offset)) \ 978 __fwd = find_fw_domain(dev_priv, offset); \ 979 __fwd; \ 980 }) 981 982 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ 983 static const struct intel_forcewake_range __gen9_fw_ranges[] = { 984 GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER), 985 GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */ 986 GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER), 987 GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER), 988 GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER), 989 GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER), 990 GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), 991 GEN_FW_RANGE(0x8000, 0x812f, FORCEWAKE_BLITTER), 992 GEN_FW_RANGE(0x8130, 0x813f, FORCEWAKE_MEDIA), 993 GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER), 994 GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER), 995 GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER), 996 GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_BLITTER), 997 GEN_FW_RANGE(0x8800, 0x89ff, FORCEWAKE_MEDIA), 998 GEN_FW_RANGE(0x8a00, 0x8bff, FORCEWAKE_BLITTER), 999 GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER), 1000 GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER), 1001 GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 1002 GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER), 1003 GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER), 1004 GEN_FW_RANGE(0xb480, 0xcfff, FORCEWAKE_BLITTER), 1005 GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA), 1006 GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_BLITTER), 1007 GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER), 1008 GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_BLITTER), 1009 GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA), 1010 GEN_FW_RANGE(0x14000, 0x19fff, FORCEWAKE_BLITTER), 1011 GEN_FW_RANGE(0x1a000, 0x1e9ff, FORCEWAKE_MEDIA), 1012 GEN_FW_RANGE(0x1ea00, 0x243ff, FORCEWAKE_BLITTER), 1013 GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER), 1014 GEN_FW_RANGE(0x24800, 0x2ffff, FORCEWAKE_BLITTER), 1015 GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA), 1016 }; 1017 1018 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ 1019 static const struct intel_forcewake_range __gen11_fw_ranges[] = { 1020 GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER), 1021 GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */ 1022 GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER), 1023 GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER), 1024 GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER), 1025 GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER), 1026 GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), 1027 GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_BLITTER), 1028 GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER), 1029 GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER), 1030 GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER), 1031 GEN_FW_RANGE(0x8500, 0x8bff, FORCEWAKE_BLITTER), 1032 GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER), 1033 GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER), 1034 GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_ALL), 1035 GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER), 1036 GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER), 1037 GEN_FW_RANGE(0xb480, 0xdfff, FORCEWAKE_BLITTER), 1038 GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER), 1039 GEN_FW_RANGE(0xe900, 0x243ff, FORCEWAKE_BLITTER), 1040 GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER), 1041 GEN_FW_RANGE(0x24800, 0x3ffff, FORCEWAKE_BLITTER), 1042 GEN_FW_RANGE(0x40000, 0x1bffff, 0), 1043 GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), 1044 GEN_FW_RANGE(0x1c4000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX1), 1045 GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), 1046 GEN_FW_RANGE(0x1cc000, 0x1cffff, FORCEWAKE_BLITTER), 1047 GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2), 1048 GEN_FW_RANGE(0x1d4000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX3), 1049 GEN_FW_RANGE(0x1d8000, 0x1dbfff, FORCEWAKE_MEDIA_VEBOX1) 1050 }; 1051 1052 static void 1053 ilk_dummy_write(struct drm_i915_private *dev_priv) 1054 { 1055 /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up 1056 * the chip from rc6 before touching it for real. MI_MODE is masked, 1057 * hence harmless to write 0 into. */ 1058 __raw_i915_write32(dev_priv, MI_MODE, 0); 1059 } 1060 1061 static void 1062 __unclaimed_reg_debug(struct drm_i915_private *dev_priv, 1063 const i915_reg_t reg, 1064 const bool read, 1065 const bool before) 1066 { 1067 if (WARN(check_for_unclaimed_mmio(dev_priv) && !before, 1068 "Unclaimed %s register 0x%x\n", 1069 read ? "read from" : "write to", 1070 i915_mmio_reg_offset(reg))) 1071 /* Only report the first N failures */ 1072 i915_modparams.mmio_debug--; 1073 } 1074 1075 static inline void 1076 unclaimed_reg_debug(struct drm_i915_private *dev_priv, 1077 const i915_reg_t reg, 1078 const bool read, 1079 const bool before) 1080 { 1081 if (likely(!i915_modparams.mmio_debug)) 1082 return; 1083 1084 __unclaimed_reg_debug(dev_priv, reg, read, before); 1085 } 1086 1087 #define GEN2_READ_HEADER(x) \ 1088 u##x val = 0; \ 1089 assert_rpm_wakelock_held(dev_priv); 1090 1091 #define GEN2_READ_FOOTER \ 1092 trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \ 1093 return val 1094 1095 #define __gen2_read(x) \ 1096 static u##x \ 1097 gen2_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \ 1098 GEN2_READ_HEADER(x); \ 1099 val = __raw_i915_read##x(dev_priv, reg); \ 1100 GEN2_READ_FOOTER; \ 1101 } 1102 1103 #define __gen5_read(x) \ 1104 static u##x \ 1105 gen5_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \ 1106 GEN2_READ_HEADER(x); \ 1107 ilk_dummy_write(dev_priv); \ 1108 val = __raw_i915_read##x(dev_priv, reg); \ 1109 GEN2_READ_FOOTER; \ 1110 } 1111 1112 __gen5_read(8) 1113 __gen5_read(16) 1114 __gen5_read(32) 1115 __gen5_read(64) 1116 __gen2_read(8) 1117 __gen2_read(16) 1118 __gen2_read(32) 1119 __gen2_read(64) 1120 1121 #undef __gen5_read 1122 #undef __gen2_read 1123 1124 #undef GEN2_READ_FOOTER 1125 #undef GEN2_READ_HEADER 1126 1127 #define GEN6_READ_HEADER(x) \ 1128 u32 offset = i915_mmio_reg_offset(reg); \ 1129 unsigned long irqflags; \ 1130 u##x val = 0; \ 1131 assert_rpm_wakelock_held(dev_priv); \ 1132 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \ 1133 unclaimed_reg_debug(dev_priv, reg, true, true) 1134 1135 #define GEN6_READ_FOOTER \ 1136 unclaimed_reg_debug(dev_priv, reg, true, false); \ 1137 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \ 1138 trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \ 1139 return val 1140 1141 static noinline void ___force_wake_auto(struct drm_i915_private *dev_priv, 1142 enum forcewake_domains fw_domains) 1143 { 1144 struct intel_uncore_forcewake_domain *domain; 1145 unsigned int tmp; 1146 1147 GEM_BUG_ON(fw_domains & ~dev_priv->uncore.fw_domains); 1148 1149 for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp) 1150 fw_domain_arm_timer(domain); 1151 1152 dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains); 1153 } 1154 1155 static inline void __force_wake_auto(struct drm_i915_private *dev_priv, 1156 enum forcewake_domains fw_domains) 1157 { 1158 if (WARN_ON(!fw_domains)) 1159 return; 1160 1161 /* Turn on all requested but inactive supported forcewake domains. */ 1162 fw_domains &= dev_priv->uncore.fw_domains; 1163 fw_domains &= ~dev_priv->uncore.fw_domains_active; 1164 1165 if (fw_domains) 1166 ___force_wake_auto(dev_priv, fw_domains); 1167 } 1168 1169 #define __gen_read(func, x) \ 1170 static u##x \ 1171 func##_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \ 1172 enum forcewake_domains fw_engine; \ 1173 GEN6_READ_HEADER(x); \ 1174 fw_engine = __##func##_reg_read_fw_domains(offset); \ 1175 if (fw_engine) \ 1176 __force_wake_auto(dev_priv, fw_engine); \ 1177 val = __raw_i915_read##x(dev_priv, reg); \ 1178 GEN6_READ_FOOTER; \ 1179 } 1180 #define __gen6_read(x) __gen_read(gen6, x) 1181 #define __fwtable_read(x) __gen_read(fwtable, x) 1182 #define __gen11_fwtable_read(x) __gen_read(gen11_fwtable, x) 1183 1184 __gen11_fwtable_read(8) 1185 __gen11_fwtable_read(16) 1186 __gen11_fwtable_read(32) 1187 __gen11_fwtable_read(64) 1188 __fwtable_read(8) 1189 __fwtable_read(16) 1190 __fwtable_read(32) 1191 __fwtable_read(64) 1192 __gen6_read(8) 1193 __gen6_read(16) 1194 __gen6_read(32) 1195 __gen6_read(64) 1196 1197 #undef __gen11_fwtable_read 1198 #undef __fwtable_read 1199 #undef __gen6_read 1200 #undef GEN6_READ_FOOTER 1201 #undef GEN6_READ_HEADER 1202 1203 #define GEN2_WRITE_HEADER \ 1204 trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \ 1205 assert_rpm_wakelock_held(dev_priv); \ 1206 1207 #define GEN2_WRITE_FOOTER 1208 1209 #define __gen2_write(x) \ 1210 static void \ 1211 gen2_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \ 1212 GEN2_WRITE_HEADER; \ 1213 __raw_i915_write##x(dev_priv, reg, val); \ 1214 GEN2_WRITE_FOOTER; \ 1215 } 1216 1217 #define __gen5_write(x) \ 1218 static void \ 1219 gen5_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \ 1220 GEN2_WRITE_HEADER; \ 1221 ilk_dummy_write(dev_priv); \ 1222 __raw_i915_write##x(dev_priv, reg, val); \ 1223 GEN2_WRITE_FOOTER; \ 1224 } 1225 1226 __gen5_write(8) 1227 __gen5_write(16) 1228 __gen5_write(32) 1229 __gen2_write(8) 1230 __gen2_write(16) 1231 __gen2_write(32) 1232 1233 #undef __gen5_write 1234 #undef __gen2_write 1235 1236 #undef GEN2_WRITE_FOOTER 1237 #undef GEN2_WRITE_HEADER 1238 1239 #define GEN6_WRITE_HEADER \ 1240 u32 offset = i915_mmio_reg_offset(reg); \ 1241 unsigned long irqflags; \ 1242 trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \ 1243 assert_rpm_wakelock_held(dev_priv); \ 1244 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \ 1245 unclaimed_reg_debug(dev_priv, reg, false, true) 1246 1247 #define GEN6_WRITE_FOOTER \ 1248 unclaimed_reg_debug(dev_priv, reg, false, false); \ 1249 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags) 1250 1251 #define __gen6_write(x) \ 1252 static void \ 1253 gen6_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \ 1254 GEN6_WRITE_HEADER; \ 1255 if (NEEDS_FORCE_WAKE(offset)) \ 1256 __gen6_gt_wait_for_fifo(dev_priv); \ 1257 __raw_i915_write##x(dev_priv, reg, val); \ 1258 GEN6_WRITE_FOOTER; \ 1259 } 1260 1261 #define __gen_write(func, x) \ 1262 static void \ 1263 func##_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \ 1264 enum forcewake_domains fw_engine; \ 1265 GEN6_WRITE_HEADER; \ 1266 fw_engine = __##func##_reg_write_fw_domains(offset); \ 1267 if (fw_engine) \ 1268 __force_wake_auto(dev_priv, fw_engine); \ 1269 __raw_i915_write##x(dev_priv, reg, val); \ 1270 GEN6_WRITE_FOOTER; \ 1271 } 1272 #define __gen8_write(x) __gen_write(gen8, x) 1273 #define __fwtable_write(x) __gen_write(fwtable, x) 1274 #define __gen11_fwtable_write(x) __gen_write(gen11_fwtable, x) 1275 1276 __gen11_fwtable_write(8) 1277 __gen11_fwtable_write(16) 1278 __gen11_fwtable_write(32) 1279 __fwtable_write(8) 1280 __fwtable_write(16) 1281 __fwtable_write(32) 1282 __gen8_write(8) 1283 __gen8_write(16) 1284 __gen8_write(32) 1285 __gen6_write(8) 1286 __gen6_write(16) 1287 __gen6_write(32) 1288 1289 #undef __gen11_fwtable_write 1290 #undef __fwtable_write 1291 #undef __gen8_write 1292 #undef __gen6_write 1293 #undef GEN6_WRITE_FOOTER 1294 #undef GEN6_WRITE_HEADER 1295 1296 #define ASSIGN_WRITE_MMIO_VFUNCS(i915, x) \ 1297 do { \ 1298 (i915)->uncore.funcs.mmio_writeb = x##_write8; \ 1299 (i915)->uncore.funcs.mmio_writew = x##_write16; \ 1300 (i915)->uncore.funcs.mmio_writel = x##_write32; \ 1301 } while (0) 1302 1303 #define ASSIGN_READ_MMIO_VFUNCS(i915, x) \ 1304 do { \ 1305 (i915)->uncore.funcs.mmio_readb = x##_read8; \ 1306 (i915)->uncore.funcs.mmio_readw = x##_read16; \ 1307 (i915)->uncore.funcs.mmio_readl = x##_read32; \ 1308 (i915)->uncore.funcs.mmio_readq = x##_read64; \ 1309 } while (0) 1310 1311 1312 static void fw_domain_init(struct drm_i915_private *dev_priv, 1313 enum forcewake_domain_id domain_id, 1314 i915_reg_t reg_set, 1315 i915_reg_t reg_ack) 1316 { 1317 struct intel_uncore_forcewake_domain *d; 1318 1319 if (WARN_ON(domain_id >= FW_DOMAIN_ID_COUNT)) 1320 return; 1321 1322 d = &dev_priv->uncore.fw_domain[domain_id]; 1323 1324 WARN_ON(d->wake_count); 1325 1326 WARN_ON(!i915_mmio_reg_valid(reg_set)); 1327 WARN_ON(!i915_mmio_reg_valid(reg_ack)); 1328 1329 d->wake_count = 0; 1330 d->reg_set = reg_set; 1331 d->reg_ack = reg_ack; 1332 1333 d->id = domain_id; 1334 1335 BUILD_BUG_ON(FORCEWAKE_RENDER != (1 << FW_DOMAIN_ID_RENDER)); 1336 BUILD_BUG_ON(FORCEWAKE_BLITTER != (1 << FW_DOMAIN_ID_BLITTER)); 1337 BUILD_BUG_ON(FORCEWAKE_MEDIA != (1 << FW_DOMAIN_ID_MEDIA)); 1338 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX0)); 1339 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX1)); 1340 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX2 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX2)); 1341 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX3 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX3)); 1342 BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX0)); 1343 BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX1)); 1344 1345 1346 d->mask = BIT(domain_id); 1347 1348 hrtimer_init(&d->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1349 d->timer.function = intel_uncore_fw_release_timer; 1350 1351 dev_priv->uncore.fw_domains |= BIT(domain_id); 1352 1353 fw_domain_reset(dev_priv, d); 1354 } 1355 1356 static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv) 1357 { 1358 if (INTEL_GEN(dev_priv) <= 5 || intel_vgpu_active(dev_priv)) 1359 return; 1360 1361 if (IS_GEN6(dev_priv)) { 1362 dev_priv->uncore.fw_reset = 0; 1363 dev_priv->uncore.fw_set = FORCEWAKE_KERNEL; 1364 dev_priv->uncore.fw_clear = 0; 1365 } else { 1366 /* WaRsClearFWBitsAtReset:bdw,skl */ 1367 dev_priv->uncore.fw_reset = _MASKED_BIT_DISABLE(0xffff); 1368 dev_priv->uncore.fw_set = _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL); 1369 dev_priv->uncore.fw_clear = _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL); 1370 } 1371 1372 if (INTEL_GEN(dev_priv) >= 11) { 1373 int i; 1374 1375 dev_priv->uncore.funcs.force_wake_get = fw_domains_get; 1376 dev_priv->uncore.funcs.force_wake_put = fw_domains_put; 1377 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, 1378 FORCEWAKE_RENDER_GEN9, 1379 FORCEWAKE_ACK_RENDER_GEN9); 1380 fw_domain_init(dev_priv, FW_DOMAIN_ID_BLITTER, 1381 FORCEWAKE_BLITTER_GEN9, 1382 FORCEWAKE_ACK_BLITTER_GEN9); 1383 for (i = 0; i < I915_MAX_VCS; i++) { 1384 if (!HAS_ENGINE(dev_priv, _VCS(i))) 1385 continue; 1386 1387 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA_VDBOX0 + i, 1388 FORCEWAKE_MEDIA_VDBOX_GEN11(i), 1389 FORCEWAKE_ACK_MEDIA_VDBOX_GEN11(i)); 1390 } 1391 for (i = 0; i < I915_MAX_VECS; i++) { 1392 if (!HAS_ENGINE(dev_priv, _VECS(i))) 1393 continue; 1394 1395 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA_VEBOX0 + i, 1396 FORCEWAKE_MEDIA_VEBOX_GEN11(i), 1397 FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(i)); 1398 } 1399 } else if (IS_GEN9(dev_priv) || IS_GEN10(dev_priv)) { 1400 dev_priv->uncore.funcs.force_wake_get = 1401 fw_domains_get_with_fallback; 1402 dev_priv->uncore.funcs.force_wake_put = fw_domains_put; 1403 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, 1404 FORCEWAKE_RENDER_GEN9, 1405 FORCEWAKE_ACK_RENDER_GEN9); 1406 fw_domain_init(dev_priv, FW_DOMAIN_ID_BLITTER, 1407 FORCEWAKE_BLITTER_GEN9, 1408 FORCEWAKE_ACK_BLITTER_GEN9); 1409 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA, 1410 FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9); 1411 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { 1412 dev_priv->uncore.funcs.force_wake_get = fw_domains_get; 1413 dev_priv->uncore.funcs.force_wake_put = fw_domains_put; 1414 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, 1415 FORCEWAKE_VLV, FORCEWAKE_ACK_VLV); 1416 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA, 1417 FORCEWAKE_MEDIA_VLV, FORCEWAKE_ACK_MEDIA_VLV); 1418 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { 1419 dev_priv->uncore.funcs.force_wake_get = 1420 fw_domains_get_with_thread_status; 1421 dev_priv->uncore.funcs.force_wake_put = fw_domains_put; 1422 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, 1423 FORCEWAKE_MT, FORCEWAKE_ACK_HSW); 1424 } else if (IS_IVYBRIDGE(dev_priv)) { 1425 u32 ecobus; 1426 1427 /* IVB configs may use multi-threaded forcewake */ 1428 1429 /* A small trick here - if the bios hasn't configured 1430 * MT forcewake, and if the device is in RC6, then 1431 * force_wake_mt_get will not wake the device and the 1432 * ECOBUS read will return zero. Which will be 1433 * (correctly) interpreted by the test below as MT 1434 * forcewake being disabled. 1435 */ 1436 dev_priv->uncore.funcs.force_wake_get = 1437 fw_domains_get_with_thread_status; 1438 dev_priv->uncore.funcs.force_wake_put = fw_domains_put; 1439 1440 /* We need to init first for ECOBUS access and then 1441 * determine later if we want to reinit, in case of MT access is 1442 * not working. In this stage we don't know which flavour this 1443 * ivb is, so it is better to reset also the gen6 fw registers 1444 * before the ecobus check. 1445 */ 1446 1447 __raw_i915_write32(dev_priv, FORCEWAKE, 0); 1448 __raw_posting_read(dev_priv, ECOBUS); 1449 1450 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, 1451 FORCEWAKE_MT, FORCEWAKE_MT_ACK); 1452 1453 spin_lock_irq(&dev_priv->uncore.lock); 1454 fw_domains_get_with_thread_status(dev_priv, FORCEWAKE_RENDER); 1455 ecobus = __raw_i915_read32(dev_priv, ECOBUS); 1456 fw_domains_put(dev_priv, FORCEWAKE_RENDER); 1457 spin_unlock_irq(&dev_priv->uncore.lock); 1458 1459 if (!(ecobus & FORCEWAKE_MT_ENABLE)) { 1460 DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n"); 1461 DRM_INFO("when using vblank-synced partial screen updates.\n"); 1462 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, 1463 FORCEWAKE, FORCEWAKE_ACK); 1464 } 1465 } else if (IS_GEN6(dev_priv)) { 1466 dev_priv->uncore.funcs.force_wake_get = 1467 fw_domains_get_with_thread_status; 1468 dev_priv->uncore.funcs.force_wake_put = fw_domains_put; 1469 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, 1470 FORCEWAKE, FORCEWAKE_ACK); 1471 } 1472 1473 /* All future platforms are expected to require complex power gating */ 1474 WARN_ON(dev_priv->uncore.fw_domains == 0); 1475 } 1476 1477 #define ASSIGN_FW_DOMAINS_TABLE(d) \ 1478 { \ 1479 dev_priv->uncore.fw_domains_table = \ 1480 (struct intel_forcewake_range *)(d); \ 1481 dev_priv->uncore.fw_domains_table_entries = ARRAY_SIZE((d)); \ 1482 } 1483 1484 static int i915_pmic_bus_access_notifier(struct notifier_block *nb, 1485 unsigned long action, void *data) 1486 { 1487 struct drm_i915_private *dev_priv = container_of(nb, 1488 struct drm_i915_private, uncore.pmic_bus_access_nb); 1489 1490 switch (action) { 1491 case MBI_PMIC_BUS_ACCESS_BEGIN: 1492 /* 1493 * forcewake all now to make sure that we don't need to do a 1494 * forcewake later which on systems where this notifier gets 1495 * called requires the punit to access to the shared pmic i2c 1496 * bus, which will be busy after this notification, leading to: 1497 * "render: timed out waiting for forcewake ack request." 1498 * errors. 1499 * 1500 * The notifier is unregistered during intel_runtime_suspend(), 1501 * so it's ok to access the HW here without holding a RPM 1502 * wake reference -> disable wakeref asserts for the time of 1503 * the access. 1504 */ 1505 disable_rpm_wakeref_asserts(dev_priv); 1506 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 1507 enable_rpm_wakeref_asserts(dev_priv); 1508 break; 1509 case MBI_PMIC_BUS_ACCESS_END: 1510 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 1511 break; 1512 } 1513 1514 return NOTIFY_OK; 1515 } 1516 1517 void intel_uncore_init(struct drm_i915_private *dev_priv) 1518 { 1519 i915_check_vgpu(dev_priv); 1520 1521 intel_uncore_edram_detect(dev_priv); 1522 intel_uncore_fw_domains_init(dev_priv); 1523 __intel_uncore_early_sanitize(dev_priv, false); 1524 1525 dev_priv->uncore.unclaimed_mmio_check = 1; 1526 dev_priv->uncore.pmic_bus_access_nb.notifier_call = 1527 i915_pmic_bus_access_notifier; 1528 1529 if (IS_GEN(dev_priv, 2, 4) || intel_vgpu_active(dev_priv)) { 1530 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen2); 1531 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen2); 1532 } else if (IS_GEN5(dev_priv)) { 1533 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen5); 1534 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen5); 1535 } else if (IS_GEN(dev_priv, 6, 7)) { 1536 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen6); 1537 1538 if (IS_VALLEYVIEW(dev_priv)) { 1539 ASSIGN_FW_DOMAINS_TABLE(__vlv_fw_ranges); 1540 ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable); 1541 } else { 1542 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6); 1543 } 1544 } else if (IS_GEN8(dev_priv)) { 1545 if (IS_CHERRYVIEW(dev_priv)) { 1546 ASSIGN_FW_DOMAINS_TABLE(__chv_fw_ranges); 1547 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable); 1548 ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable); 1549 1550 } else { 1551 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen8); 1552 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6); 1553 } 1554 } else if (IS_GEN(dev_priv, 9, 10)) { 1555 ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges); 1556 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable); 1557 ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable); 1558 } else { 1559 ASSIGN_FW_DOMAINS_TABLE(__gen11_fw_ranges); 1560 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen11_fwtable); 1561 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen11_fwtable); 1562 } 1563 1564 iosf_mbi_register_pmic_bus_access_notifier( 1565 &dev_priv->uncore.pmic_bus_access_nb); 1566 } 1567 1568 void intel_uncore_fini(struct drm_i915_private *dev_priv) 1569 { 1570 /* Paranoia: make sure we have disabled everything before we exit. */ 1571 intel_uncore_sanitize(dev_priv); 1572 1573 iosf_mbi_punit_acquire(); 1574 iosf_mbi_unregister_pmic_bus_access_notifier_unlocked( 1575 &dev_priv->uncore.pmic_bus_access_nb); 1576 intel_uncore_forcewake_reset(dev_priv, false); 1577 iosf_mbi_punit_release(); 1578 } 1579 1580 static const struct reg_whitelist { 1581 i915_reg_t offset_ldw; 1582 i915_reg_t offset_udw; 1583 u16 gen_mask; 1584 u8 size; 1585 } reg_read_whitelist[] = { { 1586 .offset_ldw = RING_TIMESTAMP(RENDER_RING_BASE), 1587 .offset_udw = RING_TIMESTAMP_UDW(RENDER_RING_BASE), 1588 .gen_mask = INTEL_GEN_MASK(4, 11), 1589 .size = 8 1590 } }; 1591 1592 int i915_reg_read_ioctl(struct drm_device *dev, 1593 void *data, struct drm_file *file) 1594 { 1595 struct drm_i915_private *dev_priv = to_i915(dev); 1596 struct drm_i915_reg_read *reg = data; 1597 struct reg_whitelist const *entry; 1598 unsigned int flags; 1599 int remain; 1600 int ret = 0; 1601 1602 entry = reg_read_whitelist; 1603 remain = ARRAY_SIZE(reg_read_whitelist); 1604 while (remain) { 1605 u32 entry_offset = i915_mmio_reg_offset(entry->offset_ldw); 1606 1607 GEM_BUG_ON(!is_power_of_2(entry->size)); 1608 GEM_BUG_ON(entry->size > 8); 1609 GEM_BUG_ON(entry_offset & (entry->size - 1)); 1610 1611 if (INTEL_INFO(dev_priv)->gen_mask & entry->gen_mask && 1612 entry_offset == (reg->offset & -entry->size)) 1613 break; 1614 entry++; 1615 remain--; 1616 } 1617 1618 if (!remain) 1619 return -EINVAL; 1620 1621 flags = reg->offset & (entry->size - 1); 1622 1623 intel_runtime_pm_get(dev_priv); 1624 if (entry->size == 8 && flags == I915_REG_READ_8B_WA) 1625 reg->val = I915_READ64_2x32(entry->offset_ldw, 1626 entry->offset_udw); 1627 else if (entry->size == 8 && flags == 0) 1628 reg->val = I915_READ64(entry->offset_ldw); 1629 else if (entry->size == 4 && flags == 0) 1630 reg->val = I915_READ(entry->offset_ldw); 1631 else if (entry->size == 2 && flags == 0) 1632 reg->val = I915_READ16(entry->offset_ldw); 1633 else if (entry->size == 1 && flags == 0) 1634 reg->val = I915_READ8(entry->offset_ldw); 1635 else 1636 ret = -EINVAL; 1637 intel_runtime_pm_put(dev_priv); 1638 1639 return ret; 1640 } 1641 1642 static void gen3_stop_engine(struct intel_engine_cs *engine) 1643 { 1644 struct drm_i915_private *dev_priv = engine->i915; 1645 const u32 base = engine->mmio_base; 1646 const i915_reg_t mode = RING_MI_MODE(base); 1647 1648 I915_WRITE_FW(mode, _MASKED_BIT_ENABLE(STOP_RING)); 1649 if (intel_wait_for_register_fw(dev_priv, 1650 mode, 1651 MODE_IDLE, 1652 MODE_IDLE, 1653 500)) 1654 DRM_DEBUG_DRIVER("%s: timed out on STOP_RING\n", 1655 engine->name); 1656 1657 I915_WRITE_FW(RING_HEAD(base), I915_READ_FW(RING_TAIL(base))); 1658 POSTING_READ_FW(RING_HEAD(base)); /* paranoia */ 1659 1660 I915_WRITE_FW(RING_HEAD(base), 0); 1661 I915_WRITE_FW(RING_TAIL(base), 0); 1662 POSTING_READ_FW(RING_TAIL(base)); 1663 1664 /* The ring must be empty before it is disabled */ 1665 I915_WRITE_FW(RING_CTL(base), 0); 1666 1667 /* Check acts as a post */ 1668 if (I915_READ_FW(RING_HEAD(base)) != 0) 1669 DRM_DEBUG_DRIVER("%s: ring head not parked\n", 1670 engine->name); 1671 } 1672 1673 static void i915_stop_engines(struct drm_i915_private *dev_priv, 1674 unsigned engine_mask) 1675 { 1676 struct intel_engine_cs *engine; 1677 enum intel_engine_id id; 1678 1679 if (INTEL_GEN(dev_priv) < 3) 1680 return; 1681 1682 for_each_engine_masked(engine, dev_priv, engine_mask, id) 1683 gen3_stop_engine(engine); 1684 } 1685 1686 static bool i915_in_reset(struct pci_dev *pdev) 1687 { 1688 u8 gdrst; 1689 1690 pci_read_config_byte(pdev, I915_GDRST, &gdrst); 1691 return gdrst & GRDOM_RESET_STATUS; 1692 } 1693 1694 static int i915_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask) 1695 { 1696 struct pci_dev *pdev = dev_priv->drm.pdev; 1697 int err; 1698 1699 /* Assert reset for at least 20 usec, and wait for acknowledgement. */ 1700 pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE); 1701 usleep_range(50, 200); 1702 err = wait_for(i915_in_reset(pdev), 500); 1703 1704 /* Clear the reset request. */ 1705 pci_write_config_byte(pdev, I915_GDRST, 0); 1706 usleep_range(50, 200); 1707 if (!err) 1708 err = wait_for(!i915_in_reset(pdev), 500); 1709 1710 return err; 1711 } 1712 1713 static bool g4x_reset_complete(struct pci_dev *pdev) 1714 { 1715 u8 gdrst; 1716 1717 pci_read_config_byte(pdev, I915_GDRST, &gdrst); 1718 return (gdrst & GRDOM_RESET_ENABLE) == 0; 1719 } 1720 1721 static int g33_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask) 1722 { 1723 struct pci_dev *pdev = dev_priv->drm.pdev; 1724 1725 pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE); 1726 return wait_for(g4x_reset_complete(pdev), 500); 1727 } 1728 1729 static int g4x_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask) 1730 { 1731 struct pci_dev *pdev = dev_priv->drm.pdev; 1732 int ret; 1733 1734 /* WaVcpClkGateDisableForMediaReset:ctg,elk */ 1735 I915_WRITE(VDECCLK_GATE_D, 1736 I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE); 1737 POSTING_READ(VDECCLK_GATE_D); 1738 1739 pci_write_config_byte(pdev, I915_GDRST, 1740 GRDOM_MEDIA | GRDOM_RESET_ENABLE); 1741 ret = wait_for(g4x_reset_complete(pdev), 500); 1742 if (ret) { 1743 DRM_DEBUG_DRIVER("Wait for media reset failed\n"); 1744 goto out; 1745 } 1746 1747 pci_write_config_byte(pdev, I915_GDRST, 1748 GRDOM_RENDER | GRDOM_RESET_ENABLE); 1749 ret = wait_for(g4x_reset_complete(pdev), 500); 1750 if (ret) { 1751 DRM_DEBUG_DRIVER("Wait for render reset failed\n"); 1752 goto out; 1753 } 1754 1755 out: 1756 pci_write_config_byte(pdev, I915_GDRST, 0); 1757 1758 I915_WRITE(VDECCLK_GATE_D, 1759 I915_READ(VDECCLK_GATE_D) & ~VCP_UNIT_CLOCK_GATE_DISABLE); 1760 POSTING_READ(VDECCLK_GATE_D); 1761 1762 return ret; 1763 } 1764 1765 static int ironlake_do_reset(struct drm_i915_private *dev_priv, 1766 unsigned engine_mask) 1767 { 1768 int ret; 1769 1770 I915_WRITE(ILK_GDSR, ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE); 1771 ret = intel_wait_for_register(dev_priv, 1772 ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0, 1773 500); 1774 if (ret) { 1775 DRM_DEBUG_DRIVER("Wait for render reset failed\n"); 1776 goto out; 1777 } 1778 1779 I915_WRITE(ILK_GDSR, ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE); 1780 ret = intel_wait_for_register(dev_priv, 1781 ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0, 1782 500); 1783 if (ret) { 1784 DRM_DEBUG_DRIVER("Wait for media reset failed\n"); 1785 goto out; 1786 } 1787 1788 out: 1789 I915_WRITE(ILK_GDSR, 0); 1790 POSTING_READ(ILK_GDSR); 1791 return ret; 1792 } 1793 1794 /* Reset the hardware domains (GENX_GRDOM_*) specified by mask */ 1795 static int gen6_hw_domain_reset(struct drm_i915_private *dev_priv, 1796 u32 hw_domain_mask) 1797 { 1798 int err; 1799 1800 /* GEN6_GDRST is not in the gt power well, no need to check 1801 * for fifo space for the write or forcewake the chip for 1802 * the read 1803 */ 1804 __raw_i915_write32(dev_priv, GEN6_GDRST, hw_domain_mask); 1805 1806 /* Wait for the device to ack the reset requests */ 1807 err = intel_wait_for_register_fw(dev_priv, 1808 GEN6_GDRST, hw_domain_mask, 0, 1809 500); 1810 if (err) 1811 DRM_DEBUG_DRIVER("Wait for 0x%08x engines reset failed\n", 1812 hw_domain_mask); 1813 1814 return err; 1815 } 1816 1817 /** 1818 * gen6_reset_engines - reset individual engines 1819 * @dev_priv: i915 device 1820 * @engine_mask: mask of intel_ring_flag() engines or ALL_ENGINES for full reset 1821 * 1822 * This function will reset the individual engines that are set in engine_mask. 1823 * If you provide ALL_ENGINES as mask, full global domain reset will be issued. 1824 * 1825 * Note: It is responsibility of the caller to handle the difference between 1826 * asking full domain reset versus reset for all available individual engines. 1827 * 1828 * Returns 0 on success, nonzero on error. 1829 */ 1830 static int gen6_reset_engines(struct drm_i915_private *dev_priv, 1831 unsigned engine_mask) 1832 { 1833 struct intel_engine_cs *engine; 1834 const u32 hw_engine_mask[I915_NUM_ENGINES] = { 1835 [RCS] = GEN6_GRDOM_RENDER, 1836 [BCS] = GEN6_GRDOM_BLT, 1837 [VCS] = GEN6_GRDOM_MEDIA, 1838 [VCS2] = GEN8_GRDOM_MEDIA2, 1839 [VECS] = GEN6_GRDOM_VECS, 1840 }; 1841 u32 hw_mask; 1842 1843 if (engine_mask == ALL_ENGINES) { 1844 hw_mask = GEN6_GRDOM_FULL; 1845 } else { 1846 unsigned int tmp; 1847 1848 hw_mask = 0; 1849 for_each_engine_masked(engine, dev_priv, engine_mask, tmp) 1850 hw_mask |= hw_engine_mask[engine->id]; 1851 } 1852 1853 return gen6_hw_domain_reset(dev_priv, hw_mask); 1854 } 1855 1856 /** 1857 * __intel_wait_for_register_fw - wait until register matches expected state 1858 * @dev_priv: the i915 device 1859 * @reg: the register to read 1860 * @mask: mask to apply to register value 1861 * @value: expected value 1862 * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait 1863 * @slow_timeout_ms: slow timeout in millisecond 1864 * @out_value: optional placeholder to hold registry value 1865 * 1866 * This routine waits until the target register @reg contains the expected 1867 * @value after applying the @mask, i.e. it waits until :: 1868 * 1869 * (I915_READ_FW(reg) & mask) == value 1870 * 1871 * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds. 1872 * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us 1873 * must be not larger than 20,0000 microseconds. 1874 * 1875 * Note that this routine assumes the caller holds forcewake asserted, it is 1876 * not suitable for very long waits. See intel_wait_for_register() if you 1877 * wish to wait without holding forcewake for the duration (i.e. you expect 1878 * the wait to be slow). 1879 * 1880 * Returns 0 if the register matches the desired condition, or -ETIMEOUT. 1881 */ 1882 int __intel_wait_for_register_fw(struct drm_i915_private *dev_priv, 1883 i915_reg_t reg, 1884 u32 mask, 1885 u32 value, 1886 unsigned int fast_timeout_us, 1887 unsigned int slow_timeout_ms, 1888 u32 *out_value) 1889 { 1890 u32 uninitialized_var(reg_value); 1891 #define done (((reg_value = I915_READ_FW(reg)) & mask) == value) 1892 int ret; 1893 1894 /* Catch any overuse of this function */ 1895 might_sleep_if(slow_timeout_ms); 1896 GEM_BUG_ON(fast_timeout_us > 20000); 1897 1898 ret = -ETIMEDOUT; 1899 if (fast_timeout_us && fast_timeout_us <= 20000) 1900 ret = _wait_for_atomic(done, fast_timeout_us, 0); 1901 if (ret && slow_timeout_ms) 1902 ret = wait_for(done, slow_timeout_ms); 1903 1904 if (out_value) 1905 *out_value = reg_value; 1906 1907 return ret; 1908 #undef done 1909 } 1910 1911 /** 1912 * __intel_wait_for_register - wait until register matches expected state 1913 * @dev_priv: the i915 device 1914 * @reg: the register to read 1915 * @mask: mask to apply to register value 1916 * @value: expected value 1917 * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait 1918 * @slow_timeout_ms: slow timeout in millisecond 1919 * @out_value: optional placeholder to hold registry value 1920 * 1921 * This routine waits until the target register @reg contains the expected 1922 * @value after applying the @mask, i.e. it waits until :: 1923 * 1924 * (I915_READ(reg) & mask) == value 1925 * 1926 * Otherwise, the wait will timeout after @timeout_ms milliseconds. 1927 * 1928 * Returns 0 if the register matches the desired condition, or -ETIMEOUT. 1929 */ 1930 int __intel_wait_for_register(struct drm_i915_private *dev_priv, 1931 i915_reg_t reg, 1932 u32 mask, 1933 u32 value, 1934 unsigned int fast_timeout_us, 1935 unsigned int slow_timeout_ms, 1936 u32 *out_value) 1937 { 1938 unsigned fw = 1939 intel_uncore_forcewake_for_reg(dev_priv, reg, FW_REG_READ); 1940 u32 reg_value; 1941 int ret; 1942 1943 might_sleep(); 1944 1945 spin_lock_irq(&dev_priv->uncore.lock); 1946 intel_uncore_forcewake_get__locked(dev_priv, fw); 1947 1948 ret = __intel_wait_for_register_fw(dev_priv, 1949 reg, mask, value, 1950 fast_timeout_us, 0, ®_value); 1951 1952 intel_uncore_forcewake_put__locked(dev_priv, fw); 1953 spin_unlock_irq(&dev_priv->uncore.lock); 1954 1955 if (ret) 1956 ret = __wait_for(reg_value = I915_READ_NOTRACE(reg), 1957 (reg_value & mask) == value, 1958 slow_timeout_ms * 1000, 10, 1000); 1959 1960 if (out_value) 1961 *out_value = reg_value; 1962 1963 return ret; 1964 } 1965 1966 static int gen8_reset_engine_start(struct intel_engine_cs *engine) 1967 { 1968 struct drm_i915_private *dev_priv = engine->i915; 1969 int ret; 1970 1971 I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base), 1972 _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET)); 1973 1974 ret = intel_wait_for_register_fw(dev_priv, 1975 RING_RESET_CTL(engine->mmio_base), 1976 RESET_CTL_READY_TO_RESET, 1977 RESET_CTL_READY_TO_RESET, 1978 700); 1979 if (ret) 1980 DRM_ERROR("%s: reset request timeout\n", engine->name); 1981 1982 return ret; 1983 } 1984 1985 static void gen8_reset_engine_cancel(struct intel_engine_cs *engine) 1986 { 1987 struct drm_i915_private *dev_priv = engine->i915; 1988 1989 I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base), 1990 _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET)); 1991 } 1992 1993 static int gen8_reset_engines(struct drm_i915_private *dev_priv, 1994 unsigned engine_mask) 1995 { 1996 struct intel_engine_cs *engine; 1997 unsigned int tmp; 1998 1999 for_each_engine_masked(engine, dev_priv, engine_mask, tmp) 2000 if (gen8_reset_engine_start(engine)) 2001 goto not_ready; 2002 2003 return gen6_reset_engines(dev_priv, engine_mask); 2004 2005 not_ready: 2006 for_each_engine_masked(engine, dev_priv, engine_mask, tmp) 2007 gen8_reset_engine_cancel(engine); 2008 2009 return -EIO; 2010 } 2011 2012 typedef int (*reset_func)(struct drm_i915_private *, unsigned engine_mask); 2013 2014 static reset_func intel_get_gpu_reset(struct drm_i915_private *dev_priv) 2015 { 2016 if (!i915_modparams.reset) 2017 return NULL; 2018 2019 if (INTEL_GEN(dev_priv) >= 8) 2020 return gen8_reset_engines; 2021 else if (INTEL_GEN(dev_priv) >= 6) 2022 return gen6_reset_engines; 2023 else if (IS_GEN5(dev_priv)) 2024 return ironlake_do_reset; 2025 else if (IS_G4X(dev_priv)) 2026 return g4x_do_reset; 2027 else if (IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) 2028 return g33_do_reset; 2029 else if (INTEL_GEN(dev_priv) >= 3) 2030 return i915_do_reset; 2031 else 2032 return NULL; 2033 } 2034 2035 int intel_gpu_reset(struct drm_i915_private *dev_priv, unsigned engine_mask) 2036 { 2037 reset_func reset = intel_get_gpu_reset(dev_priv); 2038 int retry; 2039 int ret; 2040 2041 might_sleep(); 2042 2043 /* If the power well sleeps during the reset, the reset 2044 * request may be dropped and never completes (causing -EIO). 2045 */ 2046 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 2047 for (retry = 0; retry < 3; retry++) { 2048 2049 /* We stop engines, otherwise we might get failed reset and a 2050 * dead gpu (on elk). Also as modern gpu as kbl can suffer 2051 * from system hang if batchbuffer is progressing when 2052 * the reset is issued, regardless of READY_TO_RESET ack. 2053 * Thus assume it is best to stop engines on all gens 2054 * where we have a gpu reset. 2055 * 2056 * WaMediaResetMainRingCleanup:ctg,elk (presumably) 2057 * 2058 * FIXME: Wa for more modern gens needs to be validated 2059 */ 2060 i915_stop_engines(dev_priv, engine_mask); 2061 2062 ret = -ENODEV; 2063 if (reset) 2064 ret = reset(dev_priv, engine_mask); 2065 if (ret != -ETIMEDOUT) 2066 break; 2067 2068 cond_resched(); 2069 } 2070 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 2071 2072 return ret; 2073 } 2074 2075 bool intel_has_gpu_reset(struct drm_i915_private *dev_priv) 2076 { 2077 return intel_get_gpu_reset(dev_priv) != NULL; 2078 } 2079 2080 bool intel_has_reset_engine(struct drm_i915_private *dev_priv) 2081 { 2082 return (dev_priv->info.has_reset_engine && 2083 i915_modparams.reset >= 2); 2084 } 2085 2086 int intel_reset_guc(struct drm_i915_private *dev_priv) 2087 { 2088 int ret; 2089 2090 GEM_BUG_ON(!HAS_GUC(dev_priv)); 2091 2092 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 2093 ret = gen6_hw_domain_reset(dev_priv, GEN9_GRDOM_GUC); 2094 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 2095 2096 return ret; 2097 } 2098 2099 bool intel_uncore_unclaimed_mmio(struct drm_i915_private *dev_priv) 2100 { 2101 return check_for_unclaimed_mmio(dev_priv); 2102 } 2103 2104 bool 2105 intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private *dev_priv) 2106 { 2107 if (unlikely(i915_modparams.mmio_debug || 2108 dev_priv->uncore.unclaimed_mmio_check <= 0)) 2109 return false; 2110 2111 if (unlikely(intel_uncore_unclaimed_mmio(dev_priv))) { 2112 DRM_DEBUG("Unclaimed register detected, " 2113 "enabling oneshot unclaimed register reporting. " 2114 "Please use i915.mmio_debug=N for more information.\n"); 2115 i915_modparams.mmio_debug++; 2116 dev_priv->uncore.unclaimed_mmio_check--; 2117 return true; 2118 } 2119 2120 return false; 2121 } 2122 2123 static enum forcewake_domains 2124 intel_uncore_forcewake_for_read(struct drm_i915_private *dev_priv, 2125 i915_reg_t reg) 2126 { 2127 u32 offset = i915_mmio_reg_offset(reg); 2128 enum forcewake_domains fw_domains; 2129 2130 if (INTEL_GEN(dev_priv) >= 11) { 2131 fw_domains = __gen11_fwtable_reg_read_fw_domains(offset); 2132 } else if (HAS_FWTABLE(dev_priv)) { 2133 fw_domains = __fwtable_reg_read_fw_domains(offset); 2134 } else if (INTEL_GEN(dev_priv) >= 6) { 2135 fw_domains = __gen6_reg_read_fw_domains(offset); 2136 } else { 2137 WARN_ON(!IS_GEN(dev_priv, 2, 5)); 2138 fw_domains = 0; 2139 } 2140 2141 WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains); 2142 2143 return fw_domains; 2144 } 2145 2146 static enum forcewake_domains 2147 intel_uncore_forcewake_for_write(struct drm_i915_private *dev_priv, 2148 i915_reg_t reg) 2149 { 2150 u32 offset = i915_mmio_reg_offset(reg); 2151 enum forcewake_domains fw_domains; 2152 2153 if (INTEL_GEN(dev_priv) >= 11) { 2154 fw_domains = __gen11_fwtable_reg_write_fw_domains(offset); 2155 } else if (HAS_FWTABLE(dev_priv) && !IS_VALLEYVIEW(dev_priv)) { 2156 fw_domains = __fwtable_reg_write_fw_domains(offset); 2157 } else if (IS_GEN8(dev_priv)) { 2158 fw_domains = __gen8_reg_write_fw_domains(offset); 2159 } else if (IS_GEN(dev_priv, 6, 7)) { 2160 fw_domains = FORCEWAKE_RENDER; 2161 } else { 2162 WARN_ON(!IS_GEN(dev_priv, 2, 5)); 2163 fw_domains = 0; 2164 } 2165 2166 WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains); 2167 2168 return fw_domains; 2169 } 2170 2171 /** 2172 * intel_uncore_forcewake_for_reg - which forcewake domains are needed to access 2173 * a register 2174 * @dev_priv: pointer to struct drm_i915_private 2175 * @reg: register in question 2176 * @op: operation bitmask of FW_REG_READ and/or FW_REG_WRITE 2177 * 2178 * Returns a set of forcewake domains required to be taken with for example 2179 * intel_uncore_forcewake_get for the specified register to be accessible in the 2180 * specified mode (read, write or read/write) with raw mmio accessors. 2181 * 2182 * NOTE: On Gen6 and Gen7 write forcewake domain (FORCEWAKE_RENDER) requires the 2183 * callers to do FIFO management on their own or risk losing writes. 2184 */ 2185 enum forcewake_domains 2186 intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv, 2187 i915_reg_t reg, unsigned int op) 2188 { 2189 enum forcewake_domains fw_domains = 0; 2190 2191 WARN_ON(!op); 2192 2193 if (intel_vgpu_active(dev_priv)) 2194 return 0; 2195 2196 if (op & FW_REG_READ) 2197 fw_domains = intel_uncore_forcewake_for_read(dev_priv, reg); 2198 2199 if (op & FW_REG_WRITE) 2200 fw_domains |= intel_uncore_forcewake_for_write(dev_priv, reg); 2201 2202 return fw_domains; 2203 } 2204 2205 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2206 #include "selftests/mock_uncore.c" 2207 #include "selftests/intel_uncore.c" 2208 #endif 2209