1 /* 2 * Copyright © 2013 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 */ 23 24 #include "i915_drv.h" 25 #include "intel_drv.h" 26 27 #define FORCEWAKE_ACK_TIMEOUT_MS 2 28 29 #define __raw_i915_read8(dev_priv__, reg__) readb((dev_priv__)->regs + (reg__)) 30 #define __raw_i915_write8(dev_priv__, reg__, val__) writeb(val__, (dev_priv__)->regs + (reg__)) 31 32 #define __raw_i915_read16(dev_priv__, reg__) readw((dev_priv__)->regs + (reg__)) 33 #define __raw_i915_write16(dev_priv__, reg__, val__) writew(val__, (dev_priv__)->regs + (reg__)) 34 35 #define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__)) 36 #define __raw_i915_write32(dev_priv__, reg__, val__) writel(val__, (dev_priv__)->regs + (reg__)) 37 38 #define __raw_i915_read64(dev_priv__, reg__) readq((dev_priv__)->regs + (reg__)) 39 #define __raw_i915_write64(dev_priv__, reg__, val__) writeq(val__, (dev_priv__)->regs + (reg__)) 40 41 #define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32(dev_priv__, reg__) 42 43 44 static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv) 45 { 46 u32 gt_thread_status_mask; 47 48 if (IS_HASWELL(dev_priv->dev)) 49 gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK_HSW; 50 else 51 gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK; 52 53 /* w/a for a sporadic read returning 0 by waiting for the GT 54 * thread to wake up. 55 */ 56 if (wait_for_atomic_us((__raw_i915_read32(dev_priv, GEN6_GT_THREAD_STATUS_REG) & gt_thread_status_mask) == 0, 500)) 57 DRM_ERROR("GT thread status wait timed out\n"); 58 } 59 60 static void __gen6_gt_force_wake_reset(struct drm_i915_private *dev_priv) 61 { 62 __raw_i915_write32(dev_priv, FORCEWAKE, 0); 63 /* something from same cacheline, but !FORCEWAKE */ 64 __raw_posting_read(dev_priv, ECOBUS); 65 } 66 67 static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, 68 int fw_engine) 69 { 70 if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1) == 0, 71 FORCEWAKE_ACK_TIMEOUT_MS)) 72 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n"); 73 74 __raw_i915_write32(dev_priv, FORCEWAKE, 1); 75 /* something from same cacheline, but !FORCEWAKE */ 76 __raw_posting_read(dev_priv, ECOBUS); 77 78 if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1), 79 FORCEWAKE_ACK_TIMEOUT_MS)) 80 DRM_ERROR("Timed out waiting for forcewake to ack request.\n"); 81 82 /* WaRsForcewakeWaitTC0:snb */ 83 __gen6_gt_wait_for_thread_c0(dev_priv); 84 } 85 86 static void __gen6_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv) 87 { 88 __raw_i915_write32(dev_priv, FORCEWAKE_MT, _MASKED_BIT_DISABLE(0xffff)); 89 /* something from same cacheline, but !FORCEWAKE_MT */ 90 __raw_posting_read(dev_priv, ECOBUS); 91 } 92 93 static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv, 94 int fw_engine) 95 { 96 u32 forcewake_ack; 97 98 if (IS_HASWELL(dev_priv->dev) || IS_GEN8(dev_priv->dev)) 99 forcewake_ack = FORCEWAKE_ACK_HSW; 100 else 101 forcewake_ack = FORCEWAKE_MT_ACK; 102 103 if (wait_for_atomic((__raw_i915_read32(dev_priv, forcewake_ack) & FORCEWAKE_KERNEL) == 0, 104 FORCEWAKE_ACK_TIMEOUT_MS)) 105 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n"); 106 107 __raw_i915_write32(dev_priv, FORCEWAKE_MT, 108 _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL)); 109 /* something from same cacheline, but !FORCEWAKE_MT */ 110 __raw_posting_read(dev_priv, ECOBUS); 111 112 if (wait_for_atomic((__raw_i915_read32(dev_priv, forcewake_ack) & FORCEWAKE_KERNEL), 113 FORCEWAKE_ACK_TIMEOUT_MS)) 114 DRM_ERROR("Timed out waiting for forcewake to ack request.\n"); 115 116 /* WaRsForcewakeWaitTC0:ivb,hsw */ 117 if (INTEL_INFO(dev_priv->dev)->gen < 8) 118 __gen6_gt_wait_for_thread_c0(dev_priv); 119 } 120 121 static void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv) 122 { 123 u32 gtfifodbg; 124 125 gtfifodbg = __raw_i915_read32(dev_priv, GTFIFODBG); 126 if (WARN(gtfifodbg, "GT wake FIFO error 0x%x\n", gtfifodbg)) 127 __raw_i915_write32(dev_priv, GTFIFODBG, gtfifodbg); 128 } 129 130 static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, 131 int fw_engine) 132 { 133 __raw_i915_write32(dev_priv, FORCEWAKE, 0); 134 /* something from same cacheline, but !FORCEWAKE */ 135 __raw_posting_read(dev_priv, ECOBUS); 136 gen6_gt_check_fifodbg(dev_priv); 137 } 138 139 static void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv, 140 int fw_engine) 141 { 142 __raw_i915_write32(dev_priv, FORCEWAKE_MT, 143 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); 144 /* something from same cacheline, but !FORCEWAKE_MT */ 145 __raw_posting_read(dev_priv, ECOBUS); 146 gen6_gt_check_fifodbg(dev_priv); 147 } 148 149 static int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv) 150 { 151 int ret = 0; 152 153 /* On VLV, FIFO will be shared by both SW and HW. 154 * So, we need to read the FREE_ENTRIES everytime */ 155 if (IS_VALLEYVIEW(dev_priv->dev)) 156 dev_priv->uncore.fifo_count = 157 __raw_i915_read32(dev_priv, GTFIFOCTL) & 158 GT_FIFO_FREE_ENTRIES_MASK; 159 160 if (dev_priv->uncore.fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) { 161 int loop = 500; 162 u32 fifo = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK; 163 while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) { 164 udelay(10); 165 fifo = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK; 166 } 167 if (WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES)) 168 ++ret; 169 dev_priv->uncore.fifo_count = fifo; 170 } 171 dev_priv->uncore.fifo_count--; 172 173 return ret; 174 } 175 176 static void vlv_force_wake_reset(struct drm_i915_private *dev_priv) 177 { 178 __raw_i915_write32(dev_priv, FORCEWAKE_VLV, 179 _MASKED_BIT_DISABLE(0xffff)); 180 /* something from same cacheline, but !FORCEWAKE_VLV */ 181 __raw_posting_read(dev_priv, FORCEWAKE_ACK_VLV); 182 } 183 184 static void __vlv_force_wake_get(struct drm_i915_private *dev_priv, 185 int fw_engine) 186 { 187 /* Check for Render Engine */ 188 if (FORCEWAKE_RENDER & fw_engine) { 189 if (wait_for_atomic((__raw_i915_read32(dev_priv, 190 FORCEWAKE_ACK_VLV) & 191 FORCEWAKE_KERNEL) == 0, 192 FORCEWAKE_ACK_TIMEOUT_MS)) 193 DRM_ERROR("Timed out: Render forcewake old ack to clear.\n"); 194 195 __raw_i915_write32(dev_priv, FORCEWAKE_VLV, 196 _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL)); 197 198 if (wait_for_atomic((__raw_i915_read32(dev_priv, 199 FORCEWAKE_ACK_VLV) & 200 FORCEWAKE_KERNEL), 201 FORCEWAKE_ACK_TIMEOUT_MS)) 202 DRM_ERROR("Timed out: waiting for Render to ack.\n"); 203 } 204 205 /* Check for Media Engine */ 206 if (FORCEWAKE_MEDIA & fw_engine) { 207 if (wait_for_atomic((__raw_i915_read32(dev_priv, 208 FORCEWAKE_ACK_MEDIA_VLV) & 209 FORCEWAKE_KERNEL) == 0, 210 FORCEWAKE_ACK_TIMEOUT_MS)) 211 DRM_ERROR("Timed out: Media forcewake old ack to clear.\n"); 212 213 __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV, 214 _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL)); 215 216 if (wait_for_atomic((__raw_i915_read32(dev_priv, 217 FORCEWAKE_ACK_MEDIA_VLV) & 218 FORCEWAKE_KERNEL), 219 FORCEWAKE_ACK_TIMEOUT_MS)) 220 DRM_ERROR("Timed out: waiting for media to ack.\n"); 221 } 222 223 /* WaRsForcewakeWaitTC0:vlv */ 224 __gen6_gt_wait_for_thread_c0(dev_priv); 225 226 } 227 228 static void __vlv_force_wake_put(struct drm_i915_private *dev_priv, 229 int fw_engine) 230 { 231 232 /* Check for Render Engine */ 233 if (FORCEWAKE_RENDER & fw_engine) 234 __raw_i915_write32(dev_priv, FORCEWAKE_VLV, 235 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); 236 237 238 /* Check for Media Engine */ 239 if (FORCEWAKE_MEDIA & fw_engine) 240 __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV, 241 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); 242 243 /* The below doubles as a POSTING_READ */ 244 gen6_gt_check_fifodbg(dev_priv); 245 246 } 247 248 void vlv_force_wake_get(struct drm_i915_private *dev_priv, 249 int fw_engine) 250 { 251 unsigned long irqflags; 252 253 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 254 if (FORCEWAKE_RENDER & fw_engine) { 255 if (dev_priv->uncore.fw_rendercount++ == 0) 256 dev_priv->uncore.funcs.force_wake_get(dev_priv, 257 FORCEWAKE_RENDER); 258 } 259 if (FORCEWAKE_MEDIA & fw_engine) { 260 if (dev_priv->uncore.fw_mediacount++ == 0) 261 dev_priv->uncore.funcs.force_wake_get(dev_priv, 262 FORCEWAKE_MEDIA); 263 } 264 265 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 266 } 267 268 void vlv_force_wake_put(struct drm_i915_private *dev_priv, 269 int fw_engine) 270 { 271 unsigned long irqflags; 272 273 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 274 275 if (FORCEWAKE_RENDER & fw_engine) { 276 WARN_ON(dev_priv->uncore.fw_rendercount == 0); 277 if (--dev_priv->uncore.fw_rendercount == 0) 278 dev_priv->uncore.funcs.force_wake_put(dev_priv, 279 FORCEWAKE_RENDER); 280 } 281 282 if (FORCEWAKE_MEDIA & fw_engine) { 283 WARN_ON(dev_priv->uncore.fw_mediacount == 0); 284 if (--dev_priv->uncore.fw_mediacount == 0) 285 dev_priv->uncore.funcs.force_wake_put(dev_priv, 286 FORCEWAKE_MEDIA); 287 } 288 289 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 290 } 291 292 static void gen6_force_wake_work(struct work_struct *work) 293 { 294 struct drm_i915_private *dev_priv = 295 container_of(work, typeof(*dev_priv), uncore.force_wake_work.work); 296 unsigned long irqflags; 297 298 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 299 if (--dev_priv->uncore.forcewake_count == 0) 300 dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL); 301 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 302 } 303 304 static void intel_uncore_forcewake_reset(struct drm_device *dev) 305 { 306 struct drm_i915_private *dev_priv = dev->dev_private; 307 308 if (IS_VALLEYVIEW(dev)) { 309 vlv_force_wake_reset(dev_priv); 310 } else if (INTEL_INFO(dev)->gen >= 6) { 311 __gen6_gt_force_wake_reset(dev_priv); 312 if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) 313 __gen6_gt_force_wake_mt_reset(dev_priv); 314 } 315 } 316 317 void intel_uncore_early_sanitize(struct drm_device *dev) 318 { 319 struct drm_i915_private *dev_priv = dev->dev_private; 320 321 if (HAS_FPGA_DBG_UNCLAIMED(dev)) 322 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); 323 324 if (IS_HASWELL(dev) && 325 (__raw_i915_read32(dev_priv, HSW_EDRAM_PRESENT) == 1)) { 326 /* The docs do not explain exactly how the calculation can be 327 * made. It is somewhat guessable, but for now, it's always 328 * 128MB. 329 * NB: We can't write IDICR yet because we do not have gt funcs 330 * set up */ 331 dev_priv->ellc_size = 128; 332 DRM_INFO("Found %zuMB of eLLC\n", dev_priv->ellc_size); 333 } 334 335 /* clear out old GT FIFO errors */ 336 if (IS_GEN6(dev) || IS_GEN7(dev)) 337 __raw_i915_write32(dev_priv, GTFIFODBG, 338 __raw_i915_read32(dev_priv, GTFIFODBG)); 339 340 intel_uncore_forcewake_reset(dev); 341 } 342 343 void intel_uncore_sanitize(struct drm_device *dev) 344 { 345 struct drm_i915_private *dev_priv = dev->dev_private; 346 u32 reg_val; 347 348 /* BIOS often leaves RC6 enabled, but disable it for hw init */ 349 intel_disable_gt_powersave(dev); 350 351 /* Turn off power gate, require especially for the BIOS less system */ 352 if (IS_VALLEYVIEW(dev)) { 353 354 mutex_lock(&dev_priv->rps.hw_lock); 355 reg_val = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS); 356 357 if (reg_val & (RENDER_PWRGT | MEDIA_PWRGT | DISP2D_PWRGT)) 358 vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, 0x0); 359 360 mutex_unlock(&dev_priv->rps.hw_lock); 361 362 } 363 } 364 365 /* 366 * Generally this is called implicitly by the register read function. However, 367 * if some sequence requires the GT to not power down then this function should 368 * be called at the beginning of the sequence followed by a call to 369 * gen6_gt_force_wake_put() at the end of the sequence. 370 */ 371 void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine) 372 { 373 unsigned long irqflags; 374 375 if (!dev_priv->uncore.funcs.force_wake_get) 376 return; 377 378 intel_runtime_pm_get(dev_priv); 379 380 /* Redirect to VLV specific routine */ 381 if (IS_VALLEYVIEW(dev_priv->dev)) 382 return vlv_force_wake_get(dev_priv, fw_engine); 383 384 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 385 if (dev_priv->uncore.forcewake_count++ == 0) 386 dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL); 387 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 388 } 389 390 /* 391 * see gen6_gt_force_wake_get() 392 */ 393 void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine) 394 { 395 unsigned long irqflags; 396 397 if (!dev_priv->uncore.funcs.force_wake_put) 398 return; 399 400 /* Redirect to VLV specific routine */ 401 if (IS_VALLEYVIEW(dev_priv->dev)) 402 return vlv_force_wake_put(dev_priv, fw_engine); 403 404 405 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 406 if (--dev_priv->uncore.forcewake_count == 0) { 407 dev_priv->uncore.forcewake_count++; 408 mod_delayed_work(dev_priv->wq, 409 &dev_priv->uncore.force_wake_work, 410 1); 411 } 412 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 413 414 intel_runtime_pm_put(dev_priv); 415 } 416 417 /* We give fast paths for the really cool registers */ 418 #define NEEDS_FORCE_WAKE(dev_priv, reg) \ 419 ((reg) < 0x40000 && (reg) != FORCEWAKE) 420 421 static void 422 ilk_dummy_write(struct drm_i915_private *dev_priv) 423 { 424 /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up 425 * the chip from rc6 before touching it for real. MI_MODE is masked, 426 * hence harmless to write 0 into. */ 427 __raw_i915_write32(dev_priv, MI_MODE, 0); 428 } 429 430 static void 431 hsw_unclaimed_reg_clear(struct drm_i915_private *dev_priv, u32 reg) 432 { 433 if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) { 434 DRM_ERROR("Unknown unclaimed register before writing to %x\n", 435 reg); 436 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); 437 } 438 } 439 440 static void 441 hsw_unclaimed_reg_check(struct drm_i915_private *dev_priv, u32 reg) 442 { 443 if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) { 444 DRM_ERROR("Unclaimed write to %x\n", reg); 445 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); 446 } 447 } 448 449 static void 450 assert_device_not_suspended(struct drm_i915_private *dev_priv) 451 { 452 WARN(HAS_RUNTIME_PM(dev_priv->dev) && dev_priv->pm.suspended, 453 "Device suspended\n"); 454 } 455 456 #define REG_READ_HEADER(x) \ 457 unsigned long irqflags; \ 458 u##x val = 0; \ 459 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags) 460 461 #define REG_READ_FOOTER \ 462 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \ 463 trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \ 464 return val 465 466 #define __gen4_read(x) \ 467 static u##x \ 468 gen4_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ 469 REG_READ_HEADER(x); \ 470 val = __raw_i915_read##x(dev_priv, reg); \ 471 REG_READ_FOOTER; \ 472 } 473 474 #define __gen5_read(x) \ 475 static u##x \ 476 gen5_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ 477 REG_READ_HEADER(x); \ 478 ilk_dummy_write(dev_priv); \ 479 val = __raw_i915_read##x(dev_priv, reg); \ 480 REG_READ_FOOTER; \ 481 } 482 483 #define __gen6_read(x) \ 484 static u##x \ 485 gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ 486 REG_READ_HEADER(x); \ 487 if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \ 488 if (dev_priv->uncore.forcewake_count == 0) \ 489 dev_priv->uncore.funcs.force_wake_get(dev_priv, \ 490 FORCEWAKE_ALL); \ 491 val = __raw_i915_read##x(dev_priv, reg); \ 492 if (dev_priv->uncore.forcewake_count == 0) \ 493 dev_priv->uncore.funcs.force_wake_put(dev_priv, \ 494 FORCEWAKE_ALL); \ 495 } else { \ 496 val = __raw_i915_read##x(dev_priv, reg); \ 497 } \ 498 REG_READ_FOOTER; \ 499 } 500 501 #define __vlv_read(x) \ 502 static u##x \ 503 vlv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ 504 unsigned fwengine = 0; \ 505 unsigned *fwcount; \ 506 REG_READ_HEADER(x); \ 507 if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) { \ 508 fwengine = FORCEWAKE_RENDER; \ 509 fwcount = &dev_priv->uncore.fw_rendercount; \ 510 } \ 511 else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) { \ 512 fwengine = FORCEWAKE_MEDIA; \ 513 fwcount = &dev_priv->uncore.fw_mediacount; \ 514 } \ 515 if (fwengine != 0) { \ 516 if ((*fwcount)++ == 0) \ 517 (dev_priv)->uncore.funcs.force_wake_get(dev_priv, \ 518 fwengine); \ 519 val = __raw_i915_read##x(dev_priv, reg); \ 520 if (--(*fwcount) == 0) \ 521 (dev_priv)->uncore.funcs.force_wake_put(dev_priv, \ 522 fwengine); \ 523 } else { \ 524 val = __raw_i915_read##x(dev_priv, reg); \ 525 } \ 526 REG_READ_FOOTER; \ 527 } 528 529 530 __vlv_read(8) 531 __vlv_read(16) 532 __vlv_read(32) 533 __vlv_read(64) 534 __gen6_read(8) 535 __gen6_read(16) 536 __gen6_read(32) 537 __gen6_read(64) 538 __gen5_read(8) 539 __gen5_read(16) 540 __gen5_read(32) 541 __gen5_read(64) 542 __gen4_read(8) 543 __gen4_read(16) 544 __gen4_read(32) 545 __gen4_read(64) 546 547 #undef __vlv_read 548 #undef __gen6_read 549 #undef __gen5_read 550 #undef __gen4_read 551 #undef REG_READ_FOOTER 552 #undef REG_READ_HEADER 553 554 #define REG_WRITE_HEADER \ 555 unsigned long irqflags; \ 556 trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \ 557 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags) 558 559 #define REG_WRITE_FOOTER \ 560 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags) 561 562 #define __gen4_write(x) \ 563 static void \ 564 gen4_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ 565 REG_WRITE_HEADER; \ 566 __raw_i915_write##x(dev_priv, reg, val); \ 567 REG_WRITE_FOOTER; \ 568 } 569 570 #define __gen5_write(x) \ 571 static void \ 572 gen5_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ 573 REG_WRITE_HEADER; \ 574 ilk_dummy_write(dev_priv); \ 575 __raw_i915_write##x(dev_priv, reg, val); \ 576 REG_WRITE_FOOTER; \ 577 } 578 579 #define __gen6_write(x) \ 580 static void \ 581 gen6_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ 582 u32 __fifo_ret = 0; \ 583 REG_WRITE_HEADER; \ 584 if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \ 585 __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \ 586 } \ 587 assert_device_not_suspended(dev_priv); \ 588 __raw_i915_write##x(dev_priv, reg, val); \ 589 if (unlikely(__fifo_ret)) { \ 590 gen6_gt_check_fifodbg(dev_priv); \ 591 } \ 592 REG_WRITE_FOOTER; \ 593 } 594 595 #define __hsw_write(x) \ 596 static void \ 597 hsw_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ 598 u32 __fifo_ret = 0; \ 599 REG_WRITE_HEADER; \ 600 if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \ 601 __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \ 602 } \ 603 assert_device_not_suspended(dev_priv); \ 604 hsw_unclaimed_reg_clear(dev_priv, reg); \ 605 __raw_i915_write##x(dev_priv, reg, val); \ 606 if (unlikely(__fifo_ret)) { \ 607 gen6_gt_check_fifodbg(dev_priv); \ 608 } \ 609 hsw_unclaimed_reg_check(dev_priv, reg); \ 610 REG_WRITE_FOOTER; \ 611 } 612 613 static const u32 gen8_shadowed_regs[] = { 614 FORCEWAKE_MT, 615 GEN6_RPNSWREQ, 616 GEN6_RC_VIDEO_FREQ, 617 RING_TAIL(RENDER_RING_BASE), 618 RING_TAIL(GEN6_BSD_RING_BASE), 619 RING_TAIL(VEBOX_RING_BASE), 620 RING_TAIL(BLT_RING_BASE), 621 /* TODO: Other registers are not yet used */ 622 }; 623 624 static bool is_gen8_shadowed(struct drm_i915_private *dev_priv, u32 reg) 625 { 626 int i; 627 for (i = 0; i < ARRAY_SIZE(gen8_shadowed_regs); i++) 628 if (reg == gen8_shadowed_regs[i]) 629 return true; 630 631 return false; 632 } 633 634 #define __gen8_write(x) \ 635 static void \ 636 gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ 637 bool __needs_put = reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg); \ 638 REG_WRITE_HEADER; \ 639 if (__needs_put) { \ 640 dev_priv->uncore.funcs.force_wake_get(dev_priv, \ 641 FORCEWAKE_ALL); \ 642 } \ 643 __raw_i915_write##x(dev_priv, reg, val); \ 644 if (__needs_put) { \ 645 dev_priv->uncore.funcs.force_wake_put(dev_priv, \ 646 FORCEWAKE_ALL); \ 647 } \ 648 REG_WRITE_FOOTER; \ 649 } 650 651 __gen8_write(8) 652 __gen8_write(16) 653 __gen8_write(32) 654 __gen8_write(64) 655 __hsw_write(8) 656 __hsw_write(16) 657 __hsw_write(32) 658 __hsw_write(64) 659 __gen6_write(8) 660 __gen6_write(16) 661 __gen6_write(32) 662 __gen6_write(64) 663 __gen5_write(8) 664 __gen5_write(16) 665 __gen5_write(32) 666 __gen5_write(64) 667 __gen4_write(8) 668 __gen4_write(16) 669 __gen4_write(32) 670 __gen4_write(64) 671 672 #undef __gen8_write 673 #undef __hsw_write 674 #undef __gen6_write 675 #undef __gen5_write 676 #undef __gen4_write 677 #undef REG_WRITE_FOOTER 678 #undef REG_WRITE_HEADER 679 680 void intel_uncore_init(struct drm_device *dev) 681 { 682 struct drm_i915_private *dev_priv = dev->dev_private; 683 684 INIT_DELAYED_WORK(&dev_priv->uncore.force_wake_work, 685 gen6_force_wake_work); 686 687 if (IS_VALLEYVIEW(dev)) { 688 dev_priv->uncore.funcs.force_wake_get = __vlv_force_wake_get; 689 dev_priv->uncore.funcs.force_wake_put = __vlv_force_wake_put; 690 } else if (IS_HASWELL(dev) || IS_GEN8(dev)) { 691 dev_priv->uncore.funcs.force_wake_get = __gen6_gt_force_wake_mt_get; 692 dev_priv->uncore.funcs.force_wake_put = __gen6_gt_force_wake_mt_put; 693 } else if (IS_IVYBRIDGE(dev)) { 694 u32 ecobus; 695 696 /* IVB configs may use multi-threaded forcewake */ 697 698 /* A small trick here - if the bios hasn't configured 699 * MT forcewake, and if the device is in RC6, then 700 * force_wake_mt_get will not wake the device and the 701 * ECOBUS read will return zero. Which will be 702 * (correctly) interpreted by the test below as MT 703 * forcewake being disabled. 704 */ 705 mutex_lock(&dev->struct_mutex); 706 __gen6_gt_force_wake_mt_get(dev_priv, FORCEWAKE_ALL); 707 ecobus = __raw_i915_read32(dev_priv, ECOBUS); 708 __gen6_gt_force_wake_mt_put(dev_priv, FORCEWAKE_ALL); 709 mutex_unlock(&dev->struct_mutex); 710 711 if (ecobus & FORCEWAKE_MT_ENABLE) { 712 dev_priv->uncore.funcs.force_wake_get = 713 __gen6_gt_force_wake_mt_get; 714 dev_priv->uncore.funcs.force_wake_put = 715 __gen6_gt_force_wake_mt_put; 716 } else { 717 DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n"); 718 DRM_INFO("when using vblank-synced partial screen updates.\n"); 719 dev_priv->uncore.funcs.force_wake_get = 720 __gen6_gt_force_wake_get; 721 dev_priv->uncore.funcs.force_wake_put = 722 __gen6_gt_force_wake_put; 723 } 724 } else if (IS_GEN6(dev)) { 725 dev_priv->uncore.funcs.force_wake_get = 726 __gen6_gt_force_wake_get; 727 dev_priv->uncore.funcs.force_wake_put = 728 __gen6_gt_force_wake_put; 729 } 730 731 switch (INTEL_INFO(dev)->gen) { 732 default: 733 dev_priv->uncore.funcs.mmio_writeb = gen8_write8; 734 dev_priv->uncore.funcs.mmio_writew = gen8_write16; 735 dev_priv->uncore.funcs.mmio_writel = gen8_write32; 736 dev_priv->uncore.funcs.mmio_writeq = gen8_write64; 737 dev_priv->uncore.funcs.mmio_readb = gen6_read8; 738 dev_priv->uncore.funcs.mmio_readw = gen6_read16; 739 dev_priv->uncore.funcs.mmio_readl = gen6_read32; 740 dev_priv->uncore.funcs.mmio_readq = gen6_read64; 741 break; 742 case 7: 743 case 6: 744 if (IS_HASWELL(dev)) { 745 dev_priv->uncore.funcs.mmio_writeb = hsw_write8; 746 dev_priv->uncore.funcs.mmio_writew = hsw_write16; 747 dev_priv->uncore.funcs.mmio_writel = hsw_write32; 748 dev_priv->uncore.funcs.mmio_writeq = hsw_write64; 749 } else { 750 dev_priv->uncore.funcs.mmio_writeb = gen6_write8; 751 dev_priv->uncore.funcs.mmio_writew = gen6_write16; 752 dev_priv->uncore.funcs.mmio_writel = gen6_write32; 753 dev_priv->uncore.funcs.mmio_writeq = gen6_write64; 754 } 755 756 if (IS_VALLEYVIEW(dev)) { 757 dev_priv->uncore.funcs.mmio_readb = vlv_read8; 758 dev_priv->uncore.funcs.mmio_readw = vlv_read16; 759 dev_priv->uncore.funcs.mmio_readl = vlv_read32; 760 dev_priv->uncore.funcs.mmio_readq = vlv_read64; 761 } else { 762 dev_priv->uncore.funcs.mmio_readb = gen6_read8; 763 dev_priv->uncore.funcs.mmio_readw = gen6_read16; 764 dev_priv->uncore.funcs.mmio_readl = gen6_read32; 765 dev_priv->uncore.funcs.mmio_readq = gen6_read64; 766 } 767 break; 768 case 5: 769 dev_priv->uncore.funcs.mmio_writeb = gen5_write8; 770 dev_priv->uncore.funcs.mmio_writew = gen5_write16; 771 dev_priv->uncore.funcs.mmio_writel = gen5_write32; 772 dev_priv->uncore.funcs.mmio_writeq = gen5_write64; 773 dev_priv->uncore.funcs.mmio_readb = gen5_read8; 774 dev_priv->uncore.funcs.mmio_readw = gen5_read16; 775 dev_priv->uncore.funcs.mmio_readl = gen5_read32; 776 dev_priv->uncore.funcs.mmio_readq = gen5_read64; 777 break; 778 case 4: 779 case 3: 780 case 2: 781 dev_priv->uncore.funcs.mmio_writeb = gen4_write8; 782 dev_priv->uncore.funcs.mmio_writew = gen4_write16; 783 dev_priv->uncore.funcs.mmio_writel = gen4_write32; 784 dev_priv->uncore.funcs.mmio_writeq = gen4_write64; 785 dev_priv->uncore.funcs.mmio_readb = gen4_read8; 786 dev_priv->uncore.funcs.mmio_readw = gen4_read16; 787 dev_priv->uncore.funcs.mmio_readl = gen4_read32; 788 dev_priv->uncore.funcs.mmio_readq = gen4_read64; 789 break; 790 } 791 } 792 793 void intel_uncore_fini(struct drm_device *dev) 794 { 795 struct drm_i915_private *dev_priv = dev->dev_private; 796 797 flush_delayed_work(&dev_priv->uncore.force_wake_work); 798 799 /* Paranoia: make sure we have disabled everything before we exit. */ 800 intel_uncore_sanitize(dev); 801 } 802 803 static const struct register_whitelist { 804 uint64_t offset; 805 uint32_t size; 806 uint32_t gen_bitmask; /* support gens, 0x10 for 4, 0x30 for 4 and 5, etc. */ 807 } whitelist[] = { 808 { RING_TIMESTAMP(RENDER_RING_BASE), 8, 0x1F0 }, 809 }; 810 811 int i915_reg_read_ioctl(struct drm_device *dev, 812 void *data, struct drm_file *file) 813 { 814 struct drm_i915_private *dev_priv = dev->dev_private; 815 struct drm_i915_reg_read *reg = data; 816 struct register_whitelist const *entry = whitelist; 817 int i; 818 819 for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) { 820 if (entry->offset == reg->offset && 821 (1 << INTEL_INFO(dev)->gen & entry->gen_bitmask)) 822 break; 823 } 824 825 if (i == ARRAY_SIZE(whitelist)) 826 return -EINVAL; 827 828 switch (entry->size) { 829 case 8: 830 reg->val = I915_READ64(reg->offset); 831 break; 832 case 4: 833 reg->val = I915_READ(reg->offset); 834 break; 835 case 2: 836 reg->val = I915_READ16(reg->offset); 837 break; 838 case 1: 839 reg->val = I915_READ8(reg->offset); 840 break; 841 default: 842 WARN_ON(1); 843 return -EINVAL; 844 } 845 846 return 0; 847 } 848 849 int i915_get_reset_stats_ioctl(struct drm_device *dev, 850 void *data, struct drm_file *file) 851 { 852 struct drm_i915_private *dev_priv = dev->dev_private; 853 struct drm_i915_reset_stats *args = data; 854 struct i915_ctx_hang_stats *hs; 855 int ret; 856 857 if (args->flags || args->pad) 858 return -EINVAL; 859 860 if (args->ctx_id == DEFAULT_CONTEXT_ID && !capable(CAP_SYS_ADMIN)) 861 return -EPERM; 862 863 ret = mutex_lock_interruptible(&dev->struct_mutex); 864 if (ret) 865 return ret; 866 867 hs = i915_gem_context_get_hang_stats(dev, file, args->ctx_id); 868 if (IS_ERR(hs)) { 869 mutex_unlock(&dev->struct_mutex); 870 return PTR_ERR(hs); 871 } 872 873 if (capable(CAP_SYS_ADMIN)) 874 args->reset_count = i915_reset_count(&dev_priv->gpu_error); 875 else 876 args->reset_count = 0; 877 878 args->batch_active = hs->batch_active; 879 args->batch_pending = hs->batch_pending; 880 881 mutex_unlock(&dev->struct_mutex); 882 883 return 0; 884 } 885 886 static int i965_reset_complete(struct drm_device *dev) 887 { 888 u8 gdrst; 889 pci_read_config_byte(dev->pdev, I965_GDRST, &gdrst); 890 return (gdrst & GRDOM_RESET_ENABLE) == 0; 891 } 892 893 static int i965_do_reset(struct drm_device *dev) 894 { 895 int ret; 896 897 /* 898 * Set the domains we want to reset (GRDOM/bits 2 and 3) as 899 * well as the reset bit (GR/bit 0). Setting the GR bit 900 * triggers the reset; when done, the hardware will clear it. 901 */ 902 pci_write_config_byte(dev->pdev, I965_GDRST, 903 GRDOM_RENDER | GRDOM_RESET_ENABLE); 904 ret = wait_for(i965_reset_complete(dev), 500); 905 if (ret) 906 return ret; 907 908 /* We can't reset render&media without also resetting display ... */ 909 pci_write_config_byte(dev->pdev, I965_GDRST, 910 GRDOM_MEDIA | GRDOM_RESET_ENABLE); 911 912 ret = wait_for(i965_reset_complete(dev), 500); 913 if (ret) 914 return ret; 915 916 pci_write_config_byte(dev->pdev, I965_GDRST, 0); 917 918 return 0; 919 } 920 921 static int ironlake_do_reset(struct drm_device *dev) 922 { 923 struct drm_i915_private *dev_priv = dev->dev_private; 924 u32 gdrst; 925 int ret; 926 927 gdrst = I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR); 928 gdrst &= ~GRDOM_MASK; 929 I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR, 930 gdrst | GRDOM_RENDER | GRDOM_RESET_ENABLE); 931 ret = wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500); 932 if (ret) 933 return ret; 934 935 /* We can't reset render&media without also resetting display ... */ 936 gdrst = I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR); 937 gdrst &= ~GRDOM_MASK; 938 I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR, 939 gdrst | GRDOM_MEDIA | GRDOM_RESET_ENABLE); 940 return wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500); 941 } 942 943 static int gen6_do_reset(struct drm_device *dev) 944 { 945 struct drm_i915_private *dev_priv = dev->dev_private; 946 int ret; 947 unsigned long irqflags; 948 949 /* Hold uncore.lock across reset to prevent any register access 950 * with forcewake not set correctly 951 */ 952 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 953 954 /* Reset the chip */ 955 956 /* GEN6_GDRST is not in the gt power well, no need to check 957 * for fifo space for the write or forcewake the chip for 958 * the read 959 */ 960 __raw_i915_write32(dev_priv, GEN6_GDRST, GEN6_GRDOM_FULL); 961 962 /* Spin waiting for the device to ack the reset request */ 963 ret = wait_for((__raw_i915_read32(dev_priv, GEN6_GDRST) & GEN6_GRDOM_FULL) == 0, 500); 964 965 intel_uncore_forcewake_reset(dev); 966 967 /* If reset with a user forcewake, try to restore, otherwise turn it off */ 968 if (dev_priv->uncore.forcewake_count) 969 dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL); 970 else 971 dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL); 972 973 /* Restore fifo count */ 974 dev_priv->uncore.fifo_count = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK; 975 976 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 977 return ret; 978 } 979 980 int intel_gpu_reset(struct drm_device *dev) 981 { 982 switch (INTEL_INFO(dev)->gen) { 983 case 8: 984 case 7: 985 case 6: return gen6_do_reset(dev); 986 case 5: return ironlake_do_reset(dev); 987 case 4: return i965_do_reset(dev); 988 default: return -ENODEV; 989 } 990 } 991 992 void intel_uncore_check_errors(struct drm_device *dev) 993 { 994 struct drm_i915_private *dev_priv = dev->dev_private; 995 996 if (HAS_FPGA_DBG_UNCLAIMED(dev) && 997 (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM)) { 998 DRM_ERROR("Unclaimed register before interrupt\n"); 999 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); 1000 } 1001 } 1002