1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2020 Intel Corporation 4 */ 5 6 #include <linux/kernel.h> 7 8 #include <drm/drm_print.h> 9 10 #include "i915_drv.h" 11 #include "i915_reg.h" 12 #include "i915_trace.h" 13 #include "i915_utils.h" 14 #include "intel_pm.h" 15 #include "vlv_suspend.h" 16 17 struct vlv_s0ix_state { 18 /* GAM */ 19 u32 wr_watermark; 20 u32 gfx_prio_ctrl; 21 u32 arb_mode; 22 u32 gfx_pend_tlb0; 23 u32 gfx_pend_tlb1; 24 u32 lra_limits[GEN7_LRA_LIMITS_REG_NUM]; 25 u32 media_max_req_count; 26 u32 gfx_max_req_count; 27 u32 render_hwsp; 28 u32 ecochk; 29 u32 bsd_hwsp; 30 u32 blt_hwsp; 31 u32 tlb_rd_addr; 32 33 /* MBC */ 34 u32 g3dctl; 35 u32 gsckgctl; 36 u32 mbctl; 37 38 /* GCP */ 39 u32 ucgctl1; 40 u32 ucgctl3; 41 u32 rcgctl1; 42 u32 rcgctl2; 43 u32 rstctl; 44 u32 misccpctl; 45 46 /* GPM */ 47 u32 gfxpause; 48 u32 rpdeuhwtc; 49 u32 rpdeuc; 50 u32 ecobus; 51 u32 pwrdwnupctl; 52 u32 rp_down_timeout; 53 u32 rp_deucsw; 54 u32 rcubmabdtmr; 55 u32 rcedata; 56 u32 spare2gh; 57 58 /* Display 1 CZ domain */ 59 u32 gt_imr; 60 u32 gt_ier; 61 u32 pm_imr; 62 u32 pm_ier; 63 u32 gt_scratch[GEN7_GT_SCRATCH_REG_NUM]; 64 65 /* GT SA CZ domain */ 66 u32 tilectl; 67 u32 gt_fifoctl; 68 u32 gtlc_wake_ctrl; 69 u32 gtlc_survive; 70 u32 pmwgicz; 71 72 /* Display 2 CZ domain */ 73 u32 gu_ctl0; 74 u32 gu_ctl1; 75 u32 pcbr; 76 u32 clock_gate_dis2; 77 }; 78 79 /* 80 * Save all Gunit registers that may be lost after a D3 and a subsequent 81 * S0i[R123] transition. The list of registers needing a save/restore is 82 * defined in the VLV2_S0IXRegs document. This documents marks all Gunit 83 * registers in the following way: 84 * - Driver: saved/restored by the driver 85 * - Punit : saved/restored by the Punit firmware 86 * - No, w/o marking: no need to save/restore, since the register is R/O or 87 * used internally by the HW in a way that doesn't depend 88 * keeping the content across a suspend/resume. 89 * - Debug : used for debugging 90 * 91 * We save/restore all registers marked with 'Driver', with the following 92 * exceptions: 93 * - Registers out of use, including also registers marked with 'Debug'. 94 * These have no effect on the driver's operation, so we don't save/restore 95 * them to reduce the overhead. 96 * - Registers that are fully setup by an initialization function called from 97 * the resume path. For example many clock gating and RPS/RC6 registers. 98 * - Registers that provide the right functionality with their reset defaults. 99 * 100 * TODO: Except for registers that based on the above 3 criteria can be safely 101 * ignored, we save/restore all others, practically treating the HW context as 102 * a black-box for the driver. Further investigation is needed to reduce the 103 * saved/restored registers even further, by following the same 3 criteria. 104 */ 105 static void vlv_save_gunit_s0ix_state(struct drm_i915_private *i915) 106 { 107 struct vlv_s0ix_state *s = i915->vlv_s0ix_state; 108 struct intel_uncore *uncore = &i915->uncore; 109 int i; 110 111 if (!s) 112 return; 113 114 /* GAM 0x4000-0x4770 */ 115 s->wr_watermark = intel_uncore_read(uncore, GEN7_WR_WATERMARK); 116 s->gfx_prio_ctrl = intel_uncore_read(uncore, GEN7_GFX_PRIO_CTRL); 117 s->arb_mode = intel_uncore_read(uncore, ARB_MODE); 118 s->gfx_pend_tlb0 = intel_uncore_read(uncore, GEN7_GFX_PEND_TLB0); 119 s->gfx_pend_tlb1 = intel_uncore_read(uncore, GEN7_GFX_PEND_TLB1); 120 121 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) 122 s->lra_limits[i] = intel_uncore_read(uncore, GEN7_LRA_LIMITS(i)); 123 124 s->media_max_req_count = intel_uncore_read(uncore, GEN7_MEDIA_MAX_REQ_COUNT); 125 s->gfx_max_req_count = intel_uncore_read(uncore, GEN7_GFX_MAX_REQ_COUNT); 126 127 s->render_hwsp = intel_uncore_read(uncore, RENDER_HWS_PGA_GEN7); 128 s->ecochk = intel_uncore_read(uncore, GAM_ECOCHK); 129 s->bsd_hwsp = intel_uncore_read(uncore, BSD_HWS_PGA_GEN7); 130 s->blt_hwsp = intel_uncore_read(uncore, BLT_HWS_PGA_GEN7); 131 132 s->tlb_rd_addr = intel_uncore_read(uncore, GEN7_TLB_RD_ADDR); 133 134 /* MBC 0x9024-0x91D0, 0x8500 */ 135 s->g3dctl = intel_uncore_read(uncore, VLV_G3DCTL); 136 s->gsckgctl = intel_uncore_read(uncore, VLV_GSCKGCTL); 137 s->mbctl = intel_uncore_read(uncore, GEN6_MBCTL); 138 139 /* GCP 0x9400-0x9424, 0x8100-0x810C */ 140 s->ucgctl1 = intel_uncore_read(uncore, GEN6_UCGCTL1); 141 s->ucgctl3 = intel_uncore_read(uncore, GEN6_UCGCTL3); 142 s->rcgctl1 = intel_uncore_read(uncore, GEN6_RCGCTL1); 143 s->rcgctl2 = intel_uncore_read(uncore, GEN6_RCGCTL2); 144 s->rstctl = intel_uncore_read(uncore, GEN6_RSTCTL); 145 s->misccpctl = intel_uncore_read(uncore, GEN7_MISCCPCTL); 146 147 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ 148 s->gfxpause = intel_uncore_read(uncore, GEN6_GFXPAUSE); 149 s->rpdeuhwtc = intel_uncore_read(uncore, GEN6_RPDEUHWTC); 150 s->rpdeuc = intel_uncore_read(uncore, GEN6_RPDEUC); 151 s->ecobus = intel_uncore_read(uncore, ECOBUS); 152 s->pwrdwnupctl = intel_uncore_read(uncore, VLV_PWRDWNUPCTL); 153 s->rp_down_timeout = intel_uncore_read(uncore, GEN6_RP_DOWN_TIMEOUT); 154 s->rp_deucsw = intel_uncore_read(uncore, GEN6_RPDEUCSW); 155 s->rcubmabdtmr = intel_uncore_read(uncore, GEN6_RCUBMABDTMR); 156 s->rcedata = intel_uncore_read(uncore, VLV_RCEDATA); 157 s->spare2gh = intel_uncore_read(uncore, VLV_SPAREG2H); 158 159 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ 160 s->gt_imr = intel_uncore_read(uncore, GTIMR); 161 s->gt_ier = intel_uncore_read(uncore, GTIER); 162 s->pm_imr = intel_uncore_read(uncore, GEN6_PMIMR); 163 s->pm_ier = intel_uncore_read(uncore, GEN6_PMIER); 164 165 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) 166 s->gt_scratch[i] = intel_uncore_read(uncore, GEN7_GT_SCRATCH(i)); 167 168 /* GT SA CZ domain, 0x100000-0x138124 */ 169 s->tilectl = intel_uncore_read(uncore, TILECTL); 170 s->gt_fifoctl = intel_uncore_read(uncore, GTFIFOCTL); 171 s->gtlc_wake_ctrl = intel_uncore_read(uncore, VLV_GTLC_WAKE_CTRL); 172 s->gtlc_survive = intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG); 173 s->pmwgicz = intel_uncore_read(uncore, VLV_PMWGICZ); 174 175 /* Gunit-Display CZ domain, 0x182028-0x1821CF */ 176 s->gu_ctl0 = intel_uncore_read(uncore, VLV_GU_CTL0); 177 s->gu_ctl1 = intel_uncore_read(uncore, VLV_GU_CTL1); 178 s->pcbr = intel_uncore_read(uncore, VLV_PCBR); 179 s->clock_gate_dis2 = intel_uncore_read(uncore, VLV_GUNIT_CLOCK_GATE2); 180 181 /* 182 * Not saving any of: 183 * DFT, 0x9800-0x9EC0 184 * SARB, 0xB000-0xB1FC 185 * GAC, 0x5208-0x524C, 0x14000-0x14C000 186 * PCI CFG 187 */ 188 } 189 190 static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *i915) 191 { 192 struct vlv_s0ix_state *s = i915->vlv_s0ix_state; 193 struct intel_uncore *uncore = &i915->uncore; 194 u32 val; 195 int i; 196 197 if (!s) 198 return; 199 200 /* GAM 0x4000-0x4770 */ 201 intel_uncore_write(uncore, GEN7_WR_WATERMARK, s->wr_watermark); 202 intel_uncore_write(uncore, GEN7_GFX_PRIO_CTRL, s->gfx_prio_ctrl); 203 intel_uncore_write(uncore, ARB_MODE, s->arb_mode | (0xffff << 16)); 204 intel_uncore_write(uncore, GEN7_GFX_PEND_TLB0, s->gfx_pend_tlb0); 205 intel_uncore_write(uncore, GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1); 206 207 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) 208 intel_uncore_write(uncore, GEN7_LRA_LIMITS(i), s->lra_limits[i]); 209 210 intel_uncore_write(uncore, GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count); 211 intel_uncore_write(uncore, GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count); 212 213 intel_uncore_write(uncore, RENDER_HWS_PGA_GEN7, s->render_hwsp); 214 intel_uncore_write(uncore, GAM_ECOCHK, s->ecochk); 215 intel_uncore_write(uncore, BSD_HWS_PGA_GEN7, s->bsd_hwsp); 216 intel_uncore_write(uncore, BLT_HWS_PGA_GEN7, s->blt_hwsp); 217 218 intel_uncore_write(uncore, GEN7_TLB_RD_ADDR, s->tlb_rd_addr); 219 220 /* MBC 0x9024-0x91D0, 0x8500 */ 221 intel_uncore_write(uncore, VLV_G3DCTL, s->g3dctl); 222 intel_uncore_write(uncore, VLV_GSCKGCTL, s->gsckgctl); 223 intel_uncore_write(uncore, GEN6_MBCTL, s->mbctl); 224 225 /* GCP 0x9400-0x9424, 0x8100-0x810C */ 226 intel_uncore_write(uncore, GEN6_UCGCTL1, s->ucgctl1); 227 intel_uncore_write(uncore, GEN6_UCGCTL3, s->ucgctl3); 228 intel_uncore_write(uncore, GEN6_RCGCTL1, s->rcgctl1); 229 intel_uncore_write(uncore, GEN6_RCGCTL2, s->rcgctl2); 230 intel_uncore_write(uncore, GEN6_RSTCTL, s->rstctl); 231 intel_uncore_write(uncore, GEN7_MISCCPCTL, s->misccpctl); 232 233 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ 234 intel_uncore_write(uncore, GEN6_GFXPAUSE, s->gfxpause); 235 intel_uncore_write(uncore, GEN6_RPDEUHWTC, s->rpdeuhwtc); 236 intel_uncore_write(uncore, GEN6_RPDEUC, s->rpdeuc); 237 intel_uncore_write(uncore, ECOBUS, s->ecobus); 238 intel_uncore_write(uncore, VLV_PWRDWNUPCTL, s->pwrdwnupctl); 239 intel_uncore_write(uncore, GEN6_RP_DOWN_TIMEOUT, s->rp_down_timeout); 240 intel_uncore_write(uncore, GEN6_RPDEUCSW, s->rp_deucsw); 241 intel_uncore_write(uncore, GEN6_RCUBMABDTMR, s->rcubmabdtmr); 242 intel_uncore_write(uncore, VLV_RCEDATA, s->rcedata); 243 intel_uncore_write(uncore, VLV_SPAREG2H, s->spare2gh); 244 245 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ 246 intel_uncore_write(uncore, GTIMR, s->gt_imr); 247 intel_uncore_write(uncore, GTIER, s->gt_ier); 248 intel_uncore_write(uncore, GEN6_PMIMR, s->pm_imr); 249 intel_uncore_write(uncore, GEN6_PMIER, s->pm_ier); 250 251 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) 252 intel_uncore_write(uncore, GEN7_GT_SCRATCH(i), s->gt_scratch[i]); 253 254 /* GT SA CZ domain, 0x100000-0x138124 */ 255 intel_uncore_write(uncore, TILECTL, s->tilectl); 256 intel_uncore_write(uncore, GTFIFOCTL, s->gt_fifoctl); 257 /* 258 * Preserve the GT allow wake and GFX force clock bit, they are not 259 * be restored, as they are used to control the s0ix suspend/resume 260 * sequence by the caller. 261 */ 262 val = intel_uncore_read(uncore, VLV_GTLC_WAKE_CTRL); 263 val &= VLV_GTLC_ALLOWWAKEREQ; 264 val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ; 265 intel_uncore_write(uncore, VLV_GTLC_WAKE_CTRL, val); 266 267 val = intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG); 268 val &= VLV_GFX_CLK_FORCE_ON_BIT; 269 val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT; 270 intel_uncore_write(uncore, VLV_GTLC_SURVIVABILITY_REG, val); 271 272 intel_uncore_write(uncore, VLV_PMWGICZ, s->pmwgicz); 273 274 /* Gunit-Display CZ domain, 0x182028-0x1821CF */ 275 intel_uncore_write(uncore, VLV_GU_CTL0, s->gu_ctl0); 276 intel_uncore_write(uncore, VLV_GU_CTL1, s->gu_ctl1); 277 intel_uncore_write(uncore, VLV_PCBR, s->pcbr); 278 intel_uncore_write(uncore, VLV_GUNIT_CLOCK_GATE2, s->clock_gate_dis2); 279 } 280 281 static int vlv_wait_for_pw_status(struct drm_i915_private *i915, 282 u32 mask, u32 val) 283 { 284 i915_reg_t reg = VLV_GTLC_PW_STATUS; 285 u32 reg_value; 286 int ret; 287 288 /* The HW does not like us polling for PW_STATUS frequently, so 289 * use the sleeping loop rather than risk the busy spin within 290 * intel_wait_for_register(). 291 * 292 * Transitioning between RC6 states should be at most 2ms (see 293 * valleyview_enable_rps) so use a 3ms timeout. 294 */ 295 ret = wait_for(((reg_value = 296 intel_uncore_read_notrace(&i915->uncore, reg)) & mask) 297 == val, 3); 298 299 /* just trace the final value */ 300 trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true); 301 302 return ret; 303 } 304 305 static int vlv_force_gfx_clock(struct drm_i915_private *i915, bool force_on) 306 { 307 struct intel_uncore *uncore = &i915->uncore; 308 u32 val; 309 int err; 310 311 val = intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG); 312 val &= ~VLV_GFX_CLK_FORCE_ON_BIT; 313 if (force_on) 314 val |= VLV_GFX_CLK_FORCE_ON_BIT; 315 intel_uncore_write(uncore, VLV_GTLC_SURVIVABILITY_REG, val); 316 317 if (!force_on) 318 return 0; 319 320 err = intel_wait_for_register(uncore, 321 VLV_GTLC_SURVIVABILITY_REG, 322 VLV_GFX_CLK_STATUS_BIT, 323 VLV_GFX_CLK_STATUS_BIT, 324 20); 325 if (err) 326 drm_err(&i915->drm, 327 "timeout waiting for GFX clock force-on (%08x)\n", 328 intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG)); 329 330 return err; 331 } 332 333 static int vlv_allow_gt_wake(struct drm_i915_private *i915, bool allow) 334 { 335 struct intel_uncore *uncore = &i915->uncore; 336 u32 mask; 337 u32 val; 338 int err; 339 340 val = intel_uncore_read(uncore, VLV_GTLC_WAKE_CTRL); 341 val &= ~VLV_GTLC_ALLOWWAKEREQ; 342 if (allow) 343 val |= VLV_GTLC_ALLOWWAKEREQ; 344 intel_uncore_write(uncore, VLV_GTLC_WAKE_CTRL, val); 345 intel_uncore_posting_read(uncore, VLV_GTLC_WAKE_CTRL); 346 347 mask = VLV_GTLC_ALLOWWAKEACK; 348 val = allow ? mask : 0; 349 350 err = vlv_wait_for_pw_status(i915, mask, val); 351 if (err) 352 drm_err(&i915->drm, "timeout disabling GT waking\n"); 353 354 return err; 355 } 356 357 static void vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv, 358 bool wait_for_on) 359 { 360 u32 mask; 361 u32 val; 362 363 mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK; 364 val = wait_for_on ? mask : 0; 365 366 /* 367 * RC6 transitioning can be delayed up to 2 msec (see 368 * valleyview_enable_rps), use 3 msec for safety. 369 * 370 * This can fail to turn off the rc6 if the GPU is stuck after a failed 371 * reset and we are trying to force the machine to sleep. 372 */ 373 if (vlv_wait_for_pw_status(dev_priv, mask, val)) 374 drm_dbg(&dev_priv->drm, 375 "timeout waiting for GT wells to go %s\n", 376 onoff(wait_for_on)); 377 } 378 379 static void vlv_check_no_gt_access(struct drm_i915_private *i915) 380 { 381 struct intel_uncore *uncore = &i915->uncore; 382 383 if (!(intel_uncore_read(uncore, VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR)) 384 return; 385 386 drm_dbg(&i915->drm, "GT register access while GT waking disabled\n"); 387 intel_uncore_write(uncore, VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR); 388 } 389 390 int vlv_suspend_complete(struct drm_i915_private *dev_priv) 391 { 392 u32 mask; 393 int err; 394 395 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) 396 return 0; 397 398 /* 399 * Bspec defines the following GT well on flags as debug only, so 400 * don't treat them as hard failures. 401 */ 402 vlv_wait_for_gt_wells(dev_priv, false); 403 404 mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS; 405 drm_WARN_ON(&dev_priv->drm, 406 (intel_uncore_read(&dev_priv->uncore, VLV_GTLC_WAKE_CTRL) & mask) != mask); 407 408 vlv_check_no_gt_access(dev_priv); 409 410 err = vlv_force_gfx_clock(dev_priv, true); 411 if (err) 412 goto err1; 413 414 err = vlv_allow_gt_wake(dev_priv, false); 415 if (err) 416 goto err2; 417 418 vlv_save_gunit_s0ix_state(dev_priv); 419 420 err = vlv_force_gfx_clock(dev_priv, false); 421 if (err) 422 goto err2; 423 424 return 0; 425 426 err2: 427 /* For safety always re-enable waking and disable gfx clock forcing */ 428 vlv_allow_gt_wake(dev_priv, true); 429 err1: 430 vlv_force_gfx_clock(dev_priv, false); 431 432 return err; 433 } 434 435 int vlv_resume_prepare(struct drm_i915_private *dev_priv, bool rpm_resume) 436 { 437 int err; 438 int ret; 439 440 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) 441 return 0; 442 443 /* 444 * If any of the steps fail just try to continue, that's the best we 445 * can do at this point. Return the first error code (which will also 446 * leave RPM permanently disabled). 447 */ 448 ret = vlv_force_gfx_clock(dev_priv, true); 449 450 vlv_restore_gunit_s0ix_state(dev_priv); 451 452 err = vlv_allow_gt_wake(dev_priv, true); 453 if (!ret) 454 ret = err; 455 456 err = vlv_force_gfx_clock(dev_priv, false); 457 if (!ret) 458 ret = err; 459 460 vlv_check_no_gt_access(dev_priv); 461 462 if (rpm_resume) 463 intel_init_clock_gating(dev_priv); 464 465 return ret; 466 } 467 468 int vlv_suspend_init(struct drm_i915_private *i915) 469 { 470 if (!IS_VALLEYVIEW(i915)) 471 return 0; 472 473 /* we write all the values in the struct, so no need to zero it out */ 474 i915->vlv_s0ix_state = kmalloc(sizeof(*i915->vlv_s0ix_state), 475 GFP_KERNEL); 476 if (!i915->vlv_s0ix_state) 477 return -ENOMEM; 478 479 return 0; 480 } 481 482 void vlv_suspend_cleanup(struct drm_i915_private *i915) 483 { 484 if (!i915->vlv_s0ix_state) 485 return; 486 487 kfree(i915->vlv_s0ix_state); 488 i915->vlv_s0ix_state = NULL; 489 } 490