1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2023 Intel Corporation 4 */ 5 6 #include "i915_drv.h" 7 #include "i9xx_wm.h" 8 #include "intel_atomic.h" 9 #include "intel_display.h" 10 #include "intel_display_trace.h" 11 #include "intel_mchbar_regs.h" 12 #include "intel_wm.h" 13 #include "skl_watermark.h" 14 #include "vlv_sideband.h" 15 16 /* used in computing the new watermarks state */ 17 struct intel_wm_config { 18 unsigned int num_pipes_active; 19 bool sprites_enabled; 20 bool sprites_scaled; 21 }; 22 23 struct cxsr_latency { 24 bool is_desktop : 1; 25 bool is_ddr3 : 1; 26 u16 fsb_freq; 27 u16 mem_freq; 28 u16 display_sr; 29 u16 display_hpll_disable; 30 u16 cursor_sr; 31 u16 cursor_hpll_disable; 32 }; 33 34 static const struct cxsr_latency cxsr_latency_table[] = { 35 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */ 36 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */ 37 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */ 38 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */ 39 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */ 40 41 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */ 42 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */ 43 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */ 44 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */ 45 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */ 46 47 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */ 48 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */ 49 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */ 50 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */ 51 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */ 52 53 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */ 54 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */ 55 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */ 56 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */ 57 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */ 58 59 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */ 60 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */ 61 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */ 62 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */ 63 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */ 64 65 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */ 66 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */ 67 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */ 68 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */ 69 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */ 70 }; 71 72 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop, 73 bool is_ddr3, 74 int fsb, 75 int mem) 76 { 77 const struct cxsr_latency *latency; 78 int i; 79 80 if (fsb == 0 || mem == 0) 81 return NULL; 82 83 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) { 84 latency = &cxsr_latency_table[i]; 85 if (is_desktop == latency->is_desktop && 86 is_ddr3 == latency->is_ddr3 && 87 fsb == latency->fsb_freq && mem == latency->mem_freq) 88 return latency; 89 } 90 91 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n"); 92 93 return NULL; 94 } 95 96 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable) 97 { 98 u32 val; 99 100 vlv_punit_get(dev_priv); 101 102 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2); 103 if (enable) 104 val &= ~FORCE_DDR_HIGH_FREQ; 105 else 106 val |= FORCE_DDR_HIGH_FREQ; 107 val &= ~FORCE_DDR_LOW_FREQ; 108 val |= FORCE_DDR_FREQ_REQ_ACK; 109 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val); 110 111 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) & 112 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) 113 drm_err(&dev_priv->drm, 114 "timed out waiting for Punit DDR DVFS request\n"); 115 116 vlv_punit_put(dev_priv); 117 } 118 119 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable) 120 { 121 u32 val; 122 123 vlv_punit_get(dev_priv); 124 125 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM); 126 if (enable) 127 val |= DSP_MAXFIFO_PM5_ENABLE; 128 else 129 val &= ~DSP_MAXFIFO_PM5_ENABLE; 130 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val); 131 132 vlv_punit_put(dev_priv); 133 } 134 135 #define FW_WM(value, plane) \ 136 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK) 137 138 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable) 139 { 140 bool was_enabled; 141 u32 val; 142 143 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { 144 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN; 145 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0); 146 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV); 147 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) { 148 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN; 149 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0); 150 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF); 151 } else if (IS_PINEVIEW(dev_priv)) { 152 val = intel_uncore_read(&dev_priv->uncore, DSPFW3); 153 was_enabled = val & PINEVIEW_SELF_REFRESH_EN; 154 if (enable) 155 val |= PINEVIEW_SELF_REFRESH_EN; 156 else 157 val &= ~PINEVIEW_SELF_REFRESH_EN; 158 intel_uncore_write(&dev_priv->uncore, DSPFW3, val); 159 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3); 160 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) { 161 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN; 162 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) : 163 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN); 164 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val); 165 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF); 166 } else if (IS_I915GM(dev_priv)) { 167 /* 168 * FIXME can't find a bit like this for 915G, and 169 * yet it does have the related watermark in 170 * FW_BLC_SELF. What's going on? 171 */ 172 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN; 173 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) : 174 _MASKED_BIT_DISABLE(INSTPM_SELF_EN); 175 intel_uncore_write(&dev_priv->uncore, INSTPM, val); 176 intel_uncore_posting_read(&dev_priv->uncore, INSTPM); 177 } else { 178 return false; 179 } 180 181 trace_intel_memory_cxsr(dev_priv, was_enabled, enable); 182 183 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n", 184 str_enabled_disabled(enable), 185 str_enabled_disabled(was_enabled)); 186 187 return was_enabled; 188 } 189 190 /** 191 * intel_set_memory_cxsr - Configure CxSR state 192 * @dev_priv: i915 device 193 * @enable: Allow vs. disallow CxSR 194 * 195 * Allow or disallow the system to enter a special CxSR 196 * (C-state self refresh) state. What typically happens in CxSR mode 197 * is that several display FIFOs may get combined into a single larger 198 * FIFO for a particular plane (so called max FIFO mode) to allow the 199 * system to defer memory fetches longer, and the memory will enter 200 * self refresh. 201 * 202 * Note that enabling CxSR does not guarantee that the system enter 203 * this special mode, nor does it guarantee that the system stays 204 * in that mode once entered. So this just allows/disallows the system 205 * to autonomously utilize the CxSR mode. Other factors such as core 206 * C-states will affect when/if the system actually enters/exits the 207 * CxSR mode. 208 * 209 * Note that on VLV/CHV this actually only controls the max FIFO mode, 210 * and the system is free to enter/exit memory self refresh at any time 211 * even when the use of CxSR has been disallowed. 212 * 213 * While the system is actually in the CxSR/max FIFO mode, some plane 214 * control registers will not get latched on vblank. Thus in order to 215 * guarantee the system will respond to changes in the plane registers 216 * we must always disallow CxSR prior to making changes to those registers. 217 * Unfortunately the system will re-evaluate the CxSR conditions at 218 * frame start which happens after vblank start (which is when the plane 219 * registers would get latched), so we can't proceed with the plane update 220 * during the same frame where we disallowed CxSR. 221 * 222 * Certain platforms also have a deeper HPLL SR mode. Fortunately the 223 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold 224 * the hardware w.r.t. HPLL SR when writing to plane registers. 225 * Disallowing just CxSR is sufficient. 226 */ 227 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable) 228 { 229 bool ret; 230 231 mutex_lock(&dev_priv->display.wm.wm_mutex); 232 ret = _intel_set_memory_cxsr(dev_priv, enable); 233 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) 234 dev_priv->display.wm.vlv.cxsr = enable; 235 else if (IS_G4X(dev_priv)) 236 dev_priv->display.wm.g4x.cxsr = enable; 237 mutex_unlock(&dev_priv->display.wm.wm_mutex); 238 239 return ret; 240 } 241 242 /* 243 * Latency for FIFO fetches is dependent on several factors: 244 * - memory configuration (speed, channels) 245 * - chipset 246 * - current MCH state 247 * It can be fairly high in some situations, so here we assume a fairly 248 * pessimal value. It's a tradeoff between extra memory fetches (if we 249 * set this value too high, the FIFO will fetch frequently to stay full) 250 * and power consumption (set it too low to save power and we might see 251 * FIFO underruns and display "flicker"). 252 * 253 * A value of 5us seems to be a good balance; safe for very low end 254 * platforms but not overly aggressive on lower latency configs. 255 */ 256 static const int pessimal_latency_ns = 5000; 257 258 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \ 259 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8)) 260 261 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state) 262 { 263 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 264 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 265 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state; 266 enum pipe pipe = crtc->pipe; 267 int sprite0_start, sprite1_start; 268 u32 dsparb, dsparb2, dsparb3; 269 270 switch (pipe) { 271 case PIPE_A: 272 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB); 273 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2); 274 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0); 275 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4); 276 break; 277 case PIPE_B: 278 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB); 279 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2); 280 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8); 281 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12); 282 break; 283 case PIPE_C: 284 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2); 285 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3); 286 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16); 287 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20); 288 break; 289 default: 290 MISSING_CASE(pipe); 291 return; 292 } 293 294 fifo_state->plane[PLANE_PRIMARY] = sprite0_start; 295 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start; 296 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start; 297 fifo_state->plane[PLANE_CURSOR] = 63; 298 } 299 300 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv, 301 enum i9xx_plane_id i9xx_plane) 302 { 303 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB); 304 int size; 305 306 size = dsparb & 0x7f; 307 if (i9xx_plane == PLANE_B) 308 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size; 309 310 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n", 311 dsparb, plane_name(i9xx_plane), size); 312 313 return size; 314 } 315 316 static int i830_get_fifo_size(struct drm_i915_private *dev_priv, 317 enum i9xx_plane_id i9xx_plane) 318 { 319 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB); 320 int size; 321 322 size = dsparb & 0x1ff; 323 if (i9xx_plane == PLANE_B) 324 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size; 325 size >>= 1; /* Convert to cachelines */ 326 327 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n", 328 dsparb, plane_name(i9xx_plane), size); 329 330 return size; 331 } 332 333 static int i845_get_fifo_size(struct drm_i915_private *dev_priv, 334 enum i9xx_plane_id i9xx_plane) 335 { 336 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB); 337 int size; 338 339 size = dsparb & 0x7f; 340 size >>= 2; /* Convert to cachelines */ 341 342 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n", 343 dsparb, plane_name(i9xx_plane), size); 344 345 return size; 346 } 347 348 /* Pineview has different values for various configs */ 349 static const struct intel_watermark_params pnv_display_wm = { 350 .fifo_size = PINEVIEW_DISPLAY_FIFO, 351 .max_wm = PINEVIEW_MAX_WM, 352 .default_wm = PINEVIEW_DFT_WM, 353 .guard_size = PINEVIEW_GUARD_WM, 354 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE, 355 }; 356 357 static const struct intel_watermark_params pnv_display_hplloff_wm = { 358 .fifo_size = PINEVIEW_DISPLAY_FIFO, 359 .max_wm = PINEVIEW_MAX_WM, 360 .default_wm = PINEVIEW_DFT_HPLLOFF_WM, 361 .guard_size = PINEVIEW_GUARD_WM, 362 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE, 363 }; 364 365 static const struct intel_watermark_params pnv_cursor_wm = { 366 .fifo_size = PINEVIEW_CURSOR_FIFO, 367 .max_wm = PINEVIEW_CURSOR_MAX_WM, 368 .default_wm = PINEVIEW_CURSOR_DFT_WM, 369 .guard_size = PINEVIEW_CURSOR_GUARD_WM, 370 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE, 371 }; 372 373 static const struct intel_watermark_params pnv_cursor_hplloff_wm = { 374 .fifo_size = PINEVIEW_CURSOR_FIFO, 375 .max_wm = PINEVIEW_CURSOR_MAX_WM, 376 .default_wm = PINEVIEW_CURSOR_DFT_WM, 377 .guard_size = PINEVIEW_CURSOR_GUARD_WM, 378 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE, 379 }; 380 381 static const struct intel_watermark_params i965_cursor_wm_info = { 382 .fifo_size = I965_CURSOR_FIFO, 383 .max_wm = I965_CURSOR_MAX_WM, 384 .default_wm = I965_CURSOR_DFT_WM, 385 .guard_size = 2, 386 .cacheline_size = I915_FIFO_LINE_SIZE, 387 }; 388 389 static const struct intel_watermark_params i945_wm_info = { 390 .fifo_size = I945_FIFO_SIZE, 391 .max_wm = I915_MAX_WM, 392 .default_wm = 1, 393 .guard_size = 2, 394 .cacheline_size = I915_FIFO_LINE_SIZE, 395 }; 396 397 static const struct intel_watermark_params i915_wm_info = { 398 .fifo_size = I915_FIFO_SIZE, 399 .max_wm = I915_MAX_WM, 400 .default_wm = 1, 401 .guard_size = 2, 402 .cacheline_size = I915_FIFO_LINE_SIZE, 403 }; 404 405 static const struct intel_watermark_params i830_a_wm_info = { 406 .fifo_size = I855GM_FIFO_SIZE, 407 .max_wm = I915_MAX_WM, 408 .default_wm = 1, 409 .guard_size = 2, 410 .cacheline_size = I830_FIFO_LINE_SIZE, 411 }; 412 413 static const struct intel_watermark_params i830_bc_wm_info = { 414 .fifo_size = I855GM_FIFO_SIZE, 415 .max_wm = I915_MAX_WM / 2, 416 .default_wm = 1, 417 .guard_size = 2, 418 .cacheline_size = I830_FIFO_LINE_SIZE, 419 }; 420 421 static const struct intel_watermark_params i845_wm_info = { 422 .fifo_size = I830_FIFO_SIZE, 423 .max_wm = I915_MAX_WM, 424 .default_wm = 1, 425 .guard_size = 2, 426 .cacheline_size = I830_FIFO_LINE_SIZE, 427 }; 428 429 /** 430 * intel_wm_method1 - Method 1 / "small buffer" watermark formula 431 * @pixel_rate: Pipe pixel rate in kHz 432 * @cpp: Plane bytes per pixel 433 * @latency: Memory wakeup latency in 0.1us units 434 * 435 * Compute the watermark using the method 1 or "small buffer" 436 * formula. The caller may additonally add extra cachelines 437 * to account for TLB misses and clock crossings. 438 * 439 * This method is concerned with the short term drain rate 440 * of the FIFO, ie. it does not account for blanking periods 441 * which would effectively reduce the average drain rate across 442 * a longer period. The name "small" refers to the fact the 443 * FIFO is relatively small compared to the amount of data 444 * fetched. 445 * 446 * The FIFO level vs. time graph might look something like: 447 * 448 * |\ |\ 449 * | \ | \ 450 * __---__---__ (- plane active, _ blanking) 451 * -> time 452 * 453 * or perhaps like this: 454 * 455 * |\|\ |\|\ 456 * __----__----__ (- plane active, _ blanking) 457 * -> time 458 * 459 * Returns: 460 * The watermark in bytes 461 */ 462 static unsigned int intel_wm_method1(unsigned int pixel_rate, 463 unsigned int cpp, 464 unsigned int latency) 465 { 466 u64 ret; 467 468 ret = mul_u32_u32(pixel_rate, cpp * latency); 469 ret = DIV_ROUND_UP_ULL(ret, 10000); 470 471 return ret; 472 } 473 474 /** 475 * intel_wm_method2 - Method 2 / "large buffer" watermark formula 476 * @pixel_rate: Pipe pixel rate in kHz 477 * @htotal: Pipe horizontal total 478 * @width: Plane width in pixels 479 * @cpp: Plane bytes per pixel 480 * @latency: Memory wakeup latency in 0.1us units 481 * 482 * Compute the watermark using the method 2 or "large buffer" 483 * formula. The caller may additonally add extra cachelines 484 * to account for TLB misses and clock crossings. 485 * 486 * This method is concerned with the long term drain rate 487 * of the FIFO, ie. it does account for blanking periods 488 * which effectively reduce the average drain rate across 489 * a longer period. The name "large" refers to the fact the 490 * FIFO is relatively large compared to the amount of data 491 * fetched. 492 * 493 * The FIFO level vs. time graph might look something like: 494 * 495 * |\___ |\___ 496 * | \___ | \___ 497 * | \ | \ 498 * __ --__--__--__--__--__--__ (- plane active, _ blanking) 499 * -> time 500 * 501 * Returns: 502 * The watermark in bytes 503 */ 504 static unsigned int intel_wm_method2(unsigned int pixel_rate, 505 unsigned int htotal, 506 unsigned int width, 507 unsigned int cpp, 508 unsigned int latency) 509 { 510 unsigned int ret; 511 512 /* 513 * FIXME remove once all users are computing 514 * watermarks in the correct place. 515 */ 516 if (WARN_ON_ONCE(htotal == 0)) 517 htotal = 1; 518 519 ret = (latency * pixel_rate) / (htotal * 10000); 520 ret = (ret + 1) * width * cpp; 521 522 return ret; 523 } 524 525 /** 526 * intel_calculate_wm - calculate watermark level 527 * @pixel_rate: pixel clock 528 * @wm: chip FIFO params 529 * @fifo_size: size of the FIFO buffer 530 * @cpp: bytes per pixel 531 * @latency_ns: memory latency for the platform 532 * 533 * Calculate the watermark level (the level at which the display plane will 534 * start fetching from memory again). Each chip has a different display 535 * FIFO size and allocation, so the caller needs to figure that out and pass 536 * in the correct intel_watermark_params structure. 537 * 538 * As the pixel clock runs, the FIFO will be drained at a rate that depends 539 * on the pixel size. When it reaches the watermark level, it'll start 540 * fetching FIFO line sized based chunks from memory until the FIFO fills 541 * past the watermark point. If the FIFO drains completely, a FIFO underrun 542 * will occur, and a display engine hang could result. 543 */ 544 static unsigned int intel_calculate_wm(int pixel_rate, 545 const struct intel_watermark_params *wm, 546 int fifo_size, int cpp, 547 unsigned int latency_ns) 548 { 549 int entries, wm_size; 550 551 /* 552 * Note: we need to make sure we don't overflow for various clock & 553 * latency values. 554 * clocks go from a few thousand to several hundred thousand. 555 * latency is usually a few thousand 556 */ 557 entries = intel_wm_method1(pixel_rate, cpp, 558 latency_ns / 100); 559 entries = DIV_ROUND_UP(entries, wm->cacheline_size) + 560 wm->guard_size; 561 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries); 562 563 wm_size = fifo_size - entries; 564 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size); 565 566 /* Don't promote wm_size to unsigned... */ 567 if (wm_size > wm->max_wm) 568 wm_size = wm->max_wm; 569 if (wm_size <= 0) 570 wm_size = wm->default_wm; 571 572 /* 573 * Bspec seems to indicate that the value shouldn't be lower than 574 * 'burst size + 1'. Certainly 830 is quite unhappy with low values. 575 * Lets go for 8 which is the burst size since certain platforms 576 * already use a hardcoded 8 (which is what the spec says should be 577 * done). 578 */ 579 if (wm_size <= 8) 580 wm_size = 8; 581 582 return wm_size; 583 } 584 585 static bool is_disabling(int old, int new, int threshold) 586 { 587 return old >= threshold && new < threshold; 588 } 589 590 static bool is_enabling(int old, int new, int threshold) 591 { 592 return old < threshold && new >= threshold; 593 } 594 595 static bool intel_crtc_active(struct intel_crtc *crtc) 596 { 597 /* Be paranoid as we can arrive here with only partial 598 * state retrieved from the hardware during setup. 599 * 600 * We can ditch the adjusted_mode.crtc_clock check as soon 601 * as Haswell has gained clock readout/fastboot support. 602 * 603 * We can ditch the crtc->primary->state->fb check as soon as we can 604 * properly reconstruct framebuffers. 605 * 606 * FIXME: The intel_crtc->active here should be switched to 607 * crtc->state->active once we have proper CRTC states wired up 608 * for atomic. 609 */ 610 return crtc && crtc->active && crtc->base.primary->state->fb && 611 crtc->config->hw.adjusted_mode.crtc_clock; 612 } 613 614 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv) 615 { 616 struct intel_crtc *crtc, *enabled = NULL; 617 618 for_each_intel_crtc(&dev_priv->drm, crtc) { 619 if (intel_crtc_active(crtc)) { 620 if (enabled) 621 return NULL; 622 enabled = crtc; 623 } 624 } 625 626 return enabled; 627 } 628 629 static void pnv_update_wm(struct drm_i915_private *dev_priv) 630 { 631 struct intel_crtc *crtc; 632 const struct cxsr_latency *latency; 633 u32 reg; 634 unsigned int wm; 635 636 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv), 637 dev_priv->is_ddr3, 638 dev_priv->fsb_freq, 639 dev_priv->mem_freq); 640 if (!latency) { 641 drm_dbg_kms(&dev_priv->drm, 642 "Unknown FSB/MEM found, disable CxSR\n"); 643 intel_set_memory_cxsr(dev_priv, false); 644 return; 645 } 646 647 crtc = single_enabled_crtc(dev_priv); 648 if (crtc) { 649 const struct drm_framebuffer *fb = 650 crtc->base.primary->state->fb; 651 int pixel_rate = crtc->config->pixel_rate; 652 int cpp = fb->format->cpp[0]; 653 654 /* Display SR */ 655 wm = intel_calculate_wm(pixel_rate, &pnv_display_wm, 656 pnv_display_wm.fifo_size, 657 cpp, latency->display_sr); 658 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1); 659 reg &= ~DSPFW_SR_MASK; 660 reg |= FW_WM(wm, SR); 661 intel_uncore_write(&dev_priv->uncore, DSPFW1, reg); 662 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg); 663 664 /* cursor SR */ 665 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_wm, 666 pnv_display_wm.fifo_size, 667 4, latency->cursor_sr); 668 intel_uncore_rmw(&dev_priv->uncore, DSPFW3, DSPFW_CURSOR_SR_MASK, 669 FW_WM(wm, CURSOR_SR)); 670 671 /* Display HPLL off SR */ 672 wm = intel_calculate_wm(pixel_rate, &pnv_display_hplloff_wm, 673 pnv_display_hplloff_wm.fifo_size, 674 cpp, latency->display_hpll_disable); 675 intel_uncore_rmw(&dev_priv->uncore, DSPFW3, DSPFW_HPLL_SR_MASK, FW_WM(wm, HPLL_SR)); 676 677 /* cursor HPLL off SR */ 678 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_hplloff_wm, 679 pnv_display_hplloff_wm.fifo_size, 680 4, latency->cursor_hpll_disable); 681 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3); 682 reg &= ~DSPFW_HPLL_CURSOR_MASK; 683 reg |= FW_WM(wm, HPLL_CURSOR); 684 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg); 685 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg); 686 687 intel_set_memory_cxsr(dev_priv, true); 688 } else { 689 intel_set_memory_cxsr(dev_priv, false); 690 } 691 } 692 693 /* 694 * Documentation says: 695 * "If the line size is small, the TLB fetches can get in the way of the 696 * data fetches, causing some lag in the pixel data return which is not 697 * accounted for in the above formulas. The following adjustment only 698 * needs to be applied if eight whole lines fit in the buffer at once. 699 * The WM is adjusted upwards by the difference between the FIFO size 700 * and the size of 8 whole lines. This adjustment is always performed 701 * in the actual pixel depth regardless of whether FBC is enabled or not." 702 */ 703 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp) 704 { 705 int tlb_miss = fifo_size * 64 - width * cpp * 8; 706 707 return max(0, tlb_miss); 708 } 709 710 static void g4x_write_wm_values(struct drm_i915_private *dev_priv, 711 const struct g4x_wm_values *wm) 712 { 713 enum pipe pipe; 714 715 for_each_pipe(dev_priv, pipe) 716 trace_g4x_wm(intel_crtc_for_pipe(dev_priv, pipe), wm); 717 718 intel_uncore_write(&dev_priv->uncore, DSPFW1, 719 FW_WM(wm->sr.plane, SR) | 720 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) | 721 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) | 722 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA)); 723 intel_uncore_write(&dev_priv->uncore, DSPFW2, 724 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) | 725 FW_WM(wm->sr.fbc, FBC_SR) | 726 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) | 727 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) | 728 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) | 729 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA)); 730 intel_uncore_write(&dev_priv->uncore, DSPFW3, 731 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) | 732 FW_WM(wm->sr.cursor, CURSOR_SR) | 733 FW_WM(wm->hpll.cursor, HPLL_CURSOR) | 734 FW_WM(wm->hpll.plane, HPLL_SR)); 735 736 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1); 737 } 738 739 #define FW_WM_VLV(value, plane) \ 740 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV) 741 742 static void vlv_write_wm_values(struct drm_i915_private *dev_priv, 743 const struct vlv_wm_values *wm) 744 { 745 enum pipe pipe; 746 747 for_each_pipe(dev_priv, pipe) { 748 trace_vlv_wm(intel_crtc_for_pipe(dev_priv, pipe), wm); 749 750 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe), 751 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) | 752 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) | 753 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) | 754 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT)); 755 } 756 757 /* 758 * Zero the (unused) WM1 watermarks, and also clear all the 759 * high order bits so that there are no out of bounds values 760 * present in the registers during the reprogramming. 761 */ 762 intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0); 763 intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0); 764 intel_uncore_write(&dev_priv->uncore, DSPFW4, 0); 765 intel_uncore_write(&dev_priv->uncore, DSPFW5, 0); 766 intel_uncore_write(&dev_priv->uncore, DSPFW6, 0); 767 768 intel_uncore_write(&dev_priv->uncore, DSPFW1, 769 FW_WM(wm->sr.plane, SR) | 770 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) | 771 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) | 772 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA)); 773 intel_uncore_write(&dev_priv->uncore, DSPFW2, 774 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) | 775 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) | 776 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA)); 777 intel_uncore_write(&dev_priv->uncore, DSPFW3, 778 FW_WM(wm->sr.cursor, CURSOR_SR)); 779 780 if (IS_CHERRYVIEW(dev_priv)) { 781 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV, 782 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) | 783 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC)); 784 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV, 785 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) | 786 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE)); 787 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV, 788 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) | 789 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC)); 790 intel_uncore_write(&dev_priv->uncore, DSPHOWM, 791 FW_WM(wm->sr.plane >> 9, SR_HI) | 792 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) | 793 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) | 794 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) | 795 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) | 796 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) | 797 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) | 798 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) | 799 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) | 800 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI)); 801 } else { 802 intel_uncore_write(&dev_priv->uncore, DSPFW7, 803 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) | 804 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC)); 805 intel_uncore_write(&dev_priv->uncore, DSPHOWM, 806 FW_WM(wm->sr.plane >> 9, SR_HI) | 807 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) | 808 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) | 809 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) | 810 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) | 811 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) | 812 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI)); 813 } 814 815 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1); 816 } 817 818 #undef FW_WM_VLV 819 820 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv) 821 { 822 /* all latencies in usec */ 823 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5; 824 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_SR] = 12; 825 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35; 826 827 dev_priv->display.wm.num_levels = G4X_WM_LEVEL_HPLL + 1; 828 } 829 830 static int g4x_plane_fifo_size(enum plane_id plane_id, int level) 831 { 832 /* 833 * DSPCNTR[13] supposedly controls whether the 834 * primary plane can use the FIFO space otherwise 835 * reserved for the sprite plane. It's not 100% clear 836 * what the actual FIFO size is, but it looks like we 837 * can happily set both primary and sprite watermarks 838 * up to 127 cachelines. So that would seem to mean 839 * that either DSPCNTR[13] doesn't do anything, or that 840 * the total FIFO is >= 256 cachelines in size. Either 841 * way, we don't seem to have to worry about this 842 * repartitioning as the maximum watermark value the 843 * register can hold for each plane is lower than the 844 * minimum FIFO size. 845 */ 846 switch (plane_id) { 847 case PLANE_CURSOR: 848 return 63; 849 case PLANE_PRIMARY: 850 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511; 851 case PLANE_SPRITE0: 852 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0; 853 default: 854 MISSING_CASE(plane_id); 855 return 0; 856 } 857 } 858 859 static int g4x_fbc_fifo_size(int level) 860 { 861 switch (level) { 862 case G4X_WM_LEVEL_SR: 863 return 7; 864 case G4X_WM_LEVEL_HPLL: 865 return 15; 866 default: 867 MISSING_CASE(level); 868 return 0; 869 } 870 } 871 872 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state, 873 const struct intel_plane_state *plane_state, 874 int level) 875 { 876 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); 877 struct drm_i915_private *dev_priv = to_i915(plane->base.dev); 878 const struct drm_display_mode *pipe_mode = 879 &crtc_state->hw.pipe_mode; 880 unsigned int latency = dev_priv->display.wm.pri_latency[level] * 10; 881 unsigned int pixel_rate, htotal, cpp, width, wm; 882 883 if (latency == 0) 884 return USHRT_MAX; 885 886 if (!intel_wm_plane_visible(crtc_state, plane_state)) 887 return 0; 888 889 cpp = plane_state->hw.fb->format->cpp[0]; 890 891 /* 892 * WaUse32BppForSRWM:ctg,elk 893 * 894 * The spec fails to list this restriction for the 895 * HPLL watermark, which seems a little strange. 896 * Let's use 32bpp for the HPLL watermark as well. 897 */ 898 if (plane->id == PLANE_PRIMARY && 899 level != G4X_WM_LEVEL_NORMAL) 900 cpp = max(cpp, 4u); 901 902 pixel_rate = crtc_state->pixel_rate; 903 htotal = pipe_mode->crtc_htotal; 904 width = drm_rect_width(&plane_state->uapi.src) >> 16; 905 906 if (plane->id == PLANE_CURSOR) { 907 wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency); 908 } else if (plane->id == PLANE_PRIMARY && 909 level == G4X_WM_LEVEL_NORMAL) { 910 wm = intel_wm_method1(pixel_rate, cpp, latency); 911 } else { 912 unsigned int small, large; 913 914 small = intel_wm_method1(pixel_rate, cpp, latency); 915 large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency); 916 917 wm = min(small, large); 918 } 919 920 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level), 921 width, cpp); 922 923 wm = DIV_ROUND_UP(wm, 64) + 2; 924 925 return min_t(unsigned int, wm, USHRT_MAX); 926 } 927 928 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state, 929 int level, enum plane_id plane_id, u16 value) 930 { 931 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); 932 bool dirty = false; 933 934 for (; level < dev_priv->display.wm.num_levels; level++) { 935 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level]; 936 937 dirty |= raw->plane[plane_id] != value; 938 raw->plane[plane_id] = value; 939 } 940 941 return dirty; 942 } 943 944 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state, 945 int level, u16 value) 946 { 947 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); 948 bool dirty = false; 949 950 /* NORMAL level doesn't have an FBC watermark */ 951 level = max(level, G4X_WM_LEVEL_SR); 952 953 for (; level < dev_priv->display.wm.num_levels; level++) { 954 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level]; 955 956 dirty |= raw->fbc != value; 957 raw->fbc = value; 958 } 959 960 return dirty; 961 } 962 963 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state, 964 const struct intel_plane_state *plane_state, 965 u32 pri_val); 966 967 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state, 968 const struct intel_plane_state *plane_state) 969 { 970 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); 971 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); 972 enum plane_id plane_id = plane->id; 973 bool dirty = false; 974 int level; 975 976 if (!intel_wm_plane_visible(crtc_state, plane_state)) { 977 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0); 978 if (plane_id == PLANE_PRIMARY) 979 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0); 980 goto out; 981 } 982 983 for (level = 0; level < dev_priv->display.wm.num_levels; level++) { 984 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level]; 985 int wm, max_wm; 986 987 wm = g4x_compute_wm(crtc_state, plane_state, level); 988 max_wm = g4x_plane_fifo_size(plane_id, level); 989 990 if (wm > max_wm) 991 break; 992 993 dirty |= raw->plane[plane_id] != wm; 994 raw->plane[plane_id] = wm; 995 996 if (plane_id != PLANE_PRIMARY || 997 level == G4X_WM_LEVEL_NORMAL) 998 continue; 999 1000 wm = ilk_compute_fbc_wm(crtc_state, plane_state, 1001 raw->plane[plane_id]); 1002 max_wm = g4x_fbc_fifo_size(level); 1003 1004 /* 1005 * FBC wm is not mandatory as we 1006 * can always just disable its use. 1007 */ 1008 if (wm > max_wm) 1009 wm = USHRT_MAX; 1010 1011 dirty |= raw->fbc != wm; 1012 raw->fbc = wm; 1013 } 1014 1015 /* mark watermarks as invalid */ 1016 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX); 1017 1018 if (plane_id == PLANE_PRIMARY) 1019 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX); 1020 1021 out: 1022 if (dirty) { 1023 drm_dbg_kms(&dev_priv->drm, 1024 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n", 1025 plane->base.name, 1026 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id], 1027 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id], 1028 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]); 1029 1030 if (plane_id == PLANE_PRIMARY) 1031 drm_dbg_kms(&dev_priv->drm, 1032 "FBC watermarks: SR=%d, HPLL=%d\n", 1033 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc, 1034 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc); 1035 } 1036 1037 return dirty; 1038 } 1039 1040 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state, 1041 enum plane_id plane_id, int level) 1042 { 1043 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level]; 1044 1045 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level); 1046 } 1047 1048 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, 1049 int level) 1050 { 1051 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); 1052 1053 if (level >= dev_priv->display.wm.num_levels) 1054 return false; 1055 1056 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) && 1057 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) && 1058 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level); 1059 } 1060 1061 /* mark all levels starting from 'level' as invalid */ 1062 static void g4x_invalidate_wms(struct intel_crtc *crtc, 1063 struct g4x_wm_state *wm_state, int level) 1064 { 1065 if (level <= G4X_WM_LEVEL_NORMAL) { 1066 enum plane_id plane_id; 1067 1068 for_each_plane_id_on_crtc(crtc, plane_id) 1069 wm_state->wm.plane[plane_id] = USHRT_MAX; 1070 } 1071 1072 if (level <= G4X_WM_LEVEL_SR) { 1073 wm_state->cxsr = false; 1074 wm_state->sr.cursor = USHRT_MAX; 1075 wm_state->sr.plane = USHRT_MAX; 1076 wm_state->sr.fbc = USHRT_MAX; 1077 } 1078 1079 if (level <= G4X_WM_LEVEL_HPLL) { 1080 wm_state->hpll_en = false; 1081 wm_state->hpll.cursor = USHRT_MAX; 1082 wm_state->hpll.plane = USHRT_MAX; 1083 wm_state->hpll.fbc = USHRT_MAX; 1084 } 1085 } 1086 1087 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state, 1088 int level) 1089 { 1090 if (level < G4X_WM_LEVEL_SR) 1091 return false; 1092 1093 if (level >= G4X_WM_LEVEL_SR && 1094 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR)) 1095 return false; 1096 1097 if (level >= G4X_WM_LEVEL_HPLL && 1098 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL)) 1099 return false; 1100 1101 return true; 1102 } 1103 1104 static int _g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state) 1105 { 1106 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 1107 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal; 1108 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR); 1109 const struct g4x_pipe_wm *raw; 1110 enum plane_id plane_id; 1111 int level; 1112 1113 level = G4X_WM_LEVEL_NORMAL; 1114 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level)) 1115 goto out; 1116 1117 raw = &crtc_state->wm.g4x.raw[level]; 1118 for_each_plane_id_on_crtc(crtc, plane_id) 1119 wm_state->wm.plane[plane_id] = raw->plane[plane_id]; 1120 1121 level = G4X_WM_LEVEL_SR; 1122 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level)) 1123 goto out; 1124 1125 raw = &crtc_state->wm.g4x.raw[level]; 1126 wm_state->sr.plane = raw->plane[PLANE_PRIMARY]; 1127 wm_state->sr.cursor = raw->plane[PLANE_CURSOR]; 1128 wm_state->sr.fbc = raw->fbc; 1129 1130 wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY); 1131 1132 level = G4X_WM_LEVEL_HPLL; 1133 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level)) 1134 goto out; 1135 1136 raw = &crtc_state->wm.g4x.raw[level]; 1137 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY]; 1138 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR]; 1139 wm_state->hpll.fbc = raw->fbc; 1140 1141 wm_state->hpll_en = wm_state->cxsr; 1142 1143 level++; 1144 1145 out: 1146 if (level == G4X_WM_LEVEL_NORMAL) 1147 return -EINVAL; 1148 1149 /* invalidate the higher levels */ 1150 g4x_invalidate_wms(crtc, wm_state, level); 1151 1152 /* 1153 * Determine if the FBC watermark(s) can be used. IF 1154 * this isn't the case we prefer to disable the FBC 1155 * watermark(s) rather than disable the SR/HPLL 1156 * level(s) entirely. 'level-1' is the highest valid 1157 * level here. 1158 */ 1159 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1); 1160 1161 return 0; 1162 } 1163 1164 static int g4x_compute_pipe_wm(struct intel_atomic_state *state, 1165 struct intel_crtc *crtc) 1166 { 1167 struct intel_crtc_state *crtc_state = 1168 intel_atomic_get_new_crtc_state(state, crtc); 1169 const struct intel_plane_state *old_plane_state; 1170 const struct intel_plane_state *new_plane_state; 1171 struct intel_plane *plane; 1172 unsigned int dirty = 0; 1173 int i; 1174 1175 for_each_oldnew_intel_plane_in_state(state, plane, 1176 old_plane_state, 1177 new_plane_state, i) { 1178 if (new_plane_state->hw.crtc != &crtc->base && 1179 old_plane_state->hw.crtc != &crtc->base) 1180 continue; 1181 1182 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state)) 1183 dirty |= BIT(plane->id); 1184 } 1185 1186 if (!dirty) 1187 return 0; 1188 1189 return _g4x_compute_pipe_wm(crtc_state); 1190 } 1191 1192 static int g4x_compute_intermediate_wm(struct intel_atomic_state *state, 1193 struct intel_crtc *crtc) 1194 { 1195 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 1196 struct intel_crtc_state *new_crtc_state = 1197 intel_atomic_get_new_crtc_state(state, crtc); 1198 const struct intel_crtc_state *old_crtc_state = 1199 intel_atomic_get_old_crtc_state(state, crtc); 1200 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate; 1201 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal; 1202 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal; 1203 enum plane_id plane_id; 1204 1205 if (!new_crtc_state->hw.active || 1206 intel_crtc_needs_modeset(new_crtc_state)) { 1207 *intermediate = *optimal; 1208 1209 intermediate->cxsr = false; 1210 intermediate->hpll_en = false; 1211 goto out; 1212 } 1213 1214 intermediate->cxsr = optimal->cxsr && active->cxsr && 1215 !new_crtc_state->disable_cxsr; 1216 intermediate->hpll_en = optimal->hpll_en && active->hpll_en && 1217 !new_crtc_state->disable_cxsr; 1218 intermediate->fbc_en = optimal->fbc_en && active->fbc_en; 1219 1220 for_each_plane_id_on_crtc(crtc, plane_id) { 1221 intermediate->wm.plane[plane_id] = 1222 max(optimal->wm.plane[plane_id], 1223 active->wm.plane[plane_id]); 1224 1225 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] > 1226 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL)); 1227 } 1228 1229 intermediate->sr.plane = max(optimal->sr.plane, 1230 active->sr.plane); 1231 intermediate->sr.cursor = max(optimal->sr.cursor, 1232 active->sr.cursor); 1233 intermediate->sr.fbc = max(optimal->sr.fbc, 1234 active->sr.fbc); 1235 1236 intermediate->hpll.plane = max(optimal->hpll.plane, 1237 active->hpll.plane); 1238 intermediate->hpll.cursor = max(optimal->hpll.cursor, 1239 active->hpll.cursor); 1240 intermediate->hpll.fbc = max(optimal->hpll.fbc, 1241 active->hpll.fbc); 1242 1243 drm_WARN_ON(&dev_priv->drm, 1244 (intermediate->sr.plane > 1245 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) || 1246 intermediate->sr.cursor > 1247 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) && 1248 intermediate->cxsr); 1249 drm_WARN_ON(&dev_priv->drm, 1250 (intermediate->sr.plane > 1251 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) || 1252 intermediate->sr.cursor > 1253 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) && 1254 intermediate->hpll_en); 1255 1256 drm_WARN_ON(&dev_priv->drm, 1257 intermediate->sr.fbc > g4x_fbc_fifo_size(1) && 1258 intermediate->fbc_en && intermediate->cxsr); 1259 drm_WARN_ON(&dev_priv->drm, 1260 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) && 1261 intermediate->fbc_en && intermediate->hpll_en); 1262 1263 out: 1264 /* 1265 * If our intermediate WM are identical to the final WM, then we can 1266 * omit the post-vblank programming; only update if it's different. 1267 */ 1268 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0) 1269 new_crtc_state->wm.need_postvbl_update = true; 1270 1271 return 0; 1272 } 1273 1274 static void g4x_merge_wm(struct drm_i915_private *dev_priv, 1275 struct g4x_wm_values *wm) 1276 { 1277 struct intel_crtc *crtc; 1278 int num_active_pipes = 0; 1279 1280 wm->cxsr = true; 1281 wm->hpll_en = true; 1282 wm->fbc_en = true; 1283 1284 for_each_intel_crtc(&dev_priv->drm, crtc) { 1285 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x; 1286 1287 if (!crtc->active) 1288 continue; 1289 1290 if (!wm_state->cxsr) 1291 wm->cxsr = false; 1292 if (!wm_state->hpll_en) 1293 wm->hpll_en = false; 1294 if (!wm_state->fbc_en) 1295 wm->fbc_en = false; 1296 1297 num_active_pipes++; 1298 } 1299 1300 if (num_active_pipes != 1) { 1301 wm->cxsr = false; 1302 wm->hpll_en = false; 1303 wm->fbc_en = false; 1304 } 1305 1306 for_each_intel_crtc(&dev_priv->drm, crtc) { 1307 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x; 1308 enum pipe pipe = crtc->pipe; 1309 1310 wm->pipe[pipe] = wm_state->wm; 1311 if (crtc->active && wm->cxsr) 1312 wm->sr = wm_state->sr; 1313 if (crtc->active && wm->hpll_en) 1314 wm->hpll = wm_state->hpll; 1315 } 1316 } 1317 1318 static void g4x_program_watermarks(struct drm_i915_private *dev_priv) 1319 { 1320 struct g4x_wm_values *old_wm = &dev_priv->display.wm.g4x; 1321 struct g4x_wm_values new_wm = {}; 1322 1323 g4x_merge_wm(dev_priv, &new_wm); 1324 1325 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0) 1326 return; 1327 1328 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true)) 1329 _intel_set_memory_cxsr(dev_priv, false); 1330 1331 g4x_write_wm_values(dev_priv, &new_wm); 1332 1333 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true)) 1334 _intel_set_memory_cxsr(dev_priv, true); 1335 1336 *old_wm = new_wm; 1337 } 1338 1339 static void g4x_initial_watermarks(struct intel_atomic_state *state, 1340 struct intel_crtc *crtc) 1341 { 1342 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 1343 const struct intel_crtc_state *crtc_state = 1344 intel_atomic_get_new_crtc_state(state, crtc); 1345 1346 mutex_lock(&dev_priv->display.wm.wm_mutex); 1347 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate; 1348 g4x_program_watermarks(dev_priv); 1349 mutex_unlock(&dev_priv->display.wm.wm_mutex); 1350 } 1351 1352 static void g4x_optimize_watermarks(struct intel_atomic_state *state, 1353 struct intel_crtc *crtc) 1354 { 1355 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 1356 const struct intel_crtc_state *crtc_state = 1357 intel_atomic_get_new_crtc_state(state, crtc); 1358 1359 if (!crtc_state->wm.need_postvbl_update) 1360 return; 1361 1362 mutex_lock(&dev_priv->display.wm.wm_mutex); 1363 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal; 1364 g4x_program_watermarks(dev_priv); 1365 mutex_unlock(&dev_priv->display.wm.wm_mutex); 1366 } 1367 1368 /* latency must be in 0.1us units. */ 1369 static unsigned int vlv_wm_method2(unsigned int pixel_rate, 1370 unsigned int htotal, 1371 unsigned int width, 1372 unsigned int cpp, 1373 unsigned int latency) 1374 { 1375 unsigned int ret; 1376 1377 ret = intel_wm_method2(pixel_rate, htotal, 1378 width, cpp, latency); 1379 ret = DIV_ROUND_UP(ret, 64); 1380 1381 return ret; 1382 } 1383 1384 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv) 1385 { 1386 /* all latencies in usec */ 1387 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM2] = 3; 1388 1389 dev_priv->display.wm.num_levels = VLV_WM_LEVEL_PM2 + 1; 1390 1391 if (IS_CHERRYVIEW(dev_priv)) { 1392 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM5] = 12; 1393 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33; 1394 1395 dev_priv->display.wm.num_levels = VLV_WM_LEVEL_DDR_DVFS + 1; 1396 } 1397 } 1398 1399 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state, 1400 const struct intel_plane_state *plane_state, 1401 int level) 1402 { 1403 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); 1404 struct drm_i915_private *dev_priv = to_i915(plane->base.dev); 1405 const struct drm_display_mode *pipe_mode = 1406 &crtc_state->hw.pipe_mode; 1407 unsigned int pixel_rate, htotal, cpp, width, wm; 1408 1409 if (dev_priv->display.wm.pri_latency[level] == 0) 1410 return USHRT_MAX; 1411 1412 if (!intel_wm_plane_visible(crtc_state, plane_state)) 1413 return 0; 1414 1415 cpp = plane_state->hw.fb->format->cpp[0]; 1416 pixel_rate = crtc_state->pixel_rate; 1417 htotal = pipe_mode->crtc_htotal; 1418 width = drm_rect_width(&plane_state->uapi.src) >> 16; 1419 1420 if (plane->id == PLANE_CURSOR) { 1421 /* 1422 * FIXME the formula gives values that are 1423 * too big for the cursor FIFO, and hence we 1424 * would never be able to use cursors. For 1425 * now just hardcode the watermark. 1426 */ 1427 wm = 63; 1428 } else { 1429 wm = vlv_wm_method2(pixel_rate, htotal, width, cpp, 1430 dev_priv->display.wm.pri_latency[level] * 10); 1431 } 1432 1433 return min_t(unsigned int, wm, USHRT_MAX); 1434 } 1435 1436 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes) 1437 { 1438 return (active_planes & (BIT(PLANE_SPRITE0) | 1439 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1); 1440 } 1441 1442 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state) 1443 { 1444 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 1445 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 1446 const struct g4x_pipe_wm *raw = 1447 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2]; 1448 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state; 1449 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR); 1450 int num_active_planes = hweight8(active_planes); 1451 const int fifo_size = 511; 1452 int fifo_extra, fifo_left = fifo_size; 1453 int sprite0_fifo_extra = 0; 1454 unsigned int total_rate; 1455 enum plane_id plane_id; 1456 1457 /* 1458 * When enabling sprite0 after sprite1 has already been enabled 1459 * we tend to get an underrun unless sprite0 already has some 1460 * FIFO space allcoated. Hence we always allocate at least one 1461 * cacheline for sprite0 whenever sprite1 is enabled. 1462 * 1463 * All other plane enable sequences appear immune to this problem. 1464 */ 1465 if (vlv_need_sprite0_fifo_workaround(active_planes)) 1466 sprite0_fifo_extra = 1; 1467 1468 total_rate = raw->plane[PLANE_PRIMARY] + 1469 raw->plane[PLANE_SPRITE0] + 1470 raw->plane[PLANE_SPRITE1] + 1471 sprite0_fifo_extra; 1472 1473 if (total_rate > fifo_size) 1474 return -EINVAL; 1475 1476 if (total_rate == 0) 1477 total_rate = 1; 1478 1479 for_each_plane_id_on_crtc(crtc, plane_id) { 1480 unsigned int rate; 1481 1482 if ((active_planes & BIT(plane_id)) == 0) { 1483 fifo_state->plane[plane_id] = 0; 1484 continue; 1485 } 1486 1487 rate = raw->plane[plane_id]; 1488 fifo_state->plane[plane_id] = fifo_size * rate / total_rate; 1489 fifo_left -= fifo_state->plane[plane_id]; 1490 } 1491 1492 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra; 1493 fifo_left -= sprite0_fifo_extra; 1494 1495 fifo_state->plane[PLANE_CURSOR] = 63; 1496 1497 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1); 1498 1499 /* spread the remainder evenly */ 1500 for_each_plane_id_on_crtc(crtc, plane_id) { 1501 int plane_extra; 1502 1503 if (fifo_left == 0) 1504 break; 1505 1506 if ((active_planes & BIT(plane_id)) == 0) 1507 continue; 1508 1509 plane_extra = min(fifo_extra, fifo_left); 1510 fifo_state->plane[plane_id] += plane_extra; 1511 fifo_left -= plane_extra; 1512 } 1513 1514 drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0); 1515 1516 /* give it all to the first plane if none are active */ 1517 if (active_planes == 0) { 1518 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size); 1519 fifo_state->plane[PLANE_PRIMARY] = fifo_left; 1520 } 1521 1522 return 0; 1523 } 1524 1525 /* mark all levels starting from 'level' as invalid */ 1526 static void vlv_invalidate_wms(struct intel_crtc *crtc, 1527 struct vlv_wm_state *wm_state, int level) 1528 { 1529 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 1530 1531 for (; level < dev_priv->display.wm.num_levels; level++) { 1532 enum plane_id plane_id; 1533 1534 for_each_plane_id_on_crtc(crtc, plane_id) 1535 wm_state->wm[level].plane[plane_id] = USHRT_MAX; 1536 1537 wm_state->sr[level].cursor = USHRT_MAX; 1538 wm_state->sr[level].plane = USHRT_MAX; 1539 } 1540 } 1541 1542 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size) 1543 { 1544 if (wm > fifo_size) 1545 return USHRT_MAX; 1546 else 1547 return fifo_size - wm; 1548 } 1549 1550 /* 1551 * Starting from 'level' set all higher 1552 * levels to 'value' in the "raw" watermarks. 1553 */ 1554 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state, 1555 int level, enum plane_id plane_id, u16 value) 1556 { 1557 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); 1558 bool dirty = false; 1559 1560 for (; level < dev_priv->display.wm.num_levels; level++) { 1561 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level]; 1562 1563 dirty |= raw->plane[plane_id] != value; 1564 raw->plane[plane_id] = value; 1565 } 1566 1567 return dirty; 1568 } 1569 1570 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state, 1571 const struct intel_plane_state *plane_state) 1572 { 1573 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); 1574 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); 1575 enum plane_id plane_id = plane->id; 1576 int level; 1577 bool dirty = false; 1578 1579 if (!intel_wm_plane_visible(crtc_state, plane_state)) { 1580 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0); 1581 goto out; 1582 } 1583 1584 for (level = 0; level < dev_priv->display.wm.num_levels; level++) { 1585 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level]; 1586 int wm = vlv_compute_wm_level(crtc_state, plane_state, level); 1587 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511; 1588 1589 if (wm > max_wm) 1590 break; 1591 1592 dirty |= raw->plane[plane_id] != wm; 1593 raw->plane[plane_id] = wm; 1594 } 1595 1596 /* mark all higher levels as invalid */ 1597 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX); 1598 1599 out: 1600 if (dirty) 1601 drm_dbg_kms(&dev_priv->drm, 1602 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n", 1603 plane->base.name, 1604 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id], 1605 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id], 1606 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]); 1607 1608 return dirty; 1609 } 1610 1611 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state, 1612 enum plane_id plane_id, int level) 1613 { 1614 const struct g4x_pipe_wm *raw = 1615 &crtc_state->wm.vlv.raw[level]; 1616 const struct vlv_fifo_state *fifo_state = 1617 &crtc_state->wm.vlv.fifo_state; 1618 1619 return raw->plane[plane_id] <= fifo_state->plane[plane_id]; 1620 } 1621 1622 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level) 1623 { 1624 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) && 1625 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) && 1626 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) && 1627 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level); 1628 } 1629 1630 static int _vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state) 1631 { 1632 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 1633 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 1634 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal; 1635 const struct vlv_fifo_state *fifo_state = 1636 &crtc_state->wm.vlv.fifo_state; 1637 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR); 1638 int num_active_planes = hweight8(active_planes); 1639 enum plane_id plane_id; 1640 int level; 1641 1642 /* initially allow all levels */ 1643 wm_state->num_levels = dev_priv->display.wm.num_levels; 1644 /* 1645 * Note that enabling cxsr with no primary/sprite planes 1646 * enabled can wedge the pipe. Hence we only allow cxsr 1647 * with exactly one enabled primary/sprite plane. 1648 */ 1649 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1; 1650 1651 for (level = 0; level < wm_state->num_levels; level++) { 1652 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level]; 1653 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1; 1654 1655 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level)) 1656 break; 1657 1658 for_each_plane_id_on_crtc(crtc, plane_id) { 1659 wm_state->wm[level].plane[plane_id] = 1660 vlv_invert_wm_value(raw->plane[plane_id], 1661 fifo_state->plane[plane_id]); 1662 } 1663 1664 wm_state->sr[level].plane = 1665 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY], 1666 raw->plane[PLANE_SPRITE0], 1667 raw->plane[PLANE_SPRITE1]), 1668 sr_fifo_size); 1669 1670 wm_state->sr[level].cursor = 1671 vlv_invert_wm_value(raw->plane[PLANE_CURSOR], 1672 63); 1673 } 1674 1675 if (level == 0) 1676 return -EINVAL; 1677 1678 /* limit to only levels we can actually handle */ 1679 wm_state->num_levels = level; 1680 1681 /* invalidate the higher levels */ 1682 vlv_invalidate_wms(crtc, wm_state, level); 1683 1684 return 0; 1685 } 1686 1687 static int vlv_compute_pipe_wm(struct intel_atomic_state *state, 1688 struct intel_crtc *crtc) 1689 { 1690 struct intel_crtc_state *crtc_state = 1691 intel_atomic_get_new_crtc_state(state, crtc); 1692 const struct intel_plane_state *old_plane_state; 1693 const struct intel_plane_state *new_plane_state; 1694 struct intel_plane *plane; 1695 unsigned int dirty = 0; 1696 int i; 1697 1698 for_each_oldnew_intel_plane_in_state(state, plane, 1699 old_plane_state, 1700 new_plane_state, i) { 1701 if (new_plane_state->hw.crtc != &crtc->base && 1702 old_plane_state->hw.crtc != &crtc->base) 1703 continue; 1704 1705 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state)) 1706 dirty |= BIT(plane->id); 1707 } 1708 1709 /* 1710 * DSPARB registers may have been reset due to the 1711 * power well being turned off. Make sure we restore 1712 * them to a consistent state even if no primary/sprite 1713 * planes are initially active. We also force a FIFO 1714 * recomputation so that we are sure to sanitize the 1715 * FIFO setting we took over from the BIOS even if there 1716 * are no active planes on the crtc. 1717 */ 1718 if (intel_crtc_needs_modeset(crtc_state)) 1719 dirty = ~0; 1720 1721 if (!dirty) 1722 return 0; 1723 1724 /* cursor changes don't warrant a FIFO recompute */ 1725 if (dirty & ~BIT(PLANE_CURSOR)) { 1726 const struct intel_crtc_state *old_crtc_state = 1727 intel_atomic_get_old_crtc_state(state, crtc); 1728 const struct vlv_fifo_state *old_fifo_state = 1729 &old_crtc_state->wm.vlv.fifo_state; 1730 const struct vlv_fifo_state *new_fifo_state = 1731 &crtc_state->wm.vlv.fifo_state; 1732 int ret; 1733 1734 ret = vlv_compute_fifo(crtc_state); 1735 if (ret) 1736 return ret; 1737 1738 if (intel_crtc_needs_modeset(crtc_state) || 1739 memcmp(old_fifo_state, new_fifo_state, 1740 sizeof(*new_fifo_state)) != 0) 1741 crtc_state->fifo_changed = true; 1742 } 1743 1744 return _vlv_compute_pipe_wm(crtc_state); 1745 } 1746 1747 #define VLV_FIFO(plane, value) \ 1748 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV) 1749 1750 static void vlv_atomic_update_fifo(struct intel_atomic_state *state, 1751 struct intel_crtc *crtc) 1752 { 1753 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 1754 struct intel_uncore *uncore = &dev_priv->uncore; 1755 const struct intel_crtc_state *crtc_state = 1756 intel_atomic_get_new_crtc_state(state, crtc); 1757 const struct vlv_fifo_state *fifo_state = 1758 &crtc_state->wm.vlv.fifo_state; 1759 int sprite0_start, sprite1_start, fifo_size; 1760 u32 dsparb, dsparb2, dsparb3; 1761 1762 if (!crtc_state->fifo_changed) 1763 return; 1764 1765 sprite0_start = fifo_state->plane[PLANE_PRIMARY]; 1766 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start; 1767 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start; 1768 1769 drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63); 1770 drm_WARN_ON(&dev_priv->drm, fifo_size != 511); 1771 1772 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size); 1773 1774 /* 1775 * uncore.lock serves a double purpose here. It allows us to 1776 * use the less expensive I915_{READ,WRITE}_FW() functions, and 1777 * it protects the DSPARB registers from getting clobbered by 1778 * parallel updates from multiple pipes. 1779 * 1780 * intel_pipe_update_start() has already disabled interrupts 1781 * for us, so a plain spin_lock() is sufficient here. 1782 */ 1783 spin_lock(&uncore->lock); 1784 1785 switch (crtc->pipe) { 1786 case PIPE_A: 1787 dsparb = intel_uncore_read_fw(uncore, DSPARB); 1788 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2); 1789 1790 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) | 1791 VLV_FIFO(SPRITEB, 0xff)); 1792 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) | 1793 VLV_FIFO(SPRITEB, sprite1_start)); 1794 1795 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) | 1796 VLV_FIFO(SPRITEB_HI, 0x1)); 1797 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) | 1798 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8)); 1799 1800 intel_uncore_write_fw(uncore, DSPARB, dsparb); 1801 intel_uncore_write_fw(uncore, DSPARB2, dsparb2); 1802 break; 1803 case PIPE_B: 1804 dsparb = intel_uncore_read_fw(uncore, DSPARB); 1805 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2); 1806 1807 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) | 1808 VLV_FIFO(SPRITED, 0xff)); 1809 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) | 1810 VLV_FIFO(SPRITED, sprite1_start)); 1811 1812 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) | 1813 VLV_FIFO(SPRITED_HI, 0xff)); 1814 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) | 1815 VLV_FIFO(SPRITED_HI, sprite1_start >> 8)); 1816 1817 intel_uncore_write_fw(uncore, DSPARB, dsparb); 1818 intel_uncore_write_fw(uncore, DSPARB2, dsparb2); 1819 break; 1820 case PIPE_C: 1821 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3); 1822 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2); 1823 1824 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) | 1825 VLV_FIFO(SPRITEF, 0xff)); 1826 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) | 1827 VLV_FIFO(SPRITEF, sprite1_start)); 1828 1829 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) | 1830 VLV_FIFO(SPRITEF_HI, 0xff)); 1831 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) | 1832 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8)); 1833 1834 intel_uncore_write_fw(uncore, DSPARB3, dsparb3); 1835 intel_uncore_write_fw(uncore, DSPARB2, dsparb2); 1836 break; 1837 default: 1838 break; 1839 } 1840 1841 intel_uncore_posting_read_fw(uncore, DSPARB); 1842 1843 spin_unlock(&uncore->lock); 1844 } 1845 1846 #undef VLV_FIFO 1847 1848 static int vlv_compute_intermediate_wm(struct intel_atomic_state *state, 1849 struct intel_crtc *crtc) 1850 { 1851 struct intel_crtc_state *new_crtc_state = 1852 intel_atomic_get_new_crtc_state(state, crtc); 1853 const struct intel_crtc_state *old_crtc_state = 1854 intel_atomic_get_old_crtc_state(state, crtc); 1855 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate; 1856 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal; 1857 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal; 1858 int level; 1859 1860 if (!new_crtc_state->hw.active || 1861 intel_crtc_needs_modeset(new_crtc_state)) { 1862 *intermediate = *optimal; 1863 1864 intermediate->cxsr = false; 1865 goto out; 1866 } 1867 1868 intermediate->num_levels = min(optimal->num_levels, active->num_levels); 1869 intermediate->cxsr = optimal->cxsr && active->cxsr && 1870 !new_crtc_state->disable_cxsr; 1871 1872 for (level = 0; level < intermediate->num_levels; level++) { 1873 enum plane_id plane_id; 1874 1875 for_each_plane_id_on_crtc(crtc, plane_id) { 1876 intermediate->wm[level].plane[plane_id] = 1877 min(optimal->wm[level].plane[plane_id], 1878 active->wm[level].plane[plane_id]); 1879 } 1880 1881 intermediate->sr[level].plane = min(optimal->sr[level].plane, 1882 active->sr[level].plane); 1883 intermediate->sr[level].cursor = min(optimal->sr[level].cursor, 1884 active->sr[level].cursor); 1885 } 1886 1887 vlv_invalidate_wms(crtc, intermediate, level); 1888 1889 out: 1890 /* 1891 * If our intermediate WM are identical to the final WM, then we can 1892 * omit the post-vblank programming; only update if it's different. 1893 */ 1894 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0) 1895 new_crtc_state->wm.need_postvbl_update = true; 1896 1897 return 0; 1898 } 1899 1900 static void vlv_merge_wm(struct drm_i915_private *dev_priv, 1901 struct vlv_wm_values *wm) 1902 { 1903 struct intel_crtc *crtc; 1904 int num_active_pipes = 0; 1905 1906 wm->level = dev_priv->display.wm.num_levels - 1; 1907 wm->cxsr = true; 1908 1909 for_each_intel_crtc(&dev_priv->drm, crtc) { 1910 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv; 1911 1912 if (!crtc->active) 1913 continue; 1914 1915 if (!wm_state->cxsr) 1916 wm->cxsr = false; 1917 1918 num_active_pipes++; 1919 wm->level = min_t(int, wm->level, wm_state->num_levels - 1); 1920 } 1921 1922 if (num_active_pipes != 1) 1923 wm->cxsr = false; 1924 1925 if (num_active_pipes > 1) 1926 wm->level = VLV_WM_LEVEL_PM2; 1927 1928 for_each_intel_crtc(&dev_priv->drm, crtc) { 1929 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv; 1930 enum pipe pipe = crtc->pipe; 1931 1932 wm->pipe[pipe] = wm_state->wm[wm->level]; 1933 if (crtc->active && wm->cxsr) 1934 wm->sr = wm_state->sr[wm->level]; 1935 1936 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2; 1937 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2; 1938 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2; 1939 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2; 1940 } 1941 } 1942 1943 static void vlv_program_watermarks(struct drm_i915_private *dev_priv) 1944 { 1945 struct vlv_wm_values *old_wm = &dev_priv->display.wm.vlv; 1946 struct vlv_wm_values new_wm = {}; 1947 1948 vlv_merge_wm(dev_priv, &new_wm); 1949 1950 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0) 1951 return; 1952 1953 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS)) 1954 chv_set_memory_dvfs(dev_priv, false); 1955 1956 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5)) 1957 chv_set_memory_pm5(dev_priv, false); 1958 1959 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true)) 1960 _intel_set_memory_cxsr(dev_priv, false); 1961 1962 vlv_write_wm_values(dev_priv, &new_wm); 1963 1964 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true)) 1965 _intel_set_memory_cxsr(dev_priv, true); 1966 1967 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5)) 1968 chv_set_memory_pm5(dev_priv, true); 1969 1970 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS)) 1971 chv_set_memory_dvfs(dev_priv, true); 1972 1973 *old_wm = new_wm; 1974 } 1975 1976 static void vlv_initial_watermarks(struct intel_atomic_state *state, 1977 struct intel_crtc *crtc) 1978 { 1979 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 1980 const struct intel_crtc_state *crtc_state = 1981 intel_atomic_get_new_crtc_state(state, crtc); 1982 1983 mutex_lock(&dev_priv->display.wm.wm_mutex); 1984 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate; 1985 vlv_program_watermarks(dev_priv); 1986 mutex_unlock(&dev_priv->display.wm.wm_mutex); 1987 } 1988 1989 static void vlv_optimize_watermarks(struct intel_atomic_state *state, 1990 struct intel_crtc *crtc) 1991 { 1992 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 1993 const struct intel_crtc_state *crtc_state = 1994 intel_atomic_get_new_crtc_state(state, crtc); 1995 1996 if (!crtc_state->wm.need_postvbl_update) 1997 return; 1998 1999 mutex_lock(&dev_priv->display.wm.wm_mutex); 2000 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal; 2001 vlv_program_watermarks(dev_priv); 2002 mutex_unlock(&dev_priv->display.wm.wm_mutex); 2003 } 2004 2005 static void i965_update_wm(struct drm_i915_private *dev_priv) 2006 { 2007 struct intel_crtc *crtc; 2008 int srwm = 1; 2009 int cursor_sr = 16; 2010 bool cxsr_enabled; 2011 2012 /* Calc sr entries for one plane configs */ 2013 crtc = single_enabled_crtc(dev_priv); 2014 if (crtc) { 2015 /* self-refresh has much higher latency */ 2016 static const int sr_latency_ns = 12000; 2017 const struct drm_display_mode *pipe_mode = 2018 &crtc->config->hw.pipe_mode; 2019 const struct drm_framebuffer *fb = 2020 crtc->base.primary->state->fb; 2021 int pixel_rate = crtc->config->pixel_rate; 2022 int htotal = pipe_mode->crtc_htotal; 2023 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16; 2024 int cpp = fb->format->cpp[0]; 2025 int entries; 2026 2027 entries = intel_wm_method2(pixel_rate, htotal, 2028 width, cpp, sr_latency_ns / 100); 2029 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE); 2030 srwm = I965_FIFO_SIZE - entries; 2031 if (srwm < 0) 2032 srwm = 1; 2033 srwm &= 0x1ff; 2034 drm_dbg_kms(&dev_priv->drm, 2035 "self-refresh entries: %d, wm: %d\n", 2036 entries, srwm); 2037 2038 entries = intel_wm_method2(pixel_rate, htotal, 2039 crtc->base.cursor->state->crtc_w, 4, 2040 sr_latency_ns / 100); 2041 entries = DIV_ROUND_UP(entries, 2042 i965_cursor_wm_info.cacheline_size) + 2043 i965_cursor_wm_info.guard_size; 2044 2045 cursor_sr = i965_cursor_wm_info.fifo_size - entries; 2046 if (cursor_sr > i965_cursor_wm_info.max_wm) 2047 cursor_sr = i965_cursor_wm_info.max_wm; 2048 2049 drm_dbg_kms(&dev_priv->drm, 2050 "self-refresh watermark: display plane %d " 2051 "cursor %d\n", srwm, cursor_sr); 2052 2053 cxsr_enabled = true; 2054 } else { 2055 cxsr_enabled = false; 2056 /* Turn off self refresh if both pipes are enabled */ 2057 intel_set_memory_cxsr(dev_priv, false); 2058 } 2059 2060 drm_dbg_kms(&dev_priv->drm, 2061 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n", 2062 srwm); 2063 2064 /* 965 has limitations... */ 2065 intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) | 2066 FW_WM(8, CURSORB) | 2067 FW_WM(8, PLANEB) | 2068 FW_WM(8, PLANEA)); 2069 intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) | 2070 FW_WM(8, PLANEC_OLD)); 2071 /* update cursor SR watermark */ 2072 intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR)); 2073 2074 if (cxsr_enabled) 2075 intel_set_memory_cxsr(dev_priv, true); 2076 } 2077 2078 #undef FW_WM 2079 2080 static struct intel_crtc *intel_crtc_for_plane(struct drm_i915_private *i915, 2081 enum i9xx_plane_id i9xx_plane) 2082 { 2083 struct intel_plane *plane; 2084 2085 for_each_intel_plane(&i915->drm, plane) { 2086 if (plane->id == PLANE_PRIMARY && 2087 plane->i9xx_plane == i9xx_plane) 2088 return intel_crtc_for_pipe(i915, plane->pipe); 2089 } 2090 2091 return NULL; 2092 } 2093 2094 static void i9xx_update_wm(struct drm_i915_private *dev_priv) 2095 { 2096 const struct intel_watermark_params *wm_info; 2097 u32 fwater_lo; 2098 u32 fwater_hi; 2099 int cwm, srwm = 1; 2100 int fifo_size; 2101 int planea_wm, planeb_wm; 2102 struct intel_crtc *crtc; 2103 2104 if (IS_I945GM(dev_priv)) 2105 wm_info = &i945_wm_info; 2106 else if (DISPLAY_VER(dev_priv) != 2) 2107 wm_info = &i915_wm_info; 2108 else 2109 wm_info = &i830_a_wm_info; 2110 2111 if (DISPLAY_VER(dev_priv) == 2) 2112 fifo_size = i830_get_fifo_size(dev_priv, PLANE_A); 2113 else 2114 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A); 2115 crtc = intel_crtc_for_plane(dev_priv, PLANE_A); 2116 if (intel_crtc_active(crtc)) { 2117 const struct drm_framebuffer *fb = 2118 crtc->base.primary->state->fb; 2119 int cpp; 2120 2121 if (DISPLAY_VER(dev_priv) == 2) 2122 cpp = 4; 2123 else 2124 cpp = fb->format->cpp[0]; 2125 2126 planea_wm = intel_calculate_wm(crtc->config->pixel_rate, 2127 wm_info, fifo_size, cpp, 2128 pessimal_latency_ns); 2129 } else { 2130 planea_wm = fifo_size - wm_info->guard_size; 2131 if (planea_wm > (long)wm_info->max_wm) 2132 planea_wm = wm_info->max_wm; 2133 } 2134 2135 if (DISPLAY_VER(dev_priv) == 2) 2136 wm_info = &i830_bc_wm_info; 2137 2138 if (DISPLAY_VER(dev_priv) == 2) 2139 fifo_size = i830_get_fifo_size(dev_priv, PLANE_B); 2140 else 2141 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B); 2142 crtc = intel_crtc_for_plane(dev_priv, PLANE_B); 2143 if (intel_crtc_active(crtc)) { 2144 const struct drm_framebuffer *fb = 2145 crtc->base.primary->state->fb; 2146 int cpp; 2147 2148 if (DISPLAY_VER(dev_priv) == 2) 2149 cpp = 4; 2150 else 2151 cpp = fb->format->cpp[0]; 2152 2153 planeb_wm = intel_calculate_wm(crtc->config->pixel_rate, 2154 wm_info, fifo_size, cpp, 2155 pessimal_latency_ns); 2156 } else { 2157 planeb_wm = fifo_size - wm_info->guard_size; 2158 if (planeb_wm > (long)wm_info->max_wm) 2159 planeb_wm = wm_info->max_wm; 2160 } 2161 2162 drm_dbg_kms(&dev_priv->drm, 2163 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm); 2164 2165 crtc = single_enabled_crtc(dev_priv); 2166 if (IS_I915GM(dev_priv) && crtc) { 2167 struct drm_i915_gem_object *obj; 2168 2169 obj = intel_fb_obj(crtc->base.primary->state->fb); 2170 2171 /* self-refresh seems busted with untiled */ 2172 if (!i915_gem_object_is_tiled(obj)) 2173 crtc = NULL; 2174 } 2175 2176 /* 2177 * Overlay gets an aggressive default since video jitter is bad. 2178 */ 2179 cwm = 2; 2180 2181 /* Play safe and disable self-refresh before adjusting watermarks. */ 2182 intel_set_memory_cxsr(dev_priv, false); 2183 2184 /* Calc sr entries for one plane configs */ 2185 if (HAS_FW_BLC(dev_priv) && crtc) { 2186 /* self-refresh has much higher latency */ 2187 static const int sr_latency_ns = 6000; 2188 const struct drm_display_mode *pipe_mode = 2189 &crtc->config->hw.pipe_mode; 2190 const struct drm_framebuffer *fb = 2191 crtc->base.primary->state->fb; 2192 int pixel_rate = crtc->config->pixel_rate; 2193 int htotal = pipe_mode->crtc_htotal; 2194 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16; 2195 int cpp; 2196 int entries; 2197 2198 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv)) 2199 cpp = 4; 2200 else 2201 cpp = fb->format->cpp[0]; 2202 2203 entries = intel_wm_method2(pixel_rate, htotal, width, cpp, 2204 sr_latency_ns / 100); 2205 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size); 2206 drm_dbg_kms(&dev_priv->drm, 2207 "self-refresh entries: %d\n", entries); 2208 srwm = wm_info->fifo_size - entries; 2209 if (srwm < 0) 2210 srwm = 1; 2211 2212 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) 2213 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, 2214 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff)); 2215 else 2216 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f); 2217 } 2218 2219 drm_dbg_kms(&dev_priv->drm, 2220 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n", 2221 planea_wm, planeb_wm, cwm, srwm); 2222 2223 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f); 2224 fwater_hi = (cwm & 0x1f); 2225 2226 /* Set request length to 8 cachelines per fetch */ 2227 fwater_lo = fwater_lo | (1 << 24) | (1 << 8); 2228 fwater_hi = fwater_hi | (1 << 8); 2229 2230 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo); 2231 intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi); 2232 2233 if (crtc) 2234 intel_set_memory_cxsr(dev_priv, true); 2235 } 2236 2237 static void i845_update_wm(struct drm_i915_private *dev_priv) 2238 { 2239 struct intel_crtc *crtc; 2240 u32 fwater_lo; 2241 int planea_wm; 2242 2243 crtc = single_enabled_crtc(dev_priv); 2244 if (crtc == NULL) 2245 return; 2246 2247 planea_wm = intel_calculate_wm(crtc->config->pixel_rate, 2248 &i845_wm_info, 2249 i845_get_fifo_size(dev_priv, PLANE_A), 2250 4, pessimal_latency_ns); 2251 fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff; 2252 fwater_lo |= (3<<8) | planea_wm; 2253 2254 drm_dbg_kms(&dev_priv->drm, 2255 "Setting FIFO watermarks - A: %d\n", planea_wm); 2256 2257 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo); 2258 } 2259 2260 /* latency must be in 0.1us units. */ 2261 static unsigned int ilk_wm_method1(unsigned int pixel_rate, 2262 unsigned int cpp, 2263 unsigned int latency) 2264 { 2265 unsigned int ret; 2266 2267 ret = intel_wm_method1(pixel_rate, cpp, latency); 2268 ret = DIV_ROUND_UP(ret, 64) + 2; 2269 2270 return ret; 2271 } 2272 2273 /* latency must be in 0.1us units. */ 2274 static unsigned int ilk_wm_method2(unsigned int pixel_rate, 2275 unsigned int htotal, 2276 unsigned int width, 2277 unsigned int cpp, 2278 unsigned int latency) 2279 { 2280 unsigned int ret; 2281 2282 ret = intel_wm_method2(pixel_rate, htotal, 2283 width, cpp, latency); 2284 ret = DIV_ROUND_UP(ret, 64) + 2; 2285 2286 return ret; 2287 } 2288 2289 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp) 2290 { 2291 /* 2292 * Neither of these should be possible since this function shouldn't be 2293 * called if the CRTC is off or the plane is invisible. But let's be 2294 * extra paranoid to avoid a potential divide-by-zero if we screw up 2295 * elsewhere in the driver. 2296 */ 2297 if (WARN_ON(!cpp)) 2298 return 0; 2299 if (WARN_ON(!horiz_pixels)) 2300 return 0; 2301 2302 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2; 2303 } 2304 2305 struct ilk_wm_maximums { 2306 u16 pri; 2307 u16 spr; 2308 u16 cur; 2309 u16 fbc; 2310 }; 2311 2312 /* 2313 * For both WM_PIPE and WM_LP. 2314 * mem_value must be in 0.1us units. 2315 */ 2316 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state, 2317 const struct intel_plane_state *plane_state, 2318 u32 mem_value, bool is_lp) 2319 { 2320 u32 method1, method2; 2321 int cpp; 2322 2323 if (mem_value == 0) 2324 return U32_MAX; 2325 2326 if (!intel_wm_plane_visible(crtc_state, plane_state)) 2327 return 0; 2328 2329 cpp = plane_state->hw.fb->format->cpp[0]; 2330 2331 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value); 2332 2333 if (!is_lp) 2334 return method1; 2335 2336 method2 = ilk_wm_method2(crtc_state->pixel_rate, 2337 crtc_state->hw.pipe_mode.crtc_htotal, 2338 drm_rect_width(&plane_state->uapi.src) >> 16, 2339 cpp, mem_value); 2340 2341 return min(method1, method2); 2342 } 2343 2344 /* 2345 * For both WM_PIPE and WM_LP. 2346 * mem_value must be in 0.1us units. 2347 */ 2348 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state, 2349 const struct intel_plane_state *plane_state, 2350 u32 mem_value) 2351 { 2352 u32 method1, method2; 2353 int cpp; 2354 2355 if (mem_value == 0) 2356 return U32_MAX; 2357 2358 if (!intel_wm_plane_visible(crtc_state, plane_state)) 2359 return 0; 2360 2361 cpp = plane_state->hw.fb->format->cpp[0]; 2362 2363 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value); 2364 method2 = ilk_wm_method2(crtc_state->pixel_rate, 2365 crtc_state->hw.pipe_mode.crtc_htotal, 2366 drm_rect_width(&plane_state->uapi.src) >> 16, 2367 cpp, mem_value); 2368 return min(method1, method2); 2369 } 2370 2371 /* 2372 * For both WM_PIPE and WM_LP. 2373 * mem_value must be in 0.1us units. 2374 */ 2375 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state, 2376 const struct intel_plane_state *plane_state, 2377 u32 mem_value) 2378 { 2379 int cpp; 2380 2381 if (mem_value == 0) 2382 return U32_MAX; 2383 2384 if (!intel_wm_plane_visible(crtc_state, plane_state)) 2385 return 0; 2386 2387 cpp = plane_state->hw.fb->format->cpp[0]; 2388 2389 return ilk_wm_method2(crtc_state->pixel_rate, 2390 crtc_state->hw.pipe_mode.crtc_htotal, 2391 drm_rect_width(&plane_state->uapi.src) >> 16, 2392 cpp, mem_value); 2393 } 2394 2395 /* Only for WM_LP. */ 2396 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state, 2397 const struct intel_plane_state *plane_state, 2398 u32 pri_val) 2399 { 2400 int cpp; 2401 2402 if (!intel_wm_plane_visible(crtc_state, plane_state)) 2403 return 0; 2404 2405 cpp = plane_state->hw.fb->format->cpp[0]; 2406 2407 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16, 2408 cpp); 2409 } 2410 2411 static unsigned int 2412 ilk_display_fifo_size(const struct drm_i915_private *dev_priv) 2413 { 2414 if (DISPLAY_VER(dev_priv) >= 8) 2415 return 3072; 2416 else if (DISPLAY_VER(dev_priv) >= 7) 2417 return 768; 2418 else 2419 return 512; 2420 } 2421 2422 static unsigned int 2423 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv, 2424 int level, bool is_sprite) 2425 { 2426 if (DISPLAY_VER(dev_priv) >= 8) 2427 /* BDW primary/sprite plane watermarks */ 2428 return level == 0 ? 255 : 2047; 2429 else if (DISPLAY_VER(dev_priv) >= 7) 2430 /* IVB/HSW primary/sprite plane watermarks */ 2431 return level == 0 ? 127 : 1023; 2432 else if (!is_sprite) 2433 /* ILK/SNB primary plane watermarks */ 2434 return level == 0 ? 127 : 511; 2435 else 2436 /* ILK/SNB sprite plane watermarks */ 2437 return level == 0 ? 63 : 255; 2438 } 2439 2440 static unsigned int 2441 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level) 2442 { 2443 if (DISPLAY_VER(dev_priv) >= 7) 2444 return level == 0 ? 63 : 255; 2445 else 2446 return level == 0 ? 31 : 63; 2447 } 2448 2449 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv) 2450 { 2451 if (DISPLAY_VER(dev_priv) >= 8) 2452 return 31; 2453 else 2454 return 15; 2455 } 2456 2457 /* Calculate the maximum primary/sprite plane watermark */ 2458 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv, 2459 int level, 2460 const struct intel_wm_config *config, 2461 enum intel_ddb_partitioning ddb_partitioning, 2462 bool is_sprite) 2463 { 2464 unsigned int fifo_size = ilk_display_fifo_size(dev_priv); 2465 2466 /* if sprites aren't enabled, sprites get nothing */ 2467 if (is_sprite && !config->sprites_enabled) 2468 return 0; 2469 2470 /* HSW allows LP1+ watermarks even with multiple pipes */ 2471 if (level == 0 || config->num_pipes_active > 1) { 2472 fifo_size /= INTEL_NUM_PIPES(dev_priv); 2473 2474 /* 2475 * For some reason the non self refresh 2476 * FIFO size is only half of the self 2477 * refresh FIFO size on ILK/SNB. 2478 */ 2479 if (DISPLAY_VER(dev_priv) <= 6) 2480 fifo_size /= 2; 2481 } 2482 2483 if (config->sprites_enabled) { 2484 /* level 0 is always calculated with 1:1 split */ 2485 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) { 2486 if (is_sprite) 2487 fifo_size *= 5; 2488 fifo_size /= 6; 2489 } else { 2490 fifo_size /= 2; 2491 } 2492 } 2493 2494 /* clamp to max that the registers can hold */ 2495 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite)); 2496 } 2497 2498 /* Calculate the maximum cursor plane watermark */ 2499 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv, 2500 int level, 2501 const struct intel_wm_config *config) 2502 { 2503 /* HSW LP1+ watermarks w/ multiple pipes */ 2504 if (level > 0 && config->num_pipes_active > 1) 2505 return 64; 2506 2507 /* otherwise just report max that registers can hold */ 2508 return ilk_cursor_wm_reg_max(dev_priv, level); 2509 } 2510 2511 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv, 2512 int level, 2513 const struct intel_wm_config *config, 2514 enum intel_ddb_partitioning ddb_partitioning, 2515 struct ilk_wm_maximums *max) 2516 { 2517 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false); 2518 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true); 2519 max->cur = ilk_cursor_wm_max(dev_priv, level, config); 2520 max->fbc = ilk_fbc_wm_reg_max(dev_priv); 2521 } 2522 2523 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv, 2524 int level, 2525 struct ilk_wm_maximums *max) 2526 { 2527 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false); 2528 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true); 2529 max->cur = ilk_cursor_wm_reg_max(dev_priv, level); 2530 max->fbc = ilk_fbc_wm_reg_max(dev_priv); 2531 } 2532 2533 static bool ilk_validate_wm_level(int level, 2534 const struct ilk_wm_maximums *max, 2535 struct intel_wm_level *result) 2536 { 2537 bool ret; 2538 2539 /* already determined to be invalid? */ 2540 if (!result->enable) 2541 return false; 2542 2543 result->enable = result->pri_val <= max->pri && 2544 result->spr_val <= max->spr && 2545 result->cur_val <= max->cur; 2546 2547 ret = result->enable; 2548 2549 /* 2550 * HACK until we can pre-compute everything, 2551 * and thus fail gracefully if LP0 watermarks 2552 * are exceeded... 2553 */ 2554 if (level == 0 && !result->enable) { 2555 if (result->pri_val > max->pri) 2556 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n", 2557 level, result->pri_val, max->pri); 2558 if (result->spr_val > max->spr) 2559 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n", 2560 level, result->spr_val, max->spr); 2561 if (result->cur_val > max->cur) 2562 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n", 2563 level, result->cur_val, max->cur); 2564 2565 result->pri_val = min_t(u32, result->pri_val, max->pri); 2566 result->spr_val = min_t(u32, result->spr_val, max->spr); 2567 result->cur_val = min_t(u32, result->cur_val, max->cur); 2568 result->enable = true; 2569 } 2570 2571 return ret; 2572 } 2573 2574 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv, 2575 const struct intel_crtc *crtc, 2576 int level, 2577 struct intel_crtc_state *crtc_state, 2578 const struct intel_plane_state *pristate, 2579 const struct intel_plane_state *sprstate, 2580 const struct intel_plane_state *curstate, 2581 struct intel_wm_level *result) 2582 { 2583 u16 pri_latency = dev_priv->display.wm.pri_latency[level]; 2584 u16 spr_latency = dev_priv->display.wm.spr_latency[level]; 2585 u16 cur_latency = dev_priv->display.wm.cur_latency[level]; 2586 2587 /* WM1+ latency values stored in 0.5us units */ 2588 if (level > 0) { 2589 pri_latency *= 5; 2590 spr_latency *= 5; 2591 cur_latency *= 5; 2592 } 2593 2594 if (pristate) { 2595 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate, 2596 pri_latency, level); 2597 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val); 2598 } 2599 2600 if (sprstate) 2601 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency); 2602 2603 if (curstate) 2604 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency); 2605 2606 result->enable = true; 2607 } 2608 2609 static void hsw_read_wm_latency(struct drm_i915_private *i915, u16 wm[]) 2610 { 2611 u64 sskpd; 2612 2613 i915->display.wm.num_levels = 5; 2614 2615 sskpd = intel_uncore_read64(&i915->uncore, MCH_SSKPD); 2616 2617 wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd); 2618 if (wm[0] == 0) 2619 wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd); 2620 wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd); 2621 wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd); 2622 wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd); 2623 wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd); 2624 } 2625 2626 static void snb_read_wm_latency(struct drm_i915_private *i915, u16 wm[]) 2627 { 2628 u32 sskpd; 2629 2630 i915->display.wm.num_levels = 4; 2631 2632 sskpd = intel_uncore_read(&i915->uncore, MCH_SSKPD); 2633 2634 wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd); 2635 wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd); 2636 wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd); 2637 wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd); 2638 } 2639 2640 static void ilk_read_wm_latency(struct drm_i915_private *i915, u16 wm[]) 2641 { 2642 u32 mltr; 2643 2644 i915->display.wm.num_levels = 3; 2645 2646 mltr = intel_uncore_read(&i915->uncore, MLTR_ILK); 2647 2648 /* ILK primary LP0 latency is 700 ns */ 2649 wm[0] = 7; 2650 wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr); 2651 wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr); 2652 } 2653 2654 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv, 2655 u16 wm[5]) 2656 { 2657 /* ILK sprite LP0 latency is 1300 ns */ 2658 if (DISPLAY_VER(dev_priv) == 5) 2659 wm[0] = 13; 2660 } 2661 2662 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv, 2663 u16 wm[5]) 2664 { 2665 /* ILK cursor LP0 latency is 1300 ns */ 2666 if (DISPLAY_VER(dev_priv) == 5) 2667 wm[0] = 13; 2668 } 2669 2670 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv, 2671 u16 wm[5], u16 min) 2672 { 2673 int level; 2674 2675 if (wm[0] >= min) 2676 return false; 2677 2678 wm[0] = max(wm[0], min); 2679 for (level = 1; level < dev_priv->display.wm.num_levels; level++) 2680 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5)); 2681 2682 return true; 2683 } 2684 2685 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv) 2686 { 2687 bool changed; 2688 2689 /* 2690 * The BIOS provided WM memory latency values are often 2691 * inadequate for high resolution displays. Adjust them. 2692 */ 2693 changed = ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.pri_latency, 12); 2694 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.spr_latency, 12); 2695 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.cur_latency, 12); 2696 2697 if (!changed) 2698 return; 2699 2700 drm_dbg_kms(&dev_priv->drm, 2701 "WM latency values increased to avoid potential underruns\n"); 2702 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency); 2703 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency); 2704 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency); 2705 } 2706 2707 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv) 2708 { 2709 /* 2710 * On some SNB machines (Thinkpad X220 Tablet at least) 2711 * LP3 usage can cause vblank interrupts to be lost. 2712 * The DEIIR bit will go high but it looks like the CPU 2713 * never gets interrupted. 2714 * 2715 * It's not clear whether other interrupt source could 2716 * be affected or if this is somehow limited to vblank 2717 * interrupts only. To play it safe we disable LP3 2718 * watermarks entirely. 2719 */ 2720 if (dev_priv->display.wm.pri_latency[3] == 0 && 2721 dev_priv->display.wm.spr_latency[3] == 0 && 2722 dev_priv->display.wm.cur_latency[3] == 0) 2723 return; 2724 2725 dev_priv->display.wm.pri_latency[3] = 0; 2726 dev_priv->display.wm.spr_latency[3] = 0; 2727 dev_priv->display.wm.cur_latency[3] = 0; 2728 2729 drm_dbg_kms(&dev_priv->drm, 2730 "LP3 watermarks disabled due to potential for lost interrupts\n"); 2731 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency); 2732 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency); 2733 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency); 2734 } 2735 2736 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv) 2737 { 2738 if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) 2739 hsw_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency); 2740 else if (DISPLAY_VER(dev_priv) >= 6) 2741 snb_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency); 2742 else 2743 ilk_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency); 2744 2745 memcpy(dev_priv->display.wm.spr_latency, dev_priv->display.wm.pri_latency, 2746 sizeof(dev_priv->display.wm.pri_latency)); 2747 memcpy(dev_priv->display.wm.cur_latency, dev_priv->display.wm.pri_latency, 2748 sizeof(dev_priv->display.wm.pri_latency)); 2749 2750 intel_fixup_spr_wm_latency(dev_priv, dev_priv->display.wm.spr_latency); 2751 intel_fixup_cur_wm_latency(dev_priv, dev_priv->display.wm.cur_latency); 2752 2753 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency); 2754 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency); 2755 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency); 2756 2757 if (DISPLAY_VER(dev_priv) == 6) { 2758 snb_wm_latency_quirk(dev_priv); 2759 snb_wm_lp3_irq_quirk(dev_priv); 2760 } 2761 } 2762 2763 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv, 2764 struct intel_pipe_wm *pipe_wm) 2765 { 2766 /* LP0 watermark maximums depend on this pipe alone */ 2767 const struct intel_wm_config config = { 2768 .num_pipes_active = 1, 2769 .sprites_enabled = pipe_wm->sprites_enabled, 2770 .sprites_scaled = pipe_wm->sprites_scaled, 2771 }; 2772 struct ilk_wm_maximums max; 2773 2774 /* LP0 watermarks always use 1/2 DDB partitioning */ 2775 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max); 2776 2777 /* At least LP0 must be valid */ 2778 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) { 2779 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n"); 2780 return false; 2781 } 2782 2783 return true; 2784 } 2785 2786 /* Compute new watermarks for the pipe */ 2787 static int ilk_compute_pipe_wm(struct intel_atomic_state *state, 2788 struct intel_crtc *crtc) 2789 { 2790 struct drm_i915_private *dev_priv = to_i915(state->base.dev); 2791 struct intel_crtc_state *crtc_state = 2792 intel_atomic_get_new_crtc_state(state, crtc); 2793 struct intel_pipe_wm *pipe_wm; 2794 struct intel_plane *plane; 2795 const struct intel_plane_state *plane_state; 2796 const struct intel_plane_state *pristate = NULL; 2797 const struct intel_plane_state *sprstate = NULL; 2798 const struct intel_plane_state *curstate = NULL; 2799 struct ilk_wm_maximums max; 2800 int level, usable_level; 2801 2802 pipe_wm = &crtc_state->wm.ilk.optimal; 2803 2804 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) { 2805 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY) 2806 pristate = plane_state; 2807 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY) 2808 sprstate = plane_state; 2809 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR) 2810 curstate = plane_state; 2811 } 2812 2813 pipe_wm->pipe_enabled = crtc_state->hw.active; 2814 pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0); 2815 pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0); 2816 2817 usable_level = dev_priv->display.wm.num_levels - 1; 2818 2819 /* ILK/SNB: LP2+ watermarks only w/o sprites */ 2820 if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled) 2821 usable_level = 1; 2822 2823 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */ 2824 if (pipe_wm->sprites_scaled) 2825 usable_level = 0; 2826 2827 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm)); 2828 ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state, 2829 pristate, sprstate, curstate, &pipe_wm->wm[0]); 2830 2831 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm)) 2832 return -EINVAL; 2833 2834 ilk_compute_wm_reg_maximums(dev_priv, 1, &max); 2835 2836 for (level = 1; level <= usable_level; level++) { 2837 struct intel_wm_level *wm = &pipe_wm->wm[level]; 2838 2839 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state, 2840 pristate, sprstate, curstate, wm); 2841 2842 /* 2843 * Disable any watermark level that exceeds the 2844 * register maximums since such watermarks are 2845 * always invalid. 2846 */ 2847 if (!ilk_validate_wm_level(level, &max, wm)) { 2848 memset(wm, 0, sizeof(*wm)); 2849 break; 2850 } 2851 } 2852 2853 return 0; 2854 } 2855 2856 /* 2857 * Build a set of 'intermediate' watermark values that satisfy both the old 2858 * state and the new state. These can be programmed to the hardware 2859 * immediately. 2860 */ 2861 static int ilk_compute_intermediate_wm(struct intel_atomic_state *state, 2862 struct intel_crtc *crtc) 2863 { 2864 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 2865 struct intel_crtc_state *new_crtc_state = 2866 intel_atomic_get_new_crtc_state(state, crtc); 2867 const struct intel_crtc_state *old_crtc_state = 2868 intel_atomic_get_old_crtc_state(state, crtc); 2869 struct intel_pipe_wm *a = &new_crtc_state->wm.ilk.intermediate; 2870 const struct intel_pipe_wm *b = &old_crtc_state->wm.ilk.optimal; 2871 int level; 2872 2873 /* 2874 * Start with the final, target watermarks, then combine with the 2875 * currently active watermarks to get values that are safe both before 2876 * and after the vblank. 2877 */ 2878 *a = new_crtc_state->wm.ilk.optimal; 2879 if (!new_crtc_state->hw.active || 2880 intel_crtc_needs_modeset(new_crtc_state) || 2881 state->skip_intermediate_wm) 2882 return 0; 2883 2884 a->pipe_enabled |= b->pipe_enabled; 2885 a->sprites_enabled |= b->sprites_enabled; 2886 a->sprites_scaled |= b->sprites_scaled; 2887 2888 for (level = 0; level < dev_priv->display.wm.num_levels; level++) { 2889 struct intel_wm_level *a_wm = &a->wm[level]; 2890 const struct intel_wm_level *b_wm = &b->wm[level]; 2891 2892 a_wm->enable &= b_wm->enable; 2893 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val); 2894 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val); 2895 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val); 2896 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val); 2897 } 2898 2899 /* 2900 * We need to make sure that these merged watermark values are 2901 * actually a valid configuration themselves. If they're not, 2902 * there's no safe way to transition from the old state to 2903 * the new state, so we need to fail the atomic transaction. 2904 */ 2905 if (!ilk_validate_pipe_wm(dev_priv, a)) 2906 return -EINVAL; 2907 2908 /* 2909 * If our intermediate WM are identical to the final WM, then we can 2910 * omit the post-vblank programming; only update if it's different. 2911 */ 2912 if (memcmp(a, &new_crtc_state->wm.ilk.optimal, sizeof(*a)) != 0) 2913 new_crtc_state->wm.need_postvbl_update = true; 2914 2915 return 0; 2916 } 2917 2918 /* 2919 * Merge the watermarks from all active pipes for a specific level. 2920 */ 2921 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv, 2922 int level, 2923 struct intel_wm_level *ret_wm) 2924 { 2925 const struct intel_crtc *crtc; 2926 2927 ret_wm->enable = true; 2928 2929 for_each_intel_crtc(&dev_priv->drm, crtc) { 2930 const struct intel_pipe_wm *active = &crtc->wm.active.ilk; 2931 const struct intel_wm_level *wm = &active->wm[level]; 2932 2933 if (!active->pipe_enabled) 2934 continue; 2935 2936 /* 2937 * The watermark values may have been used in the past, 2938 * so we must maintain them in the registers for some 2939 * time even if the level is now disabled. 2940 */ 2941 if (!wm->enable) 2942 ret_wm->enable = false; 2943 2944 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val); 2945 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val); 2946 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val); 2947 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val); 2948 } 2949 } 2950 2951 /* 2952 * Merge all low power watermarks for all active pipes. 2953 */ 2954 static void ilk_wm_merge(struct drm_i915_private *dev_priv, 2955 const struct intel_wm_config *config, 2956 const struct ilk_wm_maximums *max, 2957 struct intel_pipe_wm *merged) 2958 { 2959 int level, num_levels = dev_priv->display.wm.num_levels; 2960 int last_enabled_level = num_levels - 1; 2961 2962 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */ 2963 if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) && 2964 config->num_pipes_active > 1) 2965 last_enabled_level = 0; 2966 2967 /* ILK: FBC WM must be disabled always */ 2968 merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6; 2969 2970 /* merge each WM1+ level */ 2971 for (level = 1; level < num_levels; level++) { 2972 struct intel_wm_level *wm = &merged->wm[level]; 2973 2974 ilk_merge_wm_level(dev_priv, level, wm); 2975 2976 if (level > last_enabled_level) 2977 wm->enable = false; 2978 else if (!ilk_validate_wm_level(level, max, wm)) 2979 /* make sure all following levels get disabled */ 2980 last_enabled_level = level - 1; 2981 2982 /* 2983 * The spec says it is preferred to disable 2984 * FBC WMs instead of disabling a WM level. 2985 */ 2986 if (wm->fbc_val > max->fbc) { 2987 if (wm->enable) 2988 merged->fbc_wm_enabled = false; 2989 wm->fbc_val = 0; 2990 } 2991 } 2992 2993 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */ 2994 if (DISPLAY_VER(dev_priv) == 5 && HAS_FBC(dev_priv) && 2995 dev_priv->params.enable_fbc && !merged->fbc_wm_enabled) { 2996 for (level = 2; level < num_levels; level++) { 2997 struct intel_wm_level *wm = &merged->wm[level]; 2998 2999 wm->enable = false; 3000 } 3001 } 3002 } 3003 3004 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm) 3005 { 3006 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */ 3007 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable); 3008 } 3009 3010 /* The value we need to program into the WM_LPx latency field */ 3011 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv, 3012 int level) 3013 { 3014 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) 3015 return 2 * level; 3016 else 3017 return dev_priv->display.wm.pri_latency[level]; 3018 } 3019 3020 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv, 3021 const struct intel_pipe_wm *merged, 3022 enum intel_ddb_partitioning partitioning, 3023 struct ilk_wm_values *results) 3024 { 3025 struct intel_crtc *crtc; 3026 int level, wm_lp; 3027 3028 results->enable_fbc_wm = merged->fbc_wm_enabled; 3029 results->partitioning = partitioning; 3030 3031 /* LP1+ register values */ 3032 for (wm_lp = 1; wm_lp <= 3; wm_lp++) { 3033 const struct intel_wm_level *r; 3034 3035 level = ilk_wm_lp_to_level(wm_lp, merged); 3036 3037 r = &merged->wm[level]; 3038 3039 /* 3040 * Maintain the watermark values even if the level is 3041 * disabled. Doing otherwise could cause underruns. 3042 */ 3043 results->wm_lp[wm_lp - 1] = 3044 WM_LP_LATENCY(ilk_wm_lp_latency(dev_priv, level)) | 3045 WM_LP_PRIMARY(r->pri_val) | 3046 WM_LP_CURSOR(r->cur_val); 3047 3048 if (r->enable) 3049 results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE; 3050 3051 if (DISPLAY_VER(dev_priv) >= 8) 3052 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val); 3053 else 3054 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val); 3055 3056 results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val); 3057 3058 /* 3059 * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the 3060 * level is disabled. Doing otherwise could cause underruns. 3061 */ 3062 if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) { 3063 drm_WARN_ON(&dev_priv->drm, wm_lp != 1); 3064 results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE; 3065 } 3066 } 3067 3068 /* LP0 register values */ 3069 for_each_intel_crtc(&dev_priv->drm, crtc) { 3070 enum pipe pipe = crtc->pipe; 3071 const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk; 3072 const struct intel_wm_level *r = &pipe_wm->wm[0]; 3073 3074 if (drm_WARN_ON(&dev_priv->drm, !r->enable)) 3075 continue; 3076 3077 results->wm_pipe[pipe] = 3078 WM0_PIPE_PRIMARY(r->pri_val) | 3079 WM0_PIPE_SPRITE(r->spr_val) | 3080 WM0_PIPE_CURSOR(r->cur_val); 3081 } 3082 } 3083 3084 /* 3085 * Find the result with the highest level enabled. Check for enable_fbc_wm in 3086 * case both are at the same level. Prefer r1 in case they're the same. 3087 */ 3088 static struct intel_pipe_wm * 3089 ilk_find_best_result(struct drm_i915_private *dev_priv, 3090 struct intel_pipe_wm *r1, 3091 struct intel_pipe_wm *r2) 3092 { 3093 int level, level1 = 0, level2 = 0; 3094 3095 for (level = 1; level < dev_priv->display.wm.num_levels; level++) { 3096 if (r1->wm[level].enable) 3097 level1 = level; 3098 if (r2->wm[level].enable) 3099 level2 = level; 3100 } 3101 3102 if (level1 == level2) { 3103 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled) 3104 return r2; 3105 else 3106 return r1; 3107 } else if (level1 > level2) { 3108 return r1; 3109 } else { 3110 return r2; 3111 } 3112 } 3113 3114 /* dirty bits used to track which watermarks need changes */ 3115 #define WM_DIRTY_PIPE(pipe) (1 << (pipe)) 3116 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp))) 3117 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3)) 3118 #define WM_DIRTY_FBC (1 << 24) 3119 #define WM_DIRTY_DDB (1 << 25) 3120 3121 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv, 3122 const struct ilk_wm_values *old, 3123 const struct ilk_wm_values *new) 3124 { 3125 unsigned int dirty = 0; 3126 enum pipe pipe; 3127 int wm_lp; 3128 3129 for_each_pipe(dev_priv, pipe) { 3130 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) { 3131 dirty |= WM_DIRTY_PIPE(pipe); 3132 /* Must disable LP1+ watermarks too */ 3133 dirty |= WM_DIRTY_LP_ALL; 3134 } 3135 } 3136 3137 if (old->enable_fbc_wm != new->enable_fbc_wm) { 3138 dirty |= WM_DIRTY_FBC; 3139 /* Must disable LP1+ watermarks too */ 3140 dirty |= WM_DIRTY_LP_ALL; 3141 } 3142 3143 if (old->partitioning != new->partitioning) { 3144 dirty |= WM_DIRTY_DDB; 3145 /* Must disable LP1+ watermarks too */ 3146 dirty |= WM_DIRTY_LP_ALL; 3147 } 3148 3149 /* LP1+ watermarks already deemed dirty, no need to continue */ 3150 if (dirty & WM_DIRTY_LP_ALL) 3151 return dirty; 3152 3153 /* Find the lowest numbered LP1+ watermark in need of an update... */ 3154 for (wm_lp = 1; wm_lp <= 3; wm_lp++) { 3155 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] || 3156 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1]) 3157 break; 3158 } 3159 3160 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */ 3161 for (; wm_lp <= 3; wm_lp++) 3162 dirty |= WM_DIRTY_LP(wm_lp); 3163 3164 return dirty; 3165 } 3166 3167 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv, 3168 unsigned int dirty) 3169 { 3170 struct ilk_wm_values *previous = &dev_priv->display.wm.hw; 3171 bool changed = false; 3172 3173 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) { 3174 previous->wm_lp[2] &= ~WM_LP_ENABLE; 3175 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]); 3176 changed = true; 3177 } 3178 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) { 3179 previous->wm_lp[1] &= ~WM_LP_ENABLE; 3180 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]); 3181 changed = true; 3182 } 3183 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) { 3184 previous->wm_lp[0] &= ~WM_LP_ENABLE; 3185 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]); 3186 changed = true; 3187 } 3188 3189 /* 3190 * Don't touch WM_LP_SPRITE_ENABLE here. 3191 * Doing so could cause underruns. 3192 */ 3193 3194 return changed; 3195 } 3196 3197 /* 3198 * The spec says we shouldn't write when we don't need, because every write 3199 * causes WMs to be re-evaluated, expending some power. 3200 */ 3201 static void ilk_write_wm_values(struct drm_i915_private *dev_priv, 3202 struct ilk_wm_values *results) 3203 { 3204 struct ilk_wm_values *previous = &dev_priv->display.wm.hw; 3205 unsigned int dirty; 3206 3207 dirty = ilk_compute_wm_dirty(dev_priv, previous, results); 3208 if (!dirty) 3209 return; 3210 3211 _ilk_disable_lp_wm(dev_priv, dirty); 3212 3213 if (dirty & WM_DIRTY_PIPE(PIPE_A)) 3214 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]); 3215 if (dirty & WM_DIRTY_PIPE(PIPE_B)) 3216 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]); 3217 if (dirty & WM_DIRTY_PIPE(PIPE_C)) 3218 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]); 3219 3220 if (dirty & WM_DIRTY_DDB) { 3221 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) 3222 intel_uncore_rmw(&dev_priv->uncore, WM_MISC, WM_MISC_DATA_PARTITION_5_6, 3223 results->partitioning == INTEL_DDB_PART_1_2 ? 0 : 3224 WM_MISC_DATA_PARTITION_5_6); 3225 else 3226 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL2, DISP_DATA_PARTITION_5_6, 3227 results->partitioning == INTEL_DDB_PART_1_2 ? 0 : 3228 DISP_DATA_PARTITION_5_6); 3229 } 3230 3231 if (dirty & WM_DIRTY_FBC) 3232 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, DISP_FBC_WM_DIS, 3233 results->enable_fbc_wm ? 0 : DISP_FBC_WM_DIS); 3234 3235 if (dirty & WM_DIRTY_LP(1) && 3236 previous->wm_lp_spr[0] != results->wm_lp_spr[0]) 3237 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]); 3238 3239 if (DISPLAY_VER(dev_priv) >= 7) { 3240 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1]) 3241 intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]); 3242 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2]) 3243 intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]); 3244 } 3245 3246 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0]) 3247 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]); 3248 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1]) 3249 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]); 3250 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2]) 3251 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]); 3252 3253 dev_priv->display.wm.hw = *results; 3254 } 3255 3256 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv) 3257 { 3258 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL); 3259 } 3260 3261 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv, 3262 struct intel_wm_config *config) 3263 { 3264 struct intel_crtc *crtc; 3265 3266 /* Compute the currently _active_ config */ 3267 for_each_intel_crtc(&dev_priv->drm, crtc) { 3268 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk; 3269 3270 if (!wm->pipe_enabled) 3271 continue; 3272 3273 config->sprites_enabled |= wm->sprites_enabled; 3274 config->sprites_scaled |= wm->sprites_scaled; 3275 config->num_pipes_active++; 3276 } 3277 } 3278 3279 static void ilk_program_watermarks(struct drm_i915_private *dev_priv) 3280 { 3281 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm; 3282 struct ilk_wm_maximums max; 3283 struct intel_wm_config config = {}; 3284 struct ilk_wm_values results = {}; 3285 enum intel_ddb_partitioning partitioning; 3286 3287 ilk_compute_wm_config(dev_priv, &config); 3288 3289 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max); 3290 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2); 3291 3292 /* 5/6 split only in single pipe config on IVB+ */ 3293 if (DISPLAY_VER(dev_priv) >= 7 && 3294 config.num_pipes_active == 1 && config.sprites_enabled) { 3295 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max); 3296 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6); 3297 3298 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6); 3299 } else { 3300 best_lp_wm = &lp_wm_1_2; 3301 } 3302 3303 partitioning = (best_lp_wm == &lp_wm_1_2) ? 3304 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6; 3305 3306 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results); 3307 3308 ilk_write_wm_values(dev_priv, &results); 3309 } 3310 3311 static void ilk_initial_watermarks(struct intel_atomic_state *state, 3312 struct intel_crtc *crtc) 3313 { 3314 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 3315 const struct intel_crtc_state *crtc_state = 3316 intel_atomic_get_new_crtc_state(state, crtc); 3317 3318 mutex_lock(&dev_priv->display.wm.wm_mutex); 3319 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate; 3320 ilk_program_watermarks(dev_priv); 3321 mutex_unlock(&dev_priv->display.wm.wm_mutex); 3322 } 3323 3324 static void ilk_optimize_watermarks(struct intel_atomic_state *state, 3325 struct intel_crtc *crtc) 3326 { 3327 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 3328 const struct intel_crtc_state *crtc_state = 3329 intel_atomic_get_new_crtc_state(state, crtc); 3330 3331 if (!crtc_state->wm.need_postvbl_update) 3332 return; 3333 3334 mutex_lock(&dev_priv->display.wm.wm_mutex); 3335 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal; 3336 ilk_program_watermarks(dev_priv); 3337 mutex_unlock(&dev_priv->display.wm.wm_mutex); 3338 } 3339 3340 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc) 3341 { 3342 struct drm_device *dev = crtc->base.dev; 3343 struct drm_i915_private *dev_priv = to_i915(dev); 3344 struct ilk_wm_values *hw = &dev_priv->display.wm.hw; 3345 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state); 3346 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal; 3347 enum pipe pipe = crtc->pipe; 3348 3349 hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe)); 3350 3351 memset(active, 0, sizeof(*active)); 3352 3353 active->pipe_enabled = crtc->active; 3354 3355 if (active->pipe_enabled) { 3356 u32 tmp = hw->wm_pipe[pipe]; 3357 3358 /* 3359 * For active pipes LP0 watermark is marked as 3360 * enabled, and LP1+ watermaks as disabled since 3361 * we can't really reverse compute them in case 3362 * multiple pipes are active. 3363 */ 3364 active->wm[0].enable = true; 3365 active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp); 3366 active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp); 3367 active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp); 3368 } else { 3369 int level; 3370 3371 /* 3372 * For inactive pipes, all watermark levels 3373 * should be marked as enabled but zeroed, 3374 * which is what we'd compute them to. 3375 */ 3376 for (level = 0; level < dev_priv->display.wm.num_levels; level++) 3377 active->wm[level].enable = true; 3378 } 3379 3380 crtc->wm.active.ilk = *active; 3381 } 3382 3383 static int ilk_sanitize_watermarks_add_affected(struct drm_atomic_state *state) 3384 { 3385 struct drm_plane *plane; 3386 struct intel_crtc *crtc; 3387 3388 for_each_intel_crtc(state->dev, crtc) { 3389 struct intel_crtc_state *crtc_state; 3390 3391 crtc_state = intel_atomic_get_crtc_state(state, crtc); 3392 if (IS_ERR(crtc_state)) 3393 return PTR_ERR(crtc_state); 3394 3395 if (crtc_state->hw.active) { 3396 /* 3397 * Preserve the inherited flag to avoid 3398 * taking the full modeset path. 3399 */ 3400 crtc_state->inherited = true; 3401 } 3402 } 3403 3404 drm_for_each_plane(plane, state->dev) { 3405 struct drm_plane_state *plane_state; 3406 3407 plane_state = drm_atomic_get_plane_state(state, plane); 3408 if (IS_ERR(plane_state)) 3409 return PTR_ERR(plane_state); 3410 } 3411 3412 return 0; 3413 } 3414 3415 /* 3416 * Calculate what we think the watermarks should be for the state we've read 3417 * out of the hardware and then immediately program those watermarks so that 3418 * we ensure the hardware settings match our internal state. 3419 * 3420 * We can calculate what we think WM's should be by creating a duplicate of the 3421 * current state (which was constructed during hardware readout) and running it 3422 * through the atomic check code to calculate new watermark values in the 3423 * state object. 3424 */ 3425 void ilk_wm_sanitize(struct drm_i915_private *dev_priv) 3426 { 3427 struct drm_atomic_state *state; 3428 struct intel_atomic_state *intel_state; 3429 struct intel_crtc *crtc; 3430 struct intel_crtc_state *crtc_state; 3431 struct drm_modeset_acquire_ctx ctx; 3432 int ret; 3433 int i; 3434 3435 /* Only supported on platforms that use atomic watermark design */ 3436 if (!dev_priv->display.funcs.wm->optimize_watermarks) 3437 return; 3438 3439 if (drm_WARN_ON(&dev_priv->drm, DISPLAY_VER(dev_priv) >= 9)) 3440 return; 3441 3442 state = drm_atomic_state_alloc(&dev_priv->drm); 3443 if (drm_WARN_ON(&dev_priv->drm, !state)) 3444 return; 3445 3446 intel_state = to_intel_atomic_state(state); 3447 3448 drm_modeset_acquire_init(&ctx, 0); 3449 3450 retry: 3451 state->acquire_ctx = &ctx; 3452 3453 /* 3454 * Hardware readout is the only time we don't want to calculate 3455 * intermediate watermarks (since we don't trust the current 3456 * watermarks). 3457 */ 3458 if (!HAS_GMCH(dev_priv)) 3459 intel_state->skip_intermediate_wm = true; 3460 3461 ret = ilk_sanitize_watermarks_add_affected(state); 3462 if (ret) 3463 goto fail; 3464 3465 ret = intel_atomic_check(&dev_priv->drm, state); 3466 if (ret) 3467 goto fail; 3468 3469 /* Write calculated watermark values back */ 3470 for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) { 3471 crtc_state->wm.need_postvbl_update = true; 3472 intel_optimize_watermarks(intel_state, crtc); 3473 3474 to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm; 3475 } 3476 3477 fail: 3478 if (ret == -EDEADLK) { 3479 drm_atomic_state_clear(state); 3480 drm_modeset_backoff(&ctx); 3481 goto retry; 3482 } 3483 3484 /* 3485 * If we fail here, it means that the hardware appears to be 3486 * programmed in a way that shouldn't be possible, given our 3487 * understanding of watermark requirements. This might mean a 3488 * mistake in the hardware readout code or a mistake in the 3489 * watermark calculations for a given platform. Raise a WARN 3490 * so that this is noticeable. 3491 * 3492 * If this actually happens, we'll have to just leave the 3493 * BIOS-programmed watermarks untouched and hope for the best. 3494 */ 3495 drm_WARN(&dev_priv->drm, ret, 3496 "Could not determine valid watermarks for inherited state\n"); 3497 3498 drm_atomic_state_put(state); 3499 3500 drm_modeset_drop_locks(&ctx); 3501 drm_modeset_acquire_fini(&ctx); 3502 } 3503 3504 #define _FW_WM(value, plane) \ 3505 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT) 3506 #define _FW_WM_VLV(value, plane) \ 3507 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT) 3508 3509 static void g4x_read_wm_values(struct drm_i915_private *dev_priv, 3510 struct g4x_wm_values *wm) 3511 { 3512 u32 tmp; 3513 3514 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1); 3515 wm->sr.plane = _FW_WM(tmp, SR); 3516 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB); 3517 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB); 3518 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA); 3519 3520 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2); 3521 wm->fbc_en = tmp & DSPFW_FBC_SR_EN; 3522 wm->sr.fbc = _FW_WM(tmp, FBC_SR); 3523 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR); 3524 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB); 3525 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA); 3526 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA); 3527 3528 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3); 3529 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN; 3530 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR); 3531 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR); 3532 wm->hpll.plane = _FW_WM(tmp, HPLL_SR); 3533 } 3534 3535 static void vlv_read_wm_values(struct drm_i915_private *dev_priv, 3536 struct vlv_wm_values *wm) 3537 { 3538 enum pipe pipe; 3539 u32 tmp; 3540 3541 for_each_pipe(dev_priv, pipe) { 3542 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe)); 3543 3544 wm->ddl[pipe].plane[PLANE_PRIMARY] = 3545 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); 3546 wm->ddl[pipe].plane[PLANE_CURSOR] = 3547 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); 3548 wm->ddl[pipe].plane[PLANE_SPRITE0] = 3549 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); 3550 wm->ddl[pipe].plane[PLANE_SPRITE1] = 3551 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); 3552 } 3553 3554 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1); 3555 wm->sr.plane = _FW_WM(tmp, SR); 3556 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB); 3557 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB); 3558 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA); 3559 3560 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2); 3561 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB); 3562 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA); 3563 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA); 3564 3565 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3); 3566 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR); 3567 3568 if (IS_CHERRYVIEW(dev_priv)) { 3569 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV); 3570 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED); 3571 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC); 3572 3573 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV); 3574 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF); 3575 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE); 3576 3577 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV); 3578 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC); 3579 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC); 3580 3581 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM); 3582 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9; 3583 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8; 3584 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8; 3585 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8; 3586 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8; 3587 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8; 3588 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8; 3589 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8; 3590 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8; 3591 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8; 3592 } else { 3593 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7); 3594 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED); 3595 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC); 3596 3597 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM); 3598 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9; 3599 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8; 3600 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8; 3601 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8; 3602 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8; 3603 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8; 3604 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8; 3605 } 3606 } 3607 3608 #undef _FW_WM 3609 #undef _FW_WM_VLV 3610 3611 static void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv) 3612 { 3613 struct g4x_wm_values *wm = &dev_priv->display.wm.g4x; 3614 struct intel_crtc *crtc; 3615 3616 g4x_read_wm_values(dev_priv, wm); 3617 3618 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN; 3619 3620 for_each_intel_crtc(&dev_priv->drm, crtc) { 3621 struct intel_crtc_state *crtc_state = 3622 to_intel_crtc_state(crtc->base.state); 3623 struct g4x_wm_state *active = &crtc->wm.active.g4x; 3624 struct g4x_pipe_wm *raw; 3625 enum pipe pipe = crtc->pipe; 3626 enum plane_id plane_id; 3627 int level, max_level; 3628 3629 active->cxsr = wm->cxsr; 3630 active->hpll_en = wm->hpll_en; 3631 active->fbc_en = wm->fbc_en; 3632 3633 active->sr = wm->sr; 3634 active->hpll = wm->hpll; 3635 3636 for_each_plane_id_on_crtc(crtc, plane_id) { 3637 active->wm.plane[plane_id] = 3638 wm->pipe[pipe].plane[plane_id]; 3639 } 3640 3641 if (wm->cxsr && wm->hpll_en) 3642 max_level = G4X_WM_LEVEL_HPLL; 3643 else if (wm->cxsr) 3644 max_level = G4X_WM_LEVEL_SR; 3645 else 3646 max_level = G4X_WM_LEVEL_NORMAL; 3647 3648 level = G4X_WM_LEVEL_NORMAL; 3649 raw = &crtc_state->wm.g4x.raw[level]; 3650 for_each_plane_id_on_crtc(crtc, plane_id) 3651 raw->plane[plane_id] = active->wm.plane[plane_id]; 3652 3653 level = G4X_WM_LEVEL_SR; 3654 if (level > max_level) 3655 goto out; 3656 3657 raw = &crtc_state->wm.g4x.raw[level]; 3658 raw->plane[PLANE_PRIMARY] = active->sr.plane; 3659 raw->plane[PLANE_CURSOR] = active->sr.cursor; 3660 raw->plane[PLANE_SPRITE0] = 0; 3661 raw->fbc = active->sr.fbc; 3662 3663 level = G4X_WM_LEVEL_HPLL; 3664 if (level > max_level) 3665 goto out; 3666 3667 raw = &crtc_state->wm.g4x.raw[level]; 3668 raw->plane[PLANE_PRIMARY] = active->hpll.plane; 3669 raw->plane[PLANE_CURSOR] = active->hpll.cursor; 3670 raw->plane[PLANE_SPRITE0] = 0; 3671 raw->fbc = active->hpll.fbc; 3672 3673 level++; 3674 out: 3675 for_each_plane_id_on_crtc(crtc, plane_id) 3676 g4x_raw_plane_wm_set(crtc_state, level, 3677 plane_id, USHRT_MAX); 3678 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX); 3679 3680 g4x_invalidate_wms(crtc, active, level); 3681 3682 crtc_state->wm.g4x.optimal = *active; 3683 crtc_state->wm.g4x.intermediate = *active; 3684 3685 drm_dbg_kms(&dev_priv->drm, 3686 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n", 3687 pipe_name(pipe), 3688 wm->pipe[pipe].plane[PLANE_PRIMARY], 3689 wm->pipe[pipe].plane[PLANE_CURSOR], 3690 wm->pipe[pipe].plane[PLANE_SPRITE0]); 3691 } 3692 3693 drm_dbg_kms(&dev_priv->drm, 3694 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n", 3695 wm->sr.plane, wm->sr.cursor, wm->sr.fbc); 3696 drm_dbg_kms(&dev_priv->drm, 3697 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n", 3698 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc); 3699 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n", 3700 str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en), 3701 str_yes_no(wm->fbc_en)); 3702 } 3703 3704 static void g4x_wm_sanitize(struct drm_i915_private *dev_priv) 3705 { 3706 struct intel_plane *plane; 3707 struct intel_crtc *crtc; 3708 3709 mutex_lock(&dev_priv->display.wm.wm_mutex); 3710 3711 for_each_intel_plane(&dev_priv->drm, plane) { 3712 struct intel_crtc *crtc = 3713 intel_crtc_for_pipe(dev_priv, plane->pipe); 3714 struct intel_crtc_state *crtc_state = 3715 to_intel_crtc_state(crtc->base.state); 3716 struct intel_plane_state *plane_state = 3717 to_intel_plane_state(plane->base.state); 3718 enum plane_id plane_id = plane->id; 3719 int level; 3720 3721 if (plane_state->uapi.visible) 3722 continue; 3723 3724 for (level = 0; level < dev_priv->display.wm.num_levels; level++) { 3725 struct g4x_pipe_wm *raw = 3726 &crtc_state->wm.g4x.raw[level]; 3727 3728 raw->plane[plane_id] = 0; 3729 3730 if (plane_id == PLANE_PRIMARY) 3731 raw->fbc = 0; 3732 } 3733 } 3734 3735 for_each_intel_crtc(&dev_priv->drm, crtc) { 3736 struct intel_crtc_state *crtc_state = 3737 to_intel_crtc_state(crtc->base.state); 3738 int ret; 3739 3740 ret = _g4x_compute_pipe_wm(crtc_state); 3741 drm_WARN_ON(&dev_priv->drm, ret); 3742 3743 crtc_state->wm.g4x.intermediate = 3744 crtc_state->wm.g4x.optimal; 3745 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal; 3746 } 3747 3748 g4x_program_watermarks(dev_priv); 3749 3750 mutex_unlock(&dev_priv->display.wm.wm_mutex); 3751 } 3752 3753 static void g4x_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915) 3754 { 3755 g4x_wm_get_hw_state(i915); 3756 g4x_wm_sanitize(i915); 3757 } 3758 3759 static void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv) 3760 { 3761 struct vlv_wm_values *wm = &dev_priv->display.wm.vlv; 3762 struct intel_crtc *crtc; 3763 u32 val; 3764 3765 vlv_read_wm_values(dev_priv, wm); 3766 3767 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN; 3768 wm->level = VLV_WM_LEVEL_PM2; 3769 3770 if (IS_CHERRYVIEW(dev_priv)) { 3771 vlv_punit_get(dev_priv); 3772 3773 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM); 3774 if (val & DSP_MAXFIFO_PM5_ENABLE) 3775 wm->level = VLV_WM_LEVEL_PM5; 3776 3777 /* 3778 * If DDR DVFS is disabled in the BIOS, Punit 3779 * will never ack the request. So if that happens 3780 * assume we don't have to enable/disable DDR DVFS 3781 * dynamically. To test that just set the REQ_ACK 3782 * bit to poke the Punit, but don't change the 3783 * HIGH/LOW bits so that we don't actually change 3784 * the current state. 3785 */ 3786 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2); 3787 val |= FORCE_DDR_FREQ_REQ_ACK; 3788 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val); 3789 3790 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) & 3791 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) { 3792 drm_dbg_kms(&dev_priv->drm, 3793 "Punit not acking DDR DVFS request, " 3794 "assuming DDR DVFS is disabled\n"); 3795 dev_priv->display.wm.num_levels = VLV_WM_LEVEL_PM5 + 1; 3796 } else { 3797 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2); 3798 if ((val & FORCE_DDR_HIGH_FREQ) == 0) 3799 wm->level = VLV_WM_LEVEL_DDR_DVFS; 3800 } 3801 3802 vlv_punit_put(dev_priv); 3803 } 3804 3805 for_each_intel_crtc(&dev_priv->drm, crtc) { 3806 struct intel_crtc_state *crtc_state = 3807 to_intel_crtc_state(crtc->base.state); 3808 struct vlv_wm_state *active = &crtc->wm.active.vlv; 3809 const struct vlv_fifo_state *fifo_state = 3810 &crtc_state->wm.vlv.fifo_state; 3811 enum pipe pipe = crtc->pipe; 3812 enum plane_id plane_id; 3813 int level; 3814 3815 vlv_get_fifo_size(crtc_state); 3816 3817 active->num_levels = wm->level + 1; 3818 active->cxsr = wm->cxsr; 3819 3820 for (level = 0; level < active->num_levels; level++) { 3821 struct g4x_pipe_wm *raw = 3822 &crtc_state->wm.vlv.raw[level]; 3823 3824 active->sr[level].plane = wm->sr.plane; 3825 active->sr[level].cursor = wm->sr.cursor; 3826 3827 for_each_plane_id_on_crtc(crtc, plane_id) { 3828 active->wm[level].plane[plane_id] = 3829 wm->pipe[pipe].plane[plane_id]; 3830 3831 raw->plane[plane_id] = 3832 vlv_invert_wm_value(active->wm[level].plane[plane_id], 3833 fifo_state->plane[plane_id]); 3834 } 3835 } 3836 3837 for_each_plane_id_on_crtc(crtc, plane_id) 3838 vlv_raw_plane_wm_set(crtc_state, level, 3839 plane_id, USHRT_MAX); 3840 vlv_invalidate_wms(crtc, active, level); 3841 3842 crtc_state->wm.vlv.optimal = *active; 3843 crtc_state->wm.vlv.intermediate = *active; 3844 3845 drm_dbg_kms(&dev_priv->drm, 3846 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n", 3847 pipe_name(pipe), 3848 wm->pipe[pipe].plane[PLANE_PRIMARY], 3849 wm->pipe[pipe].plane[PLANE_CURSOR], 3850 wm->pipe[pipe].plane[PLANE_SPRITE0], 3851 wm->pipe[pipe].plane[PLANE_SPRITE1]); 3852 } 3853 3854 drm_dbg_kms(&dev_priv->drm, 3855 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n", 3856 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr); 3857 } 3858 3859 static void vlv_wm_sanitize(struct drm_i915_private *dev_priv) 3860 { 3861 struct intel_plane *plane; 3862 struct intel_crtc *crtc; 3863 3864 mutex_lock(&dev_priv->display.wm.wm_mutex); 3865 3866 for_each_intel_plane(&dev_priv->drm, plane) { 3867 struct intel_crtc *crtc = 3868 intel_crtc_for_pipe(dev_priv, plane->pipe); 3869 struct intel_crtc_state *crtc_state = 3870 to_intel_crtc_state(crtc->base.state); 3871 struct intel_plane_state *plane_state = 3872 to_intel_plane_state(plane->base.state); 3873 enum plane_id plane_id = plane->id; 3874 int level; 3875 3876 if (plane_state->uapi.visible) 3877 continue; 3878 3879 for (level = 0; level < dev_priv->display.wm.num_levels; level++) { 3880 struct g4x_pipe_wm *raw = 3881 &crtc_state->wm.vlv.raw[level]; 3882 3883 raw->plane[plane_id] = 0; 3884 } 3885 } 3886 3887 for_each_intel_crtc(&dev_priv->drm, crtc) { 3888 struct intel_crtc_state *crtc_state = 3889 to_intel_crtc_state(crtc->base.state); 3890 int ret; 3891 3892 ret = _vlv_compute_pipe_wm(crtc_state); 3893 drm_WARN_ON(&dev_priv->drm, ret); 3894 3895 crtc_state->wm.vlv.intermediate = 3896 crtc_state->wm.vlv.optimal; 3897 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal; 3898 } 3899 3900 vlv_program_watermarks(dev_priv); 3901 3902 mutex_unlock(&dev_priv->display.wm.wm_mutex); 3903 } 3904 3905 static void vlv_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915) 3906 { 3907 vlv_wm_get_hw_state(i915); 3908 vlv_wm_sanitize(i915); 3909 } 3910 3911 /* 3912 * FIXME should probably kill this and improve 3913 * the real watermark readout/sanitation instead 3914 */ 3915 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv) 3916 { 3917 intel_uncore_rmw(&dev_priv->uncore, WM3_LP_ILK, WM_LP_ENABLE, 0); 3918 intel_uncore_rmw(&dev_priv->uncore, WM2_LP_ILK, WM_LP_ENABLE, 0); 3919 intel_uncore_rmw(&dev_priv->uncore, WM1_LP_ILK, WM_LP_ENABLE, 0); 3920 3921 /* 3922 * Don't touch WM_LP_SPRITE_ENABLE here. 3923 * Doing so could cause underruns. 3924 */ 3925 } 3926 3927 static void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv) 3928 { 3929 struct ilk_wm_values *hw = &dev_priv->display.wm.hw; 3930 struct intel_crtc *crtc; 3931 3932 ilk_init_lp_watermarks(dev_priv); 3933 3934 for_each_intel_crtc(&dev_priv->drm, crtc) 3935 ilk_pipe_wm_get_hw_state(crtc); 3936 3937 hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK); 3938 hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK); 3939 hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK); 3940 3941 hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK); 3942 if (DISPLAY_VER(dev_priv) >= 7) { 3943 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB); 3944 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB); 3945 } 3946 3947 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) 3948 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & 3949 WM_MISC_DATA_PARTITION_5_6) ? 3950 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2; 3951 else if (IS_IVYBRIDGE(dev_priv)) 3952 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & 3953 DISP_DATA_PARTITION_5_6) ? 3954 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2; 3955 3956 hw->enable_fbc_wm = 3957 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS); 3958 } 3959 3960 static const struct intel_wm_funcs ilk_wm_funcs = { 3961 .compute_pipe_wm = ilk_compute_pipe_wm, 3962 .compute_intermediate_wm = ilk_compute_intermediate_wm, 3963 .initial_watermarks = ilk_initial_watermarks, 3964 .optimize_watermarks = ilk_optimize_watermarks, 3965 .get_hw_state = ilk_wm_get_hw_state, 3966 }; 3967 3968 static const struct intel_wm_funcs vlv_wm_funcs = { 3969 .compute_pipe_wm = vlv_compute_pipe_wm, 3970 .compute_intermediate_wm = vlv_compute_intermediate_wm, 3971 .initial_watermarks = vlv_initial_watermarks, 3972 .optimize_watermarks = vlv_optimize_watermarks, 3973 .atomic_update_watermarks = vlv_atomic_update_fifo, 3974 .get_hw_state = vlv_wm_get_hw_state_and_sanitize, 3975 }; 3976 3977 static const struct intel_wm_funcs g4x_wm_funcs = { 3978 .compute_pipe_wm = g4x_compute_pipe_wm, 3979 .compute_intermediate_wm = g4x_compute_intermediate_wm, 3980 .initial_watermarks = g4x_initial_watermarks, 3981 .optimize_watermarks = g4x_optimize_watermarks, 3982 .get_hw_state = g4x_wm_get_hw_state_and_sanitize, 3983 }; 3984 3985 static const struct intel_wm_funcs pnv_wm_funcs = { 3986 .update_wm = pnv_update_wm, 3987 }; 3988 3989 static const struct intel_wm_funcs i965_wm_funcs = { 3990 .update_wm = i965_update_wm, 3991 }; 3992 3993 static const struct intel_wm_funcs i9xx_wm_funcs = { 3994 .update_wm = i9xx_update_wm, 3995 }; 3996 3997 static const struct intel_wm_funcs i845_wm_funcs = { 3998 .update_wm = i845_update_wm, 3999 }; 4000 4001 static const struct intel_wm_funcs nop_funcs = { 4002 }; 4003 4004 void i9xx_wm_init(struct drm_i915_private *dev_priv) 4005 { 4006 /* For FIFO watermark updates */ 4007 if (HAS_PCH_SPLIT(dev_priv)) { 4008 ilk_setup_wm_latency(dev_priv); 4009 dev_priv->display.funcs.wm = &ilk_wm_funcs; 4010 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { 4011 vlv_setup_wm_latency(dev_priv); 4012 dev_priv->display.funcs.wm = &vlv_wm_funcs; 4013 } else if (IS_G4X(dev_priv)) { 4014 g4x_setup_wm_latency(dev_priv); 4015 dev_priv->display.funcs.wm = &g4x_wm_funcs; 4016 } else if (IS_PINEVIEW(dev_priv)) { 4017 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv), 4018 dev_priv->is_ddr3, 4019 dev_priv->fsb_freq, 4020 dev_priv->mem_freq)) { 4021 drm_info(&dev_priv->drm, 4022 "failed to find known CxSR latency " 4023 "(found ddr%s fsb freq %d, mem freq %d), " 4024 "disabling CxSR\n", 4025 (dev_priv->is_ddr3 == 1) ? "3" : "2", 4026 dev_priv->fsb_freq, dev_priv->mem_freq); 4027 /* Disable CxSR and never update its watermark again */ 4028 intel_set_memory_cxsr(dev_priv, false); 4029 dev_priv->display.funcs.wm = &nop_funcs; 4030 } else { 4031 dev_priv->display.funcs.wm = &pnv_wm_funcs; 4032 } 4033 } else if (DISPLAY_VER(dev_priv) == 4) { 4034 dev_priv->display.funcs.wm = &i965_wm_funcs; 4035 } else if (DISPLAY_VER(dev_priv) == 3) { 4036 dev_priv->display.funcs.wm = &i9xx_wm_funcs; 4037 } else if (DISPLAY_VER(dev_priv) == 2) { 4038 if (INTEL_NUM_PIPES(dev_priv) == 1) 4039 dev_priv->display.funcs.wm = &i845_wm_funcs; 4040 else 4041 dev_priv->display.funcs.wm = &i9xx_wm_funcs; 4042 } else { 4043 drm_err(&dev_priv->drm, 4044 "unexpected fall-through in %s\n", __func__); 4045 dev_priv->display.funcs.wm = &nop_funcs; 4046 } 4047 } 4048