1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022-2023 Intel Corporation 4 */ 5 6 #include "i915_drv.h" 7 #include "i915_reg.h" 8 #include "intel_de.h" 9 #include "intel_display_types.h" 10 #include "intel_vblank.h" 11 #include "intel_vrr.h" 12 13 /* 14 * This timing diagram depicts the video signal in and 15 * around the vertical blanking period. 16 * 17 * Assumptions about the fictitious mode used in this example: 18 * vblank_start >= 3 19 * vsync_start = vblank_start + 1 20 * vsync_end = vblank_start + 2 21 * vtotal = vblank_start + 3 22 * 23 * start of vblank: 24 * latch double buffered registers 25 * increment frame counter (ctg+) 26 * generate start of vblank interrupt (gen4+) 27 * | 28 * | frame start: 29 * | generate frame start interrupt (aka. vblank interrupt) (gmch) 30 * | may be shifted forward 1-3 extra lines via TRANSCONF 31 * | | 32 * | | start of vsync: 33 * | | generate vsync interrupt 34 * | | | 35 * ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx 36 * . \hs/ . \hs/ \hs/ \hs/ . \hs/ 37 * ----va---> <-----------------vb--------------------> <--------va------------- 38 * | | <----vs-----> | 39 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2) 40 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+) 41 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi) 42 * | | | 43 * last visible pixel first visible pixel 44 * | increment frame counter (gen3/4) 45 * pixel counter = vblank_start * htotal pixel counter = 0 (gen3/4) 46 * 47 * x = horizontal active 48 * _ = horizontal blanking 49 * hs = horizontal sync 50 * va = vertical active 51 * vb = vertical blanking 52 * vs = vertical sync 53 * vbs = vblank_start (number) 54 * 55 * Summary: 56 * - most events happen at the start of horizontal sync 57 * - frame start happens at the start of horizontal blank, 1-4 lines 58 * (depending on TRANSCONF settings) after the start of vblank 59 * - gen3/4 pixel and frame counter are synchronized with the start 60 * of horizontal active on the first line of vertical active 61 */ 62 63 /* 64 * Called from drm generic code, passed a 'crtc', which we use as a pipe index. 65 */ 66 u32 i915_get_vblank_counter(struct drm_crtc *crtc) 67 { 68 struct drm_i915_private *dev_priv = to_i915(crtc->dev); 69 struct drm_vblank_crtc *vblank = &dev_priv->drm.vblank[drm_crtc_index(crtc)]; 70 const struct drm_display_mode *mode = &vblank->hwmode; 71 enum pipe pipe = to_intel_crtc(crtc)->pipe; 72 u32 pixel, vbl_start, hsync_start, htotal; 73 u64 frame; 74 75 /* 76 * On i965gm TV output the frame counter only works up to 77 * the point when we enable the TV encoder. After that the 78 * frame counter ceases to work and reads zero. We need a 79 * vblank wait before enabling the TV encoder and so we 80 * have to enable vblank interrupts while the frame counter 81 * is still in a working state. However the core vblank code 82 * does not like us returning non-zero frame counter values 83 * when we've told it that we don't have a working frame 84 * counter. Thus we must stop non-zero values leaking out. 85 */ 86 if (!vblank->max_vblank_count) 87 return 0; 88 89 htotal = mode->crtc_htotal; 90 hsync_start = mode->crtc_hsync_start; 91 vbl_start = mode->crtc_vblank_start; 92 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 93 vbl_start = DIV_ROUND_UP(vbl_start, 2); 94 95 /* Convert to pixel count */ 96 vbl_start *= htotal; 97 98 /* Start of vblank event occurs at start of hsync */ 99 vbl_start -= htotal - hsync_start; 100 101 /* 102 * High & low register fields aren't synchronized, so make sure 103 * we get a low value that's stable across two reads of the high 104 * register. 105 */ 106 frame = intel_de_read64_2x32(dev_priv, PIPEFRAMEPIXEL(pipe), PIPEFRAME(pipe)); 107 108 pixel = frame & PIPE_PIXEL_MASK; 109 frame = (frame >> PIPE_FRAME_LOW_SHIFT) & 0xffffff; 110 111 /* 112 * The frame counter increments at beginning of active. 113 * Cook up a vblank counter by also checking the pixel 114 * counter against vblank start. 115 */ 116 return (frame + (pixel >= vbl_start)) & 0xffffff; 117 } 118 119 u32 g4x_get_vblank_counter(struct drm_crtc *crtc) 120 { 121 struct drm_i915_private *dev_priv = to_i915(crtc->dev); 122 struct drm_vblank_crtc *vblank = &dev_priv->drm.vblank[drm_crtc_index(crtc)]; 123 enum pipe pipe = to_intel_crtc(crtc)->pipe; 124 125 if (!vblank->max_vblank_count) 126 return 0; 127 128 return intel_de_read(dev_priv, PIPE_FRMCOUNT_G4X(pipe)); 129 } 130 131 static u32 intel_crtc_scanlines_since_frame_timestamp(struct intel_crtc *crtc) 132 { 133 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 134 struct drm_vblank_crtc *vblank = 135 &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)]; 136 const struct drm_display_mode *mode = &vblank->hwmode; 137 u32 htotal = mode->crtc_htotal; 138 u32 clock = mode->crtc_clock; 139 u32 scan_prev_time, scan_curr_time, scan_post_time; 140 141 /* 142 * To avoid the race condition where we might cross into the 143 * next vblank just between the PIPE_FRMTMSTMP and TIMESTAMP_CTR 144 * reads. We make sure we read PIPE_FRMTMSTMP and TIMESTAMP_CTR 145 * during the same frame. 146 */ 147 do { 148 /* 149 * This field provides read back of the display 150 * pipe frame time stamp. The time stamp value 151 * is sampled at every start of vertical blank. 152 */ 153 scan_prev_time = intel_de_read_fw(dev_priv, 154 PIPE_FRMTMSTMP(crtc->pipe)); 155 156 /* 157 * The TIMESTAMP_CTR register has the current 158 * time stamp value. 159 */ 160 scan_curr_time = intel_de_read_fw(dev_priv, IVB_TIMESTAMP_CTR); 161 162 scan_post_time = intel_de_read_fw(dev_priv, 163 PIPE_FRMTMSTMP(crtc->pipe)); 164 } while (scan_post_time != scan_prev_time); 165 166 return div_u64(mul_u32_u32(scan_curr_time - scan_prev_time, 167 clock), 1000 * htotal); 168 } 169 170 /* 171 * On certain encoders on certain platforms, pipe 172 * scanline register will not work to get the scanline, 173 * since the timings are driven from the PORT or issues 174 * with scanline register updates. 175 * This function will use Framestamp and current 176 * timestamp registers to calculate the scanline. 177 */ 178 static u32 __intel_get_crtc_scanline_from_timestamp(struct intel_crtc *crtc) 179 { 180 struct drm_vblank_crtc *vblank = 181 &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)]; 182 const struct drm_display_mode *mode = &vblank->hwmode; 183 u32 vblank_start = mode->crtc_vblank_start; 184 u32 vtotal = mode->crtc_vtotal; 185 u32 scanline; 186 187 scanline = intel_crtc_scanlines_since_frame_timestamp(crtc); 188 scanline = min(scanline, vtotal - 1); 189 scanline = (scanline + vblank_start) % vtotal; 190 191 return scanline; 192 } 193 194 /* 195 * intel_de_read_fw(), only for fast reads of display block, no need for 196 * forcewake etc. 197 */ 198 static int __intel_get_crtc_scanline(struct intel_crtc *crtc) 199 { 200 struct drm_device *dev = crtc->base.dev; 201 struct drm_i915_private *dev_priv = to_i915(dev); 202 const struct drm_display_mode *mode; 203 struct drm_vblank_crtc *vblank; 204 enum pipe pipe = crtc->pipe; 205 int position, vtotal; 206 207 if (!crtc->active) 208 return 0; 209 210 vblank = &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)]; 211 mode = &vblank->hwmode; 212 213 if (crtc->mode_flags & I915_MODE_FLAG_GET_SCANLINE_FROM_TIMESTAMP) 214 return __intel_get_crtc_scanline_from_timestamp(crtc); 215 216 vtotal = mode->crtc_vtotal; 217 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 218 vtotal /= 2; 219 220 position = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & PIPEDSL_LINE_MASK; 221 222 /* 223 * On HSW, the DSL reg (0x70000) appears to return 0 if we 224 * read it just before the start of vblank. So try it again 225 * so we don't accidentally end up spanning a vblank frame 226 * increment, causing the pipe_update_end() code to squak at us. 227 * 228 * The nature of this problem means we can't simply check the ISR 229 * bit and return the vblank start value; nor can we use the scanline 230 * debug register in the transcoder as it appears to have the same 231 * problem. We may need to extend this to include other platforms, 232 * but so far testing only shows the problem on HSW. 233 */ 234 if (HAS_DDI(dev_priv) && !position) { 235 int i, temp; 236 237 for (i = 0; i < 100; i++) { 238 udelay(1); 239 temp = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & PIPEDSL_LINE_MASK; 240 if (temp != position) { 241 position = temp; 242 break; 243 } 244 } 245 } 246 247 /* 248 * See update_scanline_offset() for the details on the 249 * scanline_offset adjustment. 250 */ 251 return (position + crtc->scanline_offset) % vtotal; 252 } 253 254 static bool i915_get_crtc_scanoutpos(struct drm_crtc *_crtc, 255 bool in_vblank_irq, 256 int *vpos, int *hpos, 257 ktime_t *stime, ktime_t *etime, 258 const struct drm_display_mode *mode) 259 { 260 struct drm_device *dev = _crtc->dev; 261 struct drm_i915_private *dev_priv = to_i915(dev); 262 struct intel_crtc *crtc = to_intel_crtc(_crtc); 263 enum pipe pipe = crtc->pipe; 264 int position; 265 int vbl_start, vbl_end, hsync_start, htotal, vtotal; 266 unsigned long irqflags; 267 bool use_scanline_counter = DISPLAY_VER(dev_priv) >= 5 || 268 IS_G4X(dev_priv) || DISPLAY_VER(dev_priv) == 2 || 269 crtc->mode_flags & I915_MODE_FLAG_USE_SCANLINE_COUNTER; 270 271 if (drm_WARN_ON(&dev_priv->drm, !mode->crtc_clock)) { 272 drm_dbg(&dev_priv->drm, 273 "trying to get scanoutpos for disabled pipe %c\n", 274 pipe_name(pipe)); 275 return false; 276 } 277 278 htotal = mode->crtc_htotal; 279 hsync_start = mode->crtc_hsync_start; 280 vtotal = mode->crtc_vtotal; 281 vbl_start = mode->crtc_vblank_start; 282 vbl_end = mode->crtc_vblank_end; 283 284 if (mode->flags & DRM_MODE_FLAG_INTERLACE) { 285 vbl_start = DIV_ROUND_UP(vbl_start, 2); 286 vbl_end /= 2; 287 vtotal /= 2; 288 } 289 290 /* 291 * Lock uncore.lock, as we will do multiple timing critical raw 292 * register reads, potentially with preemption disabled, so the 293 * following code must not block on uncore.lock. 294 */ 295 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 296 297 /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */ 298 299 /* Get optional system timestamp before query. */ 300 if (stime) 301 *stime = ktime_get(); 302 303 if (crtc->mode_flags & I915_MODE_FLAG_VRR) { 304 int scanlines = intel_crtc_scanlines_since_frame_timestamp(crtc); 305 306 position = __intel_get_crtc_scanline(crtc); 307 308 /* 309 * Already exiting vblank? If so, shift our position 310 * so it looks like we're already apporaching the full 311 * vblank end. This should make the generated timestamp 312 * more or less match when the active portion will start. 313 */ 314 if (position >= vbl_start && scanlines < position) 315 position = min(crtc->vmax_vblank_start + scanlines, vtotal - 1); 316 } else if (use_scanline_counter) { 317 /* No obvious pixelcount register. Only query vertical 318 * scanout position from Display scan line register. 319 */ 320 position = __intel_get_crtc_scanline(crtc); 321 } else { 322 /* 323 * Have access to pixelcount since start of frame. 324 * We can split this into vertical and horizontal 325 * scanout position. 326 */ 327 position = (intel_de_read_fw(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT; 328 329 /* convert to pixel counts */ 330 vbl_start *= htotal; 331 vbl_end *= htotal; 332 vtotal *= htotal; 333 334 /* 335 * In interlaced modes, the pixel counter counts all pixels, 336 * so one field will have htotal more pixels. In order to avoid 337 * the reported position from jumping backwards when the pixel 338 * counter is beyond the length of the shorter field, just 339 * clamp the position the length of the shorter field. This 340 * matches how the scanline counter based position works since 341 * the scanline counter doesn't count the two half lines. 342 */ 343 position = min(position, vtotal - 1); 344 345 /* 346 * Start of vblank interrupt is triggered at start of hsync, 347 * just prior to the first active line of vblank. However we 348 * consider lines to start at the leading edge of horizontal 349 * active. So, should we get here before we've crossed into 350 * the horizontal active of the first line in vblank, we would 351 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that, 352 * always add htotal-hsync_start to the current pixel position. 353 */ 354 position = (position + htotal - hsync_start) % vtotal; 355 } 356 357 /* Get optional system timestamp after query. */ 358 if (etime) 359 *etime = ktime_get(); 360 361 /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */ 362 363 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 364 365 /* 366 * While in vblank, position will be negative 367 * counting up towards 0 at vbl_end. And outside 368 * vblank, position will be positive counting 369 * up since vbl_end. 370 */ 371 if (position >= vbl_start) 372 position -= vbl_end; 373 else 374 position += vtotal - vbl_end; 375 376 if (use_scanline_counter) { 377 *vpos = position; 378 *hpos = 0; 379 } else { 380 *vpos = position / htotal; 381 *hpos = position - (*vpos * htotal); 382 } 383 384 return true; 385 } 386 387 bool intel_crtc_get_vblank_timestamp(struct drm_crtc *crtc, int *max_error, 388 ktime_t *vblank_time, bool in_vblank_irq) 389 { 390 return drm_crtc_vblank_helper_get_vblank_timestamp_internal( 391 crtc, max_error, vblank_time, in_vblank_irq, 392 i915_get_crtc_scanoutpos); 393 } 394 395 int intel_get_crtc_scanline(struct intel_crtc *crtc) 396 { 397 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 398 unsigned long irqflags; 399 int position; 400 401 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 402 position = __intel_get_crtc_scanline(crtc); 403 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 404 405 return position; 406 } 407 408 static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv, 409 enum pipe pipe) 410 { 411 i915_reg_t reg = PIPEDSL(pipe); 412 u32 line1, line2; 413 414 line1 = intel_de_read(dev_priv, reg) & PIPEDSL_LINE_MASK; 415 msleep(5); 416 line2 = intel_de_read(dev_priv, reg) & PIPEDSL_LINE_MASK; 417 418 return line1 != line2; 419 } 420 421 static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state) 422 { 423 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 424 enum pipe pipe = crtc->pipe; 425 426 /* Wait for the display line to settle/start moving */ 427 if (wait_for(pipe_scanline_is_moving(dev_priv, pipe) == state, 100)) 428 drm_err(&dev_priv->drm, 429 "pipe %c scanline %s wait timed out\n", 430 pipe_name(pipe), str_on_off(state)); 431 } 432 433 void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc) 434 { 435 wait_for_pipe_scanline_moving(crtc, false); 436 } 437 438 void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc) 439 { 440 wait_for_pipe_scanline_moving(crtc, true); 441 } 442 443 static int intel_crtc_scanline_offset(const struct intel_crtc_state *crtc_state) 444 { 445 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); 446 const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode; 447 448 /* 449 * The scanline counter increments at the leading edge of hsync. 450 * 451 * On most platforms it starts counting from vtotal-1 on the 452 * first active line. That means the scanline counter value is 453 * always one less than what we would expect. Ie. just after 454 * start of vblank, which also occurs at start of hsync (on the 455 * last active line), the scanline counter will read vblank_start-1. 456 * 457 * On gen2 the scanline counter starts counting from 1 instead 458 * of vtotal-1, so we have to subtract one (or rather add vtotal-1 459 * to keep the value positive), instead of adding one. 460 * 461 * On HSW+ the behaviour of the scanline counter depends on the output 462 * type. For DP ports it behaves like most other platforms, but on HDMI 463 * there's an extra 1 line difference. So we need to add two instead of 464 * one to the value. 465 * 466 * On VLV/CHV DSI the scanline counter would appear to increment 467 * approx. 1/3 of a scanline before start of vblank. Unfortunately 468 * that means we can't tell whether we're in vblank or not while 469 * we're on that particular line. We must still set scanline_offset 470 * to 1 so that the vblank timestamps come out correct when we query 471 * the scanline counter from within the vblank interrupt handler. 472 * However if queried just before the start of vblank we'll get an 473 * answer that's slightly in the future. 474 */ 475 if (DISPLAY_VER(i915) == 2) { 476 int vtotal; 477 478 vtotal = adjusted_mode->crtc_vtotal; 479 if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) 480 vtotal /= 2; 481 482 return vtotal - 1; 483 } else if (HAS_DDI(i915) && intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) { 484 return 2; 485 } else { 486 return 1; 487 } 488 } 489 490 void intel_crtc_update_active_timings(const struct intel_crtc_state *crtc_state, 491 bool vrr_enable) 492 { 493 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 494 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 495 u8 mode_flags = crtc_state->mode_flags; 496 struct drm_display_mode adjusted_mode; 497 int vmax_vblank_start = 0; 498 unsigned long irqflags; 499 500 drm_mode_init(&adjusted_mode, &crtc_state->hw.adjusted_mode); 501 502 if (vrr_enable) { 503 drm_WARN_ON(&i915->drm, (mode_flags & I915_MODE_FLAG_VRR) == 0); 504 505 adjusted_mode.crtc_vtotal = crtc_state->vrr.vmax; 506 adjusted_mode.crtc_vblank_end = crtc_state->vrr.vmax; 507 adjusted_mode.crtc_vblank_start = intel_vrr_vmin_vblank_start(crtc_state); 508 vmax_vblank_start = intel_vrr_vmax_vblank_start(crtc_state); 509 } else { 510 mode_flags &= ~I915_MODE_FLAG_VRR; 511 } 512 513 /* 514 * Belts and suspenders locking to guarantee everyone sees 100% 515 * consistent state during fastset seamless refresh rate changes. 516 * 517 * vblank_time_lock takes care of all drm_vblank.c stuff, and 518 * uncore.lock takes care of __intel_get_crtc_scanline() which 519 * may get called elsewhere as well. 520 * 521 * TODO maybe just protect everything (including 522 * __intel_get_crtc_scanline()) with vblank_time_lock? 523 * Need to audit everything to make sure it's safe. 524 */ 525 spin_lock_irqsave(&i915->drm.vblank_time_lock, irqflags); 526 spin_lock(&i915->uncore.lock); 527 528 drm_calc_timestamping_constants(&crtc->base, &adjusted_mode); 529 530 crtc->vmax_vblank_start = vmax_vblank_start; 531 532 crtc->mode_flags = mode_flags; 533 534 crtc->scanline_offset = intel_crtc_scanline_offset(crtc_state); 535 536 spin_unlock(&i915->uncore.lock); 537 spin_unlock_irqrestore(&i915->drm.vblank_time_lock, irqflags); 538 } 539