1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2020 Intel Corporation 4 */ 5 #include <linux/kernel.h> 6 #include <linux/slab.h> 7 8 #include <drm/drm_atomic_helper.h> 9 #include <drm/drm_fourcc.h> 10 #include <drm/drm_plane.h> 11 #include <drm/drm_plane_helper.h> 12 13 #include "i915_trace.h" 14 #include "i915_vgpu.h" 15 16 #include "intel_atomic.h" 17 #include "intel_atomic_plane.h" 18 #include "intel_color.h" 19 #include "intel_crtc.h" 20 #include "intel_cursor.h" 21 #include "intel_display_debugfs.h" 22 #include "intel_display_types.h" 23 #include "intel_dsi.h" 24 #include "intel_pipe_crc.h" 25 #include "intel_psr.h" 26 #include "intel_sprite.h" 27 #include "intel_vrr.h" 28 #include "i9xx_plane.h" 29 #include "skl_universal_plane.h" 30 31 static void assert_vblank_disabled(struct drm_crtc *crtc) 32 { 33 if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0)) 34 drm_crtc_vblank_put(crtc); 35 } 36 37 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc) 38 { 39 struct drm_device *dev = crtc->base.dev; 40 struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)]; 41 42 if (!crtc->active) 43 return 0; 44 45 if (!vblank->max_vblank_count) 46 return (u32)drm_crtc_accurate_vblank_count(&crtc->base); 47 48 return crtc->base.funcs->get_vblank_counter(&crtc->base); 49 } 50 51 u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state) 52 { 53 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); 54 55 /* 56 * From Gen 11, In case of dsi cmd mode, frame counter wouldnt 57 * have updated at the beginning of TE, if we want to use 58 * the hw counter, then we would find it updated in only 59 * the next TE, hence switching to sw counter. 60 */ 61 if (crtc_state->mode_flags & (I915_MODE_FLAG_DSI_USE_TE0 | 62 I915_MODE_FLAG_DSI_USE_TE1)) 63 return 0; 64 65 /* 66 * On i965gm the hardware frame counter reads 67 * zero when the TV encoder is enabled :( 68 */ 69 if (IS_I965GM(dev_priv) && 70 (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT))) 71 return 0; 72 73 if (DISPLAY_VER(dev_priv) >= 5 || IS_G4X(dev_priv)) 74 return 0xffffffff; /* full 32 bit counter */ 75 else if (DISPLAY_VER(dev_priv) >= 3) 76 return 0xffffff; /* only 24 bits of frame count */ 77 else 78 return 0; /* Gen2 doesn't have a hardware frame counter */ 79 } 80 81 void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state) 82 { 83 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 84 85 assert_vblank_disabled(&crtc->base); 86 drm_crtc_set_max_vblank_count(&crtc->base, 87 intel_crtc_max_vblank_count(crtc_state)); 88 drm_crtc_vblank_on(&crtc->base); 89 90 /* 91 * Should really happen exactly when we enable the pipe 92 * but we want the frame counters in the trace, and that 93 * requires vblank support on some platforms/outputs. 94 */ 95 trace_intel_pipe_enable(crtc); 96 } 97 98 void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state) 99 { 100 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 101 102 /* 103 * Should really happen exactly when we disable the pipe 104 * but we want the frame counters in the trace, and that 105 * requires vblank support on some platforms/outputs. 106 */ 107 trace_intel_pipe_disable(crtc); 108 109 drm_crtc_vblank_off(&crtc->base); 110 assert_vblank_disabled(&crtc->base); 111 } 112 113 struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc) 114 { 115 struct intel_crtc_state *crtc_state; 116 117 crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL); 118 119 if (crtc_state) 120 intel_crtc_state_reset(crtc_state, crtc); 121 122 return crtc_state; 123 } 124 125 void intel_crtc_state_reset(struct intel_crtc_state *crtc_state, 126 struct intel_crtc *crtc) 127 { 128 memset(crtc_state, 0, sizeof(*crtc_state)); 129 130 __drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base); 131 132 crtc_state->cpu_transcoder = INVALID_TRANSCODER; 133 crtc_state->master_transcoder = INVALID_TRANSCODER; 134 crtc_state->hsw_workaround_pipe = INVALID_PIPE; 135 crtc_state->scaler_state.scaler_id = -1; 136 crtc_state->mst_master_transcoder = INVALID_TRANSCODER; 137 } 138 139 static struct intel_crtc *intel_crtc_alloc(void) 140 { 141 struct intel_crtc_state *crtc_state; 142 struct intel_crtc *crtc; 143 144 crtc = kzalloc(sizeof(*crtc), GFP_KERNEL); 145 if (!crtc) 146 return ERR_PTR(-ENOMEM); 147 148 crtc_state = intel_crtc_state_alloc(crtc); 149 if (!crtc_state) { 150 kfree(crtc); 151 return ERR_PTR(-ENOMEM); 152 } 153 154 crtc->base.state = &crtc_state->uapi; 155 crtc->config = crtc_state; 156 157 return crtc; 158 } 159 160 static void intel_crtc_free(struct intel_crtc *crtc) 161 { 162 intel_crtc_destroy_state(&crtc->base, crtc->base.state); 163 kfree(crtc); 164 } 165 166 static void intel_crtc_destroy(struct drm_crtc *crtc) 167 { 168 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 169 170 drm_crtc_cleanup(crtc); 171 kfree(intel_crtc); 172 } 173 174 static int intel_crtc_late_register(struct drm_crtc *crtc) 175 { 176 intel_crtc_debugfs_add(crtc); 177 return 0; 178 } 179 180 #define INTEL_CRTC_FUNCS \ 181 .set_config = drm_atomic_helper_set_config, \ 182 .destroy = intel_crtc_destroy, \ 183 .page_flip = drm_atomic_helper_page_flip, \ 184 .atomic_duplicate_state = intel_crtc_duplicate_state, \ 185 .atomic_destroy_state = intel_crtc_destroy_state, \ 186 .set_crc_source = intel_crtc_set_crc_source, \ 187 .verify_crc_source = intel_crtc_verify_crc_source, \ 188 .get_crc_sources = intel_crtc_get_crc_sources, \ 189 .late_register = intel_crtc_late_register 190 191 static const struct drm_crtc_funcs bdw_crtc_funcs = { 192 INTEL_CRTC_FUNCS, 193 194 .get_vblank_counter = g4x_get_vblank_counter, 195 .enable_vblank = bdw_enable_vblank, 196 .disable_vblank = bdw_disable_vblank, 197 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, 198 }; 199 200 static const struct drm_crtc_funcs ilk_crtc_funcs = { 201 INTEL_CRTC_FUNCS, 202 203 .get_vblank_counter = g4x_get_vblank_counter, 204 .enable_vblank = ilk_enable_vblank, 205 .disable_vblank = ilk_disable_vblank, 206 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, 207 }; 208 209 static const struct drm_crtc_funcs g4x_crtc_funcs = { 210 INTEL_CRTC_FUNCS, 211 212 .get_vblank_counter = g4x_get_vblank_counter, 213 .enable_vblank = i965_enable_vblank, 214 .disable_vblank = i965_disable_vblank, 215 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, 216 }; 217 218 static const struct drm_crtc_funcs i965_crtc_funcs = { 219 INTEL_CRTC_FUNCS, 220 221 .get_vblank_counter = i915_get_vblank_counter, 222 .enable_vblank = i965_enable_vblank, 223 .disable_vblank = i965_disable_vblank, 224 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, 225 }; 226 227 static const struct drm_crtc_funcs i915gm_crtc_funcs = { 228 INTEL_CRTC_FUNCS, 229 230 .get_vblank_counter = i915_get_vblank_counter, 231 .enable_vblank = i915gm_enable_vblank, 232 .disable_vblank = i915gm_disable_vblank, 233 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, 234 }; 235 236 static const struct drm_crtc_funcs i915_crtc_funcs = { 237 INTEL_CRTC_FUNCS, 238 239 .get_vblank_counter = i915_get_vblank_counter, 240 .enable_vblank = i8xx_enable_vblank, 241 .disable_vblank = i8xx_disable_vblank, 242 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, 243 }; 244 245 static const struct drm_crtc_funcs i8xx_crtc_funcs = { 246 INTEL_CRTC_FUNCS, 247 248 /* no hw vblank counter */ 249 .enable_vblank = i8xx_enable_vblank, 250 .disable_vblank = i8xx_disable_vblank, 251 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, 252 }; 253 254 int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe) 255 { 256 struct intel_plane *primary, *cursor; 257 const struct drm_crtc_funcs *funcs; 258 struct intel_crtc *crtc; 259 int sprite, ret; 260 261 crtc = intel_crtc_alloc(); 262 if (IS_ERR(crtc)) 263 return PTR_ERR(crtc); 264 265 crtc->pipe = pipe; 266 crtc->num_scalers = RUNTIME_INFO(dev_priv)->num_scalers[pipe]; 267 268 if (DISPLAY_VER(dev_priv) >= 9) 269 primary = skl_universal_plane_create(dev_priv, pipe, 270 PLANE_PRIMARY); 271 else 272 primary = intel_primary_plane_create(dev_priv, pipe); 273 if (IS_ERR(primary)) { 274 ret = PTR_ERR(primary); 275 goto fail; 276 } 277 crtc->plane_ids_mask |= BIT(primary->id); 278 279 for_each_sprite(dev_priv, pipe, sprite) { 280 struct intel_plane *plane; 281 282 if (DISPLAY_VER(dev_priv) >= 9) 283 plane = skl_universal_plane_create(dev_priv, pipe, 284 PLANE_SPRITE0 + sprite); 285 else 286 plane = intel_sprite_plane_create(dev_priv, pipe, sprite); 287 if (IS_ERR(plane)) { 288 ret = PTR_ERR(plane); 289 goto fail; 290 } 291 crtc->plane_ids_mask |= BIT(plane->id); 292 } 293 294 cursor = intel_cursor_plane_create(dev_priv, pipe); 295 if (IS_ERR(cursor)) { 296 ret = PTR_ERR(cursor); 297 goto fail; 298 } 299 crtc->plane_ids_mask |= BIT(cursor->id); 300 301 if (HAS_GMCH(dev_priv)) { 302 if (IS_CHERRYVIEW(dev_priv) || 303 IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv)) 304 funcs = &g4x_crtc_funcs; 305 else if (DISPLAY_VER(dev_priv) == 4) 306 funcs = &i965_crtc_funcs; 307 else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv)) 308 funcs = &i915gm_crtc_funcs; 309 else if (DISPLAY_VER(dev_priv) == 3) 310 funcs = &i915_crtc_funcs; 311 else 312 funcs = &i8xx_crtc_funcs; 313 } else { 314 if (DISPLAY_VER(dev_priv) >= 8) 315 funcs = &bdw_crtc_funcs; 316 else 317 funcs = &ilk_crtc_funcs; 318 } 319 320 ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base, 321 &primary->base, &cursor->base, 322 funcs, "pipe %c", pipe_name(pipe)); 323 if (ret) 324 goto fail; 325 326 BUG_ON(pipe >= ARRAY_SIZE(dev_priv->pipe_to_crtc_mapping) || 327 dev_priv->pipe_to_crtc_mapping[pipe] != NULL); 328 dev_priv->pipe_to_crtc_mapping[pipe] = crtc; 329 330 if (DISPLAY_VER(dev_priv) < 9) { 331 enum i9xx_plane_id i9xx_plane = primary->i9xx_plane; 332 333 BUG_ON(i9xx_plane >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) || 334 dev_priv->plane_to_crtc_mapping[i9xx_plane] != NULL); 335 dev_priv->plane_to_crtc_mapping[i9xx_plane] = crtc; 336 } 337 338 if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) 339 drm_crtc_create_scaling_filter_property(&crtc->base, 340 BIT(DRM_SCALING_FILTER_DEFAULT) | 341 BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR)); 342 343 intel_color_init(crtc); 344 345 intel_crtc_crc_init(crtc); 346 347 drm_WARN_ON(&dev_priv->drm, drm_crtc_index(&crtc->base) != crtc->pipe); 348 349 return 0; 350 351 fail: 352 intel_crtc_free(crtc); 353 354 return ret; 355 } 356 357 int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode, 358 int usecs) 359 { 360 /* paranoia */ 361 if (!adjusted_mode->crtc_htotal) 362 return 1; 363 364 return DIV_ROUND_UP(usecs * adjusted_mode->crtc_clock, 365 1000 * adjusted_mode->crtc_htotal); 366 } 367 368 static int intel_mode_vblank_start(const struct drm_display_mode *mode) 369 { 370 int vblank_start = mode->crtc_vblank_start; 371 372 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 373 vblank_start = DIV_ROUND_UP(vblank_start, 2); 374 375 return vblank_start; 376 } 377 378 /** 379 * intel_pipe_update_start() - start update of a set of display registers 380 * @new_crtc_state: the new crtc state 381 * 382 * Mark the start of an update to pipe registers that should be updated 383 * atomically regarding vblank. If the next vblank will happens within 384 * the next 100 us, this function waits until the vblank passes. 385 * 386 * After a successful call to this function, interrupts will be disabled 387 * until a subsequent call to intel_pipe_update_end(). That is done to 388 * avoid random delays. 389 */ 390 void intel_pipe_update_start(const struct intel_crtc_state *new_crtc_state) 391 { 392 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc); 393 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 394 const struct drm_display_mode *adjusted_mode = &new_crtc_state->hw.adjusted_mode; 395 long timeout = msecs_to_jiffies_timeout(1); 396 int scanline, min, max, vblank_start; 397 wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base); 398 bool need_vlv_dsi_wa = (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) && 399 intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI); 400 DEFINE_WAIT(wait); 401 402 if (new_crtc_state->uapi.async_flip) 403 return; 404 405 if (new_crtc_state->vrr.enable) 406 vblank_start = intel_vrr_vmax_vblank_start(new_crtc_state); 407 else 408 vblank_start = intel_mode_vblank_start(adjusted_mode); 409 410 /* FIXME needs to be calibrated sensibly */ 411 min = vblank_start - intel_usecs_to_scanlines(adjusted_mode, 412 VBLANK_EVASION_TIME_US); 413 max = vblank_start - 1; 414 415 if (min <= 0 || max <= 0) 416 goto irq_disable; 417 418 if (drm_WARN_ON(&dev_priv->drm, drm_crtc_vblank_get(&crtc->base))) 419 goto irq_disable; 420 421 /* 422 * Wait for psr to idle out after enabling the VBL interrupts 423 * VBL interrupts will start the PSR exit and prevent a PSR 424 * re-entry as well. 425 */ 426 intel_psr_wait_for_idle(new_crtc_state); 427 428 local_irq_disable(); 429 430 crtc->debug.min_vbl = min; 431 crtc->debug.max_vbl = max; 432 trace_intel_pipe_update_start(crtc); 433 434 for (;;) { 435 /* 436 * prepare_to_wait() has a memory barrier, which guarantees 437 * other CPUs can see the task state update by the time we 438 * read the scanline. 439 */ 440 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE); 441 442 scanline = intel_get_crtc_scanline(crtc); 443 if (scanline < min || scanline > max) 444 break; 445 446 if (!timeout) { 447 drm_err(&dev_priv->drm, 448 "Potential atomic update failure on pipe %c\n", 449 pipe_name(crtc->pipe)); 450 break; 451 } 452 453 local_irq_enable(); 454 455 timeout = schedule_timeout(timeout); 456 457 local_irq_disable(); 458 } 459 460 finish_wait(wq, &wait); 461 462 drm_crtc_vblank_put(&crtc->base); 463 464 /* 465 * On VLV/CHV DSI the scanline counter would appear to 466 * increment approx. 1/3 of a scanline before start of vblank. 467 * The registers still get latched at start of vblank however. 468 * This means we must not write any registers on the first 469 * line of vblank (since not the whole line is actually in 470 * vblank). And unfortunately we can't use the interrupt to 471 * wait here since it will fire too soon. We could use the 472 * frame start interrupt instead since it will fire after the 473 * critical scanline, but that would require more changes 474 * in the interrupt code. So for now we'll just do the nasty 475 * thing and poll for the bad scanline to pass us by. 476 * 477 * FIXME figure out if BXT+ DSI suffers from this as well 478 */ 479 while (need_vlv_dsi_wa && scanline == vblank_start) 480 scanline = intel_get_crtc_scanline(crtc); 481 482 crtc->debug.scanline_start = scanline; 483 crtc->debug.start_vbl_time = ktime_get(); 484 crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc); 485 486 trace_intel_pipe_update_vblank_evaded(crtc); 487 return; 488 489 irq_disable: 490 local_irq_disable(); 491 } 492 493 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE) 494 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) 495 { 496 u64 delta = ktime_to_ns(ktime_sub(end, crtc->debug.start_vbl_time)); 497 unsigned int h; 498 499 h = ilog2(delta >> 9); 500 if (h >= ARRAY_SIZE(crtc->debug.vbl.times)) 501 h = ARRAY_SIZE(crtc->debug.vbl.times) - 1; 502 crtc->debug.vbl.times[h]++; 503 504 crtc->debug.vbl.sum += delta; 505 if (!crtc->debug.vbl.min || delta < crtc->debug.vbl.min) 506 crtc->debug.vbl.min = delta; 507 if (delta > crtc->debug.vbl.max) 508 crtc->debug.vbl.max = delta; 509 510 if (delta > 1000 * VBLANK_EVASION_TIME_US) { 511 drm_dbg_kms(crtc->base.dev, 512 "Atomic update on pipe (%c) took %lld us, max time under evasion is %u us\n", 513 pipe_name(crtc->pipe), 514 div_u64(delta, 1000), 515 VBLANK_EVASION_TIME_US); 516 crtc->debug.vbl.over++; 517 } 518 } 519 #else 520 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) {} 521 #endif 522 523 /** 524 * intel_pipe_update_end() - end update of a set of display registers 525 * @new_crtc_state: the new crtc state 526 * 527 * Mark the end of an update started with intel_pipe_update_start(). This 528 * re-enables interrupts and verifies the update was actually completed 529 * before a vblank. 530 */ 531 void intel_pipe_update_end(struct intel_crtc_state *new_crtc_state) 532 { 533 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc); 534 enum pipe pipe = crtc->pipe; 535 int scanline_end = intel_get_crtc_scanline(crtc); 536 u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc); 537 ktime_t end_vbl_time = ktime_get(); 538 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 539 540 if (new_crtc_state->uapi.async_flip) 541 return; 542 543 trace_intel_pipe_update_end(crtc, end_vbl_count, scanline_end); 544 545 /* 546 * Incase of mipi dsi command mode, we need to set frame update 547 * request for every commit. 548 */ 549 if (DISPLAY_VER(dev_priv) >= 11 && 550 intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI)) 551 icl_dsi_frame_update(new_crtc_state); 552 553 /* We're still in the vblank-evade critical section, this can't race. 554 * Would be slightly nice to just grab the vblank count and arm the 555 * event outside of the critical section - the spinlock might spin for a 556 * while ... */ 557 if (new_crtc_state->uapi.event) { 558 drm_WARN_ON(&dev_priv->drm, 559 drm_crtc_vblank_get(&crtc->base) != 0); 560 561 spin_lock(&crtc->base.dev->event_lock); 562 drm_crtc_arm_vblank_event(&crtc->base, 563 new_crtc_state->uapi.event); 564 spin_unlock(&crtc->base.dev->event_lock); 565 566 new_crtc_state->uapi.event = NULL; 567 } 568 569 local_irq_enable(); 570 571 /* Send VRR Push to terminate Vblank */ 572 intel_vrr_send_push(new_crtc_state); 573 574 if (intel_vgpu_active(dev_priv)) 575 return; 576 577 if (crtc->debug.start_vbl_count && 578 crtc->debug.start_vbl_count != end_vbl_count) { 579 drm_err(&dev_priv->drm, 580 "Atomic update failure on pipe %c (start=%u end=%u) time %lld us, min %d, max %d, scanline start %d, end %d\n", 581 pipe_name(pipe), crtc->debug.start_vbl_count, 582 end_vbl_count, 583 ktime_us_delta(end_vbl_time, 584 crtc->debug.start_vbl_time), 585 crtc->debug.min_vbl, crtc->debug.max_vbl, 586 crtc->debug.scanline_start, scanline_end); 587 } 588 589 dbg_vblank_evade(crtc, end_vbl_time); 590 } 591