1 /* 2 * Copyright (c) 2014-2018, The Linux Foundation. All rights reserved. 3 * Copyright (C) 2013 Red Hat 4 * Author: Rob Clark <robdclark@gmail.com> 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published by 8 * the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License along with 16 * this program. If not, see <http://www.gnu.org/licenses/>. 17 */ 18 19 #define pr_fmt(fmt) "[drm:%s:%d] " fmt, __func__, __LINE__ 20 #include <linux/kthread.h> 21 #include <linux/debugfs.h> 22 #include <linux/seq_file.h> 23 24 #include "msm_drv.h" 25 #include "dpu_kms.h" 26 #include <drm/drm_crtc.h> 27 #include <drm/drm_probe_helper.h> 28 #include "dpu_hwio.h" 29 #include "dpu_hw_catalog.h" 30 #include "dpu_hw_intf.h" 31 #include "dpu_hw_ctl.h" 32 #include "dpu_formats.h" 33 #include "dpu_encoder_phys.h" 34 #include "dpu_crtc.h" 35 #include "dpu_trace.h" 36 #include "dpu_core_irq.h" 37 38 #define DPU_DEBUG_ENC(e, fmt, ...) DPU_DEBUG("enc%d " fmt,\ 39 (e) ? (e)->base.base.id : -1, ##__VA_ARGS__) 40 41 #define DPU_ERROR_ENC(e, fmt, ...) DPU_ERROR("enc%d " fmt,\ 42 (e) ? (e)->base.base.id : -1, ##__VA_ARGS__) 43 44 #define DPU_DEBUG_PHYS(p, fmt, ...) DPU_DEBUG("enc%d intf%d pp%d " fmt,\ 45 (p) ? (p)->parent->base.id : -1, \ 46 (p) ? (p)->intf_idx - INTF_0 : -1, \ 47 (p) ? ((p)->hw_pp ? (p)->hw_pp->idx - PINGPONG_0 : -1) : -1, \ 48 ##__VA_ARGS__) 49 50 #define DPU_ERROR_PHYS(p, fmt, ...) DPU_ERROR("enc%d intf%d pp%d " fmt,\ 51 (p) ? (p)->parent->base.id : -1, \ 52 (p) ? (p)->intf_idx - INTF_0 : -1, \ 53 (p) ? ((p)->hw_pp ? (p)->hw_pp->idx - PINGPONG_0 : -1) : -1, \ 54 ##__VA_ARGS__) 55 56 /* 57 * Two to anticipate panels that can do cmd/vid dynamic switching 58 * plan is to create all possible physical encoder types, and switch between 59 * them at runtime 60 */ 61 #define NUM_PHYS_ENCODER_TYPES 2 62 63 #define MAX_PHYS_ENCODERS_PER_VIRTUAL \ 64 (MAX_H_TILES_PER_DISPLAY * NUM_PHYS_ENCODER_TYPES) 65 66 #define MAX_CHANNELS_PER_ENC 2 67 68 #define IDLE_SHORT_TIMEOUT 1 69 70 #define MAX_VDISPLAY_SPLIT 1080 71 72 /* timeout in frames waiting for frame done */ 73 #define DPU_ENCODER_FRAME_DONE_TIMEOUT_FRAMES 5 74 75 /** 76 * enum dpu_enc_rc_events - events for resource control state machine 77 * @DPU_ENC_RC_EVENT_KICKOFF: 78 * This event happens at NORMAL priority. 79 * Event that signals the start of the transfer. When this event is 80 * received, enable MDP/DSI core clocks. Regardless of the previous 81 * state, the resource should be in ON state at the end of this event. 82 * @DPU_ENC_RC_EVENT_FRAME_DONE: 83 * This event happens at INTERRUPT level. 84 * Event signals the end of the data transfer after the PP FRAME_DONE 85 * event. At the end of this event, a delayed work is scheduled to go to 86 * IDLE_PC state after IDLE_TIMEOUT time. 87 * @DPU_ENC_RC_EVENT_PRE_STOP: 88 * This event happens at NORMAL priority. 89 * This event, when received during the ON state, leave the RC STATE 90 * in the PRE_OFF state. It should be followed by the STOP event as 91 * part of encoder disable. 92 * If received during IDLE or OFF states, it will do nothing. 93 * @DPU_ENC_RC_EVENT_STOP: 94 * This event happens at NORMAL priority. 95 * When this event is received, disable all the MDP/DSI core clocks, and 96 * disable IRQs. It should be called from the PRE_OFF or IDLE states. 97 * IDLE is expected when IDLE_PC has run, and PRE_OFF did nothing. 98 * PRE_OFF is expected when PRE_STOP was executed during the ON state. 99 * Resource state should be in OFF at the end of the event. 100 * @DPU_ENC_RC_EVENT_ENTER_IDLE: 101 * This event happens at NORMAL priority from a work item. 102 * Event signals that there were no frame updates for IDLE_TIMEOUT time. 103 * This would disable MDP/DSI core clocks and change the resource state 104 * to IDLE. 105 */ 106 enum dpu_enc_rc_events { 107 DPU_ENC_RC_EVENT_KICKOFF = 1, 108 DPU_ENC_RC_EVENT_FRAME_DONE, 109 DPU_ENC_RC_EVENT_PRE_STOP, 110 DPU_ENC_RC_EVENT_STOP, 111 DPU_ENC_RC_EVENT_ENTER_IDLE 112 }; 113 114 /* 115 * enum dpu_enc_rc_states - states that the resource control maintains 116 * @DPU_ENC_RC_STATE_OFF: Resource is in OFF state 117 * @DPU_ENC_RC_STATE_PRE_OFF: Resource is transitioning to OFF state 118 * @DPU_ENC_RC_STATE_ON: Resource is in ON state 119 * @DPU_ENC_RC_STATE_MODESET: Resource is in modeset state 120 * @DPU_ENC_RC_STATE_IDLE: Resource is in IDLE state 121 */ 122 enum dpu_enc_rc_states { 123 DPU_ENC_RC_STATE_OFF, 124 DPU_ENC_RC_STATE_PRE_OFF, 125 DPU_ENC_RC_STATE_ON, 126 DPU_ENC_RC_STATE_IDLE 127 }; 128 129 /** 130 * struct dpu_encoder_virt - virtual encoder. Container of one or more physical 131 * encoders. Virtual encoder manages one "logical" display. Physical 132 * encoders manage one intf block, tied to a specific panel/sub-panel. 133 * Virtual encoder defers as much as possible to the physical encoders. 134 * Virtual encoder registers itself with the DRM Framework as the encoder. 135 * @base: drm_encoder base class for registration with DRM 136 * @enc_spinlock: Virtual-Encoder-Wide Spin Lock for IRQ purposes 137 * @bus_scaling_client: Client handle to the bus scaling interface 138 * @enabled: True if the encoder is active, protected by enc_lock 139 * @num_phys_encs: Actual number of physical encoders contained. 140 * @phys_encs: Container of physical encoders managed. 141 * @cur_master: Pointer to the current master in this mode. Optimization 142 * Only valid after enable. Cleared as disable. 143 * @hw_pp Handle to the pingpong blocks used for the display. No. 144 * pingpong blocks can be different than num_phys_encs. 145 * @intfs_swapped Whether or not the phys_enc interfaces have been swapped 146 * for partial update right-only cases, such as pingpong 147 * split where virtual pingpong does not generate IRQs 148 * @crtc: Pointer to the currently assigned crtc. Normally you 149 * would use crtc->state->encoder_mask to determine the 150 * link between encoder/crtc. However in this case we need 151 * to track crtc in the disable() hook which is called 152 * _after_ encoder_mask is cleared. 153 * @crtc_kickoff_cb: Callback into CRTC that will flush & start 154 * all CTL paths 155 * @crtc_kickoff_cb_data: Opaque user data given to crtc_kickoff_cb 156 * @debugfs_root: Debug file system root file node 157 * @enc_lock: Lock around physical encoder 158 * create/destroy/enable/disable 159 * @frame_busy_mask: Bitmask tracking which phys_enc we are still 160 * busy processing current command. 161 * Bit0 = phys_encs[0] etc. 162 * @crtc_frame_event_cb: callback handler for frame event 163 * @crtc_frame_event_cb_data: callback handler private data 164 * @frame_done_timeout_ms: frame done timeout in ms 165 * @frame_done_timer: watchdog timer for frame done event 166 * @vsync_event_timer: vsync timer 167 * @disp_info: local copy of msm_display_info struct 168 * @idle_pc_supported: indicate if idle power collaps is supported 169 * @rc_lock: resource control mutex lock to protect 170 * virt encoder over various state changes 171 * @rc_state: resource controller state 172 * @delayed_off_work: delayed worker to schedule disabling of 173 * clks and resources after IDLE_TIMEOUT time. 174 * @vsync_event_work: worker to handle vsync event for autorefresh 175 * @topology: topology of the display 176 * @mode_set_complete: flag to indicate modeset completion 177 * @idle_timeout: idle timeout duration in milliseconds 178 */ 179 struct dpu_encoder_virt { 180 struct drm_encoder base; 181 spinlock_t enc_spinlock; 182 uint32_t bus_scaling_client; 183 184 bool enabled; 185 186 unsigned int num_phys_encs; 187 struct dpu_encoder_phys *phys_encs[MAX_PHYS_ENCODERS_PER_VIRTUAL]; 188 struct dpu_encoder_phys *cur_master; 189 struct dpu_encoder_phys *cur_slave; 190 struct dpu_hw_pingpong *hw_pp[MAX_CHANNELS_PER_ENC]; 191 192 bool intfs_swapped; 193 194 struct drm_crtc *crtc; 195 196 struct dentry *debugfs_root; 197 struct mutex enc_lock; 198 DECLARE_BITMAP(frame_busy_mask, MAX_PHYS_ENCODERS_PER_VIRTUAL); 199 void (*crtc_frame_event_cb)(void *, u32 event); 200 void *crtc_frame_event_cb_data; 201 202 atomic_t frame_done_timeout_ms; 203 struct timer_list frame_done_timer; 204 struct timer_list vsync_event_timer; 205 206 struct msm_display_info disp_info; 207 208 bool idle_pc_supported; 209 struct mutex rc_lock; 210 enum dpu_enc_rc_states rc_state; 211 struct delayed_work delayed_off_work; 212 struct kthread_work vsync_event_work; 213 struct msm_display_topology topology; 214 bool mode_set_complete; 215 216 u32 idle_timeout; 217 }; 218 219 #define to_dpu_encoder_virt(x) container_of(x, struct dpu_encoder_virt, base) 220 221 void dpu_encoder_helper_report_irq_timeout(struct dpu_encoder_phys *phys_enc, 222 enum dpu_intr_idx intr_idx) 223 { 224 DRM_ERROR("irq timeout id=%u, intf=%d, pp=%d, intr=%d\n", 225 DRMID(phys_enc->parent), phys_enc->intf_idx - INTF_0, 226 phys_enc->hw_pp->idx - PINGPONG_0, intr_idx); 227 228 if (phys_enc->parent_ops->handle_frame_done) 229 phys_enc->parent_ops->handle_frame_done( 230 phys_enc->parent, phys_enc, 231 DPU_ENCODER_FRAME_EVENT_ERROR); 232 } 233 234 static int dpu_encoder_helper_wait_event_timeout(int32_t drm_id, 235 int32_t hw_id, struct dpu_encoder_wait_info *info); 236 237 int dpu_encoder_helper_wait_for_irq(struct dpu_encoder_phys *phys_enc, 238 enum dpu_intr_idx intr_idx, 239 struct dpu_encoder_wait_info *wait_info) 240 { 241 struct dpu_encoder_irq *irq; 242 u32 irq_status; 243 int ret; 244 245 if (!phys_enc || !wait_info || intr_idx >= INTR_IDX_MAX) { 246 DPU_ERROR("invalid params\n"); 247 return -EINVAL; 248 } 249 irq = &phys_enc->irq[intr_idx]; 250 251 /* note: do master / slave checking outside */ 252 253 /* return EWOULDBLOCK since we know the wait isn't necessary */ 254 if (phys_enc->enable_state == DPU_ENC_DISABLED) { 255 DRM_ERROR("encoder is disabled id=%u, intr=%d, hw=%d, irq=%d", 256 DRMID(phys_enc->parent), intr_idx, irq->hw_idx, 257 irq->irq_idx); 258 return -EWOULDBLOCK; 259 } 260 261 if (irq->irq_idx < 0) { 262 DRM_DEBUG_KMS("skip irq wait id=%u, intr=%d, hw=%d, irq=%s", 263 DRMID(phys_enc->parent), intr_idx, irq->hw_idx, 264 irq->name); 265 return 0; 266 } 267 268 DRM_DEBUG_KMS("id=%u, intr=%d, hw=%d, irq=%d, pp=%d, pending_cnt=%d", 269 DRMID(phys_enc->parent), intr_idx, irq->hw_idx, 270 irq->irq_idx, phys_enc->hw_pp->idx - PINGPONG_0, 271 atomic_read(wait_info->atomic_cnt)); 272 273 ret = dpu_encoder_helper_wait_event_timeout( 274 DRMID(phys_enc->parent), 275 irq->hw_idx, 276 wait_info); 277 278 if (ret <= 0) { 279 irq_status = dpu_core_irq_read(phys_enc->dpu_kms, 280 irq->irq_idx, true); 281 if (irq_status) { 282 unsigned long flags; 283 284 DRM_DEBUG_KMS("irq not triggered id=%u, intr=%d, " 285 "hw=%d, irq=%d, pp=%d, atomic_cnt=%d", 286 DRMID(phys_enc->parent), intr_idx, 287 irq->hw_idx, irq->irq_idx, 288 phys_enc->hw_pp->idx - PINGPONG_0, 289 atomic_read(wait_info->atomic_cnt)); 290 local_irq_save(flags); 291 irq->cb.func(phys_enc, irq->irq_idx); 292 local_irq_restore(flags); 293 ret = 0; 294 } else { 295 ret = -ETIMEDOUT; 296 DRM_DEBUG_KMS("irq timeout id=%u, intr=%d, " 297 "hw=%d, irq=%d, pp=%d, atomic_cnt=%d", 298 DRMID(phys_enc->parent), intr_idx, 299 irq->hw_idx, irq->irq_idx, 300 phys_enc->hw_pp->idx - PINGPONG_0, 301 atomic_read(wait_info->atomic_cnt)); 302 } 303 } else { 304 ret = 0; 305 trace_dpu_enc_irq_wait_success(DRMID(phys_enc->parent), 306 intr_idx, irq->hw_idx, irq->irq_idx, 307 phys_enc->hw_pp->idx - PINGPONG_0, 308 atomic_read(wait_info->atomic_cnt)); 309 } 310 311 return ret; 312 } 313 314 int dpu_encoder_helper_register_irq(struct dpu_encoder_phys *phys_enc, 315 enum dpu_intr_idx intr_idx) 316 { 317 struct dpu_encoder_irq *irq; 318 int ret = 0; 319 320 if (!phys_enc || intr_idx >= INTR_IDX_MAX) { 321 DPU_ERROR("invalid params\n"); 322 return -EINVAL; 323 } 324 irq = &phys_enc->irq[intr_idx]; 325 326 if (irq->irq_idx >= 0) { 327 DPU_DEBUG_PHYS(phys_enc, 328 "skipping already registered irq %s type %d\n", 329 irq->name, irq->intr_type); 330 return 0; 331 } 332 333 irq->irq_idx = dpu_core_irq_idx_lookup(phys_enc->dpu_kms, 334 irq->intr_type, irq->hw_idx); 335 if (irq->irq_idx < 0) { 336 DPU_ERROR_PHYS(phys_enc, 337 "failed to lookup IRQ index for %s type:%d\n", 338 irq->name, irq->intr_type); 339 return -EINVAL; 340 } 341 342 ret = dpu_core_irq_register_callback(phys_enc->dpu_kms, irq->irq_idx, 343 &irq->cb); 344 if (ret) { 345 DPU_ERROR_PHYS(phys_enc, 346 "failed to register IRQ callback for %s\n", 347 irq->name); 348 irq->irq_idx = -EINVAL; 349 return ret; 350 } 351 352 ret = dpu_core_irq_enable(phys_enc->dpu_kms, &irq->irq_idx, 1); 353 if (ret) { 354 DRM_ERROR("enable failed id=%u, intr=%d, hw=%d, irq=%d", 355 DRMID(phys_enc->parent), intr_idx, irq->hw_idx, 356 irq->irq_idx); 357 dpu_core_irq_unregister_callback(phys_enc->dpu_kms, 358 irq->irq_idx, &irq->cb); 359 irq->irq_idx = -EINVAL; 360 return ret; 361 } 362 363 trace_dpu_enc_irq_register_success(DRMID(phys_enc->parent), intr_idx, 364 irq->hw_idx, irq->irq_idx); 365 366 return ret; 367 } 368 369 int dpu_encoder_helper_unregister_irq(struct dpu_encoder_phys *phys_enc, 370 enum dpu_intr_idx intr_idx) 371 { 372 struct dpu_encoder_irq *irq; 373 int ret; 374 375 if (!phys_enc) { 376 DPU_ERROR("invalid encoder\n"); 377 return -EINVAL; 378 } 379 irq = &phys_enc->irq[intr_idx]; 380 381 /* silently skip irqs that weren't registered */ 382 if (irq->irq_idx < 0) { 383 DRM_ERROR("duplicate unregister id=%u, intr=%d, hw=%d, irq=%d", 384 DRMID(phys_enc->parent), intr_idx, irq->hw_idx, 385 irq->irq_idx); 386 return 0; 387 } 388 389 ret = dpu_core_irq_disable(phys_enc->dpu_kms, &irq->irq_idx, 1); 390 if (ret) { 391 DRM_ERROR("disable failed id=%u, intr=%d, hw=%d, irq=%d ret=%d", 392 DRMID(phys_enc->parent), intr_idx, irq->hw_idx, 393 irq->irq_idx, ret); 394 } 395 396 ret = dpu_core_irq_unregister_callback(phys_enc->dpu_kms, irq->irq_idx, 397 &irq->cb); 398 if (ret) { 399 DRM_ERROR("unreg cb fail id=%u, intr=%d, hw=%d, irq=%d ret=%d", 400 DRMID(phys_enc->parent), intr_idx, irq->hw_idx, 401 irq->irq_idx, ret); 402 } 403 404 trace_dpu_enc_irq_unregister_success(DRMID(phys_enc->parent), intr_idx, 405 irq->hw_idx, irq->irq_idx); 406 407 irq->irq_idx = -EINVAL; 408 409 return 0; 410 } 411 412 void dpu_encoder_get_hw_resources(struct drm_encoder *drm_enc, 413 struct dpu_encoder_hw_resources *hw_res) 414 { 415 struct dpu_encoder_virt *dpu_enc = NULL; 416 int i = 0; 417 418 dpu_enc = to_dpu_encoder_virt(drm_enc); 419 DPU_DEBUG_ENC(dpu_enc, "\n"); 420 421 /* Query resources used by phys encs, expected to be without overlap */ 422 memset(hw_res, 0, sizeof(*hw_res)); 423 424 for (i = 0; i < dpu_enc->num_phys_encs; i++) { 425 struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i]; 426 427 if (phys && phys->ops.get_hw_resources) 428 phys->ops.get_hw_resources(phys, hw_res); 429 } 430 } 431 432 static void dpu_encoder_destroy(struct drm_encoder *drm_enc) 433 { 434 struct dpu_encoder_virt *dpu_enc = NULL; 435 int i = 0; 436 437 if (!drm_enc) { 438 DPU_ERROR("invalid encoder\n"); 439 return; 440 } 441 442 dpu_enc = to_dpu_encoder_virt(drm_enc); 443 DPU_DEBUG_ENC(dpu_enc, "\n"); 444 445 mutex_lock(&dpu_enc->enc_lock); 446 447 for (i = 0; i < dpu_enc->num_phys_encs; i++) { 448 struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i]; 449 450 if (phys && phys->ops.destroy) { 451 phys->ops.destroy(phys); 452 --dpu_enc->num_phys_encs; 453 dpu_enc->phys_encs[i] = NULL; 454 } 455 } 456 457 if (dpu_enc->num_phys_encs) 458 DPU_ERROR_ENC(dpu_enc, "expected 0 num_phys_encs not %d\n", 459 dpu_enc->num_phys_encs); 460 dpu_enc->num_phys_encs = 0; 461 mutex_unlock(&dpu_enc->enc_lock); 462 463 drm_encoder_cleanup(drm_enc); 464 mutex_destroy(&dpu_enc->enc_lock); 465 } 466 467 void dpu_encoder_helper_split_config( 468 struct dpu_encoder_phys *phys_enc, 469 enum dpu_intf interface) 470 { 471 struct dpu_encoder_virt *dpu_enc; 472 struct split_pipe_cfg cfg = { 0 }; 473 struct dpu_hw_mdp *hw_mdptop; 474 struct msm_display_info *disp_info; 475 476 if (!phys_enc || !phys_enc->hw_mdptop || !phys_enc->parent) { 477 DPU_ERROR("invalid arg(s), encoder %d\n", phys_enc != 0); 478 return; 479 } 480 481 dpu_enc = to_dpu_encoder_virt(phys_enc->parent); 482 hw_mdptop = phys_enc->hw_mdptop; 483 disp_info = &dpu_enc->disp_info; 484 485 if (disp_info->intf_type != DRM_MODE_ENCODER_DSI) 486 return; 487 488 /** 489 * disable split modes since encoder will be operating in as the only 490 * encoder, either for the entire use case in the case of, for example, 491 * single DSI, or for this frame in the case of left/right only partial 492 * update. 493 */ 494 if (phys_enc->split_role == ENC_ROLE_SOLO) { 495 if (hw_mdptop->ops.setup_split_pipe) 496 hw_mdptop->ops.setup_split_pipe(hw_mdptop, &cfg); 497 return; 498 } 499 500 cfg.en = true; 501 cfg.mode = phys_enc->intf_mode; 502 cfg.intf = interface; 503 504 if (cfg.en && phys_enc->ops.needs_single_flush && 505 phys_enc->ops.needs_single_flush(phys_enc)) 506 cfg.split_flush_en = true; 507 508 if (phys_enc->split_role == ENC_ROLE_MASTER) { 509 DPU_DEBUG_ENC(dpu_enc, "enable %d\n", cfg.en); 510 511 if (hw_mdptop->ops.setup_split_pipe) 512 hw_mdptop->ops.setup_split_pipe(hw_mdptop, &cfg); 513 } 514 } 515 516 static void _dpu_encoder_adjust_mode(struct drm_connector *connector, 517 struct drm_display_mode *adj_mode) 518 { 519 struct drm_display_mode *cur_mode; 520 521 if (!connector || !adj_mode) 522 return; 523 524 list_for_each_entry(cur_mode, &connector->modes, head) { 525 if (cur_mode->vdisplay == adj_mode->vdisplay && 526 cur_mode->hdisplay == adj_mode->hdisplay && 527 drm_mode_vrefresh(cur_mode) == drm_mode_vrefresh(adj_mode)) { 528 adj_mode->private = cur_mode->private; 529 adj_mode->private_flags |= cur_mode->private_flags; 530 } 531 } 532 } 533 534 static struct msm_display_topology dpu_encoder_get_topology( 535 struct dpu_encoder_virt *dpu_enc, 536 struct dpu_kms *dpu_kms, 537 struct drm_display_mode *mode) 538 { 539 struct msm_display_topology topology; 540 int i, intf_count = 0; 541 542 for (i = 0; i < MAX_PHYS_ENCODERS_PER_VIRTUAL; i++) 543 if (dpu_enc->phys_encs[i]) 544 intf_count++; 545 546 /* User split topology for width > 1080 */ 547 topology.num_lm = (mode->vdisplay > MAX_VDISPLAY_SPLIT) ? 2 : 1; 548 topology.num_enc = 0; 549 topology.num_intf = intf_count; 550 551 return topology; 552 } 553 static int dpu_encoder_virt_atomic_check( 554 struct drm_encoder *drm_enc, 555 struct drm_crtc_state *crtc_state, 556 struct drm_connector_state *conn_state) 557 { 558 struct dpu_encoder_virt *dpu_enc; 559 struct msm_drm_private *priv; 560 struct dpu_kms *dpu_kms; 561 const struct drm_display_mode *mode; 562 struct drm_display_mode *adj_mode; 563 struct msm_display_topology topology; 564 int i = 0; 565 int ret = 0; 566 567 if (!drm_enc || !crtc_state || !conn_state) { 568 DPU_ERROR("invalid arg(s), drm_enc %d, crtc/conn state %d/%d\n", 569 drm_enc != 0, crtc_state != 0, conn_state != 0); 570 return -EINVAL; 571 } 572 573 dpu_enc = to_dpu_encoder_virt(drm_enc); 574 DPU_DEBUG_ENC(dpu_enc, "\n"); 575 576 priv = drm_enc->dev->dev_private; 577 dpu_kms = to_dpu_kms(priv->kms); 578 mode = &crtc_state->mode; 579 adj_mode = &crtc_state->adjusted_mode; 580 trace_dpu_enc_atomic_check(DRMID(drm_enc)); 581 582 /* 583 * display drivers may populate private fields of the drm display mode 584 * structure while registering possible modes of a connector with DRM. 585 * These private fields are not populated back while DRM invokes 586 * the mode_set callbacks. This module retrieves and populates the 587 * private fields of the given mode. 588 */ 589 _dpu_encoder_adjust_mode(conn_state->connector, adj_mode); 590 591 /* perform atomic check on the first physical encoder (master) */ 592 for (i = 0; i < dpu_enc->num_phys_encs; i++) { 593 struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i]; 594 595 if (phys && phys->ops.atomic_check) 596 ret = phys->ops.atomic_check(phys, crtc_state, 597 conn_state); 598 else if (phys && phys->ops.mode_fixup) 599 if (!phys->ops.mode_fixup(phys, mode, adj_mode)) 600 ret = -EINVAL; 601 602 if (ret) { 603 DPU_ERROR_ENC(dpu_enc, 604 "mode unsupported, phys idx %d\n", i); 605 break; 606 } 607 } 608 609 topology = dpu_encoder_get_topology(dpu_enc, dpu_kms, adj_mode); 610 611 /* Reserve dynamic resources now. Indicating AtomicTest phase */ 612 if (!ret) { 613 /* 614 * Avoid reserving resources when mode set is pending. Topology 615 * info may not be available to complete reservation. 616 */ 617 if (drm_atomic_crtc_needs_modeset(crtc_state) 618 && dpu_enc->mode_set_complete) { 619 ret = dpu_rm_reserve(&dpu_kms->rm, drm_enc, crtc_state, 620 topology, true); 621 dpu_enc->mode_set_complete = false; 622 } 623 } 624 625 if (!ret) 626 drm_mode_set_crtcinfo(adj_mode, 0); 627 628 trace_dpu_enc_atomic_check_flags(DRMID(drm_enc), adj_mode->flags, 629 adj_mode->private_flags); 630 631 return ret; 632 } 633 634 static void _dpu_encoder_update_vsync_source(struct dpu_encoder_virt *dpu_enc, 635 struct msm_display_info *disp_info) 636 { 637 struct dpu_vsync_source_cfg vsync_cfg = { 0 }; 638 struct msm_drm_private *priv; 639 struct dpu_kms *dpu_kms; 640 struct dpu_hw_mdp *hw_mdptop; 641 struct drm_encoder *drm_enc; 642 int i; 643 644 if (!dpu_enc || !disp_info) { 645 DPU_ERROR("invalid param dpu_enc:%d or disp_info:%d\n", 646 dpu_enc != NULL, disp_info != NULL); 647 return; 648 } else if (dpu_enc->num_phys_encs > ARRAY_SIZE(dpu_enc->hw_pp)) { 649 DPU_ERROR("invalid num phys enc %d/%d\n", 650 dpu_enc->num_phys_encs, 651 (int) ARRAY_SIZE(dpu_enc->hw_pp)); 652 return; 653 } 654 655 drm_enc = &dpu_enc->base; 656 /* this pointers are checked in virt_enable_helper */ 657 priv = drm_enc->dev->dev_private; 658 659 dpu_kms = to_dpu_kms(priv->kms); 660 if (!dpu_kms) { 661 DPU_ERROR("invalid dpu_kms\n"); 662 return; 663 } 664 665 hw_mdptop = dpu_kms->hw_mdp; 666 if (!hw_mdptop) { 667 DPU_ERROR("invalid mdptop\n"); 668 return; 669 } 670 671 if (hw_mdptop->ops.setup_vsync_source && 672 disp_info->capabilities & MSM_DISPLAY_CAP_CMD_MODE) { 673 for (i = 0; i < dpu_enc->num_phys_encs; i++) 674 vsync_cfg.ppnumber[i] = dpu_enc->hw_pp[i]->idx; 675 676 vsync_cfg.pp_count = dpu_enc->num_phys_encs; 677 if (disp_info->is_te_using_watchdog_timer) 678 vsync_cfg.vsync_source = DPU_VSYNC_SOURCE_WD_TIMER_0; 679 else 680 vsync_cfg.vsync_source = DPU_VSYNC0_SOURCE_GPIO; 681 682 hw_mdptop->ops.setup_vsync_source(hw_mdptop, &vsync_cfg); 683 } 684 } 685 686 static void _dpu_encoder_irq_control(struct drm_encoder *drm_enc, bool enable) 687 { 688 struct dpu_encoder_virt *dpu_enc; 689 int i; 690 691 if (!drm_enc) { 692 DPU_ERROR("invalid encoder\n"); 693 return; 694 } 695 696 dpu_enc = to_dpu_encoder_virt(drm_enc); 697 698 DPU_DEBUG_ENC(dpu_enc, "enable:%d\n", enable); 699 for (i = 0; i < dpu_enc->num_phys_encs; i++) { 700 struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i]; 701 702 if (phys && phys->ops.irq_control) 703 phys->ops.irq_control(phys, enable); 704 } 705 706 } 707 708 static void _dpu_encoder_resource_control_helper(struct drm_encoder *drm_enc, 709 bool enable) 710 { 711 struct msm_drm_private *priv; 712 struct dpu_kms *dpu_kms; 713 struct dpu_encoder_virt *dpu_enc; 714 715 dpu_enc = to_dpu_encoder_virt(drm_enc); 716 priv = drm_enc->dev->dev_private; 717 dpu_kms = to_dpu_kms(priv->kms); 718 719 trace_dpu_enc_rc_helper(DRMID(drm_enc), enable); 720 721 if (!dpu_enc->cur_master) { 722 DPU_ERROR("encoder master not set\n"); 723 return; 724 } 725 726 if (enable) { 727 /* enable DPU core clks */ 728 pm_runtime_get_sync(&dpu_kms->pdev->dev); 729 730 /* enable all the irq */ 731 _dpu_encoder_irq_control(drm_enc, true); 732 733 } else { 734 /* disable all the irq */ 735 _dpu_encoder_irq_control(drm_enc, false); 736 737 /* disable DPU core clks */ 738 pm_runtime_put_sync(&dpu_kms->pdev->dev); 739 } 740 741 } 742 743 static int dpu_encoder_resource_control(struct drm_encoder *drm_enc, 744 u32 sw_event) 745 { 746 struct dpu_encoder_virt *dpu_enc; 747 struct msm_drm_private *priv; 748 bool is_vid_mode = false; 749 750 if (!drm_enc || !drm_enc->dev || !drm_enc->dev->dev_private || 751 !drm_enc->crtc) { 752 DPU_ERROR("invalid parameters\n"); 753 return -EINVAL; 754 } 755 dpu_enc = to_dpu_encoder_virt(drm_enc); 756 priv = drm_enc->dev->dev_private; 757 is_vid_mode = dpu_enc->disp_info.capabilities & 758 MSM_DISPLAY_CAP_VID_MODE; 759 760 /* 761 * when idle_pc is not supported, process only KICKOFF, STOP and MODESET 762 * events and return early for other events (ie wb display). 763 */ 764 if (!dpu_enc->idle_pc_supported && 765 (sw_event != DPU_ENC_RC_EVENT_KICKOFF && 766 sw_event != DPU_ENC_RC_EVENT_STOP && 767 sw_event != DPU_ENC_RC_EVENT_PRE_STOP)) 768 return 0; 769 770 trace_dpu_enc_rc(DRMID(drm_enc), sw_event, dpu_enc->idle_pc_supported, 771 dpu_enc->rc_state, "begin"); 772 773 switch (sw_event) { 774 case DPU_ENC_RC_EVENT_KICKOFF: 775 /* cancel delayed off work, if any */ 776 if (cancel_delayed_work_sync(&dpu_enc->delayed_off_work)) 777 DPU_DEBUG_ENC(dpu_enc, "sw_event:%d, work cancelled\n", 778 sw_event); 779 780 mutex_lock(&dpu_enc->rc_lock); 781 782 /* return if the resource control is already in ON state */ 783 if (dpu_enc->rc_state == DPU_ENC_RC_STATE_ON) { 784 DRM_DEBUG_KMS("id;%u, sw_event:%d, rc in ON state\n", 785 DRMID(drm_enc), sw_event); 786 mutex_unlock(&dpu_enc->rc_lock); 787 return 0; 788 } else if (dpu_enc->rc_state != DPU_ENC_RC_STATE_OFF && 789 dpu_enc->rc_state != DPU_ENC_RC_STATE_IDLE) { 790 DRM_DEBUG_KMS("id;%u, sw_event:%d, rc in state %d\n", 791 DRMID(drm_enc), sw_event, 792 dpu_enc->rc_state); 793 mutex_unlock(&dpu_enc->rc_lock); 794 return -EINVAL; 795 } 796 797 if (is_vid_mode && dpu_enc->rc_state == DPU_ENC_RC_STATE_IDLE) 798 _dpu_encoder_irq_control(drm_enc, true); 799 else 800 _dpu_encoder_resource_control_helper(drm_enc, true); 801 802 dpu_enc->rc_state = DPU_ENC_RC_STATE_ON; 803 804 trace_dpu_enc_rc(DRMID(drm_enc), sw_event, 805 dpu_enc->idle_pc_supported, dpu_enc->rc_state, 806 "kickoff"); 807 808 mutex_unlock(&dpu_enc->rc_lock); 809 break; 810 811 case DPU_ENC_RC_EVENT_FRAME_DONE: 812 /* 813 * mutex lock is not used as this event happens at interrupt 814 * context. And locking is not required as, the other events 815 * like KICKOFF and STOP does a wait-for-idle before executing 816 * the resource_control 817 */ 818 if (dpu_enc->rc_state != DPU_ENC_RC_STATE_ON) { 819 DRM_DEBUG_KMS("id:%d, sw_event:%d,rc:%d-unexpected\n", 820 DRMID(drm_enc), sw_event, 821 dpu_enc->rc_state); 822 return -EINVAL; 823 } 824 825 /* 826 * schedule off work item only when there are no 827 * frames pending 828 */ 829 if (dpu_crtc_frame_pending(drm_enc->crtc) > 1) { 830 DRM_DEBUG_KMS("id:%d skip schedule work\n", 831 DRMID(drm_enc)); 832 return 0; 833 } 834 835 queue_delayed_work(priv->wq, &dpu_enc->delayed_off_work, 836 msecs_to_jiffies(dpu_enc->idle_timeout)); 837 838 trace_dpu_enc_rc(DRMID(drm_enc), sw_event, 839 dpu_enc->idle_pc_supported, dpu_enc->rc_state, 840 "frame done"); 841 break; 842 843 case DPU_ENC_RC_EVENT_PRE_STOP: 844 /* cancel delayed off work, if any */ 845 if (cancel_delayed_work_sync(&dpu_enc->delayed_off_work)) 846 DPU_DEBUG_ENC(dpu_enc, "sw_event:%d, work cancelled\n", 847 sw_event); 848 849 mutex_lock(&dpu_enc->rc_lock); 850 851 if (is_vid_mode && 852 dpu_enc->rc_state == DPU_ENC_RC_STATE_IDLE) { 853 _dpu_encoder_irq_control(drm_enc, true); 854 } 855 /* skip if is already OFF or IDLE, resources are off already */ 856 else if (dpu_enc->rc_state == DPU_ENC_RC_STATE_OFF || 857 dpu_enc->rc_state == DPU_ENC_RC_STATE_IDLE) { 858 DRM_DEBUG_KMS("id:%u, sw_event:%d, rc in %d state\n", 859 DRMID(drm_enc), sw_event, 860 dpu_enc->rc_state); 861 mutex_unlock(&dpu_enc->rc_lock); 862 return 0; 863 } 864 865 dpu_enc->rc_state = DPU_ENC_RC_STATE_PRE_OFF; 866 867 trace_dpu_enc_rc(DRMID(drm_enc), sw_event, 868 dpu_enc->idle_pc_supported, dpu_enc->rc_state, 869 "pre stop"); 870 871 mutex_unlock(&dpu_enc->rc_lock); 872 break; 873 874 case DPU_ENC_RC_EVENT_STOP: 875 mutex_lock(&dpu_enc->rc_lock); 876 877 /* return if the resource control is already in OFF state */ 878 if (dpu_enc->rc_state == DPU_ENC_RC_STATE_OFF) { 879 DRM_DEBUG_KMS("id: %u, sw_event:%d, rc in OFF state\n", 880 DRMID(drm_enc), sw_event); 881 mutex_unlock(&dpu_enc->rc_lock); 882 return 0; 883 } else if (dpu_enc->rc_state == DPU_ENC_RC_STATE_ON) { 884 DRM_ERROR("id: %u, sw_event:%d, rc in state %d\n", 885 DRMID(drm_enc), sw_event, dpu_enc->rc_state); 886 mutex_unlock(&dpu_enc->rc_lock); 887 return -EINVAL; 888 } 889 890 /** 891 * expect to arrive here only if in either idle state or pre-off 892 * and in IDLE state the resources are already disabled 893 */ 894 if (dpu_enc->rc_state == DPU_ENC_RC_STATE_PRE_OFF) 895 _dpu_encoder_resource_control_helper(drm_enc, false); 896 897 dpu_enc->rc_state = DPU_ENC_RC_STATE_OFF; 898 899 trace_dpu_enc_rc(DRMID(drm_enc), sw_event, 900 dpu_enc->idle_pc_supported, dpu_enc->rc_state, 901 "stop"); 902 903 mutex_unlock(&dpu_enc->rc_lock); 904 break; 905 906 case DPU_ENC_RC_EVENT_ENTER_IDLE: 907 mutex_lock(&dpu_enc->rc_lock); 908 909 if (dpu_enc->rc_state != DPU_ENC_RC_STATE_ON) { 910 DRM_ERROR("id: %u, sw_event:%d, rc:%d !ON state\n", 911 DRMID(drm_enc), sw_event, dpu_enc->rc_state); 912 mutex_unlock(&dpu_enc->rc_lock); 913 return 0; 914 } 915 916 /* 917 * if we are in ON but a frame was just kicked off, 918 * ignore the IDLE event, it's probably a stale timer event 919 */ 920 if (dpu_enc->frame_busy_mask[0]) { 921 DRM_ERROR("id:%u, sw_event:%d, rc:%d frame pending\n", 922 DRMID(drm_enc), sw_event, dpu_enc->rc_state); 923 mutex_unlock(&dpu_enc->rc_lock); 924 return 0; 925 } 926 927 if (is_vid_mode) 928 _dpu_encoder_irq_control(drm_enc, false); 929 else 930 _dpu_encoder_resource_control_helper(drm_enc, false); 931 932 dpu_enc->rc_state = DPU_ENC_RC_STATE_IDLE; 933 934 trace_dpu_enc_rc(DRMID(drm_enc), sw_event, 935 dpu_enc->idle_pc_supported, dpu_enc->rc_state, 936 "idle"); 937 938 mutex_unlock(&dpu_enc->rc_lock); 939 break; 940 941 default: 942 DRM_ERROR("id:%u, unexpected sw_event: %d\n", DRMID(drm_enc), 943 sw_event); 944 trace_dpu_enc_rc(DRMID(drm_enc), sw_event, 945 dpu_enc->idle_pc_supported, dpu_enc->rc_state, 946 "error"); 947 break; 948 } 949 950 trace_dpu_enc_rc(DRMID(drm_enc), sw_event, 951 dpu_enc->idle_pc_supported, dpu_enc->rc_state, 952 "end"); 953 return 0; 954 } 955 956 static void dpu_encoder_virt_mode_set(struct drm_encoder *drm_enc, 957 struct drm_display_mode *mode, 958 struct drm_display_mode *adj_mode) 959 { 960 struct dpu_encoder_virt *dpu_enc; 961 struct msm_drm_private *priv; 962 struct dpu_kms *dpu_kms; 963 struct list_head *connector_list; 964 struct drm_connector *conn = NULL, *conn_iter; 965 struct drm_crtc *drm_crtc; 966 struct dpu_crtc_state *cstate; 967 struct dpu_rm_hw_iter hw_iter; 968 struct msm_display_topology topology; 969 struct dpu_hw_ctl *hw_ctl[MAX_CHANNELS_PER_ENC] = { NULL }; 970 struct dpu_hw_mixer *hw_lm[MAX_CHANNELS_PER_ENC] = { NULL }; 971 int num_lm = 0, num_ctl = 0; 972 int i, j, ret; 973 974 if (!drm_enc) { 975 DPU_ERROR("invalid encoder\n"); 976 return; 977 } 978 979 dpu_enc = to_dpu_encoder_virt(drm_enc); 980 DPU_DEBUG_ENC(dpu_enc, "\n"); 981 982 priv = drm_enc->dev->dev_private; 983 dpu_kms = to_dpu_kms(priv->kms); 984 connector_list = &dpu_kms->dev->mode_config.connector_list; 985 986 trace_dpu_enc_mode_set(DRMID(drm_enc)); 987 988 list_for_each_entry(conn_iter, connector_list, head) 989 if (conn_iter->encoder == drm_enc) 990 conn = conn_iter; 991 992 if (!conn) { 993 DPU_ERROR_ENC(dpu_enc, "failed to find attached connector\n"); 994 return; 995 } else if (!conn->state) { 996 DPU_ERROR_ENC(dpu_enc, "invalid connector state\n"); 997 return; 998 } 999 1000 drm_for_each_crtc(drm_crtc, drm_enc->dev) 1001 if (drm_crtc->state->encoder_mask & drm_encoder_mask(drm_enc)) 1002 break; 1003 1004 topology = dpu_encoder_get_topology(dpu_enc, dpu_kms, adj_mode); 1005 1006 /* Reserve dynamic resources now. Indicating non-AtomicTest phase */ 1007 ret = dpu_rm_reserve(&dpu_kms->rm, drm_enc, drm_crtc->state, 1008 topology, false); 1009 if (ret) { 1010 DPU_ERROR_ENC(dpu_enc, 1011 "failed to reserve hw resources, %d\n", ret); 1012 return; 1013 } 1014 1015 dpu_rm_init_hw_iter(&hw_iter, drm_enc->base.id, DPU_HW_BLK_PINGPONG); 1016 for (i = 0; i < MAX_CHANNELS_PER_ENC; i++) { 1017 dpu_enc->hw_pp[i] = NULL; 1018 if (!dpu_rm_get_hw(&dpu_kms->rm, &hw_iter)) 1019 break; 1020 dpu_enc->hw_pp[i] = (struct dpu_hw_pingpong *) hw_iter.hw; 1021 } 1022 1023 dpu_rm_init_hw_iter(&hw_iter, drm_enc->base.id, DPU_HW_BLK_CTL); 1024 for (i = 0; i < MAX_CHANNELS_PER_ENC; i++) { 1025 if (!dpu_rm_get_hw(&dpu_kms->rm, &hw_iter)) 1026 break; 1027 hw_ctl[i] = (struct dpu_hw_ctl *)hw_iter.hw; 1028 num_ctl++; 1029 } 1030 1031 dpu_rm_init_hw_iter(&hw_iter, drm_enc->base.id, DPU_HW_BLK_LM); 1032 for (i = 0; i < MAX_CHANNELS_PER_ENC; i++) { 1033 if (!dpu_rm_get_hw(&dpu_kms->rm, &hw_iter)) 1034 break; 1035 hw_lm[i] = (struct dpu_hw_mixer *)hw_iter.hw; 1036 num_lm++; 1037 } 1038 1039 cstate = to_dpu_crtc_state(drm_crtc->state); 1040 1041 for (i = 0; i < num_lm; i++) { 1042 int ctl_idx = (i < num_ctl) ? i : (num_ctl-1); 1043 1044 cstate->mixers[i].hw_lm = hw_lm[i]; 1045 cstate->mixers[i].lm_ctl = hw_ctl[ctl_idx]; 1046 } 1047 1048 cstate->num_mixers = num_lm; 1049 1050 for (i = 0; i < dpu_enc->num_phys_encs; i++) { 1051 struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i]; 1052 1053 if (phys) { 1054 if (!dpu_enc->hw_pp[i]) { 1055 DPU_ERROR_ENC(dpu_enc, "no pp block assigned" 1056 "at idx: %d\n", i); 1057 goto error; 1058 } 1059 1060 if (!hw_ctl[i]) { 1061 DPU_ERROR_ENC(dpu_enc, "no ctl block assigned" 1062 "at idx: %d\n", i); 1063 goto error; 1064 } 1065 1066 phys->hw_pp = dpu_enc->hw_pp[i]; 1067 phys->hw_ctl = hw_ctl[i]; 1068 1069 dpu_rm_init_hw_iter(&hw_iter, drm_enc->base.id, 1070 DPU_HW_BLK_INTF); 1071 for (j = 0; j < MAX_CHANNELS_PER_ENC; j++) { 1072 struct dpu_hw_intf *hw_intf; 1073 1074 if (!dpu_rm_get_hw(&dpu_kms->rm, &hw_iter)) 1075 break; 1076 1077 hw_intf = (struct dpu_hw_intf *)hw_iter.hw; 1078 if (hw_intf->idx == phys->intf_idx) 1079 phys->hw_intf = hw_intf; 1080 } 1081 1082 if (!phys->hw_intf) { 1083 DPU_ERROR_ENC(dpu_enc, 1084 "no intf block assigned at idx: %d\n", 1085 i); 1086 goto error; 1087 } 1088 1089 phys->connector = conn->state->connector; 1090 if (phys->ops.mode_set) 1091 phys->ops.mode_set(phys, mode, adj_mode); 1092 } 1093 } 1094 1095 dpu_enc->mode_set_complete = true; 1096 1097 error: 1098 dpu_rm_release(&dpu_kms->rm, drm_enc); 1099 } 1100 1101 static void _dpu_encoder_virt_enable_helper(struct drm_encoder *drm_enc) 1102 { 1103 struct dpu_encoder_virt *dpu_enc = NULL; 1104 struct msm_drm_private *priv; 1105 struct dpu_kms *dpu_kms; 1106 1107 if (!drm_enc || !drm_enc->dev || !drm_enc->dev->dev_private) { 1108 DPU_ERROR("invalid parameters\n"); 1109 return; 1110 } 1111 1112 priv = drm_enc->dev->dev_private; 1113 dpu_kms = to_dpu_kms(priv->kms); 1114 if (!dpu_kms) { 1115 DPU_ERROR("invalid dpu_kms\n"); 1116 return; 1117 } 1118 1119 dpu_enc = to_dpu_encoder_virt(drm_enc); 1120 if (!dpu_enc || !dpu_enc->cur_master) { 1121 DPU_ERROR("invalid dpu encoder/master\n"); 1122 return; 1123 } 1124 1125 if (dpu_enc->cur_master->hw_mdptop && 1126 dpu_enc->cur_master->hw_mdptop->ops.reset_ubwc) 1127 dpu_enc->cur_master->hw_mdptop->ops.reset_ubwc( 1128 dpu_enc->cur_master->hw_mdptop, 1129 dpu_kms->catalog); 1130 1131 _dpu_encoder_update_vsync_source(dpu_enc, &dpu_enc->disp_info); 1132 } 1133 1134 void dpu_encoder_virt_runtime_resume(struct drm_encoder *drm_enc) 1135 { 1136 struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc); 1137 1138 mutex_lock(&dpu_enc->enc_lock); 1139 1140 if (!dpu_enc->enabled) 1141 goto out; 1142 1143 if (dpu_enc->cur_slave && dpu_enc->cur_slave->ops.restore) 1144 dpu_enc->cur_slave->ops.restore(dpu_enc->cur_slave); 1145 if (dpu_enc->cur_master && dpu_enc->cur_master->ops.restore) 1146 dpu_enc->cur_master->ops.restore(dpu_enc->cur_master); 1147 1148 _dpu_encoder_virt_enable_helper(drm_enc); 1149 1150 out: 1151 mutex_unlock(&dpu_enc->enc_lock); 1152 } 1153 1154 static void dpu_encoder_virt_enable(struct drm_encoder *drm_enc) 1155 { 1156 struct dpu_encoder_virt *dpu_enc = NULL; 1157 int ret = 0; 1158 struct drm_display_mode *cur_mode = NULL; 1159 1160 if (!drm_enc) { 1161 DPU_ERROR("invalid encoder\n"); 1162 return; 1163 } 1164 dpu_enc = to_dpu_encoder_virt(drm_enc); 1165 1166 mutex_lock(&dpu_enc->enc_lock); 1167 cur_mode = &dpu_enc->base.crtc->state->adjusted_mode; 1168 1169 trace_dpu_enc_enable(DRMID(drm_enc), cur_mode->hdisplay, 1170 cur_mode->vdisplay); 1171 1172 /* always enable slave encoder before master */ 1173 if (dpu_enc->cur_slave && dpu_enc->cur_slave->ops.enable) 1174 dpu_enc->cur_slave->ops.enable(dpu_enc->cur_slave); 1175 1176 if (dpu_enc->cur_master && dpu_enc->cur_master->ops.enable) 1177 dpu_enc->cur_master->ops.enable(dpu_enc->cur_master); 1178 1179 ret = dpu_encoder_resource_control(drm_enc, DPU_ENC_RC_EVENT_KICKOFF); 1180 if (ret) { 1181 DPU_ERROR_ENC(dpu_enc, "dpu resource control failed: %d\n", 1182 ret); 1183 goto out; 1184 } 1185 1186 _dpu_encoder_virt_enable_helper(drm_enc); 1187 1188 dpu_enc->enabled = true; 1189 1190 out: 1191 mutex_unlock(&dpu_enc->enc_lock); 1192 } 1193 1194 static void dpu_encoder_virt_disable(struct drm_encoder *drm_enc) 1195 { 1196 struct dpu_encoder_virt *dpu_enc = NULL; 1197 struct msm_drm_private *priv; 1198 struct dpu_kms *dpu_kms; 1199 struct drm_display_mode *mode; 1200 int i = 0; 1201 1202 if (!drm_enc) { 1203 DPU_ERROR("invalid encoder\n"); 1204 return; 1205 } else if (!drm_enc->dev) { 1206 DPU_ERROR("invalid dev\n"); 1207 return; 1208 } else if (!drm_enc->dev->dev_private) { 1209 DPU_ERROR("invalid dev_private\n"); 1210 return; 1211 } 1212 1213 dpu_enc = to_dpu_encoder_virt(drm_enc); 1214 DPU_DEBUG_ENC(dpu_enc, "\n"); 1215 1216 mutex_lock(&dpu_enc->enc_lock); 1217 dpu_enc->enabled = false; 1218 1219 mode = &drm_enc->crtc->state->adjusted_mode; 1220 1221 priv = drm_enc->dev->dev_private; 1222 dpu_kms = to_dpu_kms(priv->kms); 1223 1224 trace_dpu_enc_disable(DRMID(drm_enc)); 1225 1226 /* wait for idle */ 1227 dpu_encoder_wait_for_event(drm_enc, MSM_ENC_TX_COMPLETE); 1228 1229 dpu_encoder_resource_control(drm_enc, DPU_ENC_RC_EVENT_PRE_STOP); 1230 1231 for (i = 0; i < dpu_enc->num_phys_encs; i++) { 1232 struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i]; 1233 1234 if (phys && phys->ops.disable) 1235 phys->ops.disable(phys); 1236 } 1237 1238 /* after phys waits for frame-done, should be no more frames pending */ 1239 if (atomic_xchg(&dpu_enc->frame_done_timeout_ms, 0)) { 1240 DPU_ERROR("enc%d timeout pending\n", drm_enc->base.id); 1241 del_timer_sync(&dpu_enc->frame_done_timer); 1242 } 1243 1244 dpu_encoder_resource_control(drm_enc, DPU_ENC_RC_EVENT_STOP); 1245 1246 for (i = 0; i < dpu_enc->num_phys_encs; i++) { 1247 if (dpu_enc->phys_encs[i]) 1248 dpu_enc->phys_encs[i]->connector = NULL; 1249 } 1250 1251 DPU_DEBUG_ENC(dpu_enc, "encoder disabled\n"); 1252 1253 dpu_rm_release(&dpu_kms->rm, drm_enc); 1254 1255 mutex_unlock(&dpu_enc->enc_lock); 1256 } 1257 1258 static enum dpu_intf dpu_encoder_get_intf(struct dpu_mdss_cfg *catalog, 1259 enum dpu_intf_type type, u32 controller_id) 1260 { 1261 int i = 0; 1262 1263 for (i = 0; i < catalog->intf_count; i++) { 1264 if (catalog->intf[i].type == type 1265 && catalog->intf[i].controller_id == controller_id) { 1266 return catalog->intf[i].id; 1267 } 1268 } 1269 1270 return INTF_MAX; 1271 } 1272 1273 static void dpu_encoder_vblank_callback(struct drm_encoder *drm_enc, 1274 struct dpu_encoder_phys *phy_enc) 1275 { 1276 struct dpu_encoder_virt *dpu_enc = NULL; 1277 unsigned long lock_flags; 1278 1279 if (!drm_enc || !phy_enc) 1280 return; 1281 1282 DPU_ATRACE_BEGIN("encoder_vblank_callback"); 1283 dpu_enc = to_dpu_encoder_virt(drm_enc); 1284 1285 spin_lock_irqsave(&dpu_enc->enc_spinlock, lock_flags); 1286 if (dpu_enc->crtc) 1287 dpu_crtc_vblank_callback(dpu_enc->crtc); 1288 spin_unlock_irqrestore(&dpu_enc->enc_spinlock, lock_flags); 1289 1290 atomic_inc(&phy_enc->vsync_cnt); 1291 DPU_ATRACE_END("encoder_vblank_callback"); 1292 } 1293 1294 static void dpu_encoder_underrun_callback(struct drm_encoder *drm_enc, 1295 struct dpu_encoder_phys *phy_enc) 1296 { 1297 if (!phy_enc) 1298 return; 1299 1300 DPU_ATRACE_BEGIN("encoder_underrun_callback"); 1301 atomic_inc(&phy_enc->underrun_cnt); 1302 trace_dpu_enc_underrun_cb(DRMID(drm_enc), 1303 atomic_read(&phy_enc->underrun_cnt)); 1304 DPU_ATRACE_END("encoder_underrun_callback"); 1305 } 1306 1307 void dpu_encoder_assign_crtc(struct drm_encoder *drm_enc, struct drm_crtc *crtc) 1308 { 1309 struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc); 1310 unsigned long lock_flags; 1311 1312 spin_lock_irqsave(&dpu_enc->enc_spinlock, lock_flags); 1313 /* crtc should always be cleared before re-assigning */ 1314 WARN_ON(crtc && dpu_enc->crtc); 1315 dpu_enc->crtc = crtc; 1316 spin_unlock_irqrestore(&dpu_enc->enc_spinlock, lock_flags); 1317 } 1318 1319 void dpu_encoder_toggle_vblank_for_crtc(struct drm_encoder *drm_enc, 1320 struct drm_crtc *crtc, bool enable) 1321 { 1322 struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc); 1323 unsigned long lock_flags; 1324 int i; 1325 1326 trace_dpu_enc_vblank_cb(DRMID(drm_enc), enable); 1327 1328 spin_lock_irqsave(&dpu_enc->enc_spinlock, lock_flags); 1329 if (dpu_enc->crtc != crtc) { 1330 spin_unlock_irqrestore(&dpu_enc->enc_spinlock, lock_flags); 1331 return; 1332 } 1333 spin_unlock_irqrestore(&dpu_enc->enc_spinlock, lock_flags); 1334 1335 for (i = 0; i < dpu_enc->num_phys_encs; i++) { 1336 struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i]; 1337 1338 if (phys && phys->ops.control_vblank_irq) 1339 phys->ops.control_vblank_irq(phys, enable); 1340 } 1341 } 1342 1343 void dpu_encoder_register_frame_event_callback(struct drm_encoder *drm_enc, 1344 void (*frame_event_cb)(void *, u32 event), 1345 void *frame_event_cb_data) 1346 { 1347 struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc); 1348 unsigned long lock_flags; 1349 bool enable; 1350 1351 enable = frame_event_cb ? true : false; 1352 1353 if (!drm_enc) { 1354 DPU_ERROR("invalid encoder\n"); 1355 return; 1356 } 1357 trace_dpu_enc_frame_event_cb(DRMID(drm_enc), enable); 1358 1359 spin_lock_irqsave(&dpu_enc->enc_spinlock, lock_flags); 1360 dpu_enc->crtc_frame_event_cb = frame_event_cb; 1361 dpu_enc->crtc_frame_event_cb_data = frame_event_cb_data; 1362 spin_unlock_irqrestore(&dpu_enc->enc_spinlock, lock_flags); 1363 } 1364 1365 static void dpu_encoder_frame_done_callback( 1366 struct drm_encoder *drm_enc, 1367 struct dpu_encoder_phys *ready_phys, u32 event) 1368 { 1369 struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc); 1370 unsigned int i; 1371 1372 if (event & (DPU_ENCODER_FRAME_EVENT_DONE 1373 | DPU_ENCODER_FRAME_EVENT_ERROR 1374 | DPU_ENCODER_FRAME_EVENT_PANEL_DEAD)) { 1375 1376 if (!dpu_enc->frame_busy_mask[0]) { 1377 /** 1378 * suppress frame_done without waiter, 1379 * likely autorefresh 1380 */ 1381 trace_dpu_enc_frame_done_cb_not_busy(DRMID(drm_enc), 1382 event, ready_phys->intf_idx); 1383 return; 1384 } 1385 1386 /* One of the physical encoders has become idle */ 1387 for (i = 0; i < dpu_enc->num_phys_encs; i++) { 1388 if (dpu_enc->phys_encs[i] == ready_phys) { 1389 trace_dpu_enc_frame_done_cb(DRMID(drm_enc), i, 1390 dpu_enc->frame_busy_mask[0]); 1391 clear_bit(i, dpu_enc->frame_busy_mask); 1392 } 1393 } 1394 1395 if (!dpu_enc->frame_busy_mask[0]) { 1396 atomic_set(&dpu_enc->frame_done_timeout_ms, 0); 1397 del_timer(&dpu_enc->frame_done_timer); 1398 1399 dpu_encoder_resource_control(drm_enc, 1400 DPU_ENC_RC_EVENT_FRAME_DONE); 1401 1402 if (dpu_enc->crtc_frame_event_cb) 1403 dpu_enc->crtc_frame_event_cb( 1404 dpu_enc->crtc_frame_event_cb_data, 1405 event); 1406 } 1407 } else { 1408 if (dpu_enc->crtc_frame_event_cb) 1409 dpu_enc->crtc_frame_event_cb( 1410 dpu_enc->crtc_frame_event_cb_data, event); 1411 } 1412 } 1413 1414 static void dpu_encoder_off_work(struct work_struct *work) 1415 { 1416 struct dpu_encoder_virt *dpu_enc = container_of(work, 1417 struct dpu_encoder_virt, delayed_off_work.work); 1418 1419 if (!dpu_enc) { 1420 DPU_ERROR("invalid dpu encoder\n"); 1421 return; 1422 } 1423 1424 dpu_encoder_resource_control(&dpu_enc->base, 1425 DPU_ENC_RC_EVENT_ENTER_IDLE); 1426 1427 dpu_encoder_frame_done_callback(&dpu_enc->base, NULL, 1428 DPU_ENCODER_FRAME_EVENT_IDLE); 1429 } 1430 1431 /** 1432 * _dpu_encoder_trigger_flush - trigger flush for a physical encoder 1433 * drm_enc: Pointer to drm encoder structure 1434 * phys: Pointer to physical encoder structure 1435 * extra_flush_bits: Additional bit mask to include in flush trigger 1436 */ 1437 static void _dpu_encoder_trigger_flush(struct drm_encoder *drm_enc, 1438 struct dpu_encoder_phys *phys, uint32_t extra_flush_bits, 1439 bool async) 1440 { 1441 struct dpu_hw_ctl *ctl; 1442 int pending_kickoff_cnt; 1443 u32 ret = UINT_MAX; 1444 1445 if (!drm_enc || !phys) { 1446 DPU_ERROR("invalid argument(s), drm_enc %d, phys_enc %d\n", 1447 drm_enc != 0, phys != 0); 1448 return; 1449 } 1450 1451 if (!phys->hw_pp) { 1452 DPU_ERROR("invalid pingpong hw\n"); 1453 return; 1454 } 1455 1456 ctl = phys->hw_ctl; 1457 if (!ctl || !ctl->ops.trigger_flush) { 1458 DPU_ERROR("missing trigger cb\n"); 1459 return; 1460 } 1461 1462 if (!async) 1463 pending_kickoff_cnt = dpu_encoder_phys_inc_pending(phys); 1464 else 1465 pending_kickoff_cnt = atomic_read(&phys->pending_kickoff_cnt); 1466 1467 if (extra_flush_bits && ctl->ops.update_pending_flush) 1468 ctl->ops.update_pending_flush(ctl, extra_flush_bits); 1469 1470 ctl->ops.trigger_flush(ctl); 1471 1472 if (ctl->ops.get_pending_flush) 1473 ret = ctl->ops.get_pending_flush(ctl); 1474 1475 trace_dpu_enc_trigger_flush(DRMID(drm_enc), phys->intf_idx, 1476 pending_kickoff_cnt, ctl->idx, 1477 extra_flush_bits, ret); 1478 } 1479 1480 /** 1481 * _dpu_encoder_trigger_start - trigger start for a physical encoder 1482 * phys: Pointer to physical encoder structure 1483 */ 1484 static void _dpu_encoder_trigger_start(struct dpu_encoder_phys *phys) 1485 { 1486 if (!phys) { 1487 DPU_ERROR("invalid argument(s)\n"); 1488 return; 1489 } 1490 1491 if (!phys->hw_pp) { 1492 DPU_ERROR("invalid pingpong hw\n"); 1493 return; 1494 } 1495 1496 if (phys->ops.trigger_start && phys->enable_state != DPU_ENC_DISABLED) 1497 phys->ops.trigger_start(phys); 1498 } 1499 1500 void dpu_encoder_helper_trigger_start(struct dpu_encoder_phys *phys_enc) 1501 { 1502 struct dpu_hw_ctl *ctl; 1503 1504 if (!phys_enc) { 1505 DPU_ERROR("invalid encoder\n"); 1506 return; 1507 } 1508 1509 ctl = phys_enc->hw_ctl; 1510 if (ctl && ctl->ops.trigger_start) { 1511 ctl->ops.trigger_start(ctl); 1512 trace_dpu_enc_trigger_start(DRMID(phys_enc->parent), ctl->idx); 1513 } 1514 } 1515 1516 static int dpu_encoder_helper_wait_event_timeout( 1517 int32_t drm_id, 1518 int32_t hw_id, 1519 struct dpu_encoder_wait_info *info) 1520 { 1521 int rc = 0; 1522 s64 expected_time = ktime_to_ms(ktime_get()) + info->timeout_ms; 1523 s64 jiffies = msecs_to_jiffies(info->timeout_ms); 1524 s64 time; 1525 1526 do { 1527 rc = wait_event_timeout(*(info->wq), 1528 atomic_read(info->atomic_cnt) == 0, jiffies); 1529 time = ktime_to_ms(ktime_get()); 1530 1531 trace_dpu_enc_wait_event_timeout(drm_id, hw_id, rc, time, 1532 expected_time, 1533 atomic_read(info->atomic_cnt)); 1534 /* If we timed out, counter is valid and time is less, wait again */ 1535 } while (atomic_read(info->atomic_cnt) && (rc == 0) && 1536 (time < expected_time)); 1537 1538 return rc; 1539 } 1540 1541 static void dpu_encoder_helper_hw_reset(struct dpu_encoder_phys *phys_enc) 1542 { 1543 struct dpu_encoder_virt *dpu_enc; 1544 struct dpu_hw_ctl *ctl; 1545 int rc; 1546 1547 if (!phys_enc) { 1548 DPU_ERROR("invalid encoder\n"); 1549 return; 1550 } 1551 dpu_enc = to_dpu_encoder_virt(phys_enc->parent); 1552 ctl = phys_enc->hw_ctl; 1553 1554 if (!ctl || !ctl->ops.reset) 1555 return; 1556 1557 DRM_DEBUG_KMS("id:%u ctl %d reset\n", DRMID(phys_enc->parent), 1558 ctl->idx); 1559 1560 rc = ctl->ops.reset(ctl); 1561 if (rc) 1562 DPU_ERROR_ENC(dpu_enc, "ctl %d reset failure\n", ctl->idx); 1563 1564 phys_enc->enable_state = DPU_ENC_ENABLED; 1565 } 1566 1567 /** 1568 * _dpu_encoder_kickoff_phys - handle physical encoder kickoff 1569 * Iterate through the physical encoders and perform consolidated flush 1570 * and/or control start triggering as needed. This is done in the virtual 1571 * encoder rather than the individual physical ones in order to handle 1572 * use cases that require visibility into multiple physical encoders at 1573 * a time. 1574 * dpu_enc: Pointer to virtual encoder structure 1575 */ 1576 static void _dpu_encoder_kickoff_phys(struct dpu_encoder_virt *dpu_enc, 1577 bool async) 1578 { 1579 struct dpu_hw_ctl *ctl; 1580 uint32_t i, pending_flush; 1581 unsigned long lock_flags; 1582 1583 if (!dpu_enc) { 1584 DPU_ERROR("invalid encoder\n"); 1585 return; 1586 } 1587 1588 pending_flush = 0x0; 1589 1590 /* update pending counts and trigger kickoff ctl flush atomically */ 1591 spin_lock_irqsave(&dpu_enc->enc_spinlock, lock_flags); 1592 1593 /* don't perform flush/start operations for slave encoders */ 1594 for (i = 0; i < dpu_enc->num_phys_encs; i++) { 1595 struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i]; 1596 1597 if (!phys || phys->enable_state == DPU_ENC_DISABLED) 1598 continue; 1599 1600 ctl = phys->hw_ctl; 1601 if (!ctl) 1602 continue; 1603 1604 /* 1605 * This is cleared in frame_done worker, which isn't invoked 1606 * for async commits. So don't set this for async, since it'll 1607 * roll over to the next commit. 1608 */ 1609 if (!async && phys->split_role != ENC_ROLE_SLAVE) 1610 set_bit(i, dpu_enc->frame_busy_mask); 1611 1612 if (!phys->ops.needs_single_flush || 1613 !phys->ops.needs_single_flush(phys)) 1614 _dpu_encoder_trigger_flush(&dpu_enc->base, phys, 0x0, 1615 async); 1616 else if (ctl->ops.get_pending_flush) 1617 pending_flush |= ctl->ops.get_pending_flush(ctl); 1618 } 1619 1620 /* for split flush, combine pending flush masks and send to master */ 1621 if (pending_flush && dpu_enc->cur_master) { 1622 _dpu_encoder_trigger_flush( 1623 &dpu_enc->base, 1624 dpu_enc->cur_master, 1625 pending_flush, async); 1626 } 1627 1628 _dpu_encoder_trigger_start(dpu_enc->cur_master); 1629 1630 spin_unlock_irqrestore(&dpu_enc->enc_spinlock, lock_flags); 1631 } 1632 1633 void dpu_encoder_trigger_kickoff_pending(struct drm_encoder *drm_enc) 1634 { 1635 struct dpu_encoder_virt *dpu_enc; 1636 struct dpu_encoder_phys *phys; 1637 unsigned int i; 1638 struct dpu_hw_ctl *ctl; 1639 struct msm_display_info *disp_info; 1640 1641 if (!drm_enc) { 1642 DPU_ERROR("invalid encoder\n"); 1643 return; 1644 } 1645 dpu_enc = to_dpu_encoder_virt(drm_enc); 1646 disp_info = &dpu_enc->disp_info; 1647 1648 for (i = 0; i < dpu_enc->num_phys_encs; i++) { 1649 phys = dpu_enc->phys_encs[i]; 1650 1651 if (phys && phys->hw_ctl) { 1652 ctl = phys->hw_ctl; 1653 if (ctl->ops.clear_pending_flush) 1654 ctl->ops.clear_pending_flush(ctl); 1655 1656 /* update only for command mode primary ctl */ 1657 if ((phys == dpu_enc->cur_master) && 1658 (disp_info->capabilities & MSM_DISPLAY_CAP_CMD_MODE) 1659 && ctl->ops.trigger_pending) 1660 ctl->ops.trigger_pending(ctl); 1661 } 1662 } 1663 } 1664 1665 static u32 _dpu_encoder_calculate_linetime(struct dpu_encoder_virt *dpu_enc, 1666 struct drm_display_mode *mode) 1667 { 1668 u64 pclk_rate; 1669 u32 pclk_period; 1670 u32 line_time; 1671 1672 /* 1673 * For linetime calculation, only operate on master encoder. 1674 */ 1675 if (!dpu_enc->cur_master) 1676 return 0; 1677 1678 if (!dpu_enc->cur_master->ops.get_line_count) { 1679 DPU_ERROR("get_line_count function not defined\n"); 1680 return 0; 1681 } 1682 1683 pclk_rate = mode->clock; /* pixel clock in kHz */ 1684 if (pclk_rate == 0) { 1685 DPU_ERROR("pclk is 0, cannot calculate line time\n"); 1686 return 0; 1687 } 1688 1689 pclk_period = DIV_ROUND_UP_ULL(1000000000ull, pclk_rate); 1690 if (pclk_period == 0) { 1691 DPU_ERROR("pclk period is 0\n"); 1692 return 0; 1693 } 1694 1695 /* 1696 * Line time calculation based on Pixel clock and HTOTAL. 1697 * Final unit is in ns. 1698 */ 1699 line_time = (pclk_period * mode->htotal) / 1000; 1700 if (line_time == 0) { 1701 DPU_ERROR("line time calculation is 0\n"); 1702 return 0; 1703 } 1704 1705 DPU_DEBUG_ENC(dpu_enc, 1706 "clk_rate=%lldkHz, clk_period=%d, linetime=%dns\n", 1707 pclk_rate, pclk_period, line_time); 1708 1709 return line_time; 1710 } 1711 1712 static int _dpu_encoder_wakeup_time(struct drm_encoder *drm_enc, 1713 ktime_t *wakeup_time) 1714 { 1715 struct drm_display_mode *mode; 1716 struct dpu_encoder_virt *dpu_enc; 1717 u32 cur_line; 1718 u32 line_time; 1719 u32 vtotal, time_to_vsync; 1720 ktime_t cur_time; 1721 1722 dpu_enc = to_dpu_encoder_virt(drm_enc); 1723 1724 if (!drm_enc->crtc || !drm_enc->crtc->state) { 1725 DPU_ERROR("crtc/crtc state object is NULL\n"); 1726 return -EINVAL; 1727 } 1728 mode = &drm_enc->crtc->state->adjusted_mode; 1729 1730 line_time = _dpu_encoder_calculate_linetime(dpu_enc, mode); 1731 if (!line_time) 1732 return -EINVAL; 1733 1734 cur_line = dpu_enc->cur_master->ops.get_line_count(dpu_enc->cur_master); 1735 1736 vtotal = mode->vtotal; 1737 if (cur_line >= vtotal) 1738 time_to_vsync = line_time * vtotal; 1739 else 1740 time_to_vsync = line_time * (vtotal - cur_line); 1741 1742 if (time_to_vsync == 0) { 1743 DPU_ERROR("time to vsync should not be zero, vtotal=%d\n", 1744 vtotal); 1745 return -EINVAL; 1746 } 1747 1748 cur_time = ktime_get(); 1749 *wakeup_time = ktime_add_ns(cur_time, time_to_vsync); 1750 1751 DPU_DEBUG_ENC(dpu_enc, 1752 "cur_line=%u vtotal=%u time_to_vsync=%u, cur_time=%lld, wakeup_time=%lld\n", 1753 cur_line, vtotal, time_to_vsync, 1754 ktime_to_ms(cur_time), 1755 ktime_to_ms(*wakeup_time)); 1756 return 0; 1757 } 1758 1759 static void dpu_encoder_vsync_event_handler(struct timer_list *t) 1760 { 1761 struct dpu_encoder_virt *dpu_enc = from_timer(dpu_enc, t, 1762 vsync_event_timer); 1763 struct drm_encoder *drm_enc = &dpu_enc->base; 1764 struct msm_drm_private *priv; 1765 struct msm_drm_thread *event_thread; 1766 1767 if (!drm_enc->dev || !drm_enc->dev->dev_private || 1768 !drm_enc->crtc) { 1769 DPU_ERROR("invalid parameters\n"); 1770 return; 1771 } 1772 1773 priv = drm_enc->dev->dev_private; 1774 1775 if (drm_enc->crtc->index >= ARRAY_SIZE(priv->event_thread)) { 1776 DPU_ERROR("invalid crtc index\n"); 1777 return; 1778 } 1779 event_thread = &priv->event_thread[drm_enc->crtc->index]; 1780 if (!event_thread) { 1781 DPU_ERROR("event_thread not found for crtc:%d\n", 1782 drm_enc->crtc->index); 1783 return; 1784 } 1785 1786 del_timer(&dpu_enc->vsync_event_timer); 1787 } 1788 1789 static void dpu_encoder_vsync_event_work_handler(struct kthread_work *work) 1790 { 1791 struct dpu_encoder_virt *dpu_enc = container_of(work, 1792 struct dpu_encoder_virt, vsync_event_work); 1793 ktime_t wakeup_time; 1794 1795 if (!dpu_enc) { 1796 DPU_ERROR("invalid dpu encoder\n"); 1797 return; 1798 } 1799 1800 if (_dpu_encoder_wakeup_time(&dpu_enc->base, &wakeup_time)) 1801 return; 1802 1803 trace_dpu_enc_vsync_event_work(DRMID(&dpu_enc->base), wakeup_time); 1804 mod_timer(&dpu_enc->vsync_event_timer, 1805 nsecs_to_jiffies(ktime_to_ns(wakeup_time))); 1806 } 1807 1808 void dpu_encoder_prepare_for_kickoff(struct drm_encoder *drm_enc, bool async) 1809 { 1810 struct dpu_encoder_virt *dpu_enc; 1811 struct dpu_encoder_phys *phys; 1812 bool needs_hw_reset = false; 1813 unsigned int i; 1814 1815 if (!drm_enc) { 1816 DPU_ERROR("invalid args\n"); 1817 return; 1818 } 1819 dpu_enc = to_dpu_encoder_virt(drm_enc); 1820 1821 trace_dpu_enc_prepare_kickoff(DRMID(drm_enc)); 1822 1823 /* prepare for next kickoff, may include waiting on previous kickoff */ 1824 DPU_ATRACE_BEGIN("enc_prepare_for_kickoff"); 1825 for (i = 0; i < dpu_enc->num_phys_encs; i++) { 1826 phys = dpu_enc->phys_encs[i]; 1827 if (phys) { 1828 if (phys->ops.prepare_for_kickoff) 1829 phys->ops.prepare_for_kickoff(phys); 1830 if (phys->enable_state == DPU_ENC_ERR_NEEDS_HW_RESET) 1831 needs_hw_reset = true; 1832 } 1833 } 1834 DPU_ATRACE_END("enc_prepare_for_kickoff"); 1835 1836 dpu_encoder_resource_control(drm_enc, DPU_ENC_RC_EVENT_KICKOFF); 1837 1838 /* if any phys needs reset, reset all phys, in-order */ 1839 if (needs_hw_reset) { 1840 trace_dpu_enc_prepare_kickoff_reset(DRMID(drm_enc)); 1841 for (i = 0; i < dpu_enc->num_phys_encs; i++) { 1842 dpu_encoder_helper_hw_reset(dpu_enc->phys_encs[i]); 1843 } 1844 } 1845 } 1846 1847 void dpu_encoder_kickoff(struct drm_encoder *drm_enc, bool async) 1848 { 1849 struct dpu_encoder_virt *dpu_enc; 1850 struct dpu_encoder_phys *phys; 1851 ktime_t wakeup_time; 1852 unsigned int i; 1853 1854 if (!drm_enc) { 1855 DPU_ERROR("invalid encoder\n"); 1856 return; 1857 } 1858 DPU_ATRACE_BEGIN("encoder_kickoff"); 1859 dpu_enc = to_dpu_encoder_virt(drm_enc); 1860 1861 trace_dpu_enc_kickoff(DRMID(drm_enc)); 1862 1863 /* 1864 * Asynchronous frames don't handle FRAME_DONE events. As such, they 1865 * shouldn't enable the frame_done watchdog since it will always time 1866 * out. 1867 */ 1868 if (!async) { 1869 unsigned long timeout_ms; 1870 timeout_ms = DPU_ENCODER_FRAME_DONE_TIMEOUT_FRAMES * 1000 / 1871 drm_mode_vrefresh(&drm_enc->crtc->state->adjusted_mode); 1872 1873 atomic_set(&dpu_enc->frame_done_timeout_ms, timeout_ms); 1874 mod_timer(&dpu_enc->frame_done_timer, 1875 jiffies + msecs_to_jiffies(timeout_ms)); 1876 } 1877 1878 /* All phys encs are ready to go, trigger the kickoff */ 1879 _dpu_encoder_kickoff_phys(dpu_enc, async); 1880 1881 /* allow phys encs to handle any post-kickoff business */ 1882 for (i = 0; i < dpu_enc->num_phys_encs; i++) { 1883 phys = dpu_enc->phys_encs[i]; 1884 if (phys && phys->ops.handle_post_kickoff) 1885 phys->ops.handle_post_kickoff(phys); 1886 } 1887 1888 if (dpu_enc->disp_info.intf_type == DRM_MODE_ENCODER_DSI && 1889 !_dpu_encoder_wakeup_time(drm_enc, &wakeup_time)) { 1890 trace_dpu_enc_early_kickoff(DRMID(drm_enc), 1891 ktime_to_ms(wakeup_time)); 1892 mod_timer(&dpu_enc->vsync_event_timer, 1893 nsecs_to_jiffies(ktime_to_ns(wakeup_time))); 1894 } 1895 1896 DPU_ATRACE_END("encoder_kickoff"); 1897 } 1898 1899 void dpu_encoder_prepare_commit(struct drm_encoder *drm_enc) 1900 { 1901 struct dpu_encoder_virt *dpu_enc; 1902 struct dpu_encoder_phys *phys; 1903 int i; 1904 1905 if (!drm_enc) { 1906 DPU_ERROR("invalid encoder\n"); 1907 return; 1908 } 1909 dpu_enc = to_dpu_encoder_virt(drm_enc); 1910 1911 for (i = 0; i < dpu_enc->num_phys_encs; i++) { 1912 phys = dpu_enc->phys_encs[i]; 1913 if (phys && phys->ops.prepare_commit) 1914 phys->ops.prepare_commit(phys); 1915 } 1916 } 1917 1918 #ifdef CONFIG_DEBUG_FS 1919 static int _dpu_encoder_status_show(struct seq_file *s, void *data) 1920 { 1921 struct dpu_encoder_virt *dpu_enc = s->private; 1922 int i; 1923 1924 mutex_lock(&dpu_enc->enc_lock); 1925 for (i = 0; i < dpu_enc->num_phys_encs; i++) { 1926 struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i]; 1927 1928 if (!phys) 1929 continue; 1930 1931 seq_printf(s, "intf:%d vsync:%8d underrun:%8d ", 1932 phys->intf_idx - INTF_0, 1933 atomic_read(&phys->vsync_cnt), 1934 atomic_read(&phys->underrun_cnt)); 1935 1936 switch (phys->intf_mode) { 1937 case INTF_MODE_VIDEO: 1938 seq_puts(s, "mode: video\n"); 1939 break; 1940 case INTF_MODE_CMD: 1941 seq_puts(s, "mode: command\n"); 1942 break; 1943 default: 1944 seq_puts(s, "mode: ???\n"); 1945 break; 1946 } 1947 } 1948 mutex_unlock(&dpu_enc->enc_lock); 1949 1950 return 0; 1951 } 1952 1953 static int _dpu_encoder_debugfs_status_open(struct inode *inode, 1954 struct file *file) 1955 { 1956 return single_open(file, _dpu_encoder_status_show, inode->i_private); 1957 } 1958 1959 static int _dpu_encoder_init_debugfs(struct drm_encoder *drm_enc) 1960 { 1961 struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc); 1962 struct msm_drm_private *priv; 1963 struct dpu_kms *dpu_kms; 1964 int i; 1965 1966 static const struct file_operations debugfs_status_fops = { 1967 .open = _dpu_encoder_debugfs_status_open, 1968 .read = seq_read, 1969 .llseek = seq_lseek, 1970 .release = single_release, 1971 }; 1972 1973 char name[DPU_NAME_SIZE]; 1974 1975 if (!drm_enc->dev || !drm_enc->dev->dev_private) { 1976 DPU_ERROR("invalid encoder or kms\n"); 1977 return -EINVAL; 1978 } 1979 1980 priv = drm_enc->dev->dev_private; 1981 dpu_kms = to_dpu_kms(priv->kms); 1982 1983 snprintf(name, DPU_NAME_SIZE, "encoder%u", drm_enc->base.id); 1984 1985 /* create overall sub-directory for the encoder */ 1986 dpu_enc->debugfs_root = debugfs_create_dir(name, 1987 drm_enc->dev->primary->debugfs_root); 1988 if (!dpu_enc->debugfs_root) 1989 return -ENOMEM; 1990 1991 /* don't error check these */ 1992 debugfs_create_file("status", 0600, 1993 dpu_enc->debugfs_root, dpu_enc, &debugfs_status_fops); 1994 1995 for (i = 0; i < dpu_enc->num_phys_encs; i++) 1996 if (dpu_enc->phys_encs[i] && 1997 dpu_enc->phys_encs[i]->ops.late_register) 1998 dpu_enc->phys_encs[i]->ops.late_register( 1999 dpu_enc->phys_encs[i], 2000 dpu_enc->debugfs_root); 2001 2002 return 0; 2003 } 2004 #else 2005 static int _dpu_encoder_init_debugfs(struct drm_encoder *drm_enc) 2006 { 2007 return 0; 2008 } 2009 #endif 2010 2011 static int dpu_encoder_late_register(struct drm_encoder *encoder) 2012 { 2013 return _dpu_encoder_init_debugfs(encoder); 2014 } 2015 2016 static void dpu_encoder_early_unregister(struct drm_encoder *encoder) 2017 { 2018 struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(encoder); 2019 2020 debugfs_remove_recursive(dpu_enc->debugfs_root); 2021 } 2022 2023 static int dpu_encoder_virt_add_phys_encs( 2024 u32 display_caps, 2025 struct dpu_encoder_virt *dpu_enc, 2026 struct dpu_enc_phys_init_params *params) 2027 { 2028 struct dpu_encoder_phys *enc = NULL; 2029 2030 DPU_DEBUG_ENC(dpu_enc, "\n"); 2031 2032 /* 2033 * We may create up to NUM_PHYS_ENCODER_TYPES physical encoder types 2034 * in this function, check up-front. 2035 */ 2036 if (dpu_enc->num_phys_encs + NUM_PHYS_ENCODER_TYPES >= 2037 ARRAY_SIZE(dpu_enc->phys_encs)) { 2038 DPU_ERROR_ENC(dpu_enc, "too many physical encoders %d\n", 2039 dpu_enc->num_phys_encs); 2040 return -EINVAL; 2041 } 2042 2043 if (display_caps & MSM_DISPLAY_CAP_VID_MODE) { 2044 enc = dpu_encoder_phys_vid_init(params); 2045 2046 if (IS_ERR_OR_NULL(enc)) { 2047 DPU_ERROR_ENC(dpu_enc, "failed to init vid enc: %ld\n", 2048 PTR_ERR(enc)); 2049 return enc == 0 ? -EINVAL : PTR_ERR(enc); 2050 } 2051 2052 dpu_enc->phys_encs[dpu_enc->num_phys_encs] = enc; 2053 ++dpu_enc->num_phys_encs; 2054 } 2055 2056 if (display_caps & MSM_DISPLAY_CAP_CMD_MODE) { 2057 enc = dpu_encoder_phys_cmd_init(params); 2058 2059 if (IS_ERR_OR_NULL(enc)) { 2060 DPU_ERROR_ENC(dpu_enc, "failed to init cmd enc: %ld\n", 2061 PTR_ERR(enc)); 2062 return enc == 0 ? -EINVAL : PTR_ERR(enc); 2063 } 2064 2065 dpu_enc->phys_encs[dpu_enc->num_phys_encs] = enc; 2066 ++dpu_enc->num_phys_encs; 2067 } 2068 2069 if (params->split_role == ENC_ROLE_SLAVE) 2070 dpu_enc->cur_slave = enc; 2071 else 2072 dpu_enc->cur_master = enc; 2073 2074 return 0; 2075 } 2076 2077 static const struct dpu_encoder_virt_ops dpu_encoder_parent_ops = { 2078 .handle_vblank_virt = dpu_encoder_vblank_callback, 2079 .handle_underrun_virt = dpu_encoder_underrun_callback, 2080 .handle_frame_done = dpu_encoder_frame_done_callback, 2081 }; 2082 2083 static int dpu_encoder_setup_display(struct dpu_encoder_virt *dpu_enc, 2084 struct dpu_kms *dpu_kms, 2085 struct msm_display_info *disp_info) 2086 { 2087 int ret = 0; 2088 int i = 0; 2089 enum dpu_intf_type intf_type; 2090 struct dpu_enc_phys_init_params phys_params; 2091 2092 if (!dpu_enc || !dpu_kms) { 2093 DPU_ERROR("invalid arg(s), enc %d kms %d\n", 2094 dpu_enc != 0, dpu_kms != 0); 2095 return -EINVAL; 2096 } 2097 2098 dpu_enc->cur_master = NULL; 2099 2100 memset(&phys_params, 0, sizeof(phys_params)); 2101 phys_params.dpu_kms = dpu_kms; 2102 phys_params.parent = &dpu_enc->base; 2103 phys_params.parent_ops = &dpu_encoder_parent_ops; 2104 phys_params.enc_spinlock = &dpu_enc->enc_spinlock; 2105 2106 DPU_DEBUG("\n"); 2107 2108 switch (disp_info->intf_type) { 2109 case DRM_MODE_ENCODER_DSI: 2110 intf_type = INTF_DSI; 2111 break; 2112 default: 2113 DPU_ERROR_ENC(dpu_enc, "unsupported display interface type\n"); 2114 return -EINVAL; 2115 } 2116 2117 WARN_ON(disp_info->num_of_h_tiles < 1); 2118 2119 DPU_DEBUG("dsi_info->num_of_h_tiles %d\n", disp_info->num_of_h_tiles); 2120 2121 if ((disp_info->capabilities & MSM_DISPLAY_CAP_CMD_MODE) || 2122 (disp_info->capabilities & MSM_DISPLAY_CAP_VID_MODE)) 2123 dpu_enc->idle_pc_supported = 2124 dpu_kms->catalog->caps->has_idle_pc; 2125 2126 mutex_lock(&dpu_enc->enc_lock); 2127 for (i = 0; i < disp_info->num_of_h_tiles && !ret; i++) { 2128 /* 2129 * Left-most tile is at index 0, content is controller id 2130 * h_tile_instance_ids[2] = {0, 1}; DSI0 = left, DSI1 = right 2131 * h_tile_instance_ids[2] = {1, 0}; DSI1 = left, DSI0 = right 2132 */ 2133 u32 controller_id = disp_info->h_tile_instance[i]; 2134 2135 if (disp_info->num_of_h_tiles > 1) { 2136 if (i == 0) 2137 phys_params.split_role = ENC_ROLE_MASTER; 2138 else 2139 phys_params.split_role = ENC_ROLE_SLAVE; 2140 } else { 2141 phys_params.split_role = ENC_ROLE_SOLO; 2142 } 2143 2144 DPU_DEBUG("h_tile_instance %d = %d, split_role %d\n", 2145 i, controller_id, phys_params.split_role); 2146 2147 phys_params.intf_idx = dpu_encoder_get_intf(dpu_kms->catalog, 2148 intf_type, 2149 controller_id); 2150 if (phys_params.intf_idx == INTF_MAX) { 2151 DPU_ERROR_ENC(dpu_enc, "could not get intf: type %d, id %d\n", 2152 intf_type, controller_id); 2153 ret = -EINVAL; 2154 } 2155 2156 if (!ret) { 2157 ret = dpu_encoder_virt_add_phys_encs(disp_info->capabilities, 2158 dpu_enc, 2159 &phys_params); 2160 if (ret) 2161 DPU_ERROR_ENC(dpu_enc, "failed to add phys encs\n"); 2162 } 2163 } 2164 2165 for (i = 0; i < dpu_enc->num_phys_encs; i++) { 2166 struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i]; 2167 2168 if (phys) { 2169 atomic_set(&phys->vsync_cnt, 0); 2170 atomic_set(&phys->underrun_cnt, 0); 2171 } 2172 } 2173 mutex_unlock(&dpu_enc->enc_lock); 2174 2175 return ret; 2176 } 2177 2178 static void dpu_encoder_frame_done_timeout(struct timer_list *t) 2179 { 2180 struct dpu_encoder_virt *dpu_enc = from_timer(dpu_enc, t, 2181 frame_done_timer); 2182 struct drm_encoder *drm_enc = &dpu_enc->base; 2183 struct msm_drm_private *priv; 2184 u32 event; 2185 2186 if (!drm_enc->dev || !drm_enc->dev->dev_private) { 2187 DPU_ERROR("invalid parameters\n"); 2188 return; 2189 } 2190 priv = drm_enc->dev->dev_private; 2191 2192 if (!dpu_enc->frame_busy_mask[0] || !dpu_enc->crtc_frame_event_cb) { 2193 DRM_DEBUG_KMS("id:%u invalid timeout frame_busy_mask=%lu\n", 2194 DRMID(drm_enc), dpu_enc->frame_busy_mask[0]); 2195 return; 2196 } else if (!atomic_xchg(&dpu_enc->frame_done_timeout_ms, 0)) { 2197 DRM_DEBUG_KMS("id:%u invalid timeout\n", DRMID(drm_enc)); 2198 return; 2199 } 2200 2201 DPU_ERROR_ENC(dpu_enc, "frame done timeout\n"); 2202 2203 event = DPU_ENCODER_FRAME_EVENT_ERROR; 2204 trace_dpu_enc_frame_done_timeout(DRMID(drm_enc), event); 2205 dpu_enc->crtc_frame_event_cb(dpu_enc->crtc_frame_event_cb_data, event); 2206 } 2207 2208 static const struct drm_encoder_helper_funcs dpu_encoder_helper_funcs = { 2209 .mode_set = dpu_encoder_virt_mode_set, 2210 .disable = dpu_encoder_virt_disable, 2211 .enable = dpu_kms_encoder_enable, 2212 .atomic_check = dpu_encoder_virt_atomic_check, 2213 2214 /* This is called by dpu_kms_encoder_enable */ 2215 .commit = dpu_encoder_virt_enable, 2216 }; 2217 2218 static const struct drm_encoder_funcs dpu_encoder_funcs = { 2219 .destroy = dpu_encoder_destroy, 2220 .late_register = dpu_encoder_late_register, 2221 .early_unregister = dpu_encoder_early_unregister, 2222 }; 2223 2224 int dpu_encoder_setup(struct drm_device *dev, struct drm_encoder *enc, 2225 struct msm_display_info *disp_info) 2226 { 2227 struct msm_drm_private *priv = dev->dev_private; 2228 struct dpu_kms *dpu_kms = to_dpu_kms(priv->kms); 2229 struct drm_encoder *drm_enc = NULL; 2230 struct dpu_encoder_virt *dpu_enc = NULL; 2231 int ret = 0; 2232 2233 dpu_enc = to_dpu_encoder_virt(enc); 2234 2235 mutex_init(&dpu_enc->enc_lock); 2236 ret = dpu_encoder_setup_display(dpu_enc, dpu_kms, disp_info); 2237 if (ret) 2238 goto fail; 2239 2240 spin_lock_init(&dpu_enc->enc_spinlock); 2241 2242 atomic_set(&dpu_enc->frame_done_timeout_ms, 0); 2243 timer_setup(&dpu_enc->frame_done_timer, 2244 dpu_encoder_frame_done_timeout, 0); 2245 2246 if (disp_info->intf_type == DRM_MODE_ENCODER_DSI) 2247 timer_setup(&dpu_enc->vsync_event_timer, 2248 dpu_encoder_vsync_event_handler, 2249 0); 2250 2251 2252 mutex_init(&dpu_enc->rc_lock); 2253 INIT_DELAYED_WORK(&dpu_enc->delayed_off_work, 2254 dpu_encoder_off_work); 2255 dpu_enc->idle_timeout = IDLE_TIMEOUT; 2256 2257 kthread_init_work(&dpu_enc->vsync_event_work, 2258 dpu_encoder_vsync_event_work_handler); 2259 2260 memcpy(&dpu_enc->disp_info, disp_info, sizeof(*disp_info)); 2261 2262 DPU_DEBUG_ENC(dpu_enc, "created\n"); 2263 2264 return ret; 2265 2266 fail: 2267 DPU_ERROR("failed to create encoder\n"); 2268 if (drm_enc) 2269 dpu_encoder_destroy(drm_enc); 2270 2271 return ret; 2272 2273 2274 } 2275 2276 struct drm_encoder *dpu_encoder_init(struct drm_device *dev, 2277 int drm_enc_mode) 2278 { 2279 struct dpu_encoder_virt *dpu_enc = NULL; 2280 int rc = 0; 2281 2282 dpu_enc = devm_kzalloc(dev->dev, sizeof(*dpu_enc), GFP_KERNEL); 2283 if (!dpu_enc) 2284 return ERR_PTR(ENOMEM); 2285 2286 rc = drm_encoder_init(dev, &dpu_enc->base, &dpu_encoder_funcs, 2287 drm_enc_mode, NULL); 2288 if (rc) { 2289 devm_kfree(dev->dev, dpu_enc); 2290 return ERR_PTR(rc); 2291 } 2292 2293 drm_encoder_helper_add(&dpu_enc->base, &dpu_encoder_helper_funcs); 2294 2295 dpu_enc->enabled = false; 2296 2297 return &dpu_enc->base; 2298 } 2299 2300 int dpu_encoder_wait_for_event(struct drm_encoder *drm_enc, 2301 enum msm_event_wait event) 2302 { 2303 int (*fn_wait)(struct dpu_encoder_phys *phys_enc) = NULL; 2304 struct dpu_encoder_virt *dpu_enc = NULL; 2305 int i, ret = 0; 2306 2307 if (!drm_enc) { 2308 DPU_ERROR("invalid encoder\n"); 2309 return -EINVAL; 2310 } 2311 dpu_enc = to_dpu_encoder_virt(drm_enc); 2312 DPU_DEBUG_ENC(dpu_enc, "\n"); 2313 2314 for (i = 0; i < dpu_enc->num_phys_encs; i++) { 2315 struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i]; 2316 if (!phys) 2317 continue; 2318 2319 switch (event) { 2320 case MSM_ENC_COMMIT_DONE: 2321 fn_wait = phys->ops.wait_for_commit_done; 2322 break; 2323 case MSM_ENC_TX_COMPLETE: 2324 fn_wait = phys->ops.wait_for_tx_complete; 2325 break; 2326 case MSM_ENC_VBLANK: 2327 fn_wait = phys->ops.wait_for_vblank; 2328 break; 2329 default: 2330 DPU_ERROR_ENC(dpu_enc, "unknown wait event %d\n", 2331 event); 2332 return -EINVAL; 2333 }; 2334 2335 if (fn_wait) { 2336 DPU_ATRACE_BEGIN("wait_for_completion_event"); 2337 ret = fn_wait(phys); 2338 DPU_ATRACE_END("wait_for_completion_event"); 2339 if (ret) 2340 return ret; 2341 } 2342 } 2343 2344 return ret; 2345 } 2346 2347 enum dpu_intf_mode dpu_encoder_get_intf_mode(struct drm_encoder *encoder) 2348 { 2349 struct dpu_encoder_virt *dpu_enc = NULL; 2350 int i; 2351 2352 if (!encoder) { 2353 DPU_ERROR("invalid encoder\n"); 2354 return INTF_MODE_NONE; 2355 } 2356 dpu_enc = to_dpu_encoder_virt(encoder); 2357 2358 if (dpu_enc->cur_master) 2359 return dpu_enc->cur_master->intf_mode; 2360 2361 for (i = 0; i < dpu_enc->num_phys_encs; i++) { 2362 struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i]; 2363 2364 if (phys) 2365 return phys->intf_mode; 2366 } 2367 2368 return INTF_MODE_NONE; 2369 } 2370