1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved. 3 */ 4 5 #define pr_fmt(fmt) "[drm:%s:%d] " fmt, __func__, __LINE__ 6 7 #include <linux/debugfs.h> 8 #include <linux/errno.h> 9 #include <linux/mutex.h> 10 #include <linux/sort.h> 11 #include <linux/clk.h> 12 #include <linux/bitmap.h> 13 14 #include "dpu_kms.h" 15 #include "dpu_trace.h" 16 #include "dpu_crtc.h" 17 #include "dpu_core_perf.h" 18 19 /** 20 * enum dpu_perf_mode - performance tuning mode 21 * @DPU_PERF_MODE_NORMAL: performance controlled by user mode client 22 * @DPU_PERF_MODE_MINIMUM: performance bounded by minimum setting 23 * @DPU_PERF_MODE_FIXED: performance bounded by fixed setting 24 */ 25 enum dpu_perf_mode { 26 DPU_PERF_MODE_NORMAL, 27 DPU_PERF_MODE_MINIMUM, 28 DPU_PERF_MODE_FIXED, 29 DPU_PERF_MODE_MAX 30 }; 31 32 static struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc) 33 { 34 struct msm_drm_private *priv; 35 priv = crtc->dev->dev_private; 36 return to_dpu_kms(priv->kms); 37 } 38 39 static bool _dpu_core_video_mode_intf_connected(struct drm_crtc *crtc) 40 { 41 struct drm_crtc *tmp_crtc; 42 43 drm_for_each_crtc(tmp_crtc, crtc->dev) { 44 if ((dpu_crtc_get_intf_mode(tmp_crtc) == INTF_MODE_VIDEO) && 45 tmp_crtc->enabled) { 46 DPU_DEBUG("video interface connected crtc:%d\n", 47 tmp_crtc->base.id); 48 return true; 49 } 50 } 51 52 return false; 53 } 54 55 static void _dpu_core_perf_calc_crtc(struct dpu_kms *kms, 56 struct drm_crtc *crtc, 57 struct drm_crtc_state *state, 58 struct dpu_core_perf_params *perf) 59 { 60 struct dpu_crtc_state *dpu_cstate; 61 62 if (!kms || !kms->catalog || !crtc || !state || !perf) { 63 DPU_ERROR("invalid parameters\n"); 64 return; 65 } 66 67 dpu_cstate = to_dpu_crtc_state(state); 68 memset(perf, 0, sizeof(struct dpu_core_perf_params)); 69 70 if (!dpu_cstate->bw_control) { 71 perf->bw_ctl = kms->catalog->perf.max_bw_high * 72 1000ULL; 73 perf->max_per_pipe_ib = perf->bw_ctl; 74 perf->core_clk_rate = kms->perf.max_core_clk_rate; 75 } else if (kms->perf.perf_tune.mode == DPU_PERF_MODE_MINIMUM) { 76 perf->bw_ctl = 0; 77 perf->max_per_pipe_ib = 0; 78 perf->core_clk_rate = 0; 79 } else if (kms->perf.perf_tune.mode == DPU_PERF_MODE_FIXED) { 80 perf->bw_ctl = kms->perf.fix_core_ab_vote; 81 perf->max_per_pipe_ib = kms->perf.fix_core_ib_vote; 82 perf->core_clk_rate = kms->perf.fix_core_clk_rate; 83 } 84 85 DPU_DEBUG( 86 "crtc=%d clk_rate=%llu core_ib=%llu core_ab=%llu\n", 87 crtc->base.id, perf->core_clk_rate, 88 perf->max_per_pipe_ib, perf->bw_ctl); 89 } 90 91 int dpu_core_perf_crtc_check(struct drm_crtc *crtc, 92 struct drm_crtc_state *state) 93 { 94 u32 bw, threshold; 95 u64 bw_sum_of_intfs = 0; 96 enum dpu_crtc_client_type curr_client_type; 97 bool is_video_mode; 98 struct dpu_crtc_state *dpu_cstate; 99 struct drm_crtc *tmp_crtc; 100 struct dpu_kms *kms; 101 102 if (!crtc || !state) { 103 DPU_ERROR("invalid crtc\n"); 104 return -EINVAL; 105 } 106 107 kms = _dpu_crtc_get_kms(crtc); 108 if (!kms->catalog) { 109 DPU_ERROR("invalid parameters\n"); 110 return 0; 111 } 112 113 /* we only need bandwidth check on real-time clients (interfaces) */ 114 if (dpu_crtc_get_client_type(crtc) == NRT_CLIENT) 115 return 0; 116 117 dpu_cstate = to_dpu_crtc_state(state); 118 119 /* obtain new values */ 120 _dpu_core_perf_calc_crtc(kms, crtc, state, &dpu_cstate->new_perf); 121 122 bw_sum_of_intfs = dpu_cstate->new_perf.bw_ctl; 123 curr_client_type = dpu_crtc_get_client_type(crtc); 124 125 drm_for_each_crtc(tmp_crtc, crtc->dev) { 126 if (tmp_crtc->enabled && 127 (dpu_crtc_get_client_type(tmp_crtc) == 128 curr_client_type) && (tmp_crtc != crtc)) { 129 struct dpu_crtc_state *tmp_cstate = 130 to_dpu_crtc_state(tmp_crtc->state); 131 132 DPU_DEBUG("crtc:%d bw:%llu ctrl:%d\n", 133 tmp_crtc->base.id, tmp_cstate->new_perf.bw_ctl, 134 tmp_cstate->bw_control); 135 /* 136 * For bw check only use the bw if the 137 * atomic property has been already set 138 */ 139 if (tmp_cstate->bw_control) 140 bw_sum_of_intfs += tmp_cstate->new_perf.bw_ctl; 141 } 142 143 /* convert bandwidth to kb */ 144 bw = DIV_ROUND_UP_ULL(bw_sum_of_intfs, 1000); 145 DPU_DEBUG("calculated bandwidth=%uk\n", bw); 146 147 is_video_mode = dpu_crtc_get_intf_mode(crtc) == INTF_MODE_VIDEO; 148 threshold = (is_video_mode || 149 _dpu_core_video_mode_intf_connected(crtc)) ? 150 kms->catalog->perf.max_bw_low : 151 kms->catalog->perf.max_bw_high; 152 153 DPU_DEBUG("final threshold bw limit = %d\n", threshold); 154 155 if (!dpu_cstate->bw_control) { 156 DPU_DEBUG("bypass bandwidth check\n"); 157 } else if (!threshold) { 158 DPU_ERROR("no bandwidth limits specified\n"); 159 return -E2BIG; 160 } else if (bw > threshold) { 161 DPU_ERROR("exceeds bandwidth: %ukb > %ukb\n", bw, 162 threshold); 163 return -E2BIG; 164 } 165 } 166 167 return 0; 168 } 169 170 static int _dpu_core_perf_crtc_update_bus(struct dpu_kms *kms, 171 struct drm_crtc *crtc) 172 { 173 struct dpu_core_perf_params perf = { 0 }; 174 enum dpu_crtc_client_type curr_client_type 175 = dpu_crtc_get_client_type(crtc); 176 struct drm_crtc *tmp_crtc; 177 struct dpu_crtc_state *dpu_cstate; 178 int ret = 0; 179 180 drm_for_each_crtc(tmp_crtc, crtc->dev) { 181 if (tmp_crtc->enabled && 182 curr_client_type == 183 dpu_crtc_get_client_type(tmp_crtc)) { 184 dpu_cstate = to_dpu_crtc_state(tmp_crtc->state); 185 186 perf.max_per_pipe_ib = max(perf.max_per_pipe_ib, 187 dpu_cstate->new_perf.max_per_pipe_ib); 188 189 DPU_DEBUG("crtc=%d bw=%llu\n", tmp_crtc->base.id, 190 dpu_cstate->new_perf.bw_ctl); 191 } 192 } 193 return ret; 194 } 195 196 /** 197 * @dpu_core_perf_crtc_release_bw() - request zero bandwidth 198 * @crtc - pointer to a crtc 199 * 200 * Function checks a state variable for the crtc, if all pending commit 201 * requests are done, meaning no more bandwidth is needed, release 202 * bandwidth request. 203 */ 204 void dpu_core_perf_crtc_release_bw(struct drm_crtc *crtc) 205 { 206 struct dpu_crtc *dpu_crtc; 207 struct dpu_kms *kms; 208 209 if (!crtc) { 210 DPU_ERROR("invalid crtc\n"); 211 return; 212 } 213 214 kms = _dpu_crtc_get_kms(crtc); 215 if (!kms->catalog) { 216 DPU_ERROR("invalid kms\n"); 217 return; 218 } 219 220 dpu_crtc = to_dpu_crtc(crtc); 221 222 if (atomic_dec_return(&kms->bandwidth_ref) > 0) 223 return; 224 225 /* Release the bandwidth */ 226 if (kms->perf.enable_bw_release) { 227 trace_dpu_cmd_release_bw(crtc->base.id); 228 DPU_DEBUG("Release BW crtc=%d\n", crtc->base.id); 229 dpu_crtc->cur_perf.bw_ctl = 0; 230 _dpu_core_perf_crtc_update_bus(kms, crtc); 231 } 232 } 233 234 static int _dpu_core_perf_set_core_clk_rate(struct dpu_kms *kms, u64 rate) 235 { 236 struct dss_clk *core_clk = kms->perf.core_clk; 237 238 if (core_clk->max_rate && (rate > core_clk->max_rate)) 239 rate = core_clk->max_rate; 240 241 core_clk->rate = rate; 242 return msm_dss_clk_set_rate(core_clk, 1); 243 } 244 245 static u64 _dpu_core_perf_get_core_clk_rate(struct dpu_kms *kms) 246 { 247 u64 clk_rate = kms->perf.perf_tune.min_core_clk; 248 struct drm_crtc *crtc; 249 struct dpu_crtc_state *dpu_cstate; 250 251 drm_for_each_crtc(crtc, kms->dev) { 252 if (crtc->enabled) { 253 dpu_cstate = to_dpu_crtc_state(crtc->state); 254 clk_rate = max(dpu_cstate->new_perf.core_clk_rate, 255 clk_rate); 256 clk_rate = clk_round_rate(kms->perf.core_clk->clk, 257 clk_rate); 258 } 259 } 260 261 if (kms->perf.perf_tune.mode == DPU_PERF_MODE_FIXED) 262 clk_rate = kms->perf.fix_core_clk_rate; 263 264 DPU_DEBUG("clk:%llu\n", clk_rate); 265 266 return clk_rate; 267 } 268 269 int dpu_core_perf_crtc_update(struct drm_crtc *crtc, 270 int params_changed, bool stop_req) 271 { 272 struct dpu_core_perf_params *new, *old; 273 bool update_bus = false, update_clk = false; 274 u64 clk_rate = 0; 275 struct dpu_crtc *dpu_crtc; 276 struct dpu_crtc_state *dpu_cstate; 277 struct dpu_kms *kms; 278 int ret; 279 280 if (!crtc) { 281 DPU_ERROR("invalid crtc\n"); 282 return -EINVAL; 283 } 284 285 kms = _dpu_crtc_get_kms(crtc); 286 if (!kms->catalog) { 287 DPU_ERROR("invalid kms\n"); 288 return -EINVAL; 289 } 290 291 dpu_crtc = to_dpu_crtc(crtc); 292 dpu_cstate = to_dpu_crtc_state(crtc->state); 293 294 DPU_DEBUG("crtc:%d stop_req:%d core_clk:%llu\n", 295 crtc->base.id, stop_req, kms->perf.core_clk_rate); 296 297 old = &dpu_crtc->cur_perf; 298 new = &dpu_cstate->new_perf; 299 300 if (crtc->enabled && !stop_req) { 301 /* 302 * cases for bus bandwidth update. 303 * 1. new bandwidth vote - "ab or ib vote" is higher 304 * than current vote for update request. 305 * 2. new bandwidth vote - "ab or ib vote" is lower 306 * than current vote at end of commit or stop. 307 */ 308 if ((params_changed && ((new->bw_ctl > old->bw_ctl) || 309 (new->max_per_pipe_ib > old->max_per_pipe_ib))) || 310 (!params_changed && ((new->bw_ctl < old->bw_ctl) || 311 (new->max_per_pipe_ib < old->max_per_pipe_ib)))) { 312 DPU_DEBUG("crtc=%d p=%d new_bw=%llu,old_bw=%llu\n", 313 crtc->base.id, params_changed, 314 new->bw_ctl, old->bw_ctl); 315 old->bw_ctl = new->bw_ctl; 316 old->max_per_pipe_ib = new->max_per_pipe_ib; 317 update_bus = true; 318 } 319 320 if ((params_changed && 321 (new->core_clk_rate > old->core_clk_rate)) || 322 (!params_changed && 323 (new->core_clk_rate < old->core_clk_rate))) { 324 old->core_clk_rate = new->core_clk_rate; 325 update_clk = true; 326 } 327 } else { 328 DPU_DEBUG("crtc=%d disable\n", crtc->base.id); 329 memset(old, 0, sizeof(*old)); 330 memset(new, 0, sizeof(*new)); 331 update_bus = true; 332 update_clk = true; 333 } 334 335 trace_dpu_perf_crtc_update(crtc->base.id, new->bw_ctl, 336 new->core_clk_rate, stop_req, update_bus, update_clk); 337 338 if (update_bus) { 339 ret = _dpu_core_perf_crtc_update_bus(kms, crtc); 340 if (ret) { 341 DPU_ERROR("crtc-%d: failed to update bus bw vote\n", 342 crtc->base.id); 343 return ret; 344 } 345 } 346 347 /* 348 * Update the clock after bandwidth vote to ensure 349 * bandwidth is available before clock rate is increased. 350 */ 351 if (update_clk) { 352 clk_rate = _dpu_core_perf_get_core_clk_rate(kms); 353 354 trace_dpu_core_perf_update_clk(kms->dev, stop_req, clk_rate); 355 356 ret = _dpu_core_perf_set_core_clk_rate(kms, clk_rate); 357 if (ret) { 358 DPU_ERROR("failed to set %s clock rate %llu\n", 359 kms->perf.core_clk->clk_name, clk_rate); 360 return ret; 361 } 362 363 kms->perf.core_clk_rate = clk_rate; 364 DPU_DEBUG("update clk rate = %lld HZ\n", clk_rate); 365 } 366 return 0; 367 } 368 369 #ifdef CONFIG_DEBUG_FS 370 371 static ssize_t _dpu_core_perf_mode_write(struct file *file, 372 const char __user *user_buf, size_t count, loff_t *ppos) 373 { 374 struct dpu_core_perf *perf = file->private_data; 375 struct dpu_perf_cfg *cfg = &perf->catalog->perf; 376 u32 perf_mode = 0; 377 int ret; 378 379 ret = kstrtouint_from_user(user_buf, count, 0, &perf_mode); 380 if (ret) 381 return ret; 382 383 if (perf_mode >= DPU_PERF_MODE_MAX) 384 return -EINVAL; 385 386 if (perf_mode == DPU_PERF_MODE_FIXED) { 387 DRM_INFO("fix performance mode\n"); 388 } else if (perf_mode == DPU_PERF_MODE_MINIMUM) { 389 /* run the driver with max clk and BW vote */ 390 perf->perf_tune.min_core_clk = perf->max_core_clk_rate; 391 perf->perf_tune.min_bus_vote = 392 (u64) cfg->max_bw_high * 1000; 393 DRM_INFO("minimum performance mode\n"); 394 } else if (perf_mode == DPU_PERF_MODE_NORMAL) { 395 /* reset the perf tune params to 0 */ 396 perf->perf_tune.min_core_clk = 0; 397 perf->perf_tune.min_bus_vote = 0; 398 DRM_INFO("normal performance mode\n"); 399 } 400 perf->perf_tune.mode = perf_mode; 401 402 return count; 403 } 404 405 static ssize_t _dpu_core_perf_mode_read(struct file *file, 406 char __user *buff, size_t count, loff_t *ppos) 407 { 408 struct dpu_core_perf *perf = file->private_data; 409 int len; 410 char buf[128]; 411 412 len = scnprintf(buf, sizeof(buf), 413 "mode %d min_mdp_clk %llu min_bus_vote %llu\n", 414 perf->perf_tune.mode, 415 perf->perf_tune.min_core_clk, 416 perf->perf_tune.min_bus_vote); 417 418 return simple_read_from_buffer(buff, count, ppos, buf, len); 419 } 420 421 static const struct file_operations dpu_core_perf_mode_fops = { 422 .open = simple_open, 423 .read = _dpu_core_perf_mode_read, 424 .write = _dpu_core_perf_mode_write, 425 }; 426 427 int dpu_core_perf_debugfs_init(struct dpu_kms *dpu_kms, struct dentry *parent) 428 { 429 struct dpu_core_perf *perf = &dpu_kms->perf; 430 struct dpu_mdss_cfg *catalog = perf->catalog; 431 struct dentry *entry; 432 433 entry = debugfs_create_dir("core_perf", parent); 434 435 debugfs_create_u64("max_core_clk_rate", 0600, entry, 436 &perf->max_core_clk_rate); 437 debugfs_create_u64("core_clk_rate", 0600, entry, 438 &perf->core_clk_rate); 439 debugfs_create_u32("enable_bw_release", 0600, entry, 440 (u32 *)&perf->enable_bw_release); 441 debugfs_create_u32("threshold_low", 0600, entry, 442 (u32 *)&catalog->perf.max_bw_low); 443 debugfs_create_u32("threshold_high", 0600, entry, 444 (u32 *)&catalog->perf.max_bw_high); 445 debugfs_create_u32("min_core_ib", 0600, entry, 446 (u32 *)&catalog->perf.min_core_ib); 447 debugfs_create_u32("min_llcc_ib", 0600, entry, 448 (u32 *)&catalog->perf.min_llcc_ib); 449 debugfs_create_u32("min_dram_ib", 0600, entry, 450 (u32 *)&catalog->perf.min_dram_ib); 451 debugfs_create_file("perf_mode", 0600, entry, 452 (u32 *)perf, &dpu_core_perf_mode_fops); 453 debugfs_create_u64("fix_core_clk_rate", 0600, entry, 454 &perf->fix_core_clk_rate); 455 debugfs_create_u64("fix_core_ib_vote", 0600, entry, 456 &perf->fix_core_ib_vote); 457 debugfs_create_u64("fix_core_ab_vote", 0600, entry, 458 &perf->fix_core_ab_vote); 459 460 return 0; 461 } 462 #endif 463 464 void dpu_core_perf_destroy(struct dpu_core_perf *perf) 465 { 466 if (!perf) { 467 DPU_ERROR("invalid parameters\n"); 468 return; 469 } 470 471 perf->max_core_clk_rate = 0; 472 perf->core_clk = NULL; 473 perf->catalog = NULL; 474 perf->dev = NULL; 475 } 476 477 int dpu_core_perf_init(struct dpu_core_perf *perf, 478 struct drm_device *dev, 479 struct dpu_mdss_cfg *catalog, 480 struct dss_clk *core_clk) 481 { 482 perf->dev = dev; 483 perf->catalog = catalog; 484 perf->core_clk = core_clk; 485 486 perf->max_core_clk_rate = core_clk->max_rate; 487 if (!perf->max_core_clk_rate) { 488 DPU_DEBUG("optional max core clk rate, use default\n"); 489 perf->max_core_clk_rate = DPU_PERF_DEFAULT_MAX_CORE_CLK_RATE; 490 } 491 492 return 0; 493 } 494