1 /* 2 * Copyright (C) 2013 Red Hat 3 * Author: Rob Clark <robdclark@gmail.com> 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 as published by 7 * the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 * You should have received a copy of the GNU General Public License along with 15 * this program. If not, see <http://www.gnu.org/licenses/>. 16 */ 17 18 #include "msm_gpu.h" 19 #include "msm_gem.h" 20 #include "msm_mmu.h" 21 22 23 /* 24 * Power Management: 25 */ 26 27 #ifdef CONFIG_MSM_BUS_SCALING 28 #include <mach/board.h> 29 static void bs_init(struct msm_gpu *gpu) 30 { 31 if (gpu->bus_scale_table) { 32 gpu->bsc = msm_bus_scale_register_client(gpu->bus_scale_table); 33 DBG("bus scale client: %08x", gpu->bsc); 34 } 35 } 36 37 static void bs_fini(struct msm_gpu *gpu) 38 { 39 if (gpu->bsc) { 40 msm_bus_scale_unregister_client(gpu->bsc); 41 gpu->bsc = 0; 42 } 43 } 44 45 static void bs_set(struct msm_gpu *gpu, int idx) 46 { 47 if (gpu->bsc) { 48 DBG("set bus scaling: %d", idx); 49 msm_bus_scale_client_update_request(gpu->bsc, idx); 50 } 51 } 52 #else 53 static void bs_init(struct msm_gpu *gpu) {} 54 static void bs_fini(struct msm_gpu *gpu) {} 55 static void bs_set(struct msm_gpu *gpu, int idx) {} 56 #endif 57 58 static int enable_pwrrail(struct msm_gpu *gpu) 59 { 60 struct drm_device *dev = gpu->dev; 61 int ret = 0; 62 63 if (gpu->gpu_reg) { 64 ret = regulator_enable(gpu->gpu_reg); 65 if (ret) { 66 dev_err(dev->dev, "failed to enable 'gpu_reg': %d\n", ret); 67 return ret; 68 } 69 } 70 71 if (gpu->gpu_cx) { 72 ret = regulator_enable(gpu->gpu_cx); 73 if (ret) { 74 dev_err(dev->dev, "failed to enable 'gpu_cx': %d\n", ret); 75 return ret; 76 } 77 } 78 79 return 0; 80 } 81 82 static int disable_pwrrail(struct msm_gpu *gpu) 83 { 84 if (gpu->gpu_cx) 85 regulator_disable(gpu->gpu_cx); 86 if (gpu->gpu_reg) 87 regulator_disable(gpu->gpu_reg); 88 return 0; 89 } 90 91 static int enable_clk(struct msm_gpu *gpu) 92 { 93 struct clk *rate_clk = NULL; 94 int i; 95 96 /* NOTE: kgsl_pwrctrl_clk() ignores grp_clks[0].. */ 97 for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i > 0; i--) { 98 if (gpu->grp_clks[i]) { 99 clk_prepare(gpu->grp_clks[i]); 100 rate_clk = gpu->grp_clks[i]; 101 } 102 } 103 104 if (rate_clk && gpu->fast_rate) 105 clk_set_rate(rate_clk, gpu->fast_rate); 106 107 for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i > 0; i--) 108 if (gpu->grp_clks[i]) 109 clk_enable(gpu->grp_clks[i]); 110 111 return 0; 112 } 113 114 static int disable_clk(struct msm_gpu *gpu) 115 { 116 struct clk *rate_clk = NULL; 117 int i; 118 119 /* NOTE: kgsl_pwrctrl_clk() ignores grp_clks[0].. */ 120 for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i > 0; i--) { 121 if (gpu->grp_clks[i]) { 122 clk_disable(gpu->grp_clks[i]); 123 rate_clk = gpu->grp_clks[i]; 124 } 125 } 126 127 if (rate_clk && gpu->slow_rate) 128 clk_set_rate(rate_clk, gpu->slow_rate); 129 130 for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i > 0; i--) 131 if (gpu->grp_clks[i]) 132 clk_unprepare(gpu->grp_clks[i]); 133 134 return 0; 135 } 136 137 static int enable_axi(struct msm_gpu *gpu) 138 { 139 if (gpu->ebi1_clk) 140 clk_prepare_enable(gpu->ebi1_clk); 141 if (gpu->bus_freq) 142 bs_set(gpu, gpu->bus_freq); 143 return 0; 144 } 145 146 static int disable_axi(struct msm_gpu *gpu) 147 { 148 if (gpu->ebi1_clk) 149 clk_disable_unprepare(gpu->ebi1_clk); 150 if (gpu->bus_freq) 151 bs_set(gpu, 0); 152 return 0; 153 } 154 155 int msm_gpu_pm_resume(struct msm_gpu *gpu) 156 { 157 int ret; 158 159 DBG("%s", gpu->name); 160 161 ret = enable_pwrrail(gpu); 162 if (ret) 163 return ret; 164 165 ret = enable_clk(gpu); 166 if (ret) 167 return ret; 168 169 ret = enable_axi(gpu); 170 if (ret) 171 return ret; 172 173 return 0; 174 } 175 176 int msm_gpu_pm_suspend(struct msm_gpu *gpu) 177 { 178 int ret; 179 180 DBG("%s", gpu->name); 181 182 ret = disable_axi(gpu); 183 if (ret) 184 return ret; 185 186 ret = disable_clk(gpu); 187 if (ret) 188 return ret; 189 190 ret = disable_pwrrail(gpu); 191 if (ret) 192 return ret; 193 194 return 0; 195 } 196 197 /* 198 * Hangcheck detection for locked gpu: 199 */ 200 201 static void recover_worker(struct work_struct *work) 202 { 203 struct msm_gpu *gpu = container_of(work, struct msm_gpu, recover_work); 204 struct drm_device *dev = gpu->dev; 205 206 dev_err(dev->dev, "%s: hangcheck recover!\n", gpu->name); 207 208 mutex_lock(&dev->struct_mutex); 209 gpu->funcs->recover(gpu); 210 mutex_unlock(&dev->struct_mutex); 211 212 msm_gpu_retire(gpu); 213 } 214 215 static void hangcheck_timer_reset(struct msm_gpu *gpu) 216 { 217 DBG("%s", gpu->name); 218 mod_timer(&gpu->hangcheck_timer, 219 round_jiffies_up(jiffies + DRM_MSM_HANGCHECK_JIFFIES)); 220 } 221 222 static void hangcheck_handler(unsigned long data) 223 { 224 struct msm_gpu *gpu = (struct msm_gpu *)data; 225 struct drm_device *dev = gpu->dev; 226 struct msm_drm_private *priv = dev->dev_private; 227 uint32_t fence = gpu->funcs->last_fence(gpu); 228 229 if (fence != gpu->hangcheck_fence) { 230 /* some progress has been made.. ya! */ 231 gpu->hangcheck_fence = fence; 232 } else if (fence < gpu->submitted_fence) { 233 /* no progress and not done.. hung! */ 234 gpu->hangcheck_fence = fence; 235 dev_err(dev->dev, "%s: hangcheck detected gpu lockup!\n", 236 gpu->name); 237 dev_err(dev->dev, "%s: completed fence: %u\n", 238 gpu->name, fence); 239 dev_err(dev->dev, "%s: submitted fence: %u\n", 240 gpu->name, gpu->submitted_fence); 241 queue_work(priv->wq, &gpu->recover_work); 242 } 243 244 /* if still more pending work, reset the hangcheck timer: */ 245 if (gpu->submitted_fence > gpu->hangcheck_fence) 246 hangcheck_timer_reset(gpu); 247 248 /* workaround for missing irq: */ 249 queue_work(priv->wq, &gpu->retire_work); 250 } 251 252 /* 253 * Cmdstream submission/retirement: 254 */ 255 256 static void retire_worker(struct work_struct *work) 257 { 258 struct msm_gpu *gpu = container_of(work, struct msm_gpu, retire_work); 259 struct drm_device *dev = gpu->dev; 260 uint32_t fence = gpu->funcs->last_fence(gpu); 261 262 msm_update_fence(gpu->dev, fence); 263 264 mutex_lock(&dev->struct_mutex); 265 266 while (!list_empty(&gpu->active_list)) { 267 struct msm_gem_object *obj; 268 269 obj = list_first_entry(&gpu->active_list, 270 struct msm_gem_object, mm_list); 271 272 if ((obj->read_fence <= fence) && 273 (obj->write_fence <= fence)) { 274 /* move to inactive: */ 275 msm_gem_move_to_inactive(&obj->base); 276 msm_gem_put_iova(&obj->base, gpu->id); 277 drm_gem_object_unreference(&obj->base); 278 } else { 279 break; 280 } 281 } 282 283 mutex_unlock(&dev->struct_mutex); 284 } 285 286 /* call from irq handler to schedule work to retire bo's */ 287 void msm_gpu_retire(struct msm_gpu *gpu) 288 { 289 struct msm_drm_private *priv = gpu->dev->dev_private; 290 queue_work(priv->wq, &gpu->retire_work); 291 } 292 293 /* add bo's to gpu's ring, and kick gpu: */ 294 int msm_gpu_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit, 295 struct msm_file_private *ctx) 296 { 297 struct drm_device *dev = gpu->dev; 298 struct msm_drm_private *priv = dev->dev_private; 299 int i, ret; 300 301 mutex_lock(&dev->struct_mutex); 302 303 submit->fence = ++priv->next_fence; 304 305 gpu->submitted_fence = submit->fence; 306 307 ret = gpu->funcs->submit(gpu, submit, ctx); 308 priv->lastctx = ctx; 309 310 for (i = 0; i < submit->nr_bos; i++) { 311 struct msm_gem_object *msm_obj = submit->bos[i].obj; 312 313 /* can't happen yet.. but when we add 2d support we'll have 314 * to deal w/ cross-ring synchronization: 315 */ 316 WARN_ON(is_active(msm_obj) && (msm_obj->gpu != gpu)); 317 318 if (!is_active(msm_obj)) { 319 uint32_t iova; 320 321 /* ring takes a reference to the bo and iova: */ 322 drm_gem_object_reference(&msm_obj->base); 323 msm_gem_get_iova_locked(&msm_obj->base, 324 submit->gpu->id, &iova); 325 } 326 327 if (submit->bos[i].flags & MSM_SUBMIT_BO_READ) 328 msm_gem_move_to_active(&msm_obj->base, gpu, false, submit->fence); 329 330 if (submit->bos[i].flags & MSM_SUBMIT_BO_WRITE) 331 msm_gem_move_to_active(&msm_obj->base, gpu, true, submit->fence); 332 } 333 hangcheck_timer_reset(gpu); 334 mutex_unlock(&dev->struct_mutex); 335 336 return ret; 337 } 338 339 /* 340 * Init/Cleanup: 341 */ 342 343 static irqreturn_t irq_handler(int irq, void *data) 344 { 345 struct msm_gpu *gpu = data; 346 return gpu->funcs->irq(gpu); 347 } 348 349 static const char *clk_names[] = { 350 "src_clk", "core_clk", "iface_clk", "mem_clk", "mem_iface_clk", 351 }; 352 353 int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev, 354 struct msm_gpu *gpu, const struct msm_gpu_funcs *funcs, 355 const char *name, const char *ioname, const char *irqname, int ringsz) 356 { 357 struct iommu_domain *iommu; 358 int i, ret; 359 360 gpu->dev = drm; 361 gpu->funcs = funcs; 362 gpu->name = name; 363 364 INIT_LIST_HEAD(&gpu->active_list); 365 INIT_WORK(&gpu->retire_work, retire_worker); 366 INIT_WORK(&gpu->recover_work, recover_worker); 367 368 setup_timer(&gpu->hangcheck_timer, hangcheck_handler, 369 (unsigned long)gpu); 370 371 BUG_ON(ARRAY_SIZE(clk_names) != ARRAY_SIZE(gpu->grp_clks)); 372 373 /* Map registers: */ 374 gpu->mmio = msm_ioremap(pdev, ioname, name); 375 if (IS_ERR(gpu->mmio)) { 376 ret = PTR_ERR(gpu->mmio); 377 goto fail; 378 } 379 380 /* Get Interrupt: */ 381 gpu->irq = platform_get_irq_byname(pdev, irqname); 382 if (gpu->irq < 0) { 383 ret = gpu->irq; 384 dev_err(drm->dev, "failed to get irq: %d\n", ret); 385 goto fail; 386 } 387 388 ret = devm_request_irq(&pdev->dev, gpu->irq, irq_handler, 389 IRQF_TRIGGER_HIGH, gpu->name, gpu); 390 if (ret) { 391 dev_err(drm->dev, "failed to request IRQ%u: %d\n", gpu->irq, ret); 392 goto fail; 393 } 394 395 /* Acquire clocks: */ 396 for (i = 0; i < ARRAY_SIZE(clk_names); i++) { 397 gpu->grp_clks[i] = devm_clk_get(&pdev->dev, clk_names[i]); 398 DBG("grp_clks[%s]: %p", clk_names[i], gpu->grp_clks[i]); 399 if (IS_ERR(gpu->grp_clks[i])) 400 gpu->grp_clks[i] = NULL; 401 } 402 403 gpu->ebi1_clk = devm_clk_get(&pdev->dev, "bus_clk"); 404 DBG("ebi1_clk: %p", gpu->ebi1_clk); 405 if (IS_ERR(gpu->ebi1_clk)) 406 gpu->ebi1_clk = NULL; 407 408 /* Acquire regulators: */ 409 gpu->gpu_reg = devm_regulator_get(&pdev->dev, "vdd"); 410 DBG("gpu_reg: %p", gpu->gpu_reg); 411 if (IS_ERR(gpu->gpu_reg)) 412 gpu->gpu_reg = NULL; 413 414 gpu->gpu_cx = devm_regulator_get(&pdev->dev, "vddcx"); 415 DBG("gpu_cx: %p", gpu->gpu_cx); 416 if (IS_ERR(gpu->gpu_cx)) 417 gpu->gpu_cx = NULL; 418 419 /* Setup IOMMU.. eventually we will (I think) do this once per context 420 * and have separate page tables per context. For now, to keep things 421 * simple and to get something working, just use a single address space: 422 */ 423 iommu = iommu_domain_alloc(&platform_bus_type); 424 if (iommu) { 425 dev_info(drm->dev, "%s: using IOMMU\n", name); 426 gpu->mmu = msm_iommu_new(drm, iommu); 427 } else { 428 dev_info(drm->dev, "%s: no IOMMU, fallback to VRAM carveout!\n", name); 429 } 430 gpu->id = msm_register_mmu(drm, gpu->mmu); 431 432 /* Create ringbuffer: */ 433 gpu->rb = msm_ringbuffer_new(gpu, ringsz); 434 if (IS_ERR(gpu->rb)) { 435 ret = PTR_ERR(gpu->rb); 436 gpu->rb = NULL; 437 dev_err(drm->dev, "could not create ringbuffer: %d\n", ret); 438 goto fail; 439 } 440 441 ret = msm_gem_get_iova_locked(gpu->rb->bo, gpu->id, &gpu->rb_iova); 442 if (ret) { 443 gpu->rb_iova = 0; 444 dev_err(drm->dev, "could not map ringbuffer: %d\n", ret); 445 goto fail; 446 } 447 448 bs_init(gpu); 449 450 return 0; 451 452 fail: 453 return ret; 454 } 455 456 void msm_gpu_cleanup(struct msm_gpu *gpu) 457 { 458 DBG("%s", gpu->name); 459 460 WARN_ON(!list_empty(&gpu->active_list)); 461 462 bs_fini(gpu); 463 464 if (gpu->rb) { 465 if (gpu->rb_iova) 466 msm_gem_put_iova(gpu->rb->bo, gpu->id); 467 msm_ringbuffer_destroy(gpu->rb); 468 } 469 470 if (gpu->mmu) 471 gpu->mmu->funcs->destroy(gpu->mmu); 472 } 473