1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */ 3 4 #include <linux/regulator/consumer.h> 5 #include <linux/reset.h> 6 #include <linux/clk.h> 7 #include <linux/dma-mapping.h> 8 #include <linux/platform_device.h> 9 10 #include "lima_device.h" 11 #include "lima_gp.h" 12 #include "lima_pp.h" 13 #include "lima_mmu.h" 14 #include "lima_pmu.h" 15 #include "lima_l2_cache.h" 16 #include "lima_dlbu.h" 17 #include "lima_bcast.h" 18 #include "lima_vm.h" 19 20 struct lima_ip_desc { 21 char *name; 22 char *irq_name; 23 bool must_have[lima_gpu_num]; 24 int offset[lima_gpu_num]; 25 26 int (*init)(struct lima_ip *ip); 27 void (*fini)(struct lima_ip *ip); 28 int (*resume)(struct lima_ip *ip); 29 void (*suspend)(struct lima_ip *ip); 30 }; 31 32 #define LIMA_IP_DESC(ipname, mst0, mst1, off0, off1, func, irq) \ 33 [lima_ip_##ipname] = { \ 34 .name = #ipname, \ 35 .irq_name = irq, \ 36 .must_have = { \ 37 [lima_gpu_mali400] = mst0, \ 38 [lima_gpu_mali450] = mst1, \ 39 }, \ 40 .offset = { \ 41 [lima_gpu_mali400] = off0, \ 42 [lima_gpu_mali450] = off1, \ 43 }, \ 44 .init = lima_##func##_init, \ 45 .fini = lima_##func##_fini, \ 46 .resume = lima_##func##_resume, \ 47 .suspend = lima_##func##_suspend, \ 48 } 49 50 static struct lima_ip_desc lima_ip_desc[lima_ip_num] = { 51 LIMA_IP_DESC(pmu, false, false, 0x02000, 0x02000, pmu, "pmu"), 52 LIMA_IP_DESC(l2_cache0, true, true, 0x01000, 0x10000, l2_cache, NULL), 53 LIMA_IP_DESC(l2_cache1, false, true, -1, 0x01000, l2_cache, NULL), 54 LIMA_IP_DESC(l2_cache2, false, false, -1, 0x11000, l2_cache, NULL), 55 LIMA_IP_DESC(gp, true, true, 0x00000, 0x00000, gp, "gp"), 56 LIMA_IP_DESC(pp0, true, true, 0x08000, 0x08000, pp, "pp0"), 57 LIMA_IP_DESC(pp1, false, false, 0x0A000, 0x0A000, pp, "pp1"), 58 LIMA_IP_DESC(pp2, false, false, 0x0C000, 0x0C000, pp, "pp2"), 59 LIMA_IP_DESC(pp3, false, false, 0x0E000, 0x0E000, pp, "pp3"), 60 LIMA_IP_DESC(pp4, false, false, -1, 0x28000, pp, "pp4"), 61 LIMA_IP_DESC(pp5, false, false, -1, 0x2A000, pp, "pp5"), 62 LIMA_IP_DESC(pp6, false, false, -1, 0x2C000, pp, "pp6"), 63 LIMA_IP_DESC(pp7, false, false, -1, 0x2E000, pp, "pp7"), 64 LIMA_IP_DESC(gpmmu, true, true, 0x03000, 0x03000, mmu, "gpmmu"), 65 LIMA_IP_DESC(ppmmu0, true, true, 0x04000, 0x04000, mmu, "ppmmu0"), 66 LIMA_IP_DESC(ppmmu1, false, false, 0x05000, 0x05000, mmu, "ppmmu1"), 67 LIMA_IP_DESC(ppmmu2, false, false, 0x06000, 0x06000, mmu, "ppmmu2"), 68 LIMA_IP_DESC(ppmmu3, false, false, 0x07000, 0x07000, mmu, "ppmmu3"), 69 LIMA_IP_DESC(ppmmu4, false, false, -1, 0x1C000, mmu, "ppmmu4"), 70 LIMA_IP_DESC(ppmmu5, false, false, -1, 0x1D000, mmu, "ppmmu5"), 71 LIMA_IP_DESC(ppmmu6, false, false, -1, 0x1E000, mmu, "ppmmu6"), 72 LIMA_IP_DESC(ppmmu7, false, false, -1, 0x1F000, mmu, "ppmmu7"), 73 LIMA_IP_DESC(dlbu, false, true, -1, 0x14000, dlbu, NULL), 74 LIMA_IP_DESC(bcast, false, true, -1, 0x13000, bcast, NULL), 75 LIMA_IP_DESC(pp_bcast, false, true, -1, 0x16000, pp_bcast, "pp"), 76 LIMA_IP_DESC(ppmmu_bcast, false, true, -1, 0x15000, mmu, NULL), 77 }; 78 79 const char *lima_ip_name(struct lima_ip *ip) 80 { 81 return lima_ip_desc[ip->id].name; 82 } 83 84 static int lima_clk_enable(struct lima_device *dev) 85 { 86 int err; 87 88 err = clk_prepare_enable(dev->clk_bus); 89 if (err) 90 return err; 91 92 err = clk_prepare_enable(dev->clk_gpu); 93 if (err) 94 goto error_out0; 95 96 if (dev->reset) { 97 err = reset_control_deassert(dev->reset); 98 if (err) { 99 dev_err(dev->dev, 100 "reset controller deassert failed %d\n", err); 101 goto error_out1; 102 } 103 } 104 105 return 0; 106 107 error_out1: 108 clk_disable_unprepare(dev->clk_gpu); 109 error_out0: 110 clk_disable_unprepare(dev->clk_bus); 111 return err; 112 } 113 114 static void lima_clk_disable(struct lima_device *dev) 115 { 116 if (dev->reset) 117 reset_control_assert(dev->reset); 118 clk_disable_unprepare(dev->clk_gpu); 119 clk_disable_unprepare(dev->clk_bus); 120 } 121 122 static int lima_clk_init(struct lima_device *dev) 123 { 124 int err; 125 126 dev->clk_bus = devm_clk_get(dev->dev, "bus"); 127 if (IS_ERR(dev->clk_bus)) { 128 err = PTR_ERR(dev->clk_bus); 129 if (err != -EPROBE_DEFER) 130 dev_err(dev->dev, "get bus clk failed %d\n", err); 131 dev->clk_bus = NULL; 132 return err; 133 } 134 135 dev->clk_gpu = devm_clk_get(dev->dev, "core"); 136 if (IS_ERR(dev->clk_gpu)) { 137 err = PTR_ERR(dev->clk_gpu); 138 if (err != -EPROBE_DEFER) 139 dev_err(dev->dev, "get core clk failed %d\n", err); 140 dev->clk_gpu = NULL; 141 return err; 142 } 143 144 dev->reset = devm_reset_control_array_get_optional_shared(dev->dev); 145 if (IS_ERR(dev->reset)) { 146 err = PTR_ERR(dev->reset); 147 if (err != -EPROBE_DEFER) 148 dev_err(dev->dev, "get reset controller failed %d\n", 149 err); 150 dev->reset = NULL; 151 return err; 152 } 153 154 return lima_clk_enable(dev); 155 } 156 157 static void lima_clk_fini(struct lima_device *dev) 158 { 159 lima_clk_disable(dev); 160 } 161 162 static int lima_regulator_enable(struct lima_device *dev) 163 { 164 int ret; 165 166 if (!dev->regulator) 167 return 0; 168 169 ret = regulator_enable(dev->regulator); 170 if (ret < 0) { 171 dev_err(dev->dev, "failed to enable regulator: %d\n", ret); 172 return ret; 173 } 174 175 return 0; 176 } 177 178 static void lima_regulator_disable(struct lima_device *dev) 179 { 180 if (dev->regulator) 181 regulator_disable(dev->regulator); 182 } 183 184 static int lima_regulator_init(struct lima_device *dev) 185 { 186 int ret; 187 188 dev->regulator = devm_regulator_get_optional(dev->dev, "mali"); 189 if (IS_ERR(dev->regulator)) { 190 ret = PTR_ERR(dev->regulator); 191 dev->regulator = NULL; 192 if (ret == -ENODEV) 193 return 0; 194 if (ret != -EPROBE_DEFER) 195 dev_err(dev->dev, "failed to get regulator: %d\n", ret); 196 return ret; 197 } 198 199 return lima_regulator_enable(dev); 200 } 201 202 static void lima_regulator_fini(struct lima_device *dev) 203 { 204 lima_regulator_disable(dev); 205 } 206 207 static int lima_init_ip(struct lima_device *dev, int index) 208 { 209 struct platform_device *pdev = to_platform_device(dev->dev); 210 struct lima_ip_desc *desc = lima_ip_desc + index; 211 struct lima_ip *ip = dev->ip + index; 212 const char *irq_name = desc->irq_name; 213 int offset = desc->offset[dev->id]; 214 bool must = desc->must_have[dev->id]; 215 int err; 216 217 if (offset < 0) 218 return 0; 219 220 ip->dev = dev; 221 ip->id = index; 222 ip->iomem = dev->iomem + offset; 223 if (irq_name) { 224 err = must ? platform_get_irq_byname(pdev, irq_name) : 225 platform_get_irq_byname_optional(pdev, irq_name); 226 if (err < 0) 227 goto out; 228 ip->irq = err; 229 } 230 231 err = desc->init(ip); 232 if (!err) { 233 ip->present = true; 234 return 0; 235 } 236 237 out: 238 return must ? err : 0; 239 } 240 241 static void lima_fini_ip(struct lima_device *ldev, int index) 242 { 243 struct lima_ip_desc *desc = lima_ip_desc + index; 244 struct lima_ip *ip = ldev->ip + index; 245 246 if (ip->present) 247 desc->fini(ip); 248 } 249 250 static int lima_resume_ip(struct lima_device *ldev, int index) 251 { 252 struct lima_ip_desc *desc = lima_ip_desc + index; 253 struct lima_ip *ip = ldev->ip + index; 254 int ret = 0; 255 256 if (ip->present) 257 ret = desc->resume(ip); 258 259 return ret; 260 } 261 262 static void lima_suspend_ip(struct lima_device *ldev, int index) 263 { 264 struct lima_ip_desc *desc = lima_ip_desc + index; 265 struct lima_ip *ip = ldev->ip + index; 266 267 if (ip->present) 268 desc->suspend(ip); 269 } 270 271 static int lima_init_gp_pipe(struct lima_device *dev) 272 { 273 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp; 274 int err; 275 276 pipe->ldev = dev; 277 278 err = lima_sched_pipe_init(pipe, "gp"); 279 if (err) 280 return err; 281 282 pipe->l2_cache[pipe->num_l2_cache++] = dev->ip + lima_ip_l2_cache0; 283 pipe->mmu[pipe->num_mmu++] = dev->ip + lima_ip_gpmmu; 284 pipe->processor[pipe->num_processor++] = dev->ip + lima_ip_gp; 285 286 err = lima_gp_pipe_init(dev); 287 if (err) { 288 lima_sched_pipe_fini(pipe); 289 return err; 290 } 291 292 return 0; 293 } 294 295 static void lima_fini_gp_pipe(struct lima_device *dev) 296 { 297 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp; 298 299 lima_gp_pipe_fini(dev); 300 lima_sched_pipe_fini(pipe); 301 } 302 303 static int lima_init_pp_pipe(struct lima_device *dev) 304 { 305 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp; 306 int err, i; 307 308 pipe->ldev = dev; 309 310 err = lima_sched_pipe_init(pipe, "pp"); 311 if (err) 312 return err; 313 314 for (i = 0; i < LIMA_SCHED_PIPE_MAX_PROCESSOR; i++) { 315 struct lima_ip *pp = dev->ip + lima_ip_pp0 + i; 316 struct lima_ip *ppmmu = dev->ip + lima_ip_ppmmu0 + i; 317 struct lima_ip *l2_cache; 318 319 if (dev->id == lima_gpu_mali400) 320 l2_cache = dev->ip + lima_ip_l2_cache0; 321 else 322 l2_cache = dev->ip + lima_ip_l2_cache1 + (i >> 2); 323 324 if (pp->present && ppmmu->present && l2_cache->present) { 325 pipe->mmu[pipe->num_mmu++] = ppmmu; 326 pipe->processor[pipe->num_processor++] = pp; 327 if (!pipe->l2_cache[i >> 2]) 328 pipe->l2_cache[pipe->num_l2_cache++] = l2_cache; 329 } 330 } 331 332 if (dev->ip[lima_ip_bcast].present) { 333 pipe->bcast_processor = dev->ip + lima_ip_pp_bcast; 334 pipe->bcast_mmu = dev->ip + lima_ip_ppmmu_bcast; 335 } 336 337 err = lima_pp_pipe_init(dev); 338 if (err) { 339 lima_sched_pipe_fini(pipe); 340 return err; 341 } 342 343 return 0; 344 } 345 346 static void lima_fini_pp_pipe(struct lima_device *dev) 347 { 348 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp; 349 350 lima_pp_pipe_fini(dev); 351 lima_sched_pipe_fini(pipe); 352 } 353 354 int lima_device_init(struct lima_device *ldev) 355 { 356 struct platform_device *pdev = to_platform_device(ldev->dev); 357 int err, i; 358 359 dma_set_coherent_mask(ldev->dev, DMA_BIT_MASK(32)); 360 361 err = lima_clk_init(ldev); 362 if (err) 363 return err; 364 365 err = lima_regulator_init(ldev); 366 if (err) 367 goto err_out0; 368 369 ldev->empty_vm = lima_vm_create(ldev); 370 if (!ldev->empty_vm) { 371 err = -ENOMEM; 372 goto err_out1; 373 } 374 375 ldev->va_start = 0; 376 if (ldev->id == lima_gpu_mali450) { 377 ldev->va_end = LIMA_VA_RESERVE_START; 378 ldev->dlbu_cpu = dma_alloc_wc( 379 ldev->dev, LIMA_PAGE_SIZE, 380 &ldev->dlbu_dma, GFP_KERNEL | __GFP_NOWARN); 381 if (!ldev->dlbu_cpu) { 382 err = -ENOMEM; 383 goto err_out2; 384 } 385 } else 386 ldev->va_end = LIMA_VA_RESERVE_END; 387 388 ldev->iomem = devm_platform_ioremap_resource(pdev, 0); 389 if (IS_ERR(ldev->iomem)) { 390 dev_err(ldev->dev, "fail to ioremap iomem\n"); 391 err = PTR_ERR(ldev->iomem); 392 goto err_out3; 393 } 394 395 for (i = 0; i < lima_ip_num; i++) { 396 err = lima_init_ip(ldev, i); 397 if (err) 398 goto err_out4; 399 } 400 401 err = lima_init_gp_pipe(ldev); 402 if (err) 403 goto err_out4; 404 405 err = lima_init_pp_pipe(ldev); 406 if (err) 407 goto err_out5; 408 409 ldev->dump.magic = LIMA_DUMP_MAGIC; 410 ldev->dump.version_major = LIMA_DUMP_MAJOR; 411 ldev->dump.version_minor = LIMA_DUMP_MINOR; 412 INIT_LIST_HEAD(&ldev->error_task_list); 413 mutex_init(&ldev->error_task_list_lock); 414 415 dev_info(ldev->dev, "bus rate = %lu\n", clk_get_rate(ldev->clk_bus)); 416 dev_info(ldev->dev, "mod rate = %lu", clk_get_rate(ldev->clk_gpu)); 417 418 return 0; 419 420 err_out5: 421 lima_fini_gp_pipe(ldev); 422 err_out4: 423 while (--i >= 0) 424 lima_fini_ip(ldev, i); 425 err_out3: 426 if (ldev->dlbu_cpu) 427 dma_free_wc(ldev->dev, LIMA_PAGE_SIZE, 428 ldev->dlbu_cpu, ldev->dlbu_dma); 429 err_out2: 430 lima_vm_put(ldev->empty_vm); 431 err_out1: 432 lima_regulator_fini(ldev); 433 err_out0: 434 lima_clk_fini(ldev); 435 return err; 436 } 437 438 void lima_device_fini(struct lima_device *ldev) 439 { 440 int i; 441 struct lima_sched_error_task *et, *tmp; 442 443 list_for_each_entry_safe(et, tmp, &ldev->error_task_list, list) { 444 list_del(&et->list); 445 kvfree(et); 446 } 447 mutex_destroy(&ldev->error_task_list_lock); 448 449 lima_fini_pp_pipe(ldev); 450 lima_fini_gp_pipe(ldev); 451 452 for (i = lima_ip_num - 1; i >= 0; i--) 453 lima_fini_ip(ldev, i); 454 455 if (ldev->dlbu_cpu) 456 dma_free_wc(ldev->dev, LIMA_PAGE_SIZE, 457 ldev->dlbu_cpu, ldev->dlbu_dma); 458 459 lima_vm_put(ldev->empty_vm); 460 461 lima_regulator_fini(ldev); 462 463 lima_clk_fini(ldev); 464 } 465 466 int lima_device_resume(struct device *dev) 467 { 468 struct lima_device *ldev = dev_get_drvdata(dev); 469 int i, err; 470 471 err = lima_clk_enable(ldev); 472 if (err) { 473 dev_err(dev, "resume clk fail %d\n", err); 474 return err; 475 } 476 477 err = lima_regulator_enable(ldev); 478 if (err) { 479 dev_err(dev, "resume regulator fail %d\n", err); 480 goto err_out0; 481 } 482 483 for (i = 0; i < lima_ip_num; i++) { 484 err = lima_resume_ip(ldev, i); 485 if (err) { 486 dev_err(dev, "resume ip %d fail\n", i); 487 goto err_out1; 488 } 489 } 490 491 err = lima_devfreq_resume(&ldev->devfreq); 492 if (err) { 493 dev_err(dev, "devfreq resume fail\n"); 494 goto err_out1; 495 } 496 497 return 0; 498 499 err_out1: 500 while (--i >= 0) 501 lima_suspend_ip(ldev, i); 502 lima_regulator_disable(ldev); 503 err_out0: 504 lima_clk_disable(ldev); 505 return err; 506 } 507 508 int lima_device_suspend(struct device *dev) 509 { 510 struct lima_device *ldev = dev_get_drvdata(dev); 511 int i, err; 512 513 /* check any task running */ 514 for (i = 0; i < lima_pipe_num; i++) { 515 if (atomic_read(&ldev->pipe[i].base.hw_rq_count)) 516 return -EBUSY; 517 } 518 519 err = lima_devfreq_suspend(&ldev->devfreq); 520 if (err) { 521 dev_err(dev, "devfreq suspend fail\n"); 522 return err; 523 } 524 525 for (i = lima_ip_num - 1; i >= 0; i--) 526 lima_suspend_ip(ldev, i); 527 528 lima_regulator_disable(ldev); 529 530 lima_clk_disable(ldev); 531 532 return 0; 533 } 534