1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */ 3 4 #include <linux/regulator/consumer.h> 5 #include <linux/reset.h> 6 #include <linux/clk.h> 7 #include <linux/slab.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/platform_device.h> 10 11 #include "lima_device.h" 12 #include "lima_gp.h" 13 #include "lima_pp.h" 14 #include "lima_mmu.h" 15 #include "lima_pmu.h" 16 #include "lima_l2_cache.h" 17 #include "lima_dlbu.h" 18 #include "lima_bcast.h" 19 #include "lima_vm.h" 20 21 struct lima_ip_desc { 22 char *name; 23 char *irq_name; 24 bool must_have[lima_gpu_num]; 25 int offset[lima_gpu_num]; 26 27 int (*init)(struct lima_ip *ip); 28 void (*fini)(struct lima_ip *ip); 29 int (*resume)(struct lima_ip *ip); 30 void (*suspend)(struct lima_ip *ip); 31 }; 32 33 #define LIMA_IP_DESC(ipname, mst0, mst1, off0, off1, func, irq) \ 34 [lima_ip_##ipname] = { \ 35 .name = #ipname, \ 36 .irq_name = irq, \ 37 .must_have = { \ 38 [lima_gpu_mali400] = mst0, \ 39 [lima_gpu_mali450] = mst1, \ 40 }, \ 41 .offset = { \ 42 [lima_gpu_mali400] = off0, \ 43 [lima_gpu_mali450] = off1, \ 44 }, \ 45 .init = lima_##func##_init, \ 46 .fini = lima_##func##_fini, \ 47 .resume = lima_##func##_resume, \ 48 .suspend = lima_##func##_suspend, \ 49 } 50 51 static struct lima_ip_desc lima_ip_desc[lima_ip_num] = { 52 LIMA_IP_DESC(pmu, false, false, 0x02000, 0x02000, pmu, "pmu"), 53 LIMA_IP_DESC(l2_cache0, true, true, 0x01000, 0x10000, l2_cache, NULL), 54 LIMA_IP_DESC(l2_cache1, false, true, -1, 0x01000, l2_cache, NULL), 55 LIMA_IP_DESC(l2_cache2, false, false, -1, 0x11000, l2_cache, NULL), 56 LIMA_IP_DESC(gp, true, true, 0x00000, 0x00000, gp, "gp"), 57 LIMA_IP_DESC(pp0, true, true, 0x08000, 0x08000, pp, "pp0"), 58 LIMA_IP_DESC(pp1, false, false, 0x0A000, 0x0A000, pp, "pp1"), 59 LIMA_IP_DESC(pp2, false, false, 0x0C000, 0x0C000, pp, "pp2"), 60 LIMA_IP_DESC(pp3, false, false, 0x0E000, 0x0E000, pp, "pp3"), 61 LIMA_IP_DESC(pp4, false, false, -1, 0x28000, pp, "pp4"), 62 LIMA_IP_DESC(pp5, false, false, -1, 0x2A000, pp, "pp5"), 63 LIMA_IP_DESC(pp6, false, false, -1, 0x2C000, pp, "pp6"), 64 LIMA_IP_DESC(pp7, false, false, -1, 0x2E000, pp, "pp7"), 65 LIMA_IP_DESC(gpmmu, true, true, 0x03000, 0x03000, mmu, "gpmmu"), 66 LIMA_IP_DESC(ppmmu0, true, true, 0x04000, 0x04000, mmu, "ppmmu0"), 67 LIMA_IP_DESC(ppmmu1, false, false, 0x05000, 0x05000, mmu, "ppmmu1"), 68 LIMA_IP_DESC(ppmmu2, false, false, 0x06000, 0x06000, mmu, "ppmmu2"), 69 LIMA_IP_DESC(ppmmu3, false, false, 0x07000, 0x07000, mmu, "ppmmu3"), 70 LIMA_IP_DESC(ppmmu4, false, false, -1, 0x1C000, mmu, "ppmmu4"), 71 LIMA_IP_DESC(ppmmu5, false, false, -1, 0x1D000, mmu, "ppmmu5"), 72 LIMA_IP_DESC(ppmmu6, false, false, -1, 0x1E000, mmu, "ppmmu6"), 73 LIMA_IP_DESC(ppmmu7, false, false, -1, 0x1F000, mmu, "ppmmu7"), 74 LIMA_IP_DESC(dlbu, false, true, -1, 0x14000, dlbu, NULL), 75 LIMA_IP_DESC(bcast, false, true, -1, 0x13000, bcast, NULL), 76 LIMA_IP_DESC(pp_bcast, false, true, -1, 0x16000, pp_bcast, "pp"), 77 LIMA_IP_DESC(ppmmu_bcast, false, true, -1, 0x15000, mmu, NULL), 78 }; 79 80 const char *lima_ip_name(struct lima_ip *ip) 81 { 82 return lima_ip_desc[ip->id].name; 83 } 84 85 static int lima_clk_enable(struct lima_device *dev) 86 { 87 int err; 88 89 err = clk_prepare_enable(dev->clk_bus); 90 if (err) 91 return err; 92 93 err = clk_prepare_enable(dev->clk_gpu); 94 if (err) 95 goto error_out0; 96 97 if (dev->reset) { 98 err = reset_control_deassert(dev->reset); 99 if (err) { 100 dev_err(dev->dev, 101 "reset controller deassert failed %d\n", err); 102 goto error_out1; 103 } 104 } 105 106 return 0; 107 108 error_out1: 109 clk_disable_unprepare(dev->clk_gpu); 110 error_out0: 111 clk_disable_unprepare(dev->clk_bus); 112 return err; 113 } 114 115 static void lima_clk_disable(struct lima_device *dev) 116 { 117 if (dev->reset) 118 reset_control_assert(dev->reset); 119 clk_disable_unprepare(dev->clk_gpu); 120 clk_disable_unprepare(dev->clk_bus); 121 } 122 123 static int lima_clk_init(struct lima_device *dev) 124 { 125 int err; 126 127 dev->clk_bus = devm_clk_get(dev->dev, "bus"); 128 if (IS_ERR(dev->clk_bus)) { 129 err = PTR_ERR(dev->clk_bus); 130 if (err != -EPROBE_DEFER) 131 dev_err(dev->dev, "get bus clk failed %d\n", err); 132 dev->clk_bus = NULL; 133 return err; 134 } 135 136 dev->clk_gpu = devm_clk_get(dev->dev, "core"); 137 if (IS_ERR(dev->clk_gpu)) { 138 err = PTR_ERR(dev->clk_gpu); 139 if (err != -EPROBE_DEFER) 140 dev_err(dev->dev, "get core clk failed %d\n", err); 141 dev->clk_gpu = NULL; 142 return err; 143 } 144 145 dev->reset = devm_reset_control_array_get_optional_shared(dev->dev); 146 if (IS_ERR(dev->reset)) { 147 err = PTR_ERR(dev->reset); 148 if (err != -EPROBE_DEFER) 149 dev_err(dev->dev, "get reset controller failed %d\n", 150 err); 151 dev->reset = NULL; 152 return err; 153 } 154 155 return lima_clk_enable(dev); 156 } 157 158 static void lima_clk_fini(struct lima_device *dev) 159 { 160 lima_clk_disable(dev); 161 } 162 163 static int lima_regulator_enable(struct lima_device *dev) 164 { 165 int ret; 166 167 if (!dev->regulator) 168 return 0; 169 170 ret = regulator_enable(dev->regulator); 171 if (ret < 0) { 172 dev_err(dev->dev, "failed to enable regulator: %d\n", ret); 173 return ret; 174 } 175 176 return 0; 177 } 178 179 static void lima_regulator_disable(struct lima_device *dev) 180 { 181 if (dev->regulator) 182 regulator_disable(dev->regulator); 183 } 184 185 static int lima_regulator_init(struct lima_device *dev) 186 { 187 int ret; 188 189 dev->regulator = devm_regulator_get_optional(dev->dev, "mali"); 190 if (IS_ERR(dev->regulator)) { 191 ret = PTR_ERR(dev->regulator); 192 dev->regulator = NULL; 193 if (ret == -ENODEV) 194 return 0; 195 if (ret != -EPROBE_DEFER) 196 dev_err(dev->dev, "failed to get regulator: %d\n", ret); 197 return ret; 198 } 199 200 return lima_regulator_enable(dev); 201 } 202 203 static void lima_regulator_fini(struct lima_device *dev) 204 { 205 lima_regulator_disable(dev); 206 } 207 208 static int lima_init_ip(struct lima_device *dev, int index) 209 { 210 struct platform_device *pdev = to_platform_device(dev->dev); 211 struct lima_ip_desc *desc = lima_ip_desc + index; 212 struct lima_ip *ip = dev->ip + index; 213 const char *irq_name = desc->irq_name; 214 int offset = desc->offset[dev->id]; 215 bool must = desc->must_have[dev->id]; 216 int err; 217 218 if (offset < 0) 219 return 0; 220 221 ip->dev = dev; 222 ip->id = index; 223 ip->iomem = dev->iomem + offset; 224 if (irq_name) { 225 err = must ? platform_get_irq_byname(pdev, irq_name) : 226 platform_get_irq_byname_optional(pdev, irq_name); 227 if (err < 0) 228 goto out; 229 ip->irq = err; 230 } 231 232 err = desc->init(ip); 233 if (!err) { 234 ip->present = true; 235 return 0; 236 } 237 238 out: 239 return must ? err : 0; 240 } 241 242 static void lima_fini_ip(struct lima_device *ldev, int index) 243 { 244 struct lima_ip_desc *desc = lima_ip_desc + index; 245 struct lima_ip *ip = ldev->ip + index; 246 247 if (ip->present) 248 desc->fini(ip); 249 } 250 251 static int lima_resume_ip(struct lima_device *ldev, int index) 252 { 253 struct lima_ip_desc *desc = lima_ip_desc + index; 254 struct lima_ip *ip = ldev->ip + index; 255 int ret = 0; 256 257 if (ip->present) 258 ret = desc->resume(ip); 259 260 return ret; 261 } 262 263 static void lima_suspend_ip(struct lima_device *ldev, int index) 264 { 265 struct lima_ip_desc *desc = lima_ip_desc + index; 266 struct lima_ip *ip = ldev->ip + index; 267 268 if (ip->present) 269 desc->suspend(ip); 270 } 271 272 static int lima_init_gp_pipe(struct lima_device *dev) 273 { 274 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp; 275 int err; 276 277 pipe->ldev = dev; 278 279 err = lima_sched_pipe_init(pipe, "gp"); 280 if (err) 281 return err; 282 283 pipe->l2_cache[pipe->num_l2_cache++] = dev->ip + lima_ip_l2_cache0; 284 pipe->mmu[pipe->num_mmu++] = dev->ip + lima_ip_gpmmu; 285 pipe->processor[pipe->num_processor++] = dev->ip + lima_ip_gp; 286 287 err = lima_gp_pipe_init(dev); 288 if (err) { 289 lima_sched_pipe_fini(pipe); 290 return err; 291 } 292 293 return 0; 294 } 295 296 static void lima_fini_gp_pipe(struct lima_device *dev) 297 { 298 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp; 299 300 lima_gp_pipe_fini(dev); 301 lima_sched_pipe_fini(pipe); 302 } 303 304 static int lima_init_pp_pipe(struct lima_device *dev) 305 { 306 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp; 307 int err, i; 308 309 pipe->ldev = dev; 310 311 err = lima_sched_pipe_init(pipe, "pp"); 312 if (err) 313 return err; 314 315 for (i = 0; i < LIMA_SCHED_PIPE_MAX_PROCESSOR; i++) { 316 struct lima_ip *pp = dev->ip + lima_ip_pp0 + i; 317 struct lima_ip *ppmmu = dev->ip + lima_ip_ppmmu0 + i; 318 struct lima_ip *l2_cache; 319 320 if (dev->id == lima_gpu_mali400) 321 l2_cache = dev->ip + lima_ip_l2_cache0; 322 else 323 l2_cache = dev->ip + lima_ip_l2_cache1 + (i >> 2); 324 325 if (pp->present && ppmmu->present && l2_cache->present) { 326 pipe->mmu[pipe->num_mmu++] = ppmmu; 327 pipe->processor[pipe->num_processor++] = pp; 328 if (!pipe->l2_cache[i >> 2]) 329 pipe->l2_cache[pipe->num_l2_cache++] = l2_cache; 330 } 331 } 332 333 if (dev->ip[lima_ip_bcast].present) { 334 pipe->bcast_processor = dev->ip + lima_ip_pp_bcast; 335 pipe->bcast_mmu = dev->ip + lima_ip_ppmmu_bcast; 336 } 337 338 err = lima_pp_pipe_init(dev); 339 if (err) { 340 lima_sched_pipe_fini(pipe); 341 return err; 342 } 343 344 return 0; 345 } 346 347 static void lima_fini_pp_pipe(struct lima_device *dev) 348 { 349 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp; 350 351 lima_pp_pipe_fini(dev); 352 lima_sched_pipe_fini(pipe); 353 } 354 355 int lima_device_init(struct lima_device *ldev) 356 { 357 struct platform_device *pdev = to_platform_device(ldev->dev); 358 int err, i; 359 360 dma_set_coherent_mask(ldev->dev, DMA_BIT_MASK(32)); 361 dma_set_max_seg_size(ldev->dev, UINT_MAX); 362 363 err = lima_clk_init(ldev); 364 if (err) 365 return err; 366 367 err = lima_regulator_init(ldev); 368 if (err) 369 goto err_out0; 370 371 ldev->empty_vm = lima_vm_create(ldev); 372 if (!ldev->empty_vm) { 373 err = -ENOMEM; 374 goto err_out1; 375 } 376 377 ldev->va_start = 0; 378 if (ldev->id == lima_gpu_mali450) { 379 ldev->va_end = LIMA_VA_RESERVE_START; 380 ldev->dlbu_cpu = dma_alloc_wc( 381 ldev->dev, LIMA_PAGE_SIZE, 382 &ldev->dlbu_dma, GFP_KERNEL | __GFP_NOWARN); 383 if (!ldev->dlbu_cpu) { 384 err = -ENOMEM; 385 goto err_out2; 386 } 387 } else 388 ldev->va_end = LIMA_VA_RESERVE_END; 389 390 ldev->iomem = devm_platform_ioremap_resource(pdev, 0); 391 if (IS_ERR(ldev->iomem)) { 392 dev_err(ldev->dev, "fail to ioremap iomem\n"); 393 err = PTR_ERR(ldev->iomem); 394 goto err_out3; 395 } 396 397 for (i = 0; i < lima_ip_num; i++) { 398 err = lima_init_ip(ldev, i); 399 if (err) 400 goto err_out4; 401 } 402 403 err = lima_init_gp_pipe(ldev); 404 if (err) 405 goto err_out4; 406 407 err = lima_init_pp_pipe(ldev); 408 if (err) 409 goto err_out5; 410 411 ldev->dump.magic = LIMA_DUMP_MAGIC; 412 ldev->dump.version_major = LIMA_DUMP_MAJOR; 413 ldev->dump.version_minor = LIMA_DUMP_MINOR; 414 INIT_LIST_HEAD(&ldev->error_task_list); 415 mutex_init(&ldev->error_task_list_lock); 416 417 dev_info(ldev->dev, "bus rate = %lu\n", clk_get_rate(ldev->clk_bus)); 418 dev_info(ldev->dev, "mod rate = %lu", clk_get_rate(ldev->clk_gpu)); 419 420 return 0; 421 422 err_out5: 423 lima_fini_gp_pipe(ldev); 424 err_out4: 425 while (--i >= 0) 426 lima_fini_ip(ldev, i); 427 err_out3: 428 if (ldev->dlbu_cpu) 429 dma_free_wc(ldev->dev, LIMA_PAGE_SIZE, 430 ldev->dlbu_cpu, ldev->dlbu_dma); 431 err_out2: 432 lima_vm_put(ldev->empty_vm); 433 err_out1: 434 lima_regulator_fini(ldev); 435 err_out0: 436 lima_clk_fini(ldev); 437 return err; 438 } 439 440 void lima_device_fini(struct lima_device *ldev) 441 { 442 int i; 443 struct lima_sched_error_task *et, *tmp; 444 445 list_for_each_entry_safe(et, tmp, &ldev->error_task_list, list) { 446 list_del(&et->list); 447 kvfree(et); 448 } 449 mutex_destroy(&ldev->error_task_list_lock); 450 451 lima_fini_pp_pipe(ldev); 452 lima_fini_gp_pipe(ldev); 453 454 for (i = lima_ip_num - 1; i >= 0; i--) 455 lima_fini_ip(ldev, i); 456 457 if (ldev->dlbu_cpu) 458 dma_free_wc(ldev->dev, LIMA_PAGE_SIZE, 459 ldev->dlbu_cpu, ldev->dlbu_dma); 460 461 lima_vm_put(ldev->empty_vm); 462 463 lima_regulator_fini(ldev); 464 465 lima_clk_fini(ldev); 466 } 467 468 int lima_device_resume(struct device *dev) 469 { 470 struct lima_device *ldev = dev_get_drvdata(dev); 471 int i, err; 472 473 err = lima_clk_enable(ldev); 474 if (err) { 475 dev_err(dev, "resume clk fail %d\n", err); 476 return err; 477 } 478 479 err = lima_regulator_enable(ldev); 480 if (err) { 481 dev_err(dev, "resume regulator fail %d\n", err); 482 goto err_out0; 483 } 484 485 for (i = 0; i < lima_ip_num; i++) { 486 err = lima_resume_ip(ldev, i); 487 if (err) { 488 dev_err(dev, "resume ip %d fail\n", i); 489 goto err_out1; 490 } 491 } 492 493 err = lima_devfreq_resume(&ldev->devfreq); 494 if (err) { 495 dev_err(dev, "devfreq resume fail\n"); 496 goto err_out1; 497 } 498 499 return 0; 500 501 err_out1: 502 while (--i >= 0) 503 lima_suspend_ip(ldev, i); 504 lima_regulator_disable(ldev); 505 err_out0: 506 lima_clk_disable(ldev); 507 return err; 508 } 509 510 int lima_device_suspend(struct device *dev) 511 { 512 struct lima_device *ldev = dev_get_drvdata(dev); 513 int i, err; 514 515 /* check any task running */ 516 for (i = 0; i < lima_pipe_num; i++) { 517 if (atomic_read(&ldev->pipe[i].base.hw_rq_count)) 518 return -EBUSY; 519 } 520 521 err = lima_devfreq_suspend(&ldev->devfreq); 522 if (err) { 523 dev_err(dev, "devfreq suspend fail\n"); 524 return err; 525 } 526 527 for (i = lima_ip_num - 1; i >= 0; i--) 528 lima_suspend_ip(ldev, i); 529 530 lima_regulator_disable(ldev); 531 532 lima_clk_disable(ldev); 533 534 return 0; 535 } 536