1 /* 2 * Copyright (C) 2015 Etnaviv Project 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 as published by 6 * the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * You should have received a copy of the GNU General Public License along with 14 * this program. If not, see <http://www.gnu.org/licenses/>. 15 */ 16 17 #include <linux/component.h> 18 #include <linux/fence.h> 19 #include <linux/moduleparam.h> 20 #include <linux/of_device.h> 21 #include "etnaviv_dump.h" 22 #include "etnaviv_gpu.h" 23 #include "etnaviv_gem.h" 24 #include "etnaviv_mmu.h" 25 #include "etnaviv_iommu.h" 26 #include "etnaviv_iommu_v2.h" 27 #include "common.xml.h" 28 #include "state.xml.h" 29 #include "state_hi.xml.h" 30 #include "cmdstream.xml.h" 31 32 static const struct platform_device_id gpu_ids[] = { 33 { .name = "etnaviv-gpu,2d" }, 34 { }, 35 }; 36 37 static bool etnaviv_dump_core = true; 38 module_param_named(dump_core, etnaviv_dump_core, bool, 0600); 39 40 /* 41 * Driver functions: 42 */ 43 44 int etnaviv_gpu_get_param(struct etnaviv_gpu *gpu, u32 param, u64 *value) 45 { 46 switch (param) { 47 case ETNAVIV_PARAM_GPU_MODEL: 48 *value = gpu->identity.model; 49 break; 50 51 case ETNAVIV_PARAM_GPU_REVISION: 52 *value = gpu->identity.revision; 53 break; 54 55 case ETNAVIV_PARAM_GPU_FEATURES_0: 56 *value = gpu->identity.features; 57 break; 58 59 case ETNAVIV_PARAM_GPU_FEATURES_1: 60 *value = gpu->identity.minor_features0; 61 break; 62 63 case ETNAVIV_PARAM_GPU_FEATURES_2: 64 *value = gpu->identity.minor_features1; 65 break; 66 67 case ETNAVIV_PARAM_GPU_FEATURES_3: 68 *value = gpu->identity.minor_features2; 69 break; 70 71 case ETNAVIV_PARAM_GPU_FEATURES_4: 72 *value = gpu->identity.minor_features3; 73 break; 74 75 case ETNAVIV_PARAM_GPU_STREAM_COUNT: 76 *value = gpu->identity.stream_count; 77 break; 78 79 case ETNAVIV_PARAM_GPU_REGISTER_MAX: 80 *value = gpu->identity.register_max; 81 break; 82 83 case ETNAVIV_PARAM_GPU_THREAD_COUNT: 84 *value = gpu->identity.thread_count; 85 break; 86 87 case ETNAVIV_PARAM_GPU_VERTEX_CACHE_SIZE: 88 *value = gpu->identity.vertex_cache_size; 89 break; 90 91 case ETNAVIV_PARAM_GPU_SHADER_CORE_COUNT: 92 *value = gpu->identity.shader_core_count; 93 break; 94 95 case ETNAVIV_PARAM_GPU_PIXEL_PIPES: 96 *value = gpu->identity.pixel_pipes; 97 break; 98 99 case ETNAVIV_PARAM_GPU_VERTEX_OUTPUT_BUFFER_SIZE: 100 *value = gpu->identity.vertex_output_buffer_size; 101 break; 102 103 case ETNAVIV_PARAM_GPU_BUFFER_SIZE: 104 *value = gpu->identity.buffer_size; 105 break; 106 107 case ETNAVIV_PARAM_GPU_INSTRUCTION_COUNT: 108 *value = gpu->identity.instruction_count; 109 break; 110 111 case ETNAVIV_PARAM_GPU_NUM_CONSTANTS: 112 *value = gpu->identity.num_constants; 113 break; 114 115 default: 116 DBG("%s: invalid param: %u", dev_name(gpu->dev), param); 117 return -EINVAL; 118 } 119 120 return 0; 121 } 122 123 static void etnaviv_hw_specs(struct etnaviv_gpu *gpu) 124 { 125 if (gpu->identity.minor_features0 & 126 chipMinorFeatures0_MORE_MINOR_FEATURES) { 127 u32 specs[2]; 128 129 specs[0] = gpu_read(gpu, VIVS_HI_CHIP_SPECS); 130 specs[1] = gpu_read(gpu, VIVS_HI_CHIP_SPECS_2); 131 132 gpu->identity.stream_count = 133 (specs[0] & VIVS_HI_CHIP_SPECS_STREAM_COUNT__MASK) 134 >> VIVS_HI_CHIP_SPECS_STREAM_COUNT__SHIFT; 135 gpu->identity.register_max = 136 (specs[0] & VIVS_HI_CHIP_SPECS_REGISTER_MAX__MASK) 137 >> VIVS_HI_CHIP_SPECS_REGISTER_MAX__SHIFT; 138 gpu->identity.thread_count = 139 (specs[0] & VIVS_HI_CHIP_SPECS_THREAD_COUNT__MASK) 140 >> VIVS_HI_CHIP_SPECS_THREAD_COUNT__SHIFT; 141 gpu->identity.vertex_cache_size = 142 (specs[0] & VIVS_HI_CHIP_SPECS_VERTEX_CACHE_SIZE__MASK) 143 >> VIVS_HI_CHIP_SPECS_VERTEX_CACHE_SIZE__SHIFT; 144 gpu->identity.shader_core_count = 145 (specs[0] & VIVS_HI_CHIP_SPECS_SHADER_CORE_COUNT__MASK) 146 >> VIVS_HI_CHIP_SPECS_SHADER_CORE_COUNT__SHIFT; 147 gpu->identity.pixel_pipes = 148 (specs[0] & VIVS_HI_CHIP_SPECS_PIXEL_PIPES__MASK) 149 >> VIVS_HI_CHIP_SPECS_PIXEL_PIPES__SHIFT; 150 gpu->identity.vertex_output_buffer_size = 151 (specs[0] & VIVS_HI_CHIP_SPECS_VERTEX_OUTPUT_BUFFER_SIZE__MASK) 152 >> VIVS_HI_CHIP_SPECS_VERTEX_OUTPUT_BUFFER_SIZE__SHIFT; 153 154 gpu->identity.buffer_size = 155 (specs[1] & VIVS_HI_CHIP_SPECS_2_BUFFER_SIZE__MASK) 156 >> VIVS_HI_CHIP_SPECS_2_BUFFER_SIZE__SHIFT; 157 gpu->identity.instruction_count = 158 (specs[1] & VIVS_HI_CHIP_SPECS_2_INSTRUCTION_COUNT__MASK) 159 >> VIVS_HI_CHIP_SPECS_2_INSTRUCTION_COUNT__SHIFT; 160 gpu->identity.num_constants = 161 (specs[1] & VIVS_HI_CHIP_SPECS_2_NUM_CONSTANTS__MASK) 162 >> VIVS_HI_CHIP_SPECS_2_NUM_CONSTANTS__SHIFT; 163 } 164 165 /* Fill in the stream count if not specified */ 166 if (gpu->identity.stream_count == 0) { 167 if (gpu->identity.model >= 0x1000) 168 gpu->identity.stream_count = 4; 169 else 170 gpu->identity.stream_count = 1; 171 } 172 173 /* Convert the register max value */ 174 if (gpu->identity.register_max) 175 gpu->identity.register_max = 1 << gpu->identity.register_max; 176 else if (gpu->identity.model == 0x0400) 177 gpu->identity.register_max = 32; 178 else 179 gpu->identity.register_max = 64; 180 181 /* Convert thread count */ 182 if (gpu->identity.thread_count) 183 gpu->identity.thread_count = 1 << gpu->identity.thread_count; 184 else if (gpu->identity.model == 0x0400) 185 gpu->identity.thread_count = 64; 186 else if (gpu->identity.model == 0x0500 || 187 gpu->identity.model == 0x0530) 188 gpu->identity.thread_count = 128; 189 else 190 gpu->identity.thread_count = 256; 191 192 if (gpu->identity.vertex_cache_size == 0) 193 gpu->identity.vertex_cache_size = 8; 194 195 if (gpu->identity.shader_core_count == 0) { 196 if (gpu->identity.model >= 0x1000) 197 gpu->identity.shader_core_count = 2; 198 else 199 gpu->identity.shader_core_count = 1; 200 } 201 202 if (gpu->identity.pixel_pipes == 0) 203 gpu->identity.pixel_pipes = 1; 204 205 /* Convert virtex buffer size */ 206 if (gpu->identity.vertex_output_buffer_size) { 207 gpu->identity.vertex_output_buffer_size = 208 1 << gpu->identity.vertex_output_buffer_size; 209 } else if (gpu->identity.model == 0x0400) { 210 if (gpu->identity.revision < 0x4000) 211 gpu->identity.vertex_output_buffer_size = 512; 212 else if (gpu->identity.revision < 0x4200) 213 gpu->identity.vertex_output_buffer_size = 256; 214 else 215 gpu->identity.vertex_output_buffer_size = 128; 216 } else { 217 gpu->identity.vertex_output_buffer_size = 512; 218 } 219 220 switch (gpu->identity.instruction_count) { 221 case 0: 222 if ((gpu->identity.model == 0x2000 && 223 gpu->identity.revision == 0x5108) || 224 gpu->identity.model == 0x880) 225 gpu->identity.instruction_count = 512; 226 else 227 gpu->identity.instruction_count = 256; 228 break; 229 230 case 1: 231 gpu->identity.instruction_count = 1024; 232 break; 233 234 case 2: 235 gpu->identity.instruction_count = 2048; 236 break; 237 238 default: 239 gpu->identity.instruction_count = 256; 240 break; 241 } 242 243 if (gpu->identity.num_constants == 0) 244 gpu->identity.num_constants = 168; 245 } 246 247 static void etnaviv_hw_identify(struct etnaviv_gpu *gpu) 248 { 249 u32 chipIdentity; 250 251 chipIdentity = gpu_read(gpu, VIVS_HI_CHIP_IDENTITY); 252 253 /* Special case for older graphic cores. */ 254 if (((chipIdentity & VIVS_HI_CHIP_IDENTITY_FAMILY__MASK) 255 >> VIVS_HI_CHIP_IDENTITY_FAMILY__SHIFT) == 0x01) { 256 gpu->identity.model = 0x500; /* gc500 */ 257 gpu->identity.revision = 258 (chipIdentity & VIVS_HI_CHIP_IDENTITY_REVISION__MASK) 259 >> VIVS_HI_CHIP_IDENTITY_REVISION__SHIFT; 260 } else { 261 262 gpu->identity.model = gpu_read(gpu, VIVS_HI_CHIP_MODEL); 263 gpu->identity.revision = gpu_read(gpu, VIVS_HI_CHIP_REV); 264 265 /* 266 * !!!! HACK ALERT !!!! 267 * Because people change device IDs without letting software 268 * know about it - here is the hack to make it all look the 269 * same. Only for GC400 family. 270 */ 271 if ((gpu->identity.model & 0xff00) == 0x0400 && 272 gpu->identity.model != 0x0420) { 273 gpu->identity.model = gpu->identity.model & 0x0400; 274 } 275 276 /* Another special case */ 277 if (gpu->identity.model == 0x300 && 278 gpu->identity.revision == 0x2201) { 279 u32 chipDate = gpu_read(gpu, VIVS_HI_CHIP_DATE); 280 u32 chipTime = gpu_read(gpu, VIVS_HI_CHIP_TIME); 281 282 if (chipDate == 0x20080814 && chipTime == 0x12051100) { 283 /* 284 * This IP has an ECO; put the correct 285 * revision in it. 286 */ 287 gpu->identity.revision = 0x1051; 288 } 289 } 290 } 291 292 dev_info(gpu->dev, "model: GC%x, revision: %x\n", 293 gpu->identity.model, gpu->identity.revision); 294 295 gpu->identity.features = gpu_read(gpu, VIVS_HI_CHIP_FEATURE); 296 297 /* Disable fast clear on GC700. */ 298 if (gpu->identity.model == 0x700) 299 gpu->identity.features &= ~chipFeatures_FAST_CLEAR; 300 301 if ((gpu->identity.model == 0x500 && gpu->identity.revision < 2) || 302 (gpu->identity.model == 0x300 && gpu->identity.revision < 0x2000)) { 303 304 /* 305 * GC500 rev 1.x and GC300 rev < 2.0 doesn't have these 306 * registers. 307 */ 308 gpu->identity.minor_features0 = 0; 309 gpu->identity.minor_features1 = 0; 310 gpu->identity.minor_features2 = 0; 311 gpu->identity.minor_features3 = 0; 312 } else 313 gpu->identity.minor_features0 = 314 gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_0); 315 316 if (gpu->identity.minor_features0 & 317 chipMinorFeatures0_MORE_MINOR_FEATURES) { 318 gpu->identity.minor_features1 = 319 gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_1); 320 gpu->identity.minor_features2 = 321 gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_2); 322 gpu->identity.minor_features3 = 323 gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_3); 324 } 325 326 /* GC600 idle register reports zero bits where modules aren't present */ 327 if (gpu->identity.model == chipModel_GC600) { 328 gpu->idle_mask = VIVS_HI_IDLE_STATE_TX | 329 VIVS_HI_IDLE_STATE_RA | 330 VIVS_HI_IDLE_STATE_SE | 331 VIVS_HI_IDLE_STATE_PA | 332 VIVS_HI_IDLE_STATE_SH | 333 VIVS_HI_IDLE_STATE_PE | 334 VIVS_HI_IDLE_STATE_DE | 335 VIVS_HI_IDLE_STATE_FE; 336 } else { 337 gpu->idle_mask = ~VIVS_HI_IDLE_STATE_AXI_LP; 338 } 339 340 etnaviv_hw_specs(gpu); 341 } 342 343 static void etnaviv_gpu_load_clock(struct etnaviv_gpu *gpu, u32 clock) 344 { 345 gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, clock | 346 VIVS_HI_CLOCK_CONTROL_FSCALE_CMD_LOAD); 347 gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, clock); 348 } 349 350 static int etnaviv_hw_reset(struct etnaviv_gpu *gpu) 351 { 352 u32 control, idle; 353 unsigned long timeout; 354 bool failed = true; 355 356 /* TODO 357 * 358 * - clock gating 359 * - puls eater 360 * - what about VG? 361 */ 362 363 /* We hope that the GPU resets in under one second */ 364 timeout = jiffies + msecs_to_jiffies(1000); 365 366 while (time_is_after_jiffies(timeout)) { 367 control = VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS | 368 VIVS_HI_CLOCK_CONTROL_FSCALE_VAL(0x40); 369 370 /* enable clock */ 371 etnaviv_gpu_load_clock(gpu, control); 372 373 /* Wait for stable clock. Vivante's code waited for 1ms */ 374 usleep_range(1000, 10000); 375 376 /* isolate the GPU. */ 377 control |= VIVS_HI_CLOCK_CONTROL_ISOLATE_GPU; 378 gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control); 379 380 /* set soft reset. */ 381 control |= VIVS_HI_CLOCK_CONTROL_SOFT_RESET; 382 gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control); 383 384 /* wait for reset. */ 385 msleep(1); 386 387 /* reset soft reset bit. */ 388 control &= ~VIVS_HI_CLOCK_CONTROL_SOFT_RESET; 389 gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control); 390 391 /* reset GPU isolation. */ 392 control &= ~VIVS_HI_CLOCK_CONTROL_ISOLATE_GPU; 393 gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control); 394 395 /* read idle register. */ 396 idle = gpu_read(gpu, VIVS_HI_IDLE_STATE); 397 398 /* try reseting again if FE it not idle */ 399 if ((idle & VIVS_HI_IDLE_STATE_FE) == 0) { 400 dev_dbg(gpu->dev, "FE is not idle\n"); 401 continue; 402 } 403 404 /* read reset register. */ 405 control = gpu_read(gpu, VIVS_HI_CLOCK_CONTROL); 406 407 /* is the GPU idle? */ 408 if (((control & VIVS_HI_CLOCK_CONTROL_IDLE_3D) == 0) || 409 ((control & VIVS_HI_CLOCK_CONTROL_IDLE_2D) == 0)) { 410 dev_dbg(gpu->dev, "GPU is not idle\n"); 411 continue; 412 } 413 414 failed = false; 415 break; 416 } 417 418 if (failed) { 419 idle = gpu_read(gpu, VIVS_HI_IDLE_STATE); 420 control = gpu_read(gpu, VIVS_HI_CLOCK_CONTROL); 421 422 dev_err(gpu->dev, "GPU failed to reset: FE %sidle, 3D %sidle, 2D %sidle\n", 423 idle & VIVS_HI_IDLE_STATE_FE ? "" : "not ", 424 control & VIVS_HI_CLOCK_CONTROL_IDLE_3D ? "" : "not ", 425 control & VIVS_HI_CLOCK_CONTROL_IDLE_2D ? "" : "not "); 426 427 return -EBUSY; 428 } 429 430 /* We rely on the GPU running, so program the clock */ 431 control = VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS | 432 VIVS_HI_CLOCK_CONTROL_FSCALE_VAL(0x40); 433 434 /* enable clock */ 435 etnaviv_gpu_load_clock(gpu, control); 436 437 return 0; 438 } 439 440 static void etnaviv_gpu_hw_init(struct etnaviv_gpu *gpu) 441 { 442 u16 prefetch; 443 444 if (gpu->identity.model == chipModel_GC320 && 445 gpu_read(gpu, VIVS_HI_CHIP_TIME) != 0x2062400 && 446 (gpu->identity.revision == 0x5007 || 447 gpu->identity.revision == 0x5220)) { 448 u32 mc_memory_debug; 449 450 mc_memory_debug = gpu_read(gpu, VIVS_MC_DEBUG_MEMORY) & ~0xff; 451 452 if (gpu->identity.revision == 0x5007) 453 mc_memory_debug |= 0x0c; 454 else 455 mc_memory_debug |= 0x08; 456 457 gpu_write(gpu, VIVS_MC_DEBUG_MEMORY, mc_memory_debug); 458 } 459 460 /* 461 * Update GPU AXI cache atttribute to "cacheable, no allocate". 462 * This is necessary to prevent the iMX6 SoC locking up. 463 */ 464 gpu_write(gpu, VIVS_HI_AXI_CONFIG, 465 VIVS_HI_AXI_CONFIG_AWCACHE(2) | 466 VIVS_HI_AXI_CONFIG_ARCACHE(2)); 467 468 /* GC2000 rev 5108 needs a special bus config */ 469 if (gpu->identity.model == 0x2000 && gpu->identity.revision == 0x5108) { 470 u32 bus_config = gpu_read(gpu, VIVS_MC_BUS_CONFIG); 471 bus_config &= ~(VIVS_MC_BUS_CONFIG_FE_BUS_CONFIG__MASK | 472 VIVS_MC_BUS_CONFIG_TX_BUS_CONFIG__MASK); 473 bus_config |= VIVS_MC_BUS_CONFIG_FE_BUS_CONFIG(1) | 474 VIVS_MC_BUS_CONFIG_TX_BUS_CONFIG(0); 475 gpu_write(gpu, VIVS_MC_BUS_CONFIG, bus_config); 476 } 477 478 /* set base addresses */ 479 gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_RA, gpu->memory_base); 480 gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_FE, gpu->memory_base); 481 gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_TX, gpu->memory_base); 482 gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_PEZ, gpu->memory_base); 483 gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_PE, gpu->memory_base); 484 485 /* setup the MMU page table pointers */ 486 etnaviv_iommu_domain_restore(gpu, gpu->mmu->domain); 487 488 /* Start command processor */ 489 prefetch = etnaviv_buffer_init(gpu); 490 491 gpu_write(gpu, VIVS_HI_INTR_ENBL, ~0U); 492 gpu_write(gpu, VIVS_FE_COMMAND_ADDRESS, 493 gpu->buffer->paddr - gpu->memory_base); 494 gpu_write(gpu, VIVS_FE_COMMAND_CONTROL, 495 VIVS_FE_COMMAND_CONTROL_ENABLE | 496 VIVS_FE_COMMAND_CONTROL_PREFETCH(prefetch)); 497 } 498 499 int etnaviv_gpu_init(struct etnaviv_gpu *gpu) 500 { 501 int ret, i; 502 struct iommu_domain *iommu; 503 enum etnaviv_iommu_version version; 504 bool mmuv2; 505 506 ret = pm_runtime_get_sync(gpu->dev); 507 if (ret < 0) 508 return ret; 509 510 etnaviv_hw_identify(gpu); 511 512 if (gpu->identity.model == 0) { 513 dev_err(gpu->dev, "Unknown GPU model\n"); 514 pm_runtime_put_autosuspend(gpu->dev); 515 return -ENXIO; 516 } 517 518 ret = etnaviv_hw_reset(gpu); 519 if (ret) 520 goto fail; 521 522 /* Setup IOMMU.. eventually we will (I think) do this once per context 523 * and have separate page tables per context. For now, to keep things 524 * simple and to get something working, just use a single address space: 525 */ 526 mmuv2 = gpu->identity.minor_features1 & chipMinorFeatures1_MMU_VERSION; 527 dev_dbg(gpu->dev, "mmuv2: %d\n", mmuv2); 528 529 if (!mmuv2) { 530 iommu = etnaviv_iommu_domain_alloc(gpu); 531 version = ETNAVIV_IOMMU_V1; 532 } else { 533 iommu = etnaviv_iommu_v2_domain_alloc(gpu); 534 version = ETNAVIV_IOMMU_V2; 535 } 536 537 if (!iommu) { 538 ret = -ENOMEM; 539 goto fail; 540 } 541 542 /* TODO: we will leak here memory - fix it! */ 543 544 gpu->mmu = etnaviv_iommu_new(gpu, iommu, version); 545 if (!gpu->mmu) { 546 ret = -ENOMEM; 547 goto fail; 548 } 549 550 /* Create buffer: */ 551 gpu->buffer = etnaviv_gpu_cmdbuf_new(gpu, PAGE_SIZE, 0); 552 if (!gpu->buffer) { 553 ret = -ENOMEM; 554 dev_err(gpu->dev, "could not create command buffer\n"); 555 goto fail; 556 } 557 if (gpu->buffer->paddr - gpu->memory_base > 0x80000000) { 558 ret = -EINVAL; 559 dev_err(gpu->dev, 560 "command buffer outside valid memory window\n"); 561 goto free_buffer; 562 } 563 564 /* Setup event management */ 565 spin_lock_init(&gpu->event_spinlock); 566 init_completion(&gpu->event_free); 567 for (i = 0; i < ARRAY_SIZE(gpu->event); i++) { 568 gpu->event[i].used = false; 569 complete(&gpu->event_free); 570 } 571 572 /* Now program the hardware */ 573 mutex_lock(&gpu->lock); 574 etnaviv_gpu_hw_init(gpu); 575 mutex_unlock(&gpu->lock); 576 577 pm_runtime_mark_last_busy(gpu->dev); 578 pm_runtime_put_autosuspend(gpu->dev); 579 580 return 0; 581 582 free_buffer: 583 etnaviv_gpu_cmdbuf_free(gpu->buffer); 584 gpu->buffer = NULL; 585 fail: 586 pm_runtime_mark_last_busy(gpu->dev); 587 pm_runtime_put_autosuspend(gpu->dev); 588 589 return ret; 590 } 591 592 #ifdef CONFIG_DEBUG_FS 593 struct dma_debug { 594 u32 address[2]; 595 u32 state[2]; 596 }; 597 598 static void verify_dma(struct etnaviv_gpu *gpu, struct dma_debug *debug) 599 { 600 u32 i; 601 602 debug->address[0] = gpu_read(gpu, VIVS_FE_DMA_ADDRESS); 603 debug->state[0] = gpu_read(gpu, VIVS_FE_DMA_DEBUG_STATE); 604 605 for (i = 0; i < 500; i++) { 606 debug->address[1] = gpu_read(gpu, VIVS_FE_DMA_ADDRESS); 607 debug->state[1] = gpu_read(gpu, VIVS_FE_DMA_DEBUG_STATE); 608 609 if (debug->address[0] != debug->address[1]) 610 break; 611 612 if (debug->state[0] != debug->state[1]) 613 break; 614 } 615 } 616 617 int etnaviv_gpu_debugfs(struct etnaviv_gpu *gpu, struct seq_file *m) 618 { 619 struct dma_debug debug; 620 u32 dma_lo, dma_hi, axi, idle; 621 int ret; 622 623 seq_printf(m, "%s Status:\n", dev_name(gpu->dev)); 624 625 ret = pm_runtime_get_sync(gpu->dev); 626 if (ret < 0) 627 return ret; 628 629 dma_lo = gpu_read(gpu, VIVS_FE_DMA_LOW); 630 dma_hi = gpu_read(gpu, VIVS_FE_DMA_HIGH); 631 axi = gpu_read(gpu, VIVS_HI_AXI_STATUS); 632 idle = gpu_read(gpu, VIVS_HI_IDLE_STATE); 633 634 verify_dma(gpu, &debug); 635 636 seq_puts(m, "\tfeatures\n"); 637 seq_printf(m, "\t minor_features0: 0x%08x\n", 638 gpu->identity.minor_features0); 639 seq_printf(m, "\t minor_features1: 0x%08x\n", 640 gpu->identity.minor_features1); 641 seq_printf(m, "\t minor_features2: 0x%08x\n", 642 gpu->identity.minor_features2); 643 seq_printf(m, "\t minor_features3: 0x%08x\n", 644 gpu->identity.minor_features3); 645 646 seq_puts(m, "\tspecs\n"); 647 seq_printf(m, "\t stream_count: %d\n", 648 gpu->identity.stream_count); 649 seq_printf(m, "\t register_max: %d\n", 650 gpu->identity.register_max); 651 seq_printf(m, "\t thread_count: %d\n", 652 gpu->identity.thread_count); 653 seq_printf(m, "\t vertex_cache_size: %d\n", 654 gpu->identity.vertex_cache_size); 655 seq_printf(m, "\t shader_core_count: %d\n", 656 gpu->identity.shader_core_count); 657 seq_printf(m, "\t pixel_pipes: %d\n", 658 gpu->identity.pixel_pipes); 659 seq_printf(m, "\t vertex_output_buffer_size: %d\n", 660 gpu->identity.vertex_output_buffer_size); 661 seq_printf(m, "\t buffer_size: %d\n", 662 gpu->identity.buffer_size); 663 seq_printf(m, "\t instruction_count: %d\n", 664 gpu->identity.instruction_count); 665 seq_printf(m, "\t num_constants: %d\n", 666 gpu->identity.num_constants); 667 668 seq_printf(m, "\taxi: 0x%08x\n", axi); 669 seq_printf(m, "\tidle: 0x%08x\n", idle); 670 idle |= ~gpu->idle_mask & ~VIVS_HI_IDLE_STATE_AXI_LP; 671 if ((idle & VIVS_HI_IDLE_STATE_FE) == 0) 672 seq_puts(m, "\t FE is not idle\n"); 673 if ((idle & VIVS_HI_IDLE_STATE_DE) == 0) 674 seq_puts(m, "\t DE is not idle\n"); 675 if ((idle & VIVS_HI_IDLE_STATE_PE) == 0) 676 seq_puts(m, "\t PE is not idle\n"); 677 if ((idle & VIVS_HI_IDLE_STATE_SH) == 0) 678 seq_puts(m, "\t SH is not idle\n"); 679 if ((idle & VIVS_HI_IDLE_STATE_PA) == 0) 680 seq_puts(m, "\t PA is not idle\n"); 681 if ((idle & VIVS_HI_IDLE_STATE_SE) == 0) 682 seq_puts(m, "\t SE is not idle\n"); 683 if ((idle & VIVS_HI_IDLE_STATE_RA) == 0) 684 seq_puts(m, "\t RA is not idle\n"); 685 if ((idle & VIVS_HI_IDLE_STATE_TX) == 0) 686 seq_puts(m, "\t TX is not idle\n"); 687 if ((idle & VIVS_HI_IDLE_STATE_VG) == 0) 688 seq_puts(m, "\t VG is not idle\n"); 689 if ((idle & VIVS_HI_IDLE_STATE_IM) == 0) 690 seq_puts(m, "\t IM is not idle\n"); 691 if ((idle & VIVS_HI_IDLE_STATE_FP) == 0) 692 seq_puts(m, "\t FP is not idle\n"); 693 if ((idle & VIVS_HI_IDLE_STATE_TS) == 0) 694 seq_puts(m, "\t TS is not idle\n"); 695 if (idle & VIVS_HI_IDLE_STATE_AXI_LP) 696 seq_puts(m, "\t AXI low power mode\n"); 697 698 if (gpu->identity.features & chipFeatures_DEBUG_MODE) { 699 u32 read0 = gpu_read(gpu, VIVS_MC_DEBUG_READ0); 700 u32 read1 = gpu_read(gpu, VIVS_MC_DEBUG_READ1); 701 u32 write = gpu_read(gpu, VIVS_MC_DEBUG_WRITE); 702 703 seq_puts(m, "\tMC\n"); 704 seq_printf(m, "\t read0: 0x%08x\n", read0); 705 seq_printf(m, "\t read1: 0x%08x\n", read1); 706 seq_printf(m, "\t write: 0x%08x\n", write); 707 } 708 709 seq_puts(m, "\tDMA "); 710 711 if (debug.address[0] == debug.address[1] && 712 debug.state[0] == debug.state[1]) { 713 seq_puts(m, "seems to be stuck\n"); 714 } else if (debug.address[0] == debug.address[1]) { 715 seq_puts(m, "adress is constant\n"); 716 } else { 717 seq_puts(m, "is runing\n"); 718 } 719 720 seq_printf(m, "\t address 0: 0x%08x\n", debug.address[0]); 721 seq_printf(m, "\t address 1: 0x%08x\n", debug.address[1]); 722 seq_printf(m, "\t state 0: 0x%08x\n", debug.state[0]); 723 seq_printf(m, "\t state 1: 0x%08x\n", debug.state[1]); 724 seq_printf(m, "\t last fetch 64 bit word: 0x%08x 0x%08x\n", 725 dma_lo, dma_hi); 726 727 ret = 0; 728 729 pm_runtime_mark_last_busy(gpu->dev); 730 pm_runtime_put_autosuspend(gpu->dev); 731 732 return ret; 733 } 734 #endif 735 736 /* 737 * Power Management: 738 */ 739 static int enable_clk(struct etnaviv_gpu *gpu) 740 { 741 if (gpu->clk_core) 742 clk_prepare_enable(gpu->clk_core); 743 if (gpu->clk_shader) 744 clk_prepare_enable(gpu->clk_shader); 745 746 return 0; 747 } 748 749 static int disable_clk(struct etnaviv_gpu *gpu) 750 { 751 if (gpu->clk_core) 752 clk_disable_unprepare(gpu->clk_core); 753 if (gpu->clk_shader) 754 clk_disable_unprepare(gpu->clk_shader); 755 756 return 0; 757 } 758 759 static int enable_axi(struct etnaviv_gpu *gpu) 760 { 761 if (gpu->clk_bus) 762 clk_prepare_enable(gpu->clk_bus); 763 764 return 0; 765 } 766 767 static int disable_axi(struct etnaviv_gpu *gpu) 768 { 769 if (gpu->clk_bus) 770 clk_disable_unprepare(gpu->clk_bus); 771 772 return 0; 773 } 774 775 /* 776 * Hangcheck detection for locked gpu: 777 */ 778 static void recover_worker(struct work_struct *work) 779 { 780 struct etnaviv_gpu *gpu = container_of(work, struct etnaviv_gpu, 781 recover_work); 782 unsigned long flags; 783 unsigned int i; 784 785 dev_err(gpu->dev, "hangcheck recover!\n"); 786 787 if (pm_runtime_get_sync(gpu->dev) < 0) 788 return; 789 790 mutex_lock(&gpu->lock); 791 792 /* Only catch the first event, or when manually re-armed */ 793 if (etnaviv_dump_core) { 794 etnaviv_core_dump(gpu); 795 etnaviv_dump_core = false; 796 } 797 798 etnaviv_hw_reset(gpu); 799 800 /* complete all events, the GPU won't do it after the reset */ 801 spin_lock_irqsave(&gpu->event_spinlock, flags); 802 for (i = 0; i < ARRAY_SIZE(gpu->event); i++) { 803 if (!gpu->event[i].used) 804 continue; 805 fence_signal(gpu->event[i].fence); 806 gpu->event[i].fence = NULL; 807 gpu->event[i].used = false; 808 complete(&gpu->event_free); 809 /* 810 * Decrement the PM count for each stuck event. This is safe 811 * even in atomic context as we use ASYNC RPM here. 812 */ 813 pm_runtime_put_autosuspend(gpu->dev); 814 } 815 spin_unlock_irqrestore(&gpu->event_spinlock, flags); 816 gpu->completed_fence = gpu->active_fence; 817 818 etnaviv_gpu_hw_init(gpu); 819 gpu->switch_context = true; 820 821 mutex_unlock(&gpu->lock); 822 pm_runtime_mark_last_busy(gpu->dev); 823 pm_runtime_put_autosuspend(gpu->dev); 824 825 /* Retire the buffer objects in a work */ 826 etnaviv_queue_work(gpu->drm, &gpu->retire_work); 827 } 828 829 static void hangcheck_timer_reset(struct etnaviv_gpu *gpu) 830 { 831 DBG("%s", dev_name(gpu->dev)); 832 mod_timer(&gpu->hangcheck_timer, 833 round_jiffies_up(jiffies + DRM_ETNAVIV_HANGCHECK_JIFFIES)); 834 } 835 836 static void hangcheck_handler(unsigned long data) 837 { 838 struct etnaviv_gpu *gpu = (struct etnaviv_gpu *)data; 839 u32 fence = gpu->completed_fence; 840 bool progress = false; 841 842 if (fence != gpu->hangcheck_fence) { 843 gpu->hangcheck_fence = fence; 844 progress = true; 845 } 846 847 if (!progress) { 848 u32 dma_addr = gpu_read(gpu, VIVS_FE_DMA_ADDRESS); 849 int change = dma_addr - gpu->hangcheck_dma_addr; 850 851 if (change < 0 || change > 16) { 852 gpu->hangcheck_dma_addr = dma_addr; 853 progress = true; 854 } 855 } 856 857 if (!progress && fence_after(gpu->active_fence, fence)) { 858 dev_err(gpu->dev, "hangcheck detected gpu lockup!\n"); 859 dev_err(gpu->dev, " completed fence: %u\n", fence); 860 dev_err(gpu->dev, " active fence: %u\n", 861 gpu->active_fence); 862 etnaviv_queue_work(gpu->drm, &gpu->recover_work); 863 } 864 865 /* if still more pending work, reset the hangcheck timer: */ 866 if (fence_after(gpu->active_fence, gpu->hangcheck_fence)) 867 hangcheck_timer_reset(gpu); 868 } 869 870 static void hangcheck_disable(struct etnaviv_gpu *gpu) 871 { 872 del_timer_sync(&gpu->hangcheck_timer); 873 cancel_work_sync(&gpu->recover_work); 874 } 875 876 /* fence object management */ 877 struct etnaviv_fence { 878 struct etnaviv_gpu *gpu; 879 struct fence base; 880 }; 881 882 static inline struct etnaviv_fence *to_etnaviv_fence(struct fence *fence) 883 { 884 return container_of(fence, struct etnaviv_fence, base); 885 } 886 887 static const char *etnaviv_fence_get_driver_name(struct fence *fence) 888 { 889 return "etnaviv"; 890 } 891 892 static const char *etnaviv_fence_get_timeline_name(struct fence *fence) 893 { 894 struct etnaviv_fence *f = to_etnaviv_fence(fence); 895 896 return dev_name(f->gpu->dev); 897 } 898 899 static bool etnaviv_fence_enable_signaling(struct fence *fence) 900 { 901 return true; 902 } 903 904 static bool etnaviv_fence_signaled(struct fence *fence) 905 { 906 struct etnaviv_fence *f = to_etnaviv_fence(fence); 907 908 return fence_completed(f->gpu, f->base.seqno); 909 } 910 911 static void etnaviv_fence_release(struct fence *fence) 912 { 913 struct etnaviv_fence *f = to_etnaviv_fence(fence); 914 915 kfree_rcu(f, base.rcu); 916 } 917 918 static const struct fence_ops etnaviv_fence_ops = { 919 .get_driver_name = etnaviv_fence_get_driver_name, 920 .get_timeline_name = etnaviv_fence_get_timeline_name, 921 .enable_signaling = etnaviv_fence_enable_signaling, 922 .signaled = etnaviv_fence_signaled, 923 .wait = fence_default_wait, 924 .release = etnaviv_fence_release, 925 }; 926 927 static struct fence *etnaviv_gpu_fence_alloc(struct etnaviv_gpu *gpu) 928 { 929 struct etnaviv_fence *f; 930 931 f = kzalloc(sizeof(*f), GFP_KERNEL); 932 if (!f) 933 return NULL; 934 935 f->gpu = gpu; 936 937 fence_init(&f->base, &etnaviv_fence_ops, &gpu->fence_spinlock, 938 gpu->fence_context, ++gpu->next_fence); 939 940 return &f->base; 941 } 942 943 int etnaviv_gpu_fence_sync_obj(struct etnaviv_gem_object *etnaviv_obj, 944 unsigned int context, bool exclusive) 945 { 946 struct reservation_object *robj = etnaviv_obj->resv; 947 struct reservation_object_list *fobj; 948 struct fence *fence; 949 int i, ret; 950 951 if (!exclusive) { 952 ret = reservation_object_reserve_shared(robj); 953 if (ret) 954 return ret; 955 } 956 957 /* 958 * If we have any shared fences, then the exclusive fence 959 * should be ignored as it will already have been signalled. 960 */ 961 fobj = reservation_object_get_list(robj); 962 if (!fobj || fobj->shared_count == 0) { 963 /* Wait on any existing exclusive fence which isn't our own */ 964 fence = reservation_object_get_excl(robj); 965 if (fence && fence->context != context) { 966 ret = fence_wait(fence, true); 967 if (ret) 968 return ret; 969 } 970 } 971 972 if (!exclusive || !fobj) 973 return 0; 974 975 for (i = 0; i < fobj->shared_count; i++) { 976 fence = rcu_dereference_protected(fobj->shared[i], 977 reservation_object_held(robj)); 978 if (fence->context != context) { 979 ret = fence_wait(fence, true); 980 if (ret) 981 return ret; 982 } 983 } 984 985 return 0; 986 } 987 988 /* 989 * event management: 990 */ 991 992 static unsigned int event_alloc(struct etnaviv_gpu *gpu) 993 { 994 unsigned long ret, flags; 995 unsigned int i, event = ~0U; 996 997 ret = wait_for_completion_timeout(&gpu->event_free, 998 msecs_to_jiffies(10 * 10000)); 999 if (!ret) 1000 dev_err(gpu->dev, "wait_for_completion_timeout failed"); 1001 1002 spin_lock_irqsave(&gpu->event_spinlock, flags); 1003 1004 /* find first free event */ 1005 for (i = 0; i < ARRAY_SIZE(gpu->event); i++) { 1006 if (gpu->event[i].used == false) { 1007 gpu->event[i].used = true; 1008 event = i; 1009 break; 1010 } 1011 } 1012 1013 spin_unlock_irqrestore(&gpu->event_spinlock, flags); 1014 1015 return event; 1016 } 1017 1018 static void event_free(struct etnaviv_gpu *gpu, unsigned int event) 1019 { 1020 unsigned long flags; 1021 1022 spin_lock_irqsave(&gpu->event_spinlock, flags); 1023 1024 if (gpu->event[event].used == false) { 1025 dev_warn(gpu->dev, "event %u is already marked as free", 1026 event); 1027 spin_unlock_irqrestore(&gpu->event_spinlock, flags); 1028 } else { 1029 gpu->event[event].used = false; 1030 spin_unlock_irqrestore(&gpu->event_spinlock, flags); 1031 1032 complete(&gpu->event_free); 1033 } 1034 } 1035 1036 /* 1037 * Cmdstream submission/retirement: 1038 */ 1039 1040 struct etnaviv_cmdbuf *etnaviv_gpu_cmdbuf_new(struct etnaviv_gpu *gpu, u32 size, 1041 size_t nr_bos) 1042 { 1043 struct etnaviv_cmdbuf *cmdbuf; 1044 size_t sz = size_vstruct(nr_bos, sizeof(cmdbuf->bo[0]), 1045 sizeof(*cmdbuf)); 1046 1047 cmdbuf = kzalloc(sz, GFP_KERNEL); 1048 if (!cmdbuf) 1049 return NULL; 1050 1051 cmdbuf->vaddr = dma_alloc_writecombine(gpu->dev, size, &cmdbuf->paddr, 1052 GFP_KERNEL); 1053 if (!cmdbuf->vaddr) { 1054 kfree(cmdbuf); 1055 return NULL; 1056 } 1057 1058 cmdbuf->gpu = gpu; 1059 cmdbuf->size = size; 1060 1061 return cmdbuf; 1062 } 1063 1064 void etnaviv_gpu_cmdbuf_free(struct etnaviv_cmdbuf *cmdbuf) 1065 { 1066 dma_free_writecombine(cmdbuf->gpu->dev, cmdbuf->size, 1067 cmdbuf->vaddr, cmdbuf->paddr); 1068 kfree(cmdbuf); 1069 } 1070 1071 static void retire_worker(struct work_struct *work) 1072 { 1073 struct etnaviv_gpu *gpu = container_of(work, struct etnaviv_gpu, 1074 retire_work); 1075 u32 fence = gpu->completed_fence; 1076 struct etnaviv_cmdbuf *cmdbuf, *tmp; 1077 unsigned int i; 1078 1079 mutex_lock(&gpu->lock); 1080 list_for_each_entry_safe(cmdbuf, tmp, &gpu->active_cmd_list, node) { 1081 if (!fence_is_signaled(cmdbuf->fence)) 1082 break; 1083 1084 list_del(&cmdbuf->node); 1085 fence_put(cmdbuf->fence); 1086 1087 for (i = 0; i < cmdbuf->nr_bos; i++) { 1088 struct etnaviv_gem_object *etnaviv_obj = cmdbuf->bo[i]; 1089 1090 atomic_dec(&etnaviv_obj->gpu_active); 1091 /* drop the refcount taken in etnaviv_gpu_submit */ 1092 etnaviv_gem_put_iova(gpu, &etnaviv_obj->base); 1093 } 1094 1095 etnaviv_gpu_cmdbuf_free(cmdbuf); 1096 } 1097 1098 gpu->retired_fence = fence; 1099 1100 mutex_unlock(&gpu->lock); 1101 1102 wake_up_all(&gpu->fence_event); 1103 } 1104 1105 int etnaviv_gpu_wait_fence_interruptible(struct etnaviv_gpu *gpu, 1106 u32 fence, struct timespec *timeout) 1107 { 1108 int ret; 1109 1110 if (fence_after(fence, gpu->next_fence)) { 1111 DRM_ERROR("waiting on invalid fence: %u (of %u)\n", 1112 fence, gpu->next_fence); 1113 return -EINVAL; 1114 } 1115 1116 if (!timeout) { 1117 /* No timeout was requested: just test for completion */ 1118 ret = fence_completed(gpu, fence) ? 0 : -EBUSY; 1119 } else { 1120 unsigned long remaining = etnaviv_timeout_to_jiffies(timeout); 1121 1122 ret = wait_event_interruptible_timeout(gpu->fence_event, 1123 fence_completed(gpu, fence), 1124 remaining); 1125 if (ret == 0) { 1126 DBG("timeout waiting for fence: %u (retired: %u completed: %u)", 1127 fence, gpu->retired_fence, 1128 gpu->completed_fence); 1129 ret = -ETIMEDOUT; 1130 } else if (ret != -ERESTARTSYS) { 1131 ret = 0; 1132 } 1133 } 1134 1135 return ret; 1136 } 1137 1138 /* 1139 * Wait for an object to become inactive. This, on it's own, is not race 1140 * free: the object is moved by the retire worker off the active list, and 1141 * then the iova is put. Moreover, the object could be re-submitted just 1142 * after we notice that it's become inactive. 1143 * 1144 * Although the retirement happens under the gpu lock, we don't want to hold 1145 * that lock in this function while waiting. 1146 */ 1147 int etnaviv_gpu_wait_obj_inactive(struct etnaviv_gpu *gpu, 1148 struct etnaviv_gem_object *etnaviv_obj, struct timespec *timeout) 1149 { 1150 unsigned long remaining; 1151 long ret; 1152 1153 if (!timeout) 1154 return !is_active(etnaviv_obj) ? 0 : -EBUSY; 1155 1156 remaining = etnaviv_timeout_to_jiffies(timeout); 1157 1158 ret = wait_event_interruptible_timeout(gpu->fence_event, 1159 !is_active(etnaviv_obj), 1160 remaining); 1161 if (ret > 0) { 1162 struct etnaviv_drm_private *priv = gpu->drm->dev_private; 1163 1164 /* Synchronise with the retire worker */ 1165 flush_workqueue(priv->wq); 1166 return 0; 1167 } else if (ret == -ERESTARTSYS) { 1168 return -ERESTARTSYS; 1169 } else { 1170 return -ETIMEDOUT; 1171 } 1172 } 1173 1174 int etnaviv_gpu_pm_get_sync(struct etnaviv_gpu *gpu) 1175 { 1176 return pm_runtime_get_sync(gpu->dev); 1177 } 1178 1179 void etnaviv_gpu_pm_put(struct etnaviv_gpu *gpu) 1180 { 1181 pm_runtime_mark_last_busy(gpu->dev); 1182 pm_runtime_put_autosuspend(gpu->dev); 1183 } 1184 1185 /* add bo's to gpu's ring, and kick gpu: */ 1186 int etnaviv_gpu_submit(struct etnaviv_gpu *gpu, 1187 struct etnaviv_gem_submit *submit, struct etnaviv_cmdbuf *cmdbuf) 1188 { 1189 struct fence *fence; 1190 unsigned int event, i; 1191 int ret; 1192 1193 ret = etnaviv_gpu_pm_get_sync(gpu); 1194 if (ret < 0) 1195 return ret; 1196 1197 mutex_lock(&gpu->lock); 1198 1199 /* 1200 * TODO 1201 * 1202 * - flush 1203 * - data endian 1204 * - prefetch 1205 * 1206 */ 1207 1208 event = event_alloc(gpu); 1209 if (unlikely(event == ~0U)) { 1210 DRM_ERROR("no free event\n"); 1211 ret = -EBUSY; 1212 goto out_unlock; 1213 } 1214 1215 fence = etnaviv_gpu_fence_alloc(gpu); 1216 if (!fence) { 1217 event_free(gpu, event); 1218 ret = -ENOMEM; 1219 goto out_unlock; 1220 } 1221 1222 gpu->event[event].fence = fence; 1223 submit->fence = fence->seqno; 1224 gpu->active_fence = submit->fence; 1225 1226 if (gpu->lastctx != cmdbuf->ctx) { 1227 gpu->mmu->need_flush = true; 1228 gpu->switch_context = true; 1229 gpu->lastctx = cmdbuf->ctx; 1230 } 1231 1232 etnaviv_buffer_queue(gpu, event, cmdbuf); 1233 1234 cmdbuf->fence = fence; 1235 list_add_tail(&cmdbuf->node, &gpu->active_cmd_list); 1236 1237 /* We're committed to adding this command buffer, hold a PM reference */ 1238 pm_runtime_get_noresume(gpu->dev); 1239 1240 for (i = 0; i < submit->nr_bos; i++) { 1241 struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; 1242 u32 iova; 1243 1244 /* Each cmdbuf takes a refcount on the iova */ 1245 etnaviv_gem_get_iova(gpu, &etnaviv_obj->base, &iova); 1246 cmdbuf->bo[i] = etnaviv_obj; 1247 atomic_inc(&etnaviv_obj->gpu_active); 1248 1249 if (submit->bos[i].flags & ETNA_SUBMIT_BO_WRITE) 1250 reservation_object_add_excl_fence(etnaviv_obj->resv, 1251 fence); 1252 else 1253 reservation_object_add_shared_fence(etnaviv_obj->resv, 1254 fence); 1255 } 1256 cmdbuf->nr_bos = submit->nr_bos; 1257 hangcheck_timer_reset(gpu); 1258 ret = 0; 1259 1260 out_unlock: 1261 mutex_unlock(&gpu->lock); 1262 1263 etnaviv_gpu_pm_put(gpu); 1264 1265 return ret; 1266 } 1267 1268 /* 1269 * Init/Cleanup: 1270 */ 1271 static irqreturn_t irq_handler(int irq, void *data) 1272 { 1273 struct etnaviv_gpu *gpu = data; 1274 irqreturn_t ret = IRQ_NONE; 1275 1276 u32 intr = gpu_read(gpu, VIVS_HI_INTR_ACKNOWLEDGE); 1277 1278 if (intr != 0) { 1279 int event; 1280 1281 pm_runtime_mark_last_busy(gpu->dev); 1282 1283 dev_dbg(gpu->dev, "intr 0x%08x\n", intr); 1284 1285 if (intr & VIVS_HI_INTR_ACKNOWLEDGE_AXI_BUS_ERROR) { 1286 dev_err(gpu->dev, "AXI bus error\n"); 1287 intr &= ~VIVS_HI_INTR_ACKNOWLEDGE_AXI_BUS_ERROR; 1288 } 1289 1290 while ((event = ffs(intr)) != 0) { 1291 struct fence *fence; 1292 1293 event -= 1; 1294 1295 intr &= ~(1 << event); 1296 1297 dev_dbg(gpu->dev, "event %u\n", event); 1298 1299 fence = gpu->event[event].fence; 1300 gpu->event[event].fence = NULL; 1301 fence_signal(fence); 1302 1303 /* 1304 * Events can be processed out of order. Eg, 1305 * - allocate and queue event 0 1306 * - allocate event 1 1307 * - event 0 completes, we process it 1308 * - allocate and queue event 0 1309 * - event 1 and event 0 complete 1310 * we can end up processing event 0 first, then 1. 1311 */ 1312 if (fence_after(fence->seqno, gpu->completed_fence)) 1313 gpu->completed_fence = fence->seqno; 1314 1315 event_free(gpu, event); 1316 1317 /* 1318 * We need to balance the runtime PM count caused by 1319 * each submission. Upon submission, we increment 1320 * the runtime PM counter, and allocate one event. 1321 * So here, we put the runtime PM count for each 1322 * completed event. 1323 */ 1324 pm_runtime_put_autosuspend(gpu->dev); 1325 } 1326 1327 /* Retire the buffer objects in a work */ 1328 etnaviv_queue_work(gpu->drm, &gpu->retire_work); 1329 1330 ret = IRQ_HANDLED; 1331 } 1332 1333 return ret; 1334 } 1335 1336 static int etnaviv_gpu_clk_enable(struct etnaviv_gpu *gpu) 1337 { 1338 int ret; 1339 1340 ret = enable_clk(gpu); 1341 if (ret) 1342 return ret; 1343 1344 ret = enable_axi(gpu); 1345 if (ret) { 1346 disable_clk(gpu); 1347 return ret; 1348 } 1349 1350 return 0; 1351 } 1352 1353 static int etnaviv_gpu_clk_disable(struct etnaviv_gpu *gpu) 1354 { 1355 int ret; 1356 1357 ret = disable_axi(gpu); 1358 if (ret) 1359 return ret; 1360 1361 ret = disable_clk(gpu); 1362 if (ret) 1363 return ret; 1364 1365 return 0; 1366 } 1367 1368 static int etnaviv_gpu_hw_suspend(struct etnaviv_gpu *gpu) 1369 { 1370 if (gpu->buffer) { 1371 unsigned long timeout; 1372 1373 /* Replace the last WAIT with END */ 1374 etnaviv_buffer_end(gpu); 1375 1376 /* 1377 * We know that only the FE is busy here, this should 1378 * happen quickly (as the WAIT is only 200 cycles). If 1379 * we fail, just warn and continue. 1380 */ 1381 timeout = jiffies + msecs_to_jiffies(100); 1382 do { 1383 u32 idle = gpu_read(gpu, VIVS_HI_IDLE_STATE); 1384 1385 if ((idle & gpu->idle_mask) == gpu->idle_mask) 1386 break; 1387 1388 if (time_is_before_jiffies(timeout)) { 1389 dev_warn(gpu->dev, 1390 "timed out waiting for idle: idle=0x%x\n", 1391 idle); 1392 break; 1393 } 1394 1395 udelay(5); 1396 } while (1); 1397 } 1398 1399 return etnaviv_gpu_clk_disable(gpu); 1400 } 1401 1402 #ifdef CONFIG_PM 1403 static int etnaviv_gpu_hw_resume(struct etnaviv_gpu *gpu) 1404 { 1405 u32 clock; 1406 int ret; 1407 1408 ret = mutex_lock_killable(&gpu->lock); 1409 if (ret) 1410 return ret; 1411 1412 clock = VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS | 1413 VIVS_HI_CLOCK_CONTROL_FSCALE_VAL(0x40); 1414 1415 etnaviv_gpu_load_clock(gpu, clock); 1416 etnaviv_gpu_hw_init(gpu); 1417 1418 gpu->switch_context = true; 1419 1420 mutex_unlock(&gpu->lock); 1421 1422 return 0; 1423 } 1424 #endif 1425 1426 static int etnaviv_gpu_bind(struct device *dev, struct device *master, 1427 void *data) 1428 { 1429 struct drm_device *drm = data; 1430 struct etnaviv_drm_private *priv = drm->dev_private; 1431 struct etnaviv_gpu *gpu = dev_get_drvdata(dev); 1432 int ret; 1433 1434 #ifdef CONFIG_PM 1435 ret = pm_runtime_get_sync(gpu->dev); 1436 #else 1437 ret = etnaviv_gpu_clk_enable(gpu); 1438 #endif 1439 if (ret < 0) 1440 return ret; 1441 1442 gpu->drm = drm; 1443 gpu->fence_context = fence_context_alloc(1); 1444 spin_lock_init(&gpu->fence_spinlock); 1445 1446 INIT_LIST_HEAD(&gpu->active_cmd_list); 1447 INIT_WORK(&gpu->retire_work, retire_worker); 1448 INIT_WORK(&gpu->recover_work, recover_worker); 1449 init_waitqueue_head(&gpu->fence_event); 1450 1451 setup_timer(&gpu->hangcheck_timer, hangcheck_handler, 1452 (unsigned long)gpu); 1453 1454 priv->gpu[priv->num_gpus++] = gpu; 1455 1456 pm_runtime_mark_last_busy(gpu->dev); 1457 pm_runtime_put_autosuspend(gpu->dev); 1458 1459 return 0; 1460 } 1461 1462 static void etnaviv_gpu_unbind(struct device *dev, struct device *master, 1463 void *data) 1464 { 1465 struct etnaviv_gpu *gpu = dev_get_drvdata(dev); 1466 1467 DBG("%s", dev_name(gpu->dev)); 1468 1469 hangcheck_disable(gpu); 1470 1471 #ifdef CONFIG_PM 1472 pm_runtime_get_sync(gpu->dev); 1473 pm_runtime_put_sync_suspend(gpu->dev); 1474 #else 1475 etnaviv_gpu_hw_suspend(gpu); 1476 #endif 1477 1478 if (gpu->buffer) { 1479 etnaviv_gpu_cmdbuf_free(gpu->buffer); 1480 gpu->buffer = NULL; 1481 } 1482 1483 if (gpu->mmu) { 1484 etnaviv_iommu_destroy(gpu->mmu); 1485 gpu->mmu = NULL; 1486 } 1487 1488 gpu->drm = NULL; 1489 } 1490 1491 static const struct component_ops gpu_ops = { 1492 .bind = etnaviv_gpu_bind, 1493 .unbind = etnaviv_gpu_unbind, 1494 }; 1495 1496 static const struct of_device_id etnaviv_gpu_match[] = { 1497 { 1498 .compatible = "vivante,gc" 1499 }, 1500 { /* sentinel */ } 1501 }; 1502 1503 static int etnaviv_gpu_platform_probe(struct platform_device *pdev) 1504 { 1505 struct device *dev = &pdev->dev; 1506 struct etnaviv_gpu *gpu; 1507 int err = 0; 1508 1509 gpu = devm_kzalloc(dev, sizeof(*gpu), GFP_KERNEL); 1510 if (!gpu) 1511 return -ENOMEM; 1512 1513 gpu->dev = &pdev->dev; 1514 mutex_init(&gpu->lock); 1515 1516 /* 1517 * Set the GPU base address to the start of physical memory. This 1518 * ensures that if we have up to 2GB, the v1 MMU can address the 1519 * highest memory. This is important as command buffers may be 1520 * allocated outside of this limit. 1521 */ 1522 gpu->memory_base = PHYS_OFFSET; 1523 1524 /* Map registers: */ 1525 gpu->mmio = etnaviv_ioremap(pdev, NULL, dev_name(gpu->dev)); 1526 if (IS_ERR(gpu->mmio)) 1527 return PTR_ERR(gpu->mmio); 1528 1529 /* Get Interrupt: */ 1530 gpu->irq = platform_get_irq(pdev, 0); 1531 if (gpu->irq < 0) { 1532 err = gpu->irq; 1533 dev_err(dev, "failed to get irq: %d\n", err); 1534 goto fail; 1535 } 1536 1537 err = devm_request_irq(&pdev->dev, gpu->irq, irq_handler, 0, 1538 dev_name(gpu->dev), gpu); 1539 if (err) { 1540 dev_err(dev, "failed to request IRQ%u: %d\n", gpu->irq, err); 1541 goto fail; 1542 } 1543 1544 /* Get Clocks: */ 1545 gpu->clk_bus = devm_clk_get(&pdev->dev, "bus"); 1546 DBG("clk_bus: %p", gpu->clk_bus); 1547 if (IS_ERR(gpu->clk_bus)) 1548 gpu->clk_bus = NULL; 1549 1550 gpu->clk_core = devm_clk_get(&pdev->dev, "core"); 1551 DBG("clk_core: %p", gpu->clk_core); 1552 if (IS_ERR(gpu->clk_core)) 1553 gpu->clk_core = NULL; 1554 1555 gpu->clk_shader = devm_clk_get(&pdev->dev, "shader"); 1556 DBG("clk_shader: %p", gpu->clk_shader); 1557 if (IS_ERR(gpu->clk_shader)) 1558 gpu->clk_shader = NULL; 1559 1560 /* TODO: figure out max mapped size */ 1561 dev_set_drvdata(dev, gpu); 1562 1563 /* 1564 * We treat the device as initially suspended. The runtime PM 1565 * autosuspend delay is rather arbitary: no measurements have 1566 * yet been performed to determine an appropriate value. 1567 */ 1568 pm_runtime_use_autosuspend(gpu->dev); 1569 pm_runtime_set_autosuspend_delay(gpu->dev, 200); 1570 pm_runtime_enable(gpu->dev); 1571 1572 err = component_add(&pdev->dev, &gpu_ops); 1573 if (err < 0) { 1574 dev_err(&pdev->dev, "failed to register component: %d\n", err); 1575 goto fail; 1576 } 1577 1578 return 0; 1579 1580 fail: 1581 return err; 1582 } 1583 1584 static int etnaviv_gpu_platform_remove(struct platform_device *pdev) 1585 { 1586 component_del(&pdev->dev, &gpu_ops); 1587 pm_runtime_disable(&pdev->dev); 1588 return 0; 1589 } 1590 1591 #ifdef CONFIG_PM 1592 static int etnaviv_gpu_rpm_suspend(struct device *dev) 1593 { 1594 struct etnaviv_gpu *gpu = dev_get_drvdata(dev); 1595 u32 idle, mask; 1596 1597 /* If we have outstanding fences, we're not idle */ 1598 if (gpu->completed_fence != gpu->active_fence) 1599 return -EBUSY; 1600 1601 /* Check whether the hardware (except FE) is idle */ 1602 mask = gpu->idle_mask & ~VIVS_HI_IDLE_STATE_FE; 1603 idle = gpu_read(gpu, VIVS_HI_IDLE_STATE) & mask; 1604 if (idle != mask) 1605 return -EBUSY; 1606 1607 return etnaviv_gpu_hw_suspend(gpu); 1608 } 1609 1610 static int etnaviv_gpu_rpm_resume(struct device *dev) 1611 { 1612 struct etnaviv_gpu *gpu = dev_get_drvdata(dev); 1613 int ret; 1614 1615 ret = etnaviv_gpu_clk_enable(gpu); 1616 if (ret) 1617 return ret; 1618 1619 /* Re-initialise the basic hardware state */ 1620 if (gpu->drm && gpu->buffer) { 1621 ret = etnaviv_gpu_hw_resume(gpu); 1622 if (ret) { 1623 etnaviv_gpu_clk_disable(gpu); 1624 return ret; 1625 } 1626 } 1627 1628 return 0; 1629 } 1630 #endif 1631 1632 static const struct dev_pm_ops etnaviv_gpu_pm_ops = { 1633 SET_RUNTIME_PM_OPS(etnaviv_gpu_rpm_suspend, etnaviv_gpu_rpm_resume, 1634 NULL) 1635 }; 1636 1637 struct platform_driver etnaviv_gpu_driver = { 1638 .driver = { 1639 .name = "etnaviv-gpu", 1640 .owner = THIS_MODULE, 1641 .pm = &etnaviv_gpu_pm_ops, 1642 .of_match_table = etnaviv_gpu_match, 1643 }, 1644 .probe = etnaviv_gpu_platform_probe, 1645 .remove = etnaviv_gpu_platform_remove, 1646 .id_table = gpu_ids, 1647 }; 1648