1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2017-2018 The Linux Foundation. All rights reserved. */ 3 4 5 #include "msm_gem.h" 6 #include "msm_mmu.h" 7 #include "a6xx_gpu.h" 8 #include "a6xx_gmu.xml.h" 9 10 #include <linux/devfreq.h> 11 12 static inline bool _a6xx_check_idle(struct msm_gpu *gpu) 13 { 14 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 15 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 16 17 /* Check that the GMU is idle */ 18 if (!a6xx_gmu_isidle(&a6xx_gpu->gmu)) 19 return false; 20 21 /* Check tha the CX master is idle */ 22 if (gpu_read(gpu, REG_A6XX_RBBM_STATUS) & 23 ~A6XX_RBBM_STATUS_CP_AHB_BUSY_CX_MASTER) 24 return false; 25 26 return !(gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS) & 27 A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT); 28 } 29 30 bool a6xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring) 31 { 32 /* wait for CP to drain ringbuffer: */ 33 if (!adreno_idle(gpu, ring)) 34 return false; 35 36 if (spin_until(_a6xx_check_idle(gpu))) { 37 DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n", 38 gpu->name, __builtin_return_address(0), 39 gpu_read(gpu, REG_A6XX_RBBM_STATUS), 40 gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS), 41 gpu_read(gpu, REG_A6XX_CP_RB_RPTR), 42 gpu_read(gpu, REG_A6XX_CP_RB_WPTR)); 43 return false; 44 } 45 46 return true; 47 } 48 49 static void a6xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring) 50 { 51 uint32_t wptr; 52 unsigned long flags; 53 54 spin_lock_irqsave(&ring->lock, flags); 55 56 /* Copy the shadow to the actual register */ 57 ring->cur = ring->next; 58 59 /* Make sure to wrap wptr if we need to */ 60 wptr = get_wptr(ring); 61 62 spin_unlock_irqrestore(&ring->lock, flags); 63 64 /* Make sure everything is posted before making a decision */ 65 mb(); 66 67 gpu_write(gpu, REG_A6XX_CP_RB_WPTR, wptr); 68 } 69 70 static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit, 71 struct msm_file_private *ctx) 72 { 73 struct msm_drm_private *priv = gpu->dev->dev_private; 74 struct msm_ringbuffer *ring = submit->ring; 75 unsigned int i; 76 77 /* Invalidate CCU depth and color */ 78 OUT_PKT7(ring, CP_EVENT_WRITE, 1); 79 OUT_RING(ring, PC_CCU_INVALIDATE_DEPTH); 80 81 OUT_PKT7(ring, CP_EVENT_WRITE, 1); 82 OUT_RING(ring, PC_CCU_INVALIDATE_COLOR); 83 84 /* Submit the commands */ 85 for (i = 0; i < submit->nr_cmds; i++) { 86 switch (submit->cmd[i].type) { 87 case MSM_SUBMIT_CMD_IB_TARGET_BUF: 88 break; 89 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF: 90 if (priv->lastctx == ctx) 91 break; 92 case MSM_SUBMIT_CMD_BUF: 93 OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3); 94 OUT_RING(ring, lower_32_bits(submit->cmd[i].iova)); 95 OUT_RING(ring, upper_32_bits(submit->cmd[i].iova)); 96 OUT_RING(ring, submit->cmd[i].size); 97 break; 98 } 99 } 100 101 /* Write the fence to the scratch register */ 102 OUT_PKT4(ring, REG_A6XX_CP_SCRATCH_REG(2), 1); 103 OUT_RING(ring, submit->seqno); 104 105 /* 106 * Execute a CACHE_FLUSH_TS event. This will ensure that the 107 * timestamp is written to the memory and then triggers the interrupt 108 */ 109 OUT_PKT7(ring, CP_EVENT_WRITE, 4); 110 OUT_RING(ring, CACHE_FLUSH_TS | (1 << 31)); 111 OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence))); 112 OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence))); 113 OUT_RING(ring, submit->seqno); 114 115 a6xx_flush(gpu, ring); 116 } 117 118 static const struct { 119 u32 offset; 120 u32 value; 121 } a6xx_hwcg[] = { 122 {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x22222222}, 123 {REG_A6XX_RBBM_CLOCK_CNTL_SP1, 0x22222222}, 124 {REG_A6XX_RBBM_CLOCK_CNTL_SP2, 0x22222222}, 125 {REG_A6XX_RBBM_CLOCK_CNTL_SP3, 0x22222222}, 126 {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02022220}, 127 {REG_A6XX_RBBM_CLOCK_CNTL2_SP1, 0x02022220}, 128 {REG_A6XX_RBBM_CLOCK_CNTL2_SP2, 0x02022220}, 129 {REG_A6XX_RBBM_CLOCK_CNTL2_SP3, 0x02022220}, 130 {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, 131 {REG_A6XX_RBBM_CLOCK_DELAY_SP1, 0x00000080}, 132 {REG_A6XX_RBBM_CLOCK_DELAY_SP2, 0x00000080}, 133 {REG_A6XX_RBBM_CLOCK_DELAY_SP3, 0x00000080}, 134 {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf}, 135 {REG_A6XX_RBBM_CLOCK_HYST_SP1, 0x0000f3cf}, 136 {REG_A6XX_RBBM_CLOCK_HYST_SP2, 0x0000f3cf}, 137 {REG_A6XX_RBBM_CLOCK_HYST_SP3, 0x0000f3cf}, 138 {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222}, 139 {REG_A6XX_RBBM_CLOCK_CNTL_TP1, 0x02222222}, 140 {REG_A6XX_RBBM_CLOCK_CNTL_TP2, 0x02222222}, 141 {REG_A6XX_RBBM_CLOCK_CNTL_TP3, 0x02222222}, 142 {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, 143 {REG_A6XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, 144 {REG_A6XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222}, 145 {REG_A6XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222}, 146 {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, 147 {REG_A6XX_RBBM_CLOCK_CNTL3_TP1, 0x22222222}, 148 {REG_A6XX_RBBM_CLOCK_CNTL3_TP2, 0x22222222}, 149 {REG_A6XX_RBBM_CLOCK_CNTL3_TP3, 0x22222222}, 150 {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, 151 {REG_A6XX_RBBM_CLOCK_CNTL4_TP1, 0x00022222}, 152 {REG_A6XX_RBBM_CLOCK_CNTL4_TP2, 0x00022222}, 153 {REG_A6XX_RBBM_CLOCK_CNTL4_TP3, 0x00022222}, 154 {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, 155 {REG_A6XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, 156 {REG_A6XX_RBBM_CLOCK_HYST_TP2, 0x77777777}, 157 {REG_A6XX_RBBM_CLOCK_HYST_TP3, 0x77777777}, 158 {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, 159 {REG_A6XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, 160 {REG_A6XX_RBBM_CLOCK_HYST2_TP2, 0x77777777}, 161 {REG_A6XX_RBBM_CLOCK_HYST2_TP3, 0x77777777}, 162 {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, 163 {REG_A6XX_RBBM_CLOCK_HYST3_TP1, 0x77777777}, 164 {REG_A6XX_RBBM_CLOCK_HYST3_TP2, 0x77777777}, 165 {REG_A6XX_RBBM_CLOCK_HYST3_TP3, 0x77777777}, 166 {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, 167 {REG_A6XX_RBBM_CLOCK_HYST4_TP1, 0x00077777}, 168 {REG_A6XX_RBBM_CLOCK_HYST4_TP2, 0x00077777}, 169 {REG_A6XX_RBBM_CLOCK_HYST4_TP3, 0x00077777}, 170 {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, 171 {REG_A6XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, 172 {REG_A6XX_RBBM_CLOCK_DELAY_TP2, 0x11111111}, 173 {REG_A6XX_RBBM_CLOCK_DELAY_TP3, 0x11111111}, 174 {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, 175 {REG_A6XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, 176 {REG_A6XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111}, 177 {REG_A6XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111}, 178 {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, 179 {REG_A6XX_RBBM_CLOCK_DELAY3_TP1, 0x11111111}, 180 {REG_A6XX_RBBM_CLOCK_DELAY3_TP2, 0x11111111}, 181 {REG_A6XX_RBBM_CLOCK_DELAY3_TP3, 0x11111111}, 182 {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, 183 {REG_A6XX_RBBM_CLOCK_DELAY4_TP1, 0x00011111}, 184 {REG_A6XX_RBBM_CLOCK_DELAY4_TP2, 0x00011111}, 185 {REG_A6XX_RBBM_CLOCK_DELAY4_TP3, 0x00011111}, 186 {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, 187 {REG_A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, 188 {REG_A6XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, 189 {REG_A6XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, 190 {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, 191 {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, 192 {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, 193 {REG_A6XX_RBBM_CLOCK_CNTL_RB1, 0x22222222}, 194 {REG_A6XX_RBBM_CLOCK_CNTL_RB2, 0x22222222}, 195 {REG_A6XX_RBBM_CLOCK_CNTL_RB3, 0x22222222}, 196 {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x00002222}, 197 {REG_A6XX_RBBM_CLOCK_CNTL2_RB1, 0x00002222}, 198 {REG_A6XX_RBBM_CLOCK_CNTL2_RB2, 0x00002222}, 199 {REG_A6XX_RBBM_CLOCK_CNTL2_RB3, 0x00002222}, 200 {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, 201 {REG_A6XX_RBBM_CLOCK_CNTL_CCU1, 0x00002220}, 202 {REG_A6XX_RBBM_CLOCK_CNTL_CCU2, 0x00002220}, 203 {REG_A6XX_RBBM_CLOCK_CNTL_CCU3, 0x00002220}, 204 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040f00}, 205 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU1, 0x00040f00}, 206 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU2, 0x00040f00}, 207 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU3, 0x00040f00}, 208 {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05022022}, 209 {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, 210 {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, 211 {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, 212 {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, 213 {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222}, 214 {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, 215 {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, 216 {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, 217 {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, 218 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, 219 {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, 220 {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, 221 {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, 222 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, 223 {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, 224 {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, 225 {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, 226 {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555} 227 }; 228 229 static void a6xx_set_hwcg(struct msm_gpu *gpu, bool state) 230 { 231 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 232 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 233 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; 234 unsigned int i; 235 u32 val; 236 237 val = gpu_read(gpu, REG_A6XX_RBBM_CLOCK_CNTL); 238 239 /* Don't re-program the registers if they are already correct */ 240 if ((!state && !val) || (state && (val == 0x8aa8aa02))) 241 return; 242 243 /* Disable SP clock before programming HWCG registers */ 244 gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 1, 0); 245 246 for (i = 0; i < ARRAY_SIZE(a6xx_hwcg); i++) 247 gpu_write(gpu, a6xx_hwcg[i].offset, 248 state ? a6xx_hwcg[i].value : 0); 249 250 /* Enable SP clock */ 251 gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 0, 1); 252 253 gpu_write(gpu, REG_A6XX_RBBM_CLOCK_CNTL, state ? 0x8aa8aa02 : 0); 254 } 255 256 static int a6xx_cp_init(struct msm_gpu *gpu) 257 { 258 struct msm_ringbuffer *ring = gpu->rb[0]; 259 260 OUT_PKT7(ring, CP_ME_INIT, 8); 261 262 OUT_RING(ring, 0x0000002f); 263 264 /* Enable multiple hardware contexts */ 265 OUT_RING(ring, 0x00000003); 266 267 /* Enable error detection */ 268 OUT_RING(ring, 0x20000000); 269 270 /* Don't enable header dump */ 271 OUT_RING(ring, 0x00000000); 272 OUT_RING(ring, 0x00000000); 273 274 /* No workarounds enabled */ 275 OUT_RING(ring, 0x00000000); 276 277 /* Pad rest of the cmds with 0's */ 278 OUT_RING(ring, 0x00000000); 279 OUT_RING(ring, 0x00000000); 280 281 a6xx_flush(gpu, ring); 282 return a6xx_idle(gpu, ring) ? 0 : -EINVAL; 283 } 284 285 static int a6xx_ucode_init(struct msm_gpu *gpu) 286 { 287 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 288 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 289 290 if (!a6xx_gpu->sqe_bo) { 291 a6xx_gpu->sqe_bo = adreno_fw_create_bo(gpu, 292 adreno_gpu->fw[ADRENO_FW_SQE], &a6xx_gpu->sqe_iova); 293 294 if (IS_ERR(a6xx_gpu->sqe_bo)) { 295 int ret = PTR_ERR(a6xx_gpu->sqe_bo); 296 297 a6xx_gpu->sqe_bo = NULL; 298 DRM_DEV_ERROR(&gpu->pdev->dev, 299 "Could not allocate SQE ucode: %d\n", ret); 300 301 return ret; 302 } 303 } 304 305 gpu_write64(gpu, REG_A6XX_CP_SQE_INSTR_BASE_LO, 306 REG_A6XX_CP_SQE_INSTR_BASE_HI, a6xx_gpu->sqe_iova); 307 308 return 0; 309 } 310 311 #define A6XX_INT_MASK (A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR | \ 312 A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW | \ 313 A6XX_RBBM_INT_0_MASK_CP_HW_ERROR | \ 314 A6XX_RBBM_INT_0_MASK_CP_IB2 | \ 315 A6XX_RBBM_INT_0_MASK_CP_IB1 | \ 316 A6XX_RBBM_INT_0_MASK_CP_RB | \ 317 A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \ 318 A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW | \ 319 A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT | \ 320 A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \ 321 A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR) 322 323 static int a6xx_hw_init(struct msm_gpu *gpu) 324 { 325 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 326 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 327 int ret; 328 329 /* Make sure the GMU keeps the GPU on while we set it up */ 330 a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); 331 332 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_CNTL, 0); 333 334 /* 335 * Disable the trusted memory range - we don't actually supported secure 336 * memory rendering at this point in time and we don't want to block off 337 * part of the virtual memory space. 338 */ 339 gpu_write64(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO, 340 REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI, 0x00000000); 341 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000); 342 343 /* enable hardware clockgating */ 344 a6xx_set_hwcg(gpu, true); 345 346 /* VBIF start */ 347 gpu_write(gpu, REG_A6XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009); 348 gpu_write(gpu, REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x3); 349 350 /* Make all blocks contribute to the GPU BUSY perf counter */ 351 gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xffffffff); 352 353 /* Disable L2 bypass in the UCHE */ 354 gpu_write(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX_LO, 0xffffffc0); 355 gpu_write(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX_HI, 0x0001ffff); 356 gpu_write(gpu, REG_A6XX_UCHE_TRAP_BASE_LO, 0xfffff000); 357 gpu_write(gpu, REG_A6XX_UCHE_TRAP_BASE_HI, 0x0001ffff); 358 gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_LO, 0xfffff000); 359 gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_HI, 0x0001ffff); 360 361 /* Set the GMEM VA range [0x100000:0x100000 + gpu->gmem - 1] */ 362 gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MIN_LO, 363 REG_A6XX_UCHE_GMEM_RANGE_MIN_HI, 0x00100000); 364 365 gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MAX_LO, 366 REG_A6XX_UCHE_GMEM_RANGE_MAX_HI, 367 0x00100000 + adreno_gpu->gmem - 1); 368 369 gpu_write(gpu, REG_A6XX_UCHE_FILTER_CNTL, 0x804); 370 gpu_write(gpu, REG_A6XX_UCHE_CACHE_WAYS, 0x4); 371 372 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x010000c0); 373 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362c); 374 375 /* Setting the mem pool size */ 376 gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 128); 377 378 /* Setting the primFifo thresholds default values */ 379 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, (0x300 << 11)); 380 381 /* Set the AHB default slave response to "ERROR" */ 382 gpu_write(gpu, REG_A6XX_CP_AHB_CNTL, 0x1); 383 384 /* Turn on performance counters */ 385 gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_CNTL, 0x1); 386 387 /* Select CP0 to always count cycles */ 388 gpu_write(gpu, REG_A6XX_CP_PERFCTR_CP_SEL_0, PERF_CP_ALWAYS_COUNT); 389 390 /* FIXME: not sure if this should live here or in a6xx_gmu.c */ 391 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_MASK, 392 0xff000000); 393 gmu_rmw(&a6xx_gpu->gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0, 394 0xff, 0x20); 395 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 396 0x01); 397 398 gpu_write(gpu, REG_A6XX_RB_NC_MODE_CNTL, 2 << 1); 399 gpu_write(gpu, REG_A6XX_TPL1_NC_MODE_CNTL, 2 << 1); 400 gpu_write(gpu, REG_A6XX_SP_NC_MODE_CNTL, 2 << 1); 401 gpu_write(gpu, REG_A6XX_UCHE_MODE_CNTL, 2 << 21); 402 403 /* Enable fault detection */ 404 gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, 405 (1 << 30) | 0x1fffff); 406 407 gpu_write(gpu, REG_A6XX_UCHE_CLIENT_PF, 1); 408 409 /* Protect registers from the CP */ 410 gpu_write(gpu, REG_A6XX_CP_PROTECT_CNTL, 0x00000003); 411 412 gpu_write(gpu, REG_A6XX_CP_PROTECT(0), 413 A6XX_PROTECT_RDONLY(0x600, 0x51)); 414 gpu_write(gpu, REG_A6XX_CP_PROTECT(1), A6XX_PROTECT_RW(0xae50, 0x2)); 415 gpu_write(gpu, REG_A6XX_CP_PROTECT(2), A6XX_PROTECT_RW(0x9624, 0x13)); 416 gpu_write(gpu, REG_A6XX_CP_PROTECT(3), A6XX_PROTECT_RW(0x8630, 0x8)); 417 gpu_write(gpu, REG_A6XX_CP_PROTECT(4), A6XX_PROTECT_RW(0x9e70, 0x1)); 418 gpu_write(gpu, REG_A6XX_CP_PROTECT(5), A6XX_PROTECT_RW(0x9e78, 0x187)); 419 gpu_write(gpu, REG_A6XX_CP_PROTECT(6), A6XX_PROTECT_RW(0xf000, 0x810)); 420 gpu_write(gpu, REG_A6XX_CP_PROTECT(7), 421 A6XX_PROTECT_RDONLY(0xfc00, 0x3)); 422 gpu_write(gpu, REG_A6XX_CP_PROTECT(8), A6XX_PROTECT_RW(0x50e, 0x0)); 423 gpu_write(gpu, REG_A6XX_CP_PROTECT(9), A6XX_PROTECT_RDONLY(0x50f, 0x0)); 424 gpu_write(gpu, REG_A6XX_CP_PROTECT(10), A6XX_PROTECT_RW(0x510, 0x0)); 425 gpu_write(gpu, REG_A6XX_CP_PROTECT(11), 426 A6XX_PROTECT_RDONLY(0x0, 0x4f9)); 427 gpu_write(gpu, REG_A6XX_CP_PROTECT(12), 428 A6XX_PROTECT_RDONLY(0x501, 0xa)); 429 gpu_write(gpu, REG_A6XX_CP_PROTECT(13), 430 A6XX_PROTECT_RDONLY(0x511, 0x44)); 431 gpu_write(gpu, REG_A6XX_CP_PROTECT(14), A6XX_PROTECT_RW(0xe00, 0xe)); 432 gpu_write(gpu, REG_A6XX_CP_PROTECT(15), A6XX_PROTECT_RW(0x8e00, 0x0)); 433 gpu_write(gpu, REG_A6XX_CP_PROTECT(16), A6XX_PROTECT_RW(0x8e50, 0xf)); 434 gpu_write(gpu, REG_A6XX_CP_PROTECT(17), A6XX_PROTECT_RW(0xbe02, 0x0)); 435 gpu_write(gpu, REG_A6XX_CP_PROTECT(18), 436 A6XX_PROTECT_RW(0xbe20, 0x11f3)); 437 gpu_write(gpu, REG_A6XX_CP_PROTECT(19), A6XX_PROTECT_RW(0x800, 0x82)); 438 gpu_write(gpu, REG_A6XX_CP_PROTECT(20), A6XX_PROTECT_RW(0x8a0, 0x8)); 439 gpu_write(gpu, REG_A6XX_CP_PROTECT(21), A6XX_PROTECT_RW(0x8ab, 0x19)); 440 gpu_write(gpu, REG_A6XX_CP_PROTECT(22), A6XX_PROTECT_RW(0x900, 0x4d)); 441 gpu_write(gpu, REG_A6XX_CP_PROTECT(23), A6XX_PROTECT_RW(0x98d, 0x76)); 442 gpu_write(gpu, REG_A6XX_CP_PROTECT(24), 443 A6XX_PROTECT_RDONLY(0x980, 0x4)); 444 gpu_write(gpu, REG_A6XX_CP_PROTECT(25), A6XX_PROTECT_RW(0xa630, 0x0)); 445 446 /* Enable interrupts */ 447 gpu_write(gpu, REG_A6XX_RBBM_INT_0_MASK, A6XX_INT_MASK); 448 449 ret = adreno_hw_init(gpu); 450 if (ret) 451 goto out; 452 453 ret = a6xx_ucode_init(gpu); 454 if (ret) 455 goto out; 456 457 /* Always come up on rb 0 */ 458 a6xx_gpu->cur_ring = gpu->rb[0]; 459 460 /* Enable the SQE_to start the CP engine */ 461 gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 1); 462 463 ret = a6xx_cp_init(gpu); 464 if (ret) 465 goto out; 466 467 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TRUST_CNTL, 0x0); 468 469 out: 470 /* 471 * Tell the GMU that we are done touching the GPU and it can start power 472 * management 473 */ 474 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); 475 476 /* Take the GMU out of its special boot mode */ 477 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_BOOT_SLUMBER); 478 479 return ret; 480 } 481 482 static void a6xx_dump(struct msm_gpu *gpu) 483 { 484 dev_info(&gpu->pdev->dev, "status: %08x\n", 485 gpu_read(gpu, REG_A6XX_RBBM_STATUS)); 486 adreno_dump(gpu); 487 } 488 489 #define VBIF_RESET_ACK_TIMEOUT 100 490 #define VBIF_RESET_ACK_MASK 0x00f0 491 492 static void a6xx_recover(struct msm_gpu *gpu) 493 { 494 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 495 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 496 int i; 497 498 adreno_dump_info(gpu); 499 500 for (i = 0; i < 8; i++) 501 dev_info(&gpu->pdev->dev, "CP_SCRATCH_REG%d: %u\n", i, 502 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(i))); 503 504 if (hang_debug) 505 a6xx_dump(gpu); 506 507 /* 508 * Turn off keep alive that might have been enabled by the hang 509 * interrupt 510 */ 511 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 0); 512 513 gpu->funcs->pm_suspend(gpu); 514 gpu->funcs->pm_resume(gpu); 515 516 msm_gpu_hw_init(gpu); 517 } 518 519 static int a6xx_fault_handler(void *arg, unsigned long iova, int flags) 520 { 521 struct msm_gpu *gpu = arg; 522 523 pr_warn_ratelimited("*** gpu fault: iova=%08lx, flags=%d (%u,%u,%u,%u)\n", 524 iova, flags, 525 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(4)), 526 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(5)), 527 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(6)), 528 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(7))); 529 530 return -EFAULT; 531 } 532 533 static void a6xx_cp_hw_err_irq(struct msm_gpu *gpu) 534 { 535 u32 status = gpu_read(gpu, REG_A6XX_CP_INTERRUPT_STATUS); 536 537 if (status & A6XX_CP_INT_CP_OPCODE_ERROR) { 538 u32 val; 539 540 gpu_write(gpu, REG_A6XX_CP_SQE_STAT_ADDR, 1); 541 val = gpu_read(gpu, REG_A6XX_CP_SQE_STAT_DATA); 542 dev_err_ratelimited(&gpu->pdev->dev, 543 "CP | opcode error | possible opcode=0x%8.8X\n", 544 val); 545 } 546 547 if (status & A6XX_CP_INT_CP_UCODE_ERROR) 548 dev_err_ratelimited(&gpu->pdev->dev, 549 "CP ucode error interrupt\n"); 550 551 if (status & A6XX_CP_INT_CP_HW_FAULT_ERROR) 552 dev_err_ratelimited(&gpu->pdev->dev, "CP | HW fault | status=0x%8.8X\n", 553 gpu_read(gpu, REG_A6XX_CP_HW_FAULT)); 554 555 if (status & A6XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) { 556 u32 val = gpu_read(gpu, REG_A6XX_CP_PROTECT_STATUS); 557 558 dev_err_ratelimited(&gpu->pdev->dev, 559 "CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n", 560 val & (1 << 20) ? "READ" : "WRITE", 561 (val & 0x3ffff), val); 562 } 563 564 if (status & A6XX_CP_INT_CP_AHB_ERROR) 565 dev_err_ratelimited(&gpu->pdev->dev, "CP AHB error interrupt\n"); 566 567 if (status & A6XX_CP_INT_CP_VSD_PARITY_ERROR) 568 dev_err_ratelimited(&gpu->pdev->dev, "CP VSD decoder parity error\n"); 569 570 if (status & A6XX_CP_INT_CP_ILLEGAL_INSTR_ERROR) 571 dev_err_ratelimited(&gpu->pdev->dev, "CP illegal instruction error\n"); 572 573 } 574 575 static void a6xx_fault_detect_irq(struct msm_gpu *gpu) 576 { 577 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 578 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 579 struct drm_device *dev = gpu->dev; 580 struct msm_drm_private *priv = dev->dev_private; 581 struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu); 582 583 /* 584 * Force the GPU to stay on until after we finish 585 * collecting information 586 */ 587 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 1); 588 589 DRM_DEV_ERROR(&gpu->pdev->dev, 590 "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n", 591 ring ? ring->id : -1, ring ? ring->seqno : 0, 592 gpu_read(gpu, REG_A6XX_RBBM_STATUS), 593 gpu_read(gpu, REG_A6XX_CP_RB_RPTR), 594 gpu_read(gpu, REG_A6XX_CP_RB_WPTR), 595 gpu_read64(gpu, REG_A6XX_CP_IB1_BASE, REG_A6XX_CP_IB1_BASE_HI), 596 gpu_read(gpu, REG_A6XX_CP_IB1_REM_SIZE), 597 gpu_read64(gpu, REG_A6XX_CP_IB2_BASE, REG_A6XX_CP_IB2_BASE_HI), 598 gpu_read(gpu, REG_A6XX_CP_IB2_REM_SIZE)); 599 600 /* Turn off the hangcheck timer to keep it from bothering us */ 601 del_timer(&gpu->hangcheck_timer); 602 603 queue_work(priv->wq, &gpu->recover_work); 604 } 605 606 static irqreturn_t a6xx_irq(struct msm_gpu *gpu) 607 { 608 u32 status = gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS); 609 610 gpu_write(gpu, REG_A6XX_RBBM_INT_CLEAR_CMD, status); 611 612 if (status & A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT) 613 a6xx_fault_detect_irq(gpu); 614 615 if (status & A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR) 616 dev_err_ratelimited(&gpu->pdev->dev, "CP | AHB bus error\n"); 617 618 if (status & A6XX_RBBM_INT_0_MASK_CP_HW_ERROR) 619 a6xx_cp_hw_err_irq(gpu); 620 621 if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW) 622 dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB ASYNC overflow\n"); 623 624 if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW) 625 dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB bus overflow\n"); 626 627 if (status & A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS) 628 dev_err_ratelimited(&gpu->pdev->dev, "UCHE | Out of bounds access\n"); 629 630 if (status & A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS) 631 msm_gpu_retire(gpu); 632 633 return IRQ_HANDLED; 634 } 635 636 static const u32 a6xx_register_offsets[REG_ADRENO_REGISTER_MAX] = { 637 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_A6XX_CP_RB_BASE), 638 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE_HI, REG_A6XX_CP_RB_BASE_HI), 639 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR, 640 REG_A6XX_CP_RB_RPTR_ADDR_LO), 641 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR_HI, 642 REG_A6XX_CP_RB_RPTR_ADDR_HI), 643 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR, REG_A6XX_CP_RB_RPTR), 644 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_WPTR, REG_A6XX_CP_RB_WPTR), 645 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_CNTL, REG_A6XX_CP_RB_CNTL), 646 }; 647 648 static const u32 a6xx_registers[] = { 649 0x0000, 0x0002, 0x0010, 0x0010, 0x0012, 0x0012, 0x0018, 0x001b, 650 0x001e, 0x0032, 0x0038, 0x003c, 0x0042, 0x0042, 0x0044, 0x0044, 651 0x0047, 0x0047, 0x0056, 0x0056, 0x00ad, 0x00ae, 0x00b0, 0x00fb, 652 0x0100, 0x011d, 0x0200, 0x020d, 0x0210, 0x0213, 0x0218, 0x023d, 653 0x0400, 0x04f9, 0x0500, 0x0500, 0x0505, 0x050b, 0x050e, 0x0511, 654 0x0533, 0x0533, 0x0540, 0x0555, 0x0800, 0x0808, 0x0810, 0x0813, 655 0x0820, 0x0821, 0x0823, 0x0827, 0x0830, 0x0833, 0x0840, 0x0843, 656 0x084f, 0x086f, 0x0880, 0x088a, 0x08a0, 0x08ab, 0x08c0, 0x08c4, 657 0x08d0, 0x08dd, 0x08f0, 0x08f3, 0x0900, 0x0903, 0x0908, 0x0911, 658 0x0928, 0x093e, 0x0942, 0x094d, 0x0980, 0x0984, 0x098d, 0x0996, 659 0x0998, 0x099e, 0x09a0, 0x09a6, 0x09a8, 0x09ae, 0x09b0, 0x09b1, 660 0x09c2, 0x09c8, 0x0a00, 0x0a03, 0x0c00, 0x0c04, 0x0c06, 0x0c06, 661 0x0c10, 0x0cd9, 0x0e00, 0x0e0e, 0x0e10, 0x0e13, 0x0e17, 0x0e19, 662 0x0e1c, 0x0e2b, 0x0e30, 0x0e32, 0x0e38, 0x0e39, 0x8600, 0x8601, 663 0x8610, 0x861b, 0x8620, 0x8620, 0x8628, 0x862b, 0x8630, 0x8637, 664 0x8e01, 0x8e01, 0x8e04, 0x8e05, 0x8e07, 0x8e08, 0x8e0c, 0x8e0c, 665 0x8e10, 0x8e1c, 0x8e20, 0x8e25, 0x8e28, 0x8e28, 0x8e2c, 0x8e2f, 666 0x8e3b, 0x8e3e, 0x8e40, 0x8e43, 0x8e50, 0x8e5e, 0x8e70, 0x8e77, 667 0x9600, 0x9604, 0x9624, 0x9637, 0x9e00, 0x9e01, 0x9e03, 0x9e0e, 668 0x9e11, 0x9e16, 0x9e19, 0x9e19, 0x9e1c, 0x9e1c, 0x9e20, 0x9e23, 669 0x9e30, 0x9e31, 0x9e34, 0x9e34, 0x9e70, 0x9e72, 0x9e78, 0x9e79, 670 0x9e80, 0x9fff, 0xa600, 0xa601, 0xa603, 0xa603, 0xa60a, 0xa60a, 671 0xa610, 0xa617, 0xa630, 0xa630, 672 ~0 673 }; 674 675 static int a6xx_pm_resume(struct msm_gpu *gpu) 676 { 677 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 678 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 679 int ret; 680 681 ret = a6xx_gmu_resume(a6xx_gpu); 682 683 gpu->needs_hw_init = true; 684 685 msm_gpu_resume_devfreq(gpu); 686 687 return ret; 688 } 689 690 static int a6xx_pm_suspend(struct msm_gpu *gpu) 691 { 692 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 693 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 694 695 devfreq_suspend_device(gpu->devfreq.devfreq); 696 697 /* 698 * Make sure the GMU is idle before continuing (because some transitions 699 * may use VBIF 700 */ 701 a6xx_gmu_wait_for_idle(a6xx_gpu); 702 703 /* Clear the VBIF pipe before shutting down */ 704 /* FIXME: This accesses the GPU - do we need to make sure it is on? */ 705 gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0xf); 706 spin_until((gpu_read(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL1) & 0xf) == 0xf); 707 gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0); 708 709 return a6xx_gmu_stop(a6xx_gpu); 710 } 711 712 static int a6xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value) 713 { 714 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 715 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 716 717 /* Force the GPU power on so we can read this register */ 718 a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); 719 720 *value = gpu_read64(gpu, REG_A6XX_RBBM_PERFCTR_CP_0_LO, 721 REG_A6XX_RBBM_PERFCTR_CP_0_HI); 722 723 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); 724 return 0; 725 } 726 727 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP) 728 static void a6xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state, 729 struct drm_printer *p) 730 { 731 adreno_show(gpu, state, p); 732 } 733 #endif 734 735 static struct msm_ringbuffer *a6xx_active_ring(struct msm_gpu *gpu) 736 { 737 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 738 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 739 740 return a6xx_gpu->cur_ring; 741 } 742 743 static void a6xx_destroy(struct msm_gpu *gpu) 744 { 745 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 746 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 747 748 if (a6xx_gpu->sqe_bo) { 749 if (a6xx_gpu->sqe_iova) 750 msm_gem_put_iova(a6xx_gpu->sqe_bo, gpu->aspace); 751 drm_gem_object_put_unlocked(a6xx_gpu->sqe_bo); 752 } 753 754 a6xx_gmu_remove(a6xx_gpu); 755 756 adreno_gpu_cleanup(adreno_gpu); 757 kfree(a6xx_gpu); 758 } 759 760 static unsigned long a6xx_gpu_busy(struct msm_gpu *gpu) 761 { 762 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 763 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 764 u64 busy_cycles, busy_time; 765 766 busy_cycles = gmu_read64(&a6xx_gpu->gmu, 767 REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_L, 768 REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H); 769 770 busy_time = (busy_cycles - gpu->devfreq.busy_cycles) * 10; 771 do_div(busy_time, 192); 772 773 gpu->devfreq.busy_cycles = busy_cycles; 774 775 if (WARN_ON(busy_time > ~0LU)) 776 return ~0LU; 777 778 return (unsigned long)busy_time; 779 } 780 781 static const struct adreno_gpu_funcs funcs = { 782 .base = { 783 .get_param = adreno_get_param, 784 .hw_init = a6xx_hw_init, 785 .pm_suspend = a6xx_pm_suspend, 786 .pm_resume = a6xx_pm_resume, 787 .recover = a6xx_recover, 788 .submit = a6xx_submit, 789 .flush = a6xx_flush, 790 .active_ring = a6xx_active_ring, 791 .irq = a6xx_irq, 792 .destroy = a6xx_destroy, 793 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP) 794 .show = a6xx_show, 795 #endif 796 .gpu_busy = a6xx_gpu_busy, 797 .gpu_get_freq = a6xx_gmu_get_freq, 798 .gpu_set_freq = a6xx_gmu_set_freq, 799 }, 800 .get_timestamp = a6xx_get_timestamp, 801 }; 802 803 struct msm_gpu *a6xx_gpu_init(struct drm_device *dev) 804 { 805 struct msm_drm_private *priv = dev->dev_private; 806 struct platform_device *pdev = priv->gpu_pdev; 807 struct device_node *node; 808 struct a6xx_gpu *a6xx_gpu; 809 struct adreno_gpu *adreno_gpu; 810 struct msm_gpu *gpu; 811 int ret; 812 813 a6xx_gpu = kzalloc(sizeof(*a6xx_gpu), GFP_KERNEL); 814 if (!a6xx_gpu) 815 return ERR_PTR(-ENOMEM); 816 817 adreno_gpu = &a6xx_gpu->base; 818 gpu = &adreno_gpu->base; 819 820 adreno_gpu->registers = a6xx_registers; 821 adreno_gpu->reg_offsets = a6xx_register_offsets; 822 823 ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 1); 824 if (ret) { 825 a6xx_destroy(&(a6xx_gpu->base.base)); 826 return ERR_PTR(ret); 827 } 828 829 /* Check if there is a GMU phandle and set it up */ 830 node = of_parse_phandle(pdev->dev.of_node, "qcom,gmu", 0); 831 832 /* FIXME: How do we gracefully handle this? */ 833 BUG_ON(!node); 834 835 ret = a6xx_gmu_probe(a6xx_gpu, node); 836 if (ret) { 837 a6xx_destroy(&(a6xx_gpu->base.base)); 838 return ERR_PTR(ret); 839 } 840 841 if (gpu->aspace) 842 msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu, 843 a6xx_fault_handler); 844 845 return gpu; 846 } 847