1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2016-2017 The Linux Foundation. All rights reserved. 3 */ 4 5 #include <linux/kernel.h> 6 #include <linux/types.h> 7 #include <linux/cpumask.h> 8 #include <linux/qcom_scm.h> 9 #include <linux/pm_opp.h> 10 #include <linux/nvmem-consumer.h> 11 #include <linux/slab.h> 12 #include "msm_gem.h" 13 #include "msm_mmu.h" 14 #include "a5xx_gpu.h" 15 16 extern bool hang_debug; 17 static void a5xx_dump(struct msm_gpu *gpu); 18 19 #define GPU_PAS_ID 13 20 21 static void a5xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring) 22 { 23 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 24 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 25 uint32_t wptr; 26 unsigned long flags; 27 28 spin_lock_irqsave(&ring->lock, flags); 29 30 /* Copy the shadow to the actual register */ 31 ring->cur = ring->next; 32 33 /* Make sure to wrap wptr if we need to */ 34 wptr = get_wptr(ring); 35 36 spin_unlock_irqrestore(&ring->lock, flags); 37 38 /* Make sure everything is posted before making a decision */ 39 mb(); 40 41 /* Update HW if this is the current ring and we are not in preempt */ 42 if (a5xx_gpu->cur_ring == ring && !a5xx_in_preempt(a5xx_gpu)) 43 gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr); 44 } 45 46 static void a5xx_submit_in_rb(struct msm_gpu *gpu, struct msm_gem_submit *submit, 47 struct msm_file_private *ctx) 48 { 49 struct msm_drm_private *priv = gpu->dev->dev_private; 50 struct msm_ringbuffer *ring = submit->ring; 51 struct msm_gem_object *obj; 52 uint32_t *ptr, dwords; 53 unsigned int i; 54 55 for (i = 0; i < submit->nr_cmds; i++) { 56 switch (submit->cmd[i].type) { 57 case MSM_SUBMIT_CMD_IB_TARGET_BUF: 58 break; 59 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF: 60 if (priv->lastctx == ctx) 61 break; 62 case MSM_SUBMIT_CMD_BUF: 63 /* copy commands into RB: */ 64 obj = submit->bos[submit->cmd[i].idx].obj; 65 dwords = submit->cmd[i].size; 66 67 ptr = msm_gem_get_vaddr(&obj->base); 68 69 /* _get_vaddr() shouldn't fail at this point, 70 * since we've already mapped it once in 71 * submit_reloc() 72 */ 73 if (WARN_ON(!ptr)) 74 return; 75 76 for (i = 0; i < dwords; i++) { 77 /* normally the OUT_PKTn() would wait 78 * for space for the packet. But since 79 * we just OUT_RING() the whole thing, 80 * need to call adreno_wait_ring() 81 * ourself: 82 */ 83 adreno_wait_ring(ring, 1); 84 OUT_RING(ring, ptr[i]); 85 } 86 87 msm_gem_put_vaddr(&obj->base); 88 89 break; 90 } 91 } 92 93 a5xx_flush(gpu, ring); 94 a5xx_preempt_trigger(gpu); 95 96 /* we might not necessarily have a cmd from userspace to 97 * trigger an event to know that submit has completed, so 98 * do this manually: 99 */ 100 a5xx_idle(gpu, ring); 101 ring->memptrs->fence = submit->seqno; 102 msm_gpu_retire(gpu); 103 } 104 105 static void a5xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit, 106 struct msm_file_private *ctx) 107 { 108 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 109 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 110 struct msm_drm_private *priv = gpu->dev->dev_private; 111 struct msm_ringbuffer *ring = submit->ring; 112 unsigned int i, ibs = 0; 113 114 if (IS_ENABLED(CONFIG_DRM_MSM_GPU_SUDO) && submit->in_rb) { 115 priv->lastctx = NULL; 116 a5xx_submit_in_rb(gpu, submit, ctx); 117 return; 118 } 119 120 OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1); 121 OUT_RING(ring, 0x02); 122 123 /* Turn off protected mode to write to special registers */ 124 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1); 125 OUT_RING(ring, 0); 126 127 /* Set the save preemption record for the ring/command */ 128 OUT_PKT4(ring, REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 2); 129 OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[submit->ring->id])); 130 OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[submit->ring->id])); 131 132 /* Turn back on protected mode */ 133 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1); 134 OUT_RING(ring, 1); 135 136 /* Enable local preemption for finegrain preemption */ 137 OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1); 138 OUT_RING(ring, 0x02); 139 140 /* Allow CP_CONTEXT_SWITCH_YIELD packets in the IB2 */ 141 OUT_PKT7(ring, CP_YIELD_ENABLE, 1); 142 OUT_RING(ring, 0x02); 143 144 /* Submit the commands */ 145 for (i = 0; i < submit->nr_cmds; i++) { 146 switch (submit->cmd[i].type) { 147 case MSM_SUBMIT_CMD_IB_TARGET_BUF: 148 break; 149 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF: 150 if (priv->lastctx == ctx) 151 break; 152 case MSM_SUBMIT_CMD_BUF: 153 OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3); 154 OUT_RING(ring, lower_32_bits(submit->cmd[i].iova)); 155 OUT_RING(ring, upper_32_bits(submit->cmd[i].iova)); 156 OUT_RING(ring, submit->cmd[i].size); 157 ibs++; 158 break; 159 } 160 } 161 162 /* 163 * Write the render mode to NULL (0) to indicate to the CP that the IBs 164 * are done rendering - otherwise a lucky preemption would start 165 * replaying from the last checkpoint 166 */ 167 OUT_PKT7(ring, CP_SET_RENDER_MODE, 5); 168 OUT_RING(ring, 0); 169 OUT_RING(ring, 0); 170 OUT_RING(ring, 0); 171 OUT_RING(ring, 0); 172 OUT_RING(ring, 0); 173 174 /* Turn off IB level preemptions */ 175 OUT_PKT7(ring, CP_YIELD_ENABLE, 1); 176 OUT_RING(ring, 0x01); 177 178 /* Write the fence to the scratch register */ 179 OUT_PKT4(ring, REG_A5XX_CP_SCRATCH_REG(2), 1); 180 OUT_RING(ring, submit->seqno); 181 182 /* 183 * Execute a CACHE_FLUSH_TS event. This will ensure that the 184 * timestamp is written to the memory and then triggers the interrupt 185 */ 186 OUT_PKT7(ring, CP_EVENT_WRITE, 4); 187 OUT_RING(ring, CACHE_FLUSH_TS | (1 << 31)); 188 OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence))); 189 OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence))); 190 OUT_RING(ring, submit->seqno); 191 192 /* Yield the floor on command completion */ 193 OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4); 194 /* 195 * If dword[2:1] are non zero, they specify an address for the CP to 196 * write the value of dword[3] to on preemption complete. Write 0 to 197 * skip the write 198 */ 199 OUT_RING(ring, 0x00); 200 OUT_RING(ring, 0x00); 201 /* Data value - not used if the address above is 0 */ 202 OUT_RING(ring, 0x01); 203 /* Set bit 0 to trigger an interrupt on preempt complete */ 204 OUT_RING(ring, 0x01); 205 206 a5xx_flush(gpu, ring); 207 208 /* Check to see if we need to start preemption */ 209 a5xx_preempt_trigger(gpu); 210 } 211 212 static const struct { 213 u32 offset; 214 u32 value; 215 } a5xx_hwcg[] = { 216 {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, 217 {REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222}, 218 {REG_A5XX_RBBM_CLOCK_CNTL_SP2, 0x02222222}, 219 {REG_A5XX_RBBM_CLOCK_CNTL_SP3, 0x02222222}, 220 {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, 221 {REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220}, 222 {REG_A5XX_RBBM_CLOCK_CNTL2_SP2, 0x02222220}, 223 {REG_A5XX_RBBM_CLOCK_CNTL2_SP3, 0x02222220}, 224 {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, 225 {REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF}, 226 {REG_A5XX_RBBM_CLOCK_HYST_SP2, 0x0000F3CF}, 227 {REG_A5XX_RBBM_CLOCK_HYST_SP3, 0x0000F3CF}, 228 {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, 229 {REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080}, 230 {REG_A5XX_RBBM_CLOCK_DELAY_SP2, 0x00000080}, 231 {REG_A5XX_RBBM_CLOCK_DELAY_SP3, 0x00000080}, 232 {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, 233 {REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222}, 234 {REG_A5XX_RBBM_CLOCK_CNTL_TP2, 0x22222222}, 235 {REG_A5XX_RBBM_CLOCK_CNTL_TP3, 0x22222222}, 236 {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, 237 {REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, 238 {REG_A5XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222}, 239 {REG_A5XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222}, 240 {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, 241 {REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222}, 242 {REG_A5XX_RBBM_CLOCK_CNTL3_TP2, 0x00002222}, 243 {REG_A5XX_RBBM_CLOCK_CNTL3_TP3, 0x00002222}, 244 {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, 245 {REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, 246 {REG_A5XX_RBBM_CLOCK_HYST_TP2, 0x77777777}, 247 {REG_A5XX_RBBM_CLOCK_HYST_TP3, 0x77777777}, 248 {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, 249 {REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, 250 {REG_A5XX_RBBM_CLOCK_HYST2_TP2, 0x77777777}, 251 {REG_A5XX_RBBM_CLOCK_HYST2_TP3, 0x77777777}, 252 {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, 253 {REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777}, 254 {REG_A5XX_RBBM_CLOCK_HYST3_TP2, 0x00007777}, 255 {REG_A5XX_RBBM_CLOCK_HYST3_TP3, 0x00007777}, 256 {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, 257 {REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, 258 {REG_A5XX_RBBM_CLOCK_DELAY_TP2, 0x11111111}, 259 {REG_A5XX_RBBM_CLOCK_DELAY_TP3, 0x11111111}, 260 {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, 261 {REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, 262 {REG_A5XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111}, 263 {REG_A5XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111}, 264 {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, 265 {REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111}, 266 {REG_A5XX_RBBM_CLOCK_DELAY3_TP2, 0x00001111}, 267 {REG_A5XX_RBBM_CLOCK_DELAY3_TP3, 0x00001111}, 268 {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, 269 {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, 270 {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, 271 {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, 272 {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444}, 273 {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, 274 {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, 275 {REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222}, 276 {REG_A5XX_RBBM_CLOCK_CNTL_RB2, 0x22222222}, 277 {REG_A5XX_RBBM_CLOCK_CNTL_RB3, 0x22222222}, 278 {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, 279 {REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222}, 280 {REG_A5XX_RBBM_CLOCK_CNTL2_RB2, 0x00222222}, 281 {REG_A5XX_RBBM_CLOCK_CNTL2_RB3, 0x00222222}, 282 {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, 283 {REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220}, 284 {REG_A5XX_RBBM_CLOCK_CNTL_CCU2, 0x00022220}, 285 {REG_A5XX_RBBM_CLOCK_CNTL_CCU3, 0x00022220}, 286 {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, 287 {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, 288 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, 289 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404}, 290 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU2, 0x04040404}, 291 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU3, 0x04040404}, 292 {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, 293 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, 294 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002}, 295 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2, 0x00000002}, 296 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3, 0x00000002}, 297 {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, 298 {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, 299 {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, 300 {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, 301 {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, 302 {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, 303 {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, 304 {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, 305 {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, 306 {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, 307 {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222} 308 }; 309 310 void a5xx_set_hwcg(struct msm_gpu *gpu, bool state) 311 { 312 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 313 unsigned int i; 314 315 for (i = 0; i < ARRAY_SIZE(a5xx_hwcg); i++) 316 gpu_write(gpu, a5xx_hwcg[i].offset, 317 state ? a5xx_hwcg[i].value : 0); 318 319 if (adreno_is_a540(adreno_gpu)) { 320 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_DELAY_GPMU, state ? 0x00000770 : 0); 321 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_HYST_GPMU, state ? 0x00000004 : 0); 322 } 323 324 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, state ? 0xAAA8AA00 : 0); 325 gpu_write(gpu, REG_A5XX_RBBM_ISDB_CNT, state ? 0x182 : 0x180); 326 } 327 328 static int a5xx_me_init(struct msm_gpu *gpu) 329 { 330 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 331 struct msm_ringbuffer *ring = gpu->rb[0]; 332 333 OUT_PKT7(ring, CP_ME_INIT, 8); 334 335 OUT_RING(ring, 0x0000002F); 336 337 /* Enable multiple hardware contexts */ 338 OUT_RING(ring, 0x00000003); 339 340 /* Enable error detection */ 341 OUT_RING(ring, 0x20000000); 342 343 /* Don't enable header dump */ 344 OUT_RING(ring, 0x00000000); 345 OUT_RING(ring, 0x00000000); 346 347 /* Specify workarounds for various microcode issues */ 348 if (adreno_is_a530(adreno_gpu)) { 349 /* Workaround for token end syncs 350 * Force a WFI after every direct-render 3D mode draw and every 351 * 2D mode 3 draw 352 */ 353 OUT_RING(ring, 0x0000000B); 354 } else { 355 /* No workarounds enabled */ 356 OUT_RING(ring, 0x00000000); 357 } 358 359 OUT_RING(ring, 0x00000000); 360 OUT_RING(ring, 0x00000000); 361 362 gpu->funcs->flush(gpu, ring); 363 return a5xx_idle(gpu, ring) ? 0 : -EINVAL; 364 } 365 366 static int a5xx_preempt_start(struct msm_gpu *gpu) 367 { 368 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 369 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 370 struct msm_ringbuffer *ring = gpu->rb[0]; 371 372 if (gpu->nr_rings == 1) 373 return 0; 374 375 /* Turn off protected mode to write to special registers */ 376 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1); 377 OUT_RING(ring, 0); 378 379 /* Set the save preemption record for the ring/command */ 380 OUT_PKT4(ring, REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 2); 381 OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[ring->id])); 382 OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[ring->id])); 383 384 /* Turn back on protected mode */ 385 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1); 386 OUT_RING(ring, 1); 387 388 OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1); 389 OUT_RING(ring, 0x00); 390 391 OUT_PKT7(ring, CP_PREEMPT_ENABLE_LOCAL, 1); 392 OUT_RING(ring, 0x01); 393 394 OUT_PKT7(ring, CP_YIELD_ENABLE, 1); 395 OUT_RING(ring, 0x01); 396 397 /* Yield the floor on command completion */ 398 OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4); 399 OUT_RING(ring, 0x00); 400 OUT_RING(ring, 0x00); 401 OUT_RING(ring, 0x01); 402 OUT_RING(ring, 0x01); 403 404 gpu->funcs->flush(gpu, ring); 405 406 return a5xx_idle(gpu, ring) ? 0 : -EINVAL; 407 } 408 409 static int a5xx_ucode_init(struct msm_gpu *gpu) 410 { 411 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 412 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 413 int ret; 414 415 if (!a5xx_gpu->pm4_bo) { 416 a5xx_gpu->pm4_bo = adreno_fw_create_bo(gpu, 417 adreno_gpu->fw[ADRENO_FW_PM4], &a5xx_gpu->pm4_iova); 418 419 420 if (IS_ERR(a5xx_gpu->pm4_bo)) { 421 ret = PTR_ERR(a5xx_gpu->pm4_bo); 422 a5xx_gpu->pm4_bo = NULL; 423 DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PM4: %d\n", 424 ret); 425 return ret; 426 } 427 428 msm_gem_object_set_name(a5xx_gpu->pm4_bo, "pm4fw"); 429 } 430 431 if (!a5xx_gpu->pfp_bo) { 432 a5xx_gpu->pfp_bo = adreno_fw_create_bo(gpu, 433 adreno_gpu->fw[ADRENO_FW_PFP], &a5xx_gpu->pfp_iova); 434 435 if (IS_ERR(a5xx_gpu->pfp_bo)) { 436 ret = PTR_ERR(a5xx_gpu->pfp_bo); 437 a5xx_gpu->pfp_bo = NULL; 438 DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PFP: %d\n", 439 ret); 440 return ret; 441 } 442 443 msm_gem_object_set_name(a5xx_gpu->pfp_bo, "pfpfw"); 444 } 445 446 gpu_write64(gpu, REG_A5XX_CP_ME_INSTR_BASE_LO, 447 REG_A5XX_CP_ME_INSTR_BASE_HI, a5xx_gpu->pm4_iova); 448 449 gpu_write64(gpu, REG_A5XX_CP_PFP_INSTR_BASE_LO, 450 REG_A5XX_CP_PFP_INSTR_BASE_HI, a5xx_gpu->pfp_iova); 451 452 return 0; 453 } 454 455 #define SCM_GPU_ZAP_SHADER_RESUME 0 456 457 static int a5xx_zap_shader_resume(struct msm_gpu *gpu) 458 { 459 int ret; 460 461 ret = qcom_scm_set_remote_state(SCM_GPU_ZAP_SHADER_RESUME, GPU_PAS_ID); 462 if (ret) 463 DRM_ERROR("%s: zap-shader resume failed: %d\n", 464 gpu->name, ret); 465 466 return ret; 467 } 468 469 static int a5xx_zap_shader_init(struct msm_gpu *gpu) 470 { 471 static bool loaded; 472 int ret; 473 474 /* 475 * If the zap shader is already loaded into memory we just need to kick 476 * the remote processor to reinitialize it 477 */ 478 if (loaded) 479 return a5xx_zap_shader_resume(gpu); 480 481 ret = adreno_zap_shader_load(gpu, GPU_PAS_ID); 482 483 loaded = !ret; 484 return ret; 485 } 486 487 #define A5XX_INT_MASK (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \ 488 A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \ 489 A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \ 490 A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \ 491 A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \ 492 A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW | \ 493 A5XX_RBBM_INT_0_MASK_CP_HW_ERROR | \ 494 A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT | \ 495 A5XX_RBBM_INT_0_MASK_CP_SW | \ 496 A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \ 497 A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \ 498 A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP) 499 500 static int a5xx_hw_init(struct msm_gpu *gpu) 501 { 502 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 503 int ret; 504 505 gpu_write(gpu, REG_A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003); 506 507 if (adreno_is_a540(adreno_gpu)) 508 gpu_write(gpu, REG_A5XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009); 509 510 /* Make all blocks contribute to the GPU BUSY perf counter */ 511 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF); 512 513 /* Enable RBBM error reporting bits */ 514 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL0, 0x00000001); 515 516 if (adreno_gpu->info->quirks & ADRENO_QUIRK_FAULT_DETECT_MASK) { 517 /* 518 * Mask out the activity signals from RB1-3 to avoid false 519 * positives 520 */ 521 522 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11, 523 0xF0000000); 524 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12, 525 0xFFFFFFFF); 526 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13, 527 0xFFFFFFFF); 528 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14, 529 0xFFFFFFFF); 530 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15, 531 0xFFFFFFFF); 532 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16, 533 0xFFFFFFFF); 534 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17, 535 0xFFFFFFFF); 536 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18, 537 0xFFFFFFFF); 538 } 539 540 /* Enable fault detection */ 541 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_INT_CNTL, 542 (1 << 30) | 0xFFFF); 543 544 /* Turn on performance counters */ 545 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_CNTL, 0x01); 546 547 /* Select CP0 to always count cycles */ 548 gpu_write(gpu, REG_A5XX_CP_PERFCTR_CP_SEL_0, PERF_CP_ALWAYS_COUNT); 549 550 /* Select RBBM0 to countable 6 to get the busy status for devfreq */ 551 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_SEL_0, 6); 552 553 /* Increase VFD cache access so LRZ and other data gets evicted less */ 554 gpu_write(gpu, REG_A5XX_UCHE_CACHE_WAYS, 0x02); 555 556 /* Disable L2 bypass in the UCHE */ 557 gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_LO, 0xFFFF0000); 558 gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_HI, 0x0001FFFF); 559 gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_LO, 0xFFFF0000); 560 gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_HI, 0x0001FFFF); 561 562 /* Set the GMEM VA range (0 to gpu->gmem) */ 563 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_LO, 0x00100000); 564 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_HI, 0x00000000); 565 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_LO, 566 0x00100000 + adreno_gpu->gmem - 1); 567 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_HI, 0x00000000); 568 569 gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x40); 570 if (adreno_is_a530(adreno_gpu)) 571 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x40); 572 if (adreno_is_a540(adreno_gpu)) 573 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400); 574 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060); 575 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16); 576 577 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, (0x400 << 11 | 0x300 << 22)); 578 579 if (adreno_gpu->info->quirks & ADRENO_QUIRK_TWO_PASS_USE_WFI) 580 gpu_rmw(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8)); 581 582 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0xc0200100); 583 584 /* Enable USE_RETENTION_FLOPS */ 585 gpu_write(gpu, REG_A5XX_CP_CHICKEN_DBG, 0x02000000); 586 587 /* Enable ME/PFP split notification */ 588 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL1, 0xA6FFFFFF); 589 590 /* Enable HWCG */ 591 a5xx_set_hwcg(gpu, true); 592 593 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL2, 0x0000003F); 594 595 /* Set the highest bank bit */ 596 gpu_write(gpu, REG_A5XX_TPL1_MODE_CNTL, 2 << 7); 597 gpu_write(gpu, REG_A5XX_RB_MODE_CNTL, 2 << 1); 598 if (adreno_is_a540(adreno_gpu)) 599 gpu_write(gpu, REG_A5XX_UCHE_DBG_ECO_CNTL_2, 2); 600 601 /* Protect registers from the CP */ 602 gpu_write(gpu, REG_A5XX_CP_PROTECT_CNTL, 0x00000007); 603 604 /* RBBM */ 605 gpu_write(gpu, REG_A5XX_CP_PROTECT(0), ADRENO_PROTECT_RW(0x04, 4)); 606 gpu_write(gpu, REG_A5XX_CP_PROTECT(1), ADRENO_PROTECT_RW(0x08, 8)); 607 gpu_write(gpu, REG_A5XX_CP_PROTECT(2), ADRENO_PROTECT_RW(0x10, 16)); 608 gpu_write(gpu, REG_A5XX_CP_PROTECT(3), ADRENO_PROTECT_RW(0x20, 32)); 609 gpu_write(gpu, REG_A5XX_CP_PROTECT(4), ADRENO_PROTECT_RW(0x40, 64)); 610 gpu_write(gpu, REG_A5XX_CP_PROTECT(5), ADRENO_PROTECT_RW(0x80, 64)); 611 612 /* Content protect */ 613 gpu_write(gpu, REG_A5XX_CP_PROTECT(6), 614 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO, 615 16)); 616 gpu_write(gpu, REG_A5XX_CP_PROTECT(7), 617 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TRUST_CNTL, 2)); 618 619 /* CP */ 620 gpu_write(gpu, REG_A5XX_CP_PROTECT(8), ADRENO_PROTECT_RW(0x800, 64)); 621 gpu_write(gpu, REG_A5XX_CP_PROTECT(9), ADRENO_PROTECT_RW(0x840, 8)); 622 gpu_write(gpu, REG_A5XX_CP_PROTECT(10), ADRENO_PROTECT_RW(0x880, 32)); 623 gpu_write(gpu, REG_A5XX_CP_PROTECT(11), ADRENO_PROTECT_RW(0xAA0, 1)); 624 625 /* RB */ 626 gpu_write(gpu, REG_A5XX_CP_PROTECT(12), ADRENO_PROTECT_RW(0xCC0, 1)); 627 gpu_write(gpu, REG_A5XX_CP_PROTECT(13), ADRENO_PROTECT_RW(0xCF0, 2)); 628 629 /* VPC */ 630 gpu_write(gpu, REG_A5XX_CP_PROTECT(14), ADRENO_PROTECT_RW(0xE68, 8)); 631 gpu_write(gpu, REG_A5XX_CP_PROTECT(15), ADRENO_PROTECT_RW(0xE70, 4)); 632 633 /* UCHE */ 634 gpu_write(gpu, REG_A5XX_CP_PROTECT(16), ADRENO_PROTECT_RW(0xE80, 16)); 635 636 if (adreno_is_a530(adreno_gpu)) 637 gpu_write(gpu, REG_A5XX_CP_PROTECT(17), 638 ADRENO_PROTECT_RW(0x10000, 0x8000)); 639 640 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_CNTL, 0); 641 /* 642 * Disable the trusted memory range - we don't actually supported secure 643 * memory rendering at this point in time and we don't want to block off 644 * part of the virtual memory space. 645 */ 646 gpu_write64(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO, 647 REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI, 0x00000000); 648 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000); 649 650 /* Put the GPU into 64 bit by default */ 651 gpu_write(gpu, REG_A5XX_CP_ADDR_MODE_CNTL, 0x1); 652 gpu_write(gpu, REG_A5XX_VSC_ADDR_MODE_CNTL, 0x1); 653 gpu_write(gpu, REG_A5XX_GRAS_ADDR_MODE_CNTL, 0x1); 654 gpu_write(gpu, REG_A5XX_RB_ADDR_MODE_CNTL, 0x1); 655 gpu_write(gpu, REG_A5XX_PC_ADDR_MODE_CNTL, 0x1); 656 gpu_write(gpu, REG_A5XX_HLSQ_ADDR_MODE_CNTL, 0x1); 657 gpu_write(gpu, REG_A5XX_VFD_ADDR_MODE_CNTL, 0x1); 658 gpu_write(gpu, REG_A5XX_VPC_ADDR_MODE_CNTL, 0x1); 659 gpu_write(gpu, REG_A5XX_UCHE_ADDR_MODE_CNTL, 0x1); 660 gpu_write(gpu, REG_A5XX_SP_ADDR_MODE_CNTL, 0x1); 661 gpu_write(gpu, REG_A5XX_TPL1_ADDR_MODE_CNTL, 0x1); 662 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1); 663 664 /* 665 * VPC corner case with local memory load kill leads to corrupt 666 * internal state. Normal Disable does not work for all a5x chips. 667 * So do the following setting to disable it. 668 */ 669 if (adreno_gpu->info->quirks & ADRENO_QUIRK_LMLOADKILL_DISABLE) { 670 gpu_rmw(gpu, REG_A5XX_VPC_DBG_ECO_CNTL, 0, BIT(23)); 671 gpu_rmw(gpu, REG_A5XX_HLSQ_DBG_ECO_CNTL, BIT(18), 0); 672 } 673 674 ret = adreno_hw_init(gpu); 675 if (ret) 676 return ret; 677 678 a5xx_preempt_hw_init(gpu); 679 680 a5xx_gpmu_ucode_init(gpu); 681 682 ret = a5xx_ucode_init(gpu); 683 if (ret) 684 return ret; 685 686 /* Disable the interrupts through the initial bringup stage */ 687 gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK, A5XX_INT_MASK); 688 689 /* Clear ME_HALT to start the micro engine */ 690 gpu_write(gpu, REG_A5XX_CP_PFP_ME_CNTL, 0); 691 ret = a5xx_me_init(gpu); 692 if (ret) 693 return ret; 694 695 ret = a5xx_power_init(gpu); 696 if (ret) 697 return ret; 698 699 /* 700 * Send a pipeline event stat to get misbehaving counters to start 701 * ticking correctly 702 */ 703 if (adreno_is_a530(adreno_gpu)) { 704 OUT_PKT7(gpu->rb[0], CP_EVENT_WRITE, 1); 705 OUT_RING(gpu->rb[0], 0x0F); 706 707 gpu->funcs->flush(gpu, gpu->rb[0]); 708 if (!a5xx_idle(gpu, gpu->rb[0])) 709 return -EINVAL; 710 } 711 712 /* 713 * Try to load a zap shader into the secure world. If successful 714 * we can use the CP to switch out of secure mode. If not then we 715 * have no resource but to try to switch ourselves out manually. If we 716 * guessed wrong then access to the RBBM_SECVID_TRUST_CNTL register will 717 * be blocked and a permissions violation will soon follow. 718 */ 719 ret = a5xx_zap_shader_init(gpu); 720 if (!ret) { 721 OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1); 722 OUT_RING(gpu->rb[0], 0x00000000); 723 724 gpu->funcs->flush(gpu, gpu->rb[0]); 725 if (!a5xx_idle(gpu, gpu->rb[0])) 726 return -EINVAL; 727 } else { 728 /* Print a warning so if we die, we know why */ 729 dev_warn_once(gpu->dev->dev, 730 "Zap shader not enabled - using SECVID_TRUST_CNTL instead\n"); 731 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TRUST_CNTL, 0x0); 732 } 733 734 /* Last step - yield the ringbuffer */ 735 a5xx_preempt_start(gpu); 736 737 return 0; 738 } 739 740 static void a5xx_recover(struct msm_gpu *gpu) 741 { 742 int i; 743 744 adreno_dump_info(gpu); 745 746 for (i = 0; i < 8; i++) { 747 printk("CP_SCRATCH_REG%d: %u\n", i, 748 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(i))); 749 } 750 751 if (hang_debug) 752 a5xx_dump(gpu); 753 754 gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 1); 755 gpu_read(gpu, REG_A5XX_RBBM_SW_RESET_CMD); 756 gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 0); 757 adreno_recover(gpu); 758 } 759 760 static void a5xx_destroy(struct msm_gpu *gpu) 761 { 762 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 763 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 764 765 DBG("%s", gpu->name); 766 767 a5xx_preempt_fini(gpu); 768 769 if (a5xx_gpu->pm4_bo) { 770 msm_gem_unpin_iova(a5xx_gpu->pm4_bo, gpu->aspace); 771 drm_gem_object_put_unlocked(a5xx_gpu->pm4_bo); 772 } 773 774 if (a5xx_gpu->pfp_bo) { 775 msm_gem_unpin_iova(a5xx_gpu->pfp_bo, gpu->aspace); 776 drm_gem_object_put_unlocked(a5xx_gpu->pfp_bo); 777 } 778 779 if (a5xx_gpu->gpmu_bo) { 780 msm_gem_unpin_iova(a5xx_gpu->gpmu_bo, gpu->aspace); 781 drm_gem_object_put_unlocked(a5xx_gpu->gpmu_bo); 782 } 783 784 adreno_gpu_cleanup(adreno_gpu); 785 kfree(a5xx_gpu); 786 } 787 788 static inline bool _a5xx_check_idle(struct msm_gpu *gpu) 789 { 790 if (gpu_read(gpu, REG_A5XX_RBBM_STATUS) & ~A5XX_RBBM_STATUS_HI_BUSY) 791 return false; 792 793 /* 794 * Nearly every abnormality ends up pausing the GPU and triggering a 795 * fault so we can safely just watch for this one interrupt to fire 796 */ 797 return !(gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS) & 798 A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT); 799 } 800 801 bool a5xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring) 802 { 803 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 804 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 805 806 if (ring != a5xx_gpu->cur_ring) { 807 WARN(1, "Tried to idle a non-current ringbuffer\n"); 808 return false; 809 } 810 811 /* wait for CP to drain ringbuffer: */ 812 if (!adreno_idle(gpu, ring)) 813 return false; 814 815 if (spin_until(_a5xx_check_idle(gpu))) { 816 DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n", 817 gpu->name, __builtin_return_address(0), 818 gpu_read(gpu, REG_A5XX_RBBM_STATUS), 819 gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS), 820 gpu_read(gpu, REG_A5XX_CP_RB_RPTR), 821 gpu_read(gpu, REG_A5XX_CP_RB_WPTR)); 822 return false; 823 } 824 825 return true; 826 } 827 828 static int a5xx_fault_handler(void *arg, unsigned long iova, int flags) 829 { 830 struct msm_gpu *gpu = arg; 831 pr_warn_ratelimited("*** gpu fault: iova=%08lx, flags=%d (%u,%u,%u,%u)\n", 832 iova, flags, 833 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(4)), 834 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(5)), 835 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(6)), 836 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(7))); 837 838 return -EFAULT; 839 } 840 841 static void a5xx_cp_err_irq(struct msm_gpu *gpu) 842 { 843 u32 status = gpu_read(gpu, REG_A5XX_CP_INTERRUPT_STATUS); 844 845 if (status & A5XX_CP_INT_CP_OPCODE_ERROR) { 846 u32 val; 847 848 gpu_write(gpu, REG_A5XX_CP_PFP_STAT_ADDR, 0); 849 850 /* 851 * REG_A5XX_CP_PFP_STAT_DATA is indexed, and we want index 1 so 852 * read it twice 853 */ 854 855 gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA); 856 val = gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA); 857 858 dev_err_ratelimited(gpu->dev->dev, "CP | opcode error | possible opcode=0x%8.8X\n", 859 val); 860 } 861 862 if (status & A5XX_CP_INT_CP_HW_FAULT_ERROR) 863 dev_err_ratelimited(gpu->dev->dev, "CP | HW fault | status=0x%8.8X\n", 864 gpu_read(gpu, REG_A5XX_CP_HW_FAULT)); 865 866 if (status & A5XX_CP_INT_CP_DMA_ERROR) 867 dev_err_ratelimited(gpu->dev->dev, "CP | DMA error\n"); 868 869 if (status & A5XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) { 870 u32 val = gpu_read(gpu, REG_A5XX_CP_PROTECT_STATUS); 871 872 dev_err_ratelimited(gpu->dev->dev, 873 "CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n", 874 val & (1 << 24) ? "WRITE" : "READ", 875 (val & 0xFFFFF) >> 2, val); 876 } 877 878 if (status & A5XX_CP_INT_CP_AHB_ERROR) { 879 u32 status = gpu_read(gpu, REG_A5XX_CP_AHB_FAULT); 880 const char *access[16] = { "reserved", "reserved", 881 "timestamp lo", "timestamp hi", "pfp read", "pfp write", 882 "", "", "me read", "me write", "", "", "crashdump read", 883 "crashdump write" }; 884 885 dev_err_ratelimited(gpu->dev->dev, 886 "CP | AHB error | addr=%X access=%s error=%d | status=0x%8.8X\n", 887 status & 0xFFFFF, access[(status >> 24) & 0xF], 888 (status & (1 << 31)), status); 889 } 890 } 891 892 static void a5xx_rbbm_err_irq(struct msm_gpu *gpu, u32 status) 893 { 894 if (status & A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR) { 895 u32 val = gpu_read(gpu, REG_A5XX_RBBM_AHB_ERROR_STATUS); 896 897 dev_err_ratelimited(gpu->dev->dev, 898 "RBBM | AHB bus error | %s | addr=0x%X | ports=0x%X:0x%X\n", 899 val & (1 << 28) ? "WRITE" : "READ", 900 (val & 0xFFFFF) >> 2, (val >> 20) & 0x3, 901 (val >> 24) & 0xF); 902 903 /* Clear the error */ 904 gpu_write(gpu, REG_A5XX_RBBM_AHB_CMD, (1 << 4)); 905 906 /* Clear the interrupt */ 907 gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD, 908 A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR); 909 } 910 911 if (status & A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT) 912 dev_err_ratelimited(gpu->dev->dev, "RBBM | AHB transfer timeout\n"); 913 914 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT) 915 dev_err_ratelimited(gpu->dev->dev, "RBBM | ME master split | status=0x%X\n", 916 gpu_read(gpu, REG_A5XX_RBBM_AHB_ME_SPLIT_STATUS)); 917 918 if (status & A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT) 919 dev_err_ratelimited(gpu->dev->dev, "RBBM | PFP master split | status=0x%X\n", 920 gpu_read(gpu, REG_A5XX_RBBM_AHB_PFP_SPLIT_STATUS)); 921 922 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT) 923 dev_err_ratelimited(gpu->dev->dev, "RBBM | ETS master split | status=0x%X\n", 924 gpu_read(gpu, REG_A5XX_RBBM_AHB_ETS_SPLIT_STATUS)); 925 926 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW) 927 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB ASYNC overflow\n"); 928 929 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW) 930 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB bus overflow\n"); 931 } 932 933 static void a5xx_uche_err_irq(struct msm_gpu *gpu) 934 { 935 uint64_t addr = (uint64_t) gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_HI); 936 937 addr |= gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_LO); 938 939 dev_err_ratelimited(gpu->dev->dev, "UCHE | Out of bounds access | addr=0x%llX\n", 940 addr); 941 } 942 943 static void a5xx_gpmu_err_irq(struct msm_gpu *gpu) 944 { 945 dev_err_ratelimited(gpu->dev->dev, "GPMU | voltage droop\n"); 946 } 947 948 static void a5xx_fault_detect_irq(struct msm_gpu *gpu) 949 { 950 struct drm_device *dev = gpu->dev; 951 struct msm_drm_private *priv = dev->dev_private; 952 struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu); 953 954 DRM_DEV_ERROR(dev->dev, "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n", 955 ring ? ring->id : -1, ring ? ring->seqno : 0, 956 gpu_read(gpu, REG_A5XX_RBBM_STATUS), 957 gpu_read(gpu, REG_A5XX_CP_RB_RPTR), 958 gpu_read(gpu, REG_A5XX_CP_RB_WPTR), 959 gpu_read64(gpu, REG_A5XX_CP_IB1_BASE, REG_A5XX_CP_IB1_BASE_HI), 960 gpu_read(gpu, REG_A5XX_CP_IB1_BUFSZ), 961 gpu_read64(gpu, REG_A5XX_CP_IB2_BASE, REG_A5XX_CP_IB2_BASE_HI), 962 gpu_read(gpu, REG_A5XX_CP_IB2_BUFSZ)); 963 964 /* Turn off the hangcheck timer to keep it from bothering us */ 965 del_timer(&gpu->hangcheck_timer); 966 967 queue_work(priv->wq, &gpu->recover_work); 968 } 969 970 #define RBBM_ERROR_MASK \ 971 (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \ 972 A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \ 973 A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \ 974 A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \ 975 A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \ 976 A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW) 977 978 static irqreturn_t a5xx_irq(struct msm_gpu *gpu) 979 { 980 u32 status = gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS); 981 982 /* 983 * Clear all the interrupts except RBBM_AHB_ERROR - if we clear it 984 * before the source is cleared the interrupt will storm. 985 */ 986 gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD, 987 status & ~A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR); 988 989 /* Pass status to a5xx_rbbm_err_irq because we've already cleared it */ 990 if (status & RBBM_ERROR_MASK) 991 a5xx_rbbm_err_irq(gpu, status); 992 993 if (status & A5XX_RBBM_INT_0_MASK_CP_HW_ERROR) 994 a5xx_cp_err_irq(gpu); 995 996 if (status & A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT) 997 a5xx_fault_detect_irq(gpu); 998 999 if (status & A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS) 1000 a5xx_uche_err_irq(gpu); 1001 1002 if (status & A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP) 1003 a5xx_gpmu_err_irq(gpu); 1004 1005 if (status & A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS) { 1006 a5xx_preempt_trigger(gpu); 1007 msm_gpu_retire(gpu); 1008 } 1009 1010 if (status & A5XX_RBBM_INT_0_MASK_CP_SW) 1011 a5xx_preempt_irq(gpu); 1012 1013 return IRQ_HANDLED; 1014 } 1015 1016 static const u32 a5xx_register_offsets[REG_ADRENO_REGISTER_MAX] = { 1017 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_A5XX_CP_RB_BASE), 1018 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE_HI, REG_A5XX_CP_RB_BASE_HI), 1019 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR, REG_A5XX_CP_RB_RPTR_ADDR), 1020 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR_HI, 1021 REG_A5XX_CP_RB_RPTR_ADDR_HI), 1022 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR, REG_A5XX_CP_RB_RPTR), 1023 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_WPTR, REG_A5XX_CP_RB_WPTR), 1024 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_CNTL, REG_A5XX_CP_RB_CNTL), 1025 }; 1026 1027 static const u32 a5xx_registers[] = { 1028 0x0000, 0x0002, 0x0004, 0x0020, 0x0022, 0x0026, 0x0029, 0x002B, 1029 0x002E, 0x0035, 0x0038, 0x0042, 0x0044, 0x0044, 0x0047, 0x0095, 1030 0x0097, 0x00BB, 0x03A0, 0x0464, 0x0469, 0x046F, 0x04D2, 0x04D3, 1031 0x04E0, 0x0533, 0x0540, 0x0555, 0x0800, 0x081A, 0x081F, 0x0841, 1032 0x0860, 0x0860, 0x0880, 0x08A0, 0x0B00, 0x0B12, 0x0B15, 0x0B28, 1033 0x0B78, 0x0B7F, 0x0BB0, 0x0BBD, 0x0BC0, 0x0BC6, 0x0BD0, 0x0C53, 1034 0x0C60, 0x0C61, 0x0C80, 0x0C82, 0x0C84, 0x0C85, 0x0C90, 0x0C98, 1035 0x0CA0, 0x0CA0, 0x0CB0, 0x0CB2, 0x2180, 0x2185, 0x2580, 0x2585, 1036 0x0CC1, 0x0CC1, 0x0CC4, 0x0CC7, 0x0CCC, 0x0CCC, 0x0CD0, 0x0CD8, 1037 0x0CE0, 0x0CE5, 0x0CE8, 0x0CE8, 0x0CEC, 0x0CF1, 0x0CFB, 0x0D0E, 1038 0x2100, 0x211E, 0x2140, 0x2145, 0x2500, 0x251E, 0x2540, 0x2545, 1039 0x0D10, 0x0D17, 0x0D20, 0x0D23, 0x0D30, 0x0D30, 0x20C0, 0x20C0, 1040 0x24C0, 0x24C0, 0x0E40, 0x0E43, 0x0E4A, 0x0E4A, 0x0E50, 0x0E57, 1041 0x0E60, 0x0E7C, 0x0E80, 0x0E8E, 0x0E90, 0x0E96, 0x0EA0, 0x0EA8, 1042 0x0EB0, 0x0EB2, 0xE140, 0xE147, 0xE150, 0xE187, 0xE1A0, 0xE1A9, 1043 0xE1B0, 0xE1B6, 0xE1C0, 0xE1C7, 0xE1D0, 0xE1D1, 0xE200, 0xE201, 1044 0xE210, 0xE21C, 0xE240, 0xE268, 0xE000, 0xE006, 0xE010, 0xE09A, 1045 0xE0A0, 0xE0A4, 0xE0AA, 0xE0EB, 0xE100, 0xE105, 0xE380, 0xE38F, 1046 0xE3B0, 0xE3B0, 0xE400, 0xE405, 0xE408, 0xE4E9, 0xE4F0, 0xE4F0, 1047 0xE280, 0xE280, 0xE282, 0xE2A3, 0xE2A5, 0xE2C2, 0xE940, 0xE947, 1048 0xE950, 0xE987, 0xE9A0, 0xE9A9, 0xE9B0, 0xE9B6, 0xE9C0, 0xE9C7, 1049 0xE9D0, 0xE9D1, 0xEA00, 0xEA01, 0xEA10, 0xEA1C, 0xEA40, 0xEA68, 1050 0xE800, 0xE806, 0xE810, 0xE89A, 0xE8A0, 0xE8A4, 0xE8AA, 0xE8EB, 1051 0xE900, 0xE905, 0xEB80, 0xEB8F, 0xEBB0, 0xEBB0, 0xEC00, 0xEC05, 1052 0xEC08, 0xECE9, 0xECF0, 0xECF0, 0xEA80, 0xEA80, 0xEA82, 0xEAA3, 1053 0xEAA5, 0xEAC2, 0xA800, 0xA800, 0xA820, 0xA828, 0xA840, 0xA87D, 1054 0XA880, 0xA88D, 0xA890, 0xA8A3, 0xA8D0, 0xA8D8, 0xA8E0, 0xA8F5, 1055 0xAC60, 0xAC60, ~0, 1056 }; 1057 1058 static void a5xx_dump(struct msm_gpu *gpu) 1059 { 1060 DRM_DEV_INFO(gpu->dev->dev, "status: %08x\n", 1061 gpu_read(gpu, REG_A5XX_RBBM_STATUS)); 1062 adreno_dump(gpu); 1063 } 1064 1065 static int a5xx_pm_resume(struct msm_gpu *gpu) 1066 { 1067 int ret; 1068 1069 /* Turn on the core power */ 1070 ret = msm_gpu_pm_resume(gpu); 1071 if (ret) 1072 return ret; 1073 1074 /* Turn the RBCCU domain first to limit the chances of voltage droop */ 1075 gpu_write(gpu, REG_A5XX_GPMU_RBCCU_POWER_CNTL, 0x778000); 1076 1077 /* Wait 3 usecs before polling */ 1078 udelay(3); 1079 1080 ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS, 1081 (1 << 20), (1 << 20)); 1082 if (ret) { 1083 DRM_ERROR("%s: timeout waiting for RBCCU GDSC enable: %X\n", 1084 gpu->name, 1085 gpu_read(gpu, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS)); 1086 return ret; 1087 } 1088 1089 /* Turn on the SP domain */ 1090 gpu_write(gpu, REG_A5XX_GPMU_SP_POWER_CNTL, 0x778000); 1091 ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_SP_PWR_CLK_STATUS, 1092 (1 << 20), (1 << 20)); 1093 if (ret) 1094 DRM_ERROR("%s: timeout waiting for SP GDSC enable\n", 1095 gpu->name); 1096 1097 return ret; 1098 } 1099 1100 static int a5xx_pm_suspend(struct msm_gpu *gpu) 1101 { 1102 /* Clear the VBIF pipe before shutting down */ 1103 gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0xF); 1104 spin_until((gpu_read(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL1) & 0xF) == 0xF); 1105 1106 gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0); 1107 1108 /* 1109 * Reset the VBIF before power collapse to avoid issue with FIFO 1110 * entries 1111 */ 1112 gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x003C0000); 1113 gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x00000000); 1114 1115 return msm_gpu_pm_suspend(gpu); 1116 } 1117 1118 static int a5xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value) 1119 { 1120 *value = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_CP_0_LO, 1121 REG_A5XX_RBBM_PERFCTR_CP_0_HI); 1122 1123 return 0; 1124 } 1125 1126 struct a5xx_crashdumper { 1127 void *ptr; 1128 struct drm_gem_object *bo; 1129 u64 iova; 1130 }; 1131 1132 struct a5xx_gpu_state { 1133 struct msm_gpu_state base; 1134 u32 *hlsqregs; 1135 }; 1136 1137 static int a5xx_crashdumper_init(struct msm_gpu *gpu, 1138 struct a5xx_crashdumper *dumper) 1139 { 1140 dumper->ptr = msm_gem_kernel_new_locked(gpu->dev, 1141 SZ_1M, MSM_BO_UNCACHED, gpu->aspace, 1142 &dumper->bo, &dumper->iova); 1143 1144 if (!IS_ERR(dumper->ptr)) 1145 msm_gem_object_set_name(dumper->bo, "crashdump"); 1146 1147 return PTR_ERR_OR_ZERO(dumper->ptr); 1148 } 1149 1150 static int a5xx_crashdumper_run(struct msm_gpu *gpu, 1151 struct a5xx_crashdumper *dumper) 1152 { 1153 u32 val; 1154 1155 if (IS_ERR_OR_NULL(dumper->ptr)) 1156 return -EINVAL; 1157 1158 gpu_write64(gpu, REG_A5XX_CP_CRASH_SCRIPT_BASE_LO, 1159 REG_A5XX_CP_CRASH_SCRIPT_BASE_HI, dumper->iova); 1160 1161 gpu_write(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, 1); 1162 1163 return gpu_poll_timeout(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, val, 1164 val & 0x04, 100, 10000); 1165 } 1166 1167 /* 1168 * These are a list of the registers that need to be read through the HLSQ 1169 * aperture through the crashdumper. These are not nominally accessible from 1170 * the CPU on a secure platform. 1171 */ 1172 static const struct { 1173 u32 type; 1174 u32 regoffset; 1175 u32 count; 1176 } a5xx_hlsq_aperture_regs[] = { 1177 { 0x35, 0xe00, 0x32 }, /* HSLQ non-context */ 1178 { 0x31, 0x2080, 0x1 }, /* HLSQ 2D context 0 */ 1179 { 0x33, 0x2480, 0x1 }, /* HLSQ 2D context 1 */ 1180 { 0x32, 0xe780, 0x62 }, /* HLSQ 3D context 0 */ 1181 { 0x34, 0xef80, 0x62 }, /* HLSQ 3D context 1 */ 1182 { 0x3f, 0x0ec0, 0x40 }, /* SP non-context */ 1183 { 0x3d, 0x2040, 0x1 }, /* SP 2D context 0 */ 1184 { 0x3b, 0x2440, 0x1 }, /* SP 2D context 1 */ 1185 { 0x3e, 0xe580, 0x170 }, /* SP 3D context 0 */ 1186 { 0x3c, 0xed80, 0x170 }, /* SP 3D context 1 */ 1187 { 0x3a, 0x0f00, 0x1c }, /* TP non-context */ 1188 { 0x38, 0x2000, 0xa }, /* TP 2D context 0 */ 1189 { 0x36, 0x2400, 0xa }, /* TP 2D context 1 */ 1190 { 0x39, 0xe700, 0x80 }, /* TP 3D context 0 */ 1191 { 0x37, 0xef00, 0x80 }, /* TP 3D context 1 */ 1192 }; 1193 1194 static void a5xx_gpu_state_get_hlsq_regs(struct msm_gpu *gpu, 1195 struct a5xx_gpu_state *a5xx_state) 1196 { 1197 struct a5xx_crashdumper dumper = { 0 }; 1198 u32 offset, count = 0; 1199 u64 *ptr; 1200 int i; 1201 1202 if (a5xx_crashdumper_init(gpu, &dumper)) 1203 return; 1204 1205 /* The script will be written at offset 0 */ 1206 ptr = dumper.ptr; 1207 1208 /* Start writing the data at offset 256k */ 1209 offset = dumper.iova + (256 * SZ_1K); 1210 1211 /* Count how many additional registers to get from the HLSQ aperture */ 1212 for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) 1213 count += a5xx_hlsq_aperture_regs[i].count; 1214 1215 a5xx_state->hlsqregs = kcalloc(count, sizeof(u32), GFP_KERNEL); 1216 if (!a5xx_state->hlsqregs) 1217 return; 1218 1219 /* Build the crashdump script */ 1220 for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) { 1221 u32 type = a5xx_hlsq_aperture_regs[i].type; 1222 u32 c = a5xx_hlsq_aperture_regs[i].count; 1223 1224 /* Write the register to select the desired bank */ 1225 *ptr++ = ((u64) type << 8); 1226 *ptr++ = (((u64) REG_A5XX_HLSQ_DBG_READ_SEL) << 44) | 1227 (1 << 21) | 1; 1228 1229 *ptr++ = offset; 1230 *ptr++ = (((u64) REG_A5XX_HLSQ_DBG_AHB_READ_APERTURE) << 44) 1231 | c; 1232 1233 offset += c * sizeof(u32); 1234 } 1235 1236 /* Write two zeros to close off the script */ 1237 *ptr++ = 0; 1238 *ptr++ = 0; 1239 1240 if (a5xx_crashdumper_run(gpu, &dumper)) { 1241 kfree(a5xx_state->hlsqregs); 1242 msm_gem_kernel_put(dumper.bo, gpu->aspace, true); 1243 return; 1244 } 1245 1246 /* Copy the data from the crashdumper to the state */ 1247 memcpy(a5xx_state->hlsqregs, dumper.ptr + (256 * SZ_1K), 1248 count * sizeof(u32)); 1249 1250 msm_gem_kernel_put(dumper.bo, gpu->aspace, true); 1251 } 1252 1253 static struct msm_gpu_state *a5xx_gpu_state_get(struct msm_gpu *gpu) 1254 { 1255 struct a5xx_gpu_state *a5xx_state = kzalloc(sizeof(*a5xx_state), 1256 GFP_KERNEL); 1257 1258 if (!a5xx_state) 1259 return ERR_PTR(-ENOMEM); 1260 1261 /* Temporarily disable hardware clock gating before reading the hw */ 1262 a5xx_set_hwcg(gpu, false); 1263 1264 /* First get the generic state from the adreno core */ 1265 adreno_gpu_state_get(gpu, &(a5xx_state->base)); 1266 1267 a5xx_state->base.rbbm_status = gpu_read(gpu, REG_A5XX_RBBM_STATUS); 1268 1269 /* Get the HLSQ regs with the help of the crashdumper */ 1270 a5xx_gpu_state_get_hlsq_regs(gpu, a5xx_state); 1271 1272 a5xx_set_hwcg(gpu, true); 1273 1274 return &a5xx_state->base; 1275 } 1276 1277 static void a5xx_gpu_state_destroy(struct kref *kref) 1278 { 1279 struct msm_gpu_state *state = container_of(kref, 1280 struct msm_gpu_state, ref); 1281 struct a5xx_gpu_state *a5xx_state = container_of(state, 1282 struct a5xx_gpu_state, base); 1283 1284 kfree(a5xx_state->hlsqregs); 1285 1286 adreno_gpu_state_destroy(state); 1287 kfree(a5xx_state); 1288 } 1289 1290 int a5xx_gpu_state_put(struct msm_gpu_state *state) 1291 { 1292 if (IS_ERR_OR_NULL(state)) 1293 return 1; 1294 1295 return kref_put(&state->ref, a5xx_gpu_state_destroy); 1296 } 1297 1298 1299 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP) 1300 void a5xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state, 1301 struct drm_printer *p) 1302 { 1303 int i, j; 1304 u32 pos = 0; 1305 struct a5xx_gpu_state *a5xx_state = container_of(state, 1306 struct a5xx_gpu_state, base); 1307 1308 if (IS_ERR_OR_NULL(state)) 1309 return; 1310 1311 adreno_show(gpu, state, p); 1312 1313 /* Dump the additional a5xx HLSQ registers */ 1314 if (!a5xx_state->hlsqregs) 1315 return; 1316 1317 drm_printf(p, "registers-hlsq:\n"); 1318 1319 for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) { 1320 u32 o = a5xx_hlsq_aperture_regs[i].regoffset; 1321 u32 c = a5xx_hlsq_aperture_regs[i].count; 1322 1323 for (j = 0; j < c; j++, pos++, o++) { 1324 /* 1325 * To keep the crashdump simple we pull the entire range 1326 * for each register type but not all of the registers 1327 * in the range are valid. Fortunately invalid registers 1328 * stick out like a sore thumb with a value of 1329 * 0xdeadbeef 1330 */ 1331 if (a5xx_state->hlsqregs[pos] == 0xdeadbeef) 1332 continue; 1333 1334 drm_printf(p, " - { offset: 0x%04x, value: 0x%08x }\n", 1335 o << 2, a5xx_state->hlsqregs[pos]); 1336 } 1337 } 1338 } 1339 #endif 1340 1341 static struct msm_ringbuffer *a5xx_active_ring(struct msm_gpu *gpu) 1342 { 1343 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1344 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 1345 1346 return a5xx_gpu->cur_ring; 1347 } 1348 1349 static unsigned long a5xx_gpu_busy(struct msm_gpu *gpu) 1350 { 1351 u64 busy_cycles, busy_time; 1352 1353 busy_cycles = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_0_LO, 1354 REG_A5XX_RBBM_PERFCTR_RBBM_0_HI); 1355 1356 busy_time = busy_cycles - gpu->devfreq.busy_cycles; 1357 do_div(busy_time, clk_get_rate(gpu->core_clk) / 1000000); 1358 1359 gpu->devfreq.busy_cycles = busy_cycles; 1360 1361 if (WARN_ON(busy_time > ~0LU)) 1362 return ~0LU; 1363 1364 return (unsigned long)busy_time; 1365 } 1366 1367 static const struct adreno_gpu_funcs funcs = { 1368 .base = { 1369 .get_param = adreno_get_param, 1370 .hw_init = a5xx_hw_init, 1371 .pm_suspend = a5xx_pm_suspend, 1372 .pm_resume = a5xx_pm_resume, 1373 .recover = a5xx_recover, 1374 .submit = a5xx_submit, 1375 .flush = a5xx_flush, 1376 .active_ring = a5xx_active_ring, 1377 .irq = a5xx_irq, 1378 .destroy = a5xx_destroy, 1379 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP) 1380 .show = a5xx_show, 1381 #endif 1382 #if defined(CONFIG_DEBUG_FS) 1383 .debugfs_init = a5xx_debugfs_init, 1384 #endif 1385 .gpu_busy = a5xx_gpu_busy, 1386 .gpu_state_get = a5xx_gpu_state_get, 1387 .gpu_state_put = a5xx_gpu_state_put, 1388 }, 1389 .get_timestamp = a5xx_get_timestamp, 1390 }; 1391 1392 static void check_speed_bin(struct device *dev) 1393 { 1394 struct nvmem_cell *cell; 1395 u32 bin, val; 1396 1397 cell = nvmem_cell_get(dev, "speed_bin"); 1398 1399 /* If a nvmem cell isn't defined, nothing to do */ 1400 if (IS_ERR(cell)) 1401 return; 1402 1403 bin = *((u32 *) nvmem_cell_read(cell, NULL)); 1404 nvmem_cell_put(cell); 1405 1406 val = (1 << bin); 1407 1408 dev_pm_opp_set_supported_hw(dev, &val, 1); 1409 } 1410 1411 struct msm_gpu *a5xx_gpu_init(struct drm_device *dev) 1412 { 1413 struct msm_drm_private *priv = dev->dev_private; 1414 struct platform_device *pdev = priv->gpu_pdev; 1415 struct a5xx_gpu *a5xx_gpu = NULL; 1416 struct adreno_gpu *adreno_gpu; 1417 struct msm_gpu *gpu; 1418 int ret; 1419 1420 if (!pdev) { 1421 DRM_DEV_ERROR(dev->dev, "No A5XX device is defined\n"); 1422 return ERR_PTR(-ENXIO); 1423 } 1424 1425 a5xx_gpu = kzalloc(sizeof(*a5xx_gpu), GFP_KERNEL); 1426 if (!a5xx_gpu) 1427 return ERR_PTR(-ENOMEM); 1428 1429 adreno_gpu = &a5xx_gpu->base; 1430 gpu = &adreno_gpu->base; 1431 1432 adreno_gpu->registers = a5xx_registers; 1433 adreno_gpu->reg_offsets = a5xx_register_offsets; 1434 1435 a5xx_gpu->lm_leakage = 0x4E001A; 1436 1437 check_speed_bin(&pdev->dev); 1438 1439 ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 4); 1440 if (ret) { 1441 a5xx_destroy(&(a5xx_gpu->base.base)); 1442 return ERR_PTR(ret); 1443 } 1444 1445 if (gpu->aspace) 1446 msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu, a5xx_fault_handler); 1447 1448 /* Set up the preemption specific bits and pieces for each ringbuffer */ 1449 a5xx_preempt_init(gpu); 1450 1451 return gpu; 1452 } 1453