1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2016-2017 The Linux Foundation. All rights reserved. 3 */ 4 5 #include <linux/kernel.h> 6 #include <linux/types.h> 7 #include <linux/cpumask.h> 8 #include <linux/firmware/qcom/qcom_scm.h> 9 #include <linux/pm_opp.h> 10 #include <linux/nvmem-consumer.h> 11 #include <linux/slab.h> 12 #include "msm_gem.h" 13 #include "msm_mmu.h" 14 #include "a5xx_gpu.h" 15 16 extern bool hang_debug; 17 static void a5xx_dump(struct msm_gpu *gpu); 18 19 #define GPU_PAS_ID 13 20 21 static void update_shadow_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring) 22 { 23 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 24 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 25 26 if (a5xx_gpu->has_whereami) { 27 OUT_PKT7(ring, CP_WHERE_AM_I, 2); 28 OUT_RING(ring, lower_32_bits(shadowptr(a5xx_gpu, ring))); 29 OUT_RING(ring, upper_32_bits(shadowptr(a5xx_gpu, ring))); 30 } 31 } 32 33 void a5xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring, 34 bool sync) 35 { 36 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 37 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 38 uint32_t wptr; 39 unsigned long flags; 40 41 /* 42 * Most flush operations need to issue a WHERE_AM_I opcode to sync up 43 * the rptr shadow 44 */ 45 if (sync) 46 update_shadow_rptr(gpu, ring); 47 48 spin_lock_irqsave(&ring->preempt_lock, flags); 49 50 /* Copy the shadow to the actual register */ 51 ring->cur = ring->next; 52 53 /* Make sure to wrap wptr if we need to */ 54 wptr = get_wptr(ring); 55 56 spin_unlock_irqrestore(&ring->preempt_lock, flags); 57 58 /* Make sure everything is posted before making a decision */ 59 mb(); 60 61 /* Update HW if this is the current ring and we are not in preempt */ 62 if (a5xx_gpu->cur_ring == ring && !a5xx_in_preempt(a5xx_gpu)) 63 gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr); 64 } 65 66 static void a5xx_submit_in_rb(struct msm_gpu *gpu, struct msm_gem_submit *submit) 67 { 68 struct msm_ringbuffer *ring = submit->ring; 69 struct drm_gem_object *obj; 70 uint32_t *ptr, dwords; 71 unsigned int i; 72 73 for (i = 0; i < submit->nr_cmds; i++) { 74 switch (submit->cmd[i].type) { 75 case MSM_SUBMIT_CMD_IB_TARGET_BUF: 76 break; 77 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF: 78 if (gpu->cur_ctx_seqno == submit->queue->ctx->seqno) 79 break; 80 fallthrough; 81 case MSM_SUBMIT_CMD_BUF: 82 /* copy commands into RB: */ 83 obj = submit->bos[submit->cmd[i].idx].obj; 84 dwords = submit->cmd[i].size; 85 86 ptr = msm_gem_get_vaddr(obj); 87 88 /* _get_vaddr() shouldn't fail at this point, 89 * since we've already mapped it once in 90 * submit_reloc() 91 */ 92 if (WARN_ON(IS_ERR_OR_NULL(ptr))) 93 return; 94 95 for (i = 0; i < dwords; i++) { 96 /* normally the OUT_PKTn() would wait 97 * for space for the packet. But since 98 * we just OUT_RING() the whole thing, 99 * need to call adreno_wait_ring() 100 * ourself: 101 */ 102 adreno_wait_ring(ring, 1); 103 OUT_RING(ring, ptr[i]); 104 } 105 106 msm_gem_put_vaddr(obj); 107 108 break; 109 } 110 } 111 112 a5xx_flush(gpu, ring, true); 113 a5xx_preempt_trigger(gpu); 114 115 /* we might not necessarily have a cmd from userspace to 116 * trigger an event to know that submit has completed, so 117 * do this manually: 118 */ 119 a5xx_idle(gpu, ring); 120 ring->memptrs->fence = submit->seqno; 121 msm_gpu_retire(gpu); 122 } 123 124 static void a5xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) 125 { 126 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 127 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 128 struct msm_ringbuffer *ring = submit->ring; 129 unsigned int i, ibs = 0; 130 131 if (IS_ENABLED(CONFIG_DRM_MSM_GPU_SUDO) && submit->in_rb) { 132 gpu->cur_ctx_seqno = 0; 133 a5xx_submit_in_rb(gpu, submit); 134 return; 135 } 136 137 OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1); 138 OUT_RING(ring, 0x02); 139 140 /* Turn off protected mode to write to special registers */ 141 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1); 142 OUT_RING(ring, 0); 143 144 /* Set the save preemption record for the ring/command */ 145 OUT_PKT4(ring, REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 2); 146 OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[submit->ring->id])); 147 OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[submit->ring->id])); 148 149 /* Turn back on protected mode */ 150 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1); 151 OUT_RING(ring, 1); 152 153 /* 154 * Disable local preemption by default because it requires 155 * user-space to be aware of it and provide additional handling 156 * to restore rendering state or do various flushes on switch. 157 */ 158 OUT_PKT7(ring, CP_PREEMPT_ENABLE_LOCAL, 1); 159 OUT_RING(ring, 0x0); 160 161 /* Allow CP_CONTEXT_SWITCH_YIELD packets in the IB2 */ 162 OUT_PKT7(ring, CP_YIELD_ENABLE, 1); 163 OUT_RING(ring, 0x02); 164 165 /* Submit the commands */ 166 for (i = 0; i < submit->nr_cmds; i++) { 167 switch (submit->cmd[i].type) { 168 case MSM_SUBMIT_CMD_IB_TARGET_BUF: 169 break; 170 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF: 171 if (gpu->cur_ctx_seqno == submit->queue->ctx->seqno) 172 break; 173 fallthrough; 174 case MSM_SUBMIT_CMD_BUF: 175 OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3); 176 OUT_RING(ring, lower_32_bits(submit->cmd[i].iova)); 177 OUT_RING(ring, upper_32_bits(submit->cmd[i].iova)); 178 OUT_RING(ring, submit->cmd[i].size); 179 ibs++; 180 break; 181 } 182 183 /* 184 * Periodically update shadow-wptr if needed, so that we 185 * can see partial progress of submits with large # of 186 * cmds.. otherwise we could needlessly stall waiting for 187 * ringbuffer state, simply due to looking at a shadow 188 * rptr value that has not been updated 189 */ 190 if ((ibs % 32) == 0) 191 update_shadow_rptr(gpu, ring); 192 } 193 194 /* 195 * Write the render mode to NULL (0) to indicate to the CP that the IBs 196 * are done rendering - otherwise a lucky preemption would start 197 * replaying from the last checkpoint 198 */ 199 OUT_PKT7(ring, CP_SET_RENDER_MODE, 5); 200 OUT_RING(ring, 0); 201 OUT_RING(ring, 0); 202 OUT_RING(ring, 0); 203 OUT_RING(ring, 0); 204 OUT_RING(ring, 0); 205 206 /* Turn off IB level preemptions */ 207 OUT_PKT7(ring, CP_YIELD_ENABLE, 1); 208 OUT_RING(ring, 0x01); 209 210 /* Write the fence to the scratch register */ 211 OUT_PKT4(ring, REG_A5XX_CP_SCRATCH_REG(2), 1); 212 OUT_RING(ring, submit->seqno); 213 214 /* 215 * Execute a CACHE_FLUSH_TS event. This will ensure that the 216 * timestamp is written to the memory and then triggers the interrupt 217 */ 218 OUT_PKT7(ring, CP_EVENT_WRITE, 4); 219 OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(CACHE_FLUSH_TS) | 220 CP_EVENT_WRITE_0_IRQ); 221 OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence))); 222 OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence))); 223 OUT_RING(ring, submit->seqno); 224 225 /* Yield the floor on command completion */ 226 OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4); 227 /* 228 * If dword[2:1] are non zero, they specify an address for the CP to 229 * write the value of dword[3] to on preemption complete. Write 0 to 230 * skip the write 231 */ 232 OUT_RING(ring, 0x00); 233 OUT_RING(ring, 0x00); 234 /* Data value - not used if the address above is 0 */ 235 OUT_RING(ring, 0x01); 236 /* Set bit 0 to trigger an interrupt on preempt complete */ 237 OUT_RING(ring, 0x01); 238 239 /* A WHERE_AM_I packet is not needed after a YIELD */ 240 a5xx_flush(gpu, ring, false); 241 242 /* Check to see if we need to start preemption */ 243 a5xx_preempt_trigger(gpu); 244 } 245 246 static const struct adreno_five_hwcg_regs { 247 u32 offset; 248 u32 value; 249 } a5xx_hwcg[] = { 250 {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, 251 {REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222}, 252 {REG_A5XX_RBBM_CLOCK_CNTL_SP2, 0x02222222}, 253 {REG_A5XX_RBBM_CLOCK_CNTL_SP3, 0x02222222}, 254 {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, 255 {REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220}, 256 {REG_A5XX_RBBM_CLOCK_CNTL2_SP2, 0x02222220}, 257 {REG_A5XX_RBBM_CLOCK_CNTL2_SP3, 0x02222220}, 258 {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, 259 {REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF}, 260 {REG_A5XX_RBBM_CLOCK_HYST_SP2, 0x0000F3CF}, 261 {REG_A5XX_RBBM_CLOCK_HYST_SP3, 0x0000F3CF}, 262 {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, 263 {REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080}, 264 {REG_A5XX_RBBM_CLOCK_DELAY_SP2, 0x00000080}, 265 {REG_A5XX_RBBM_CLOCK_DELAY_SP3, 0x00000080}, 266 {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, 267 {REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222}, 268 {REG_A5XX_RBBM_CLOCK_CNTL_TP2, 0x22222222}, 269 {REG_A5XX_RBBM_CLOCK_CNTL_TP3, 0x22222222}, 270 {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, 271 {REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, 272 {REG_A5XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222}, 273 {REG_A5XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222}, 274 {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, 275 {REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222}, 276 {REG_A5XX_RBBM_CLOCK_CNTL3_TP2, 0x00002222}, 277 {REG_A5XX_RBBM_CLOCK_CNTL3_TP3, 0x00002222}, 278 {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, 279 {REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, 280 {REG_A5XX_RBBM_CLOCK_HYST_TP2, 0x77777777}, 281 {REG_A5XX_RBBM_CLOCK_HYST_TP3, 0x77777777}, 282 {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, 283 {REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, 284 {REG_A5XX_RBBM_CLOCK_HYST2_TP2, 0x77777777}, 285 {REG_A5XX_RBBM_CLOCK_HYST2_TP3, 0x77777777}, 286 {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, 287 {REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777}, 288 {REG_A5XX_RBBM_CLOCK_HYST3_TP2, 0x00007777}, 289 {REG_A5XX_RBBM_CLOCK_HYST3_TP3, 0x00007777}, 290 {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, 291 {REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, 292 {REG_A5XX_RBBM_CLOCK_DELAY_TP2, 0x11111111}, 293 {REG_A5XX_RBBM_CLOCK_DELAY_TP3, 0x11111111}, 294 {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, 295 {REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, 296 {REG_A5XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111}, 297 {REG_A5XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111}, 298 {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, 299 {REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111}, 300 {REG_A5XX_RBBM_CLOCK_DELAY3_TP2, 0x00001111}, 301 {REG_A5XX_RBBM_CLOCK_DELAY3_TP3, 0x00001111}, 302 {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, 303 {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, 304 {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, 305 {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, 306 {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444}, 307 {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, 308 {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, 309 {REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222}, 310 {REG_A5XX_RBBM_CLOCK_CNTL_RB2, 0x22222222}, 311 {REG_A5XX_RBBM_CLOCK_CNTL_RB3, 0x22222222}, 312 {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, 313 {REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222}, 314 {REG_A5XX_RBBM_CLOCK_CNTL2_RB2, 0x00222222}, 315 {REG_A5XX_RBBM_CLOCK_CNTL2_RB3, 0x00222222}, 316 {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, 317 {REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220}, 318 {REG_A5XX_RBBM_CLOCK_CNTL_CCU2, 0x00022220}, 319 {REG_A5XX_RBBM_CLOCK_CNTL_CCU3, 0x00022220}, 320 {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, 321 {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, 322 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, 323 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404}, 324 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU2, 0x04040404}, 325 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU3, 0x04040404}, 326 {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, 327 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, 328 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002}, 329 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2, 0x00000002}, 330 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3, 0x00000002}, 331 {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, 332 {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, 333 {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, 334 {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, 335 {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, 336 {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, 337 {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, 338 {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, 339 {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, 340 {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, 341 {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222} 342 }, a50x_hwcg[] = { 343 {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, 344 {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, 345 {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, 346 {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, 347 {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, 348 {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, 349 {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, 350 {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, 351 {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, 352 {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, 353 {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, 354 {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, 355 {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, 356 {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, 357 {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, 358 {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, 359 {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, 360 {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00FFFFF4}, 361 {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, 362 {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, 363 {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, 364 {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, 365 {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, 366 {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, 367 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, 368 {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, 369 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, 370 {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, 371 {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, 372 {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, 373 {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, 374 {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, 375 {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, 376 {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, 377 {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, 378 {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, 379 {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, 380 {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, 381 }, a512_hwcg[] = { 382 {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, 383 {REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222}, 384 {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, 385 {REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220}, 386 {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, 387 {REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF}, 388 {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, 389 {REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080}, 390 {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, 391 {REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222}, 392 {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, 393 {REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, 394 {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, 395 {REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222}, 396 {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, 397 {REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, 398 {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, 399 {REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, 400 {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, 401 {REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777}, 402 {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, 403 {REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, 404 {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, 405 {REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, 406 {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, 407 {REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111}, 408 {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, 409 {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, 410 {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, 411 {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, 412 {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444}, 413 {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, 414 {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, 415 {REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222}, 416 {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, 417 {REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222}, 418 {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, 419 {REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220}, 420 {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, 421 {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, 422 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, 423 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404}, 424 {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, 425 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, 426 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002}, 427 {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, 428 {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, 429 {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, 430 {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, 431 {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, 432 {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, 433 {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, 434 {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, 435 {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, 436 {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, 437 {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, 438 }; 439 440 void a5xx_set_hwcg(struct msm_gpu *gpu, bool state) 441 { 442 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 443 const struct adreno_five_hwcg_regs *regs; 444 unsigned int i, sz; 445 446 if (adreno_is_a506(adreno_gpu) || adreno_is_a508(adreno_gpu)) { 447 regs = a50x_hwcg; 448 sz = ARRAY_SIZE(a50x_hwcg); 449 } else if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu)) { 450 regs = a512_hwcg; 451 sz = ARRAY_SIZE(a512_hwcg); 452 } else { 453 regs = a5xx_hwcg; 454 sz = ARRAY_SIZE(a5xx_hwcg); 455 } 456 457 for (i = 0; i < sz; i++) 458 gpu_write(gpu, regs[i].offset, 459 state ? regs[i].value : 0); 460 461 if (adreno_is_a540(adreno_gpu)) { 462 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_DELAY_GPMU, state ? 0x00000770 : 0); 463 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_HYST_GPMU, state ? 0x00000004 : 0); 464 } 465 466 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, state ? 0xAAA8AA00 : 0); 467 gpu_write(gpu, REG_A5XX_RBBM_ISDB_CNT, state ? 0x182 : 0x180); 468 } 469 470 static int a5xx_me_init(struct msm_gpu *gpu) 471 { 472 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 473 struct msm_ringbuffer *ring = gpu->rb[0]; 474 475 OUT_PKT7(ring, CP_ME_INIT, 8); 476 477 OUT_RING(ring, 0x0000002F); 478 479 /* Enable multiple hardware contexts */ 480 OUT_RING(ring, 0x00000003); 481 482 /* Enable error detection */ 483 OUT_RING(ring, 0x20000000); 484 485 /* Don't enable header dump */ 486 OUT_RING(ring, 0x00000000); 487 OUT_RING(ring, 0x00000000); 488 489 /* Specify workarounds for various microcode issues */ 490 if (adreno_is_a506(adreno_gpu) || adreno_is_a530(adreno_gpu)) { 491 /* Workaround for token end syncs 492 * Force a WFI after every direct-render 3D mode draw and every 493 * 2D mode 3 draw 494 */ 495 OUT_RING(ring, 0x0000000B); 496 } else if (adreno_is_a510(adreno_gpu)) { 497 /* Workaround for token and syncs */ 498 OUT_RING(ring, 0x00000001); 499 } else { 500 /* No workarounds enabled */ 501 OUT_RING(ring, 0x00000000); 502 } 503 504 OUT_RING(ring, 0x00000000); 505 OUT_RING(ring, 0x00000000); 506 507 a5xx_flush(gpu, ring, true); 508 return a5xx_idle(gpu, ring) ? 0 : -EINVAL; 509 } 510 511 static int a5xx_preempt_start(struct msm_gpu *gpu) 512 { 513 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 514 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 515 struct msm_ringbuffer *ring = gpu->rb[0]; 516 517 if (gpu->nr_rings == 1) 518 return 0; 519 520 /* Turn off protected mode to write to special registers */ 521 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1); 522 OUT_RING(ring, 0); 523 524 /* Set the save preemption record for the ring/command */ 525 OUT_PKT4(ring, REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 2); 526 OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[ring->id])); 527 OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[ring->id])); 528 529 /* Turn back on protected mode */ 530 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1); 531 OUT_RING(ring, 1); 532 533 OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1); 534 OUT_RING(ring, 0x00); 535 536 OUT_PKT7(ring, CP_PREEMPT_ENABLE_LOCAL, 1); 537 OUT_RING(ring, 0x01); 538 539 OUT_PKT7(ring, CP_YIELD_ENABLE, 1); 540 OUT_RING(ring, 0x01); 541 542 /* Yield the floor on command completion */ 543 OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4); 544 OUT_RING(ring, 0x00); 545 OUT_RING(ring, 0x00); 546 OUT_RING(ring, 0x01); 547 OUT_RING(ring, 0x01); 548 549 /* The WHERE_AMI_I packet is not needed after a YIELD is issued */ 550 a5xx_flush(gpu, ring, false); 551 552 return a5xx_idle(gpu, ring) ? 0 : -EINVAL; 553 } 554 555 static void a5xx_ucode_check_version(struct a5xx_gpu *a5xx_gpu, 556 struct drm_gem_object *obj) 557 { 558 u32 *buf = msm_gem_get_vaddr(obj); 559 560 if (IS_ERR(buf)) 561 return; 562 563 /* 564 * If the lowest nibble is 0xa that is an indication that this microcode 565 * has been patched. The actual version is in dword [3] but we only care 566 * about the patchlevel which is the lowest nibble of dword [3] 567 */ 568 if (((buf[0] & 0xf) == 0xa) && (buf[2] & 0xf) >= 1) 569 a5xx_gpu->has_whereami = true; 570 571 msm_gem_put_vaddr(obj); 572 } 573 574 static int a5xx_ucode_load(struct msm_gpu *gpu) 575 { 576 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 577 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 578 int ret; 579 580 if (!a5xx_gpu->pm4_bo) { 581 a5xx_gpu->pm4_bo = adreno_fw_create_bo(gpu, 582 adreno_gpu->fw[ADRENO_FW_PM4], &a5xx_gpu->pm4_iova); 583 584 585 if (IS_ERR(a5xx_gpu->pm4_bo)) { 586 ret = PTR_ERR(a5xx_gpu->pm4_bo); 587 a5xx_gpu->pm4_bo = NULL; 588 DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PM4: %d\n", 589 ret); 590 return ret; 591 } 592 593 msm_gem_object_set_name(a5xx_gpu->pm4_bo, "pm4fw"); 594 } 595 596 if (!a5xx_gpu->pfp_bo) { 597 a5xx_gpu->pfp_bo = adreno_fw_create_bo(gpu, 598 adreno_gpu->fw[ADRENO_FW_PFP], &a5xx_gpu->pfp_iova); 599 600 if (IS_ERR(a5xx_gpu->pfp_bo)) { 601 ret = PTR_ERR(a5xx_gpu->pfp_bo); 602 a5xx_gpu->pfp_bo = NULL; 603 DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PFP: %d\n", 604 ret); 605 return ret; 606 } 607 608 msm_gem_object_set_name(a5xx_gpu->pfp_bo, "pfpfw"); 609 a5xx_ucode_check_version(a5xx_gpu, a5xx_gpu->pfp_bo); 610 } 611 612 if (a5xx_gpu->has_whereami) { 613 if (!a5xx_gpu->shadow_bo) { 614 a5xx_gpu->shadow = msm_gem_kernel_new(gpu->dev, 615 sizeof(u32) * gpu->nr_rings, 616 MSM_BO_WC | MSM_BO_MAP_PRIV, 617 gpu->aspace, &a5xx_gpu->shadow_bo, 618 &a5xx_gpu->shadow_iova); 619 620 if (IS_ERR(a5xx_gpu->shadow)) 621 return PTR_ERR(a5xx_gpu->shadow); 622 623 msm_gem_object_set_name(a5xx_gpu->shadow_bo, "shadow"); 624 } 625 } else if (gpu->nr_rings > 1) { 626 /* Disable preemption if WHERE_AM_I isn't available */ 627 a5xx_preempt_fini(gpu); 628 gpu->nr_rings = 1; 629 } 630 631 return 0; 632 } 633 634 #define SCM_GPU_ZAP_SHADER_RESUME 0 635 636 static int a5xx_zap_shader_resume(struct msm_gpu *gpu) 637 { 638 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 639 int ret; 640 641 /* 642 * Adreno 506 have CPZ Retention feature and doesn't require 643 * to resume zap shader 644 */ 645 if (adreno_is_a506(adreno_gpu)) 646 return 0; 647 648 ret = qcom_scm_set_remote_state(SCM_GPU_ZAP_SHADER_RESUME, GPU_PAS_ID); 649 if (ret) 650 DRM_ERROR("%s: zap-shader resume failed: %d\n", 651 gpu->name, ret); 652 653 return ret; 654 } 655 656 static int a5xx_zap_shader_init(struct msm_gpu *gpu) 657 { 658 static bool loaded; 659 int ret; 660 661 /* 662 * If the zap shader is already loaded into memory we just need to kick 663 * the remote processor to reinitialize it 664 */ 665 if (loaded) 666 return a5xx_zap_shader_resume(gpu); 667 668 ret = adreno_zap_shader_load(gpu, GPU_PAS_ID); 669 670 loaded = !ret; 671 return ret; 672 } 673 674 #define A5XX_INT_MASK (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \ 675 A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \ 676 A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \ 677 A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \ 678 A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \ 679 A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW | \ 680 A5XX_RBBM_INT_0_MASK_CP_HW_ERROR | \ 681 A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT | \ 682 A5XX_RBBM_INT_0_MASK_CP_SW | \ 683 A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \ 684 A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \ 685 A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP) 686 687 static int a5xx_hw_init(struct msm_gpu *gpu) 688 { 689 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 690 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 691 u32 regbit; 692 int ret; 693 694 gpu_write(gpu, REG_A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003); 695 696 if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu) || 697 adreno_is_a540(adreno_gpu)) 698 gpu_write(gpu, REG_A5XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009); 699 700 /* Make all blocks contribute to the GPU BUSY perf counter */ 701 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF); 702 703 /* Enable RBBM error reporting bits */ 704 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL0, 0x00000001); 705 706 if (adreno_gpu->info->quirks & ADRENO_QUIRK_FAULT_DETECT_MASK) { 707 /* 708 * Mask out the activity signals from RB1-3 to avoid false 709 * positives 710 */ 711 712 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11, 713 0xF0000000); 714 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12, 715 0xFFFFFFFF); 716 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13, 717 0xFFFFFFFF); 718 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14, 719 0xFFFFFFFF); 720 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15, 721 0xFFFFFFFF); 722 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16, 723 0xFFFFFFFF); 724 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17, 725 0xFFFFFFFF); 726 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18, 727 0xFFFFFFFF); 728 } 729 730 /* Enable fault detection */ 731 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_INT_CNTL, 732 (1 << 30) | 0xFFFF); 733 734 /* Turn on performance counters */ 735 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_CNTL, 0x01); 736 737 /* Select CP0 to always count cycles */ 738 gpu_write(gpu, REG_A5XX_CP_PERFCTR_CP_SEL_0, PERF_CP_ALWAYS_COUNT); 739 740 /* Select RBBM0 to countable 6 to get the busy status for devfreq */ 741 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_SEL_0, 6); 742 743 /* Increase VFD cache access so LRZ and other data gets evicted less */ 744 gpu_write(gpu, REG_A5XX_UCHE_CACHE_WAYS, 0x02); 745 746 /* Disable L2 bypass in the UCHE */ 747 gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_LO, 0xFFFF0000); 748 gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_HI, 0x0001FFFF); 749 gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_LO, 0xFFFF0000); 750 gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_HI, 0x0001FFFF); 751 752 /* Set the GMEM VA range (0 to gpu->gmem) */ 753 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_LO, 0x00100000); 754 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_HI, 0x00000000); 755 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_LO, 756 0x00100000 + adreno_gpu->info->gmem - 1); 757 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_HI, 0x00000000); 758 759 if (adreno_is_a506(adreno_gpu) || adreno_is_a508(adreno_gpu) || 760 adreno_is_a510(adreno_gpu)) { 761 gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x20); 762 if (adreno_is_a506(adreno_gpu) || adreno_is_a508(adreno_gpu)) 763 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400); 764 else 765 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x20); 766 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x40000030); 767 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x20100D0A); 768 } else { 769 gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x40); 770 if (adreno_is_a530(adreno_gpu)) 771 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x40); 772 else 773 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400); 774 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060); 775 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16); 776 } 777 778 if (adreno_is_a506(adreno_gpu) || adreno_is_a508(adreno_gpu)) 779 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 780 (0x100 << 11 | 0x100 << 22)); 781 else if (adreno_is_a509(adreno_gpu) || adreno_is_a510(adreno_gpu) || 782 adreno_is_a512(adreno_gpu)) 783 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 784 (0x200 << 11 | 0x200 << 22)); 785 else 786 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 787 (0x400 << 11 | 0x300 << 22)); 788 789 if (adreno_gpu->info->quirks & ADRENO_QUIRK_TWO_PASS_USE_WFI) 790 gpu_rmw(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8)); 791 792 /* 793 * Disable the RB sampler datapath DP2 clock gating optimization 794 * for 1-SP GPUs, as it is enabled by default. 795 */ 796 if (adreno_is_a506(adreno_gpu) || adreno_is_a508(adreno_gpu) || 797 adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu)) 798 gpu_rmw(gpu, REG_A5XX_RB_DBG_ECO_CNTL, 0, (1 << 9)); 799 800 /* Disable UCHE global filter as SP can invalidate/flush independently */ 801 gpu_write(gpu, REG_A5XX_UCHE_MODE_CNTL, BIT(29)); 802 803 /* Enable USE_RETENTION_FLOPS */ 804 gpu_write(gpu, REG_A5XX_CP_CHICKEN_DBG, 0x02000000); 805 806 /* Enable ME/PFP split notification */ 807 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL1, 0xA6FFFFFF); 808 809 /* 810 * In A5x, CCU can send context_done event of a particular context to 811 * UCHE which ultimately reaches CP even when there is valid 812 * transaction of that context inside CCU. This can let CP to program 813 * config registers, which will make the "valid transaction" inside 814 * CCU to be interpreted differently. This can cause gpu fault. This 815 * bug is fixed in latest A510 revision. To enable this bug fix - 816 * bit[11] of RB_DBG_ECO_CNTL need to be set to 0, default is 1 817 * (disable). For older A510 version this bit is unused. 818 */ 819 if (adreno_is_a510(adreno_gpu)) 820 gpu_rmw(gpu, REG_A5XX_RB_DBG_ECO_CNTL, (1 << 11), 0); 821 822 /* Enable HWCG */ 823 a5xx_set_hwcg(gpu, true); 824 825 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL2, 0x0000003F); 826 827 /* Set the highest bank bit */ 828 if (adreno_is_a540(adreno_gpu) || adreno_is_a530(adreno_gpu)) 829 regbit = 2; 830 else 831 regbit = 1; 832 833 gpu_write(gpu, REG_A5XX_TPL1_MODE_CNTL, regbit << 7); 834 gpu_write(gpu, REG_A5XX_RB_MODE_CNTL, regbit << 1); 835 836 if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu) || 837 adreno_is_a540(adreno_gpu)) 838 gpu_write(gpu, REG_A5XX_UCHE_DBG_ECO_CNTL_2, regbit); 839 840 /* Disable All flat shading optimization (ALLFLATOPTDIS) */ 841 gpu_rmw(gpu, REG_A5XX_VPC_DBG_ECO_CNTL, 0, (1 << 10)); 842 843 /* Protect registers from the CP */ 844 gpu_write(gpu, REG_A5XX_CP_PROTECT_CNTL, 0x00000007); 845 846 /* RBBM */ 847 gpu_write(gpu, REG_A5XX_CP_PROTECT(0), ADRENO_PROTECT_RW(0x04, 4)); 848 gpu_write(gpu, REG_A5XX_CP_PROTECT(1), ADRENO_PROTECT_RW(0x08, 8)); 849 gpu_write(gpu, REG_A5XX_CP_PROTECT(2), ADRENO_PROTECT_RW(0x10, 16)); 850 gpu_write(gpu, REG_A5XX_CP_PROTECT(3), ADRENO_PROTECT_RW(0x20, 32)); 851 gpu_write(gpu, REG_A5XX_CP_PROTECT(4), ADRENO_PROTECT_RW(0x40, 64)); 852 gpu_write(gpu, REG_A5XX_CP_PROTECT(5), ADRENO_PROTECT_RW(0x80, 64)); 853 854 /* Content protect */ 855 gpu_write(gpu, REG_A5XX_CP_PROTECT(6), 856 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO, 857 16)); 858 gpu_write(gpu, REG_A5XX_CP_PROTECT(7), 859 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TRUST_CNTL, 2)); 860 861 /* CP */ 862 gpu_write(gpu, REG_A5XX_CP_PROTECT(8), ADRENO_PROTECT_RW(0x800, 64)); 863 gpu_write(gpu, REG_A5XX_CP_PROTECT(9), ADRENO_PROTECT_RW(0x840, 8)); 864 gpu_write(gpu, REG_A5XX_CP_PROTECT(10), ADRENO_PROTECT_RW(0x880, 32)); 865 gpu_write(gpu, REG_A5XX_CP_PROTECT(11), ADRENO_PROTECT_RW(0xAA0, 1)); 866 867 /* RB */ 868 gpu_write(gpu, REG_A5XX_CP_PROTECT(12), ADRENO_PROTECT_RW(0xCC0, 1)); 869 gpu_write(gpu, REG_A5XX_CP_PROTECT(13), ADRENO_PROTECT_RW(0xCF0, 2)); 870 871 /* VPC */ 872 gpu_write(gpu, REG_A5XX_CP_PROTECT(14), ADRENO_PROTECT_RW(0xE68, 8)); 873 gpu_write(gpu, REG_A5XX_CP_PROTECT(15), ADRENO_PROTECT_RW(0xE70, 16)); 874 875 /* UCHE */ 876 gpu_write(gpu, REG_A5XX_CP_PROTECT(16), ADRENO_PROTECT_RW(0xE80, 16)); 877 878 /* SMMU */ 879 gpu_write(gpu, REG_A5XX_CP_PROTECT(17), 880 ADRENO_PROTECT_RW(0x10000, 0x8000)); 881 882 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_CNTL, 0); 883 /* 884 * Disable the trusted memory range - we don't actually supported secure 885 * memory rendering at this point in time and we don't want to block off 886 * part of the virtual memory space. 887 */ 888 gpu_write64(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO, 0x00000000); 889 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000); 890 891 /* Put the GPU into 64 bit by default */ 892 gpu_write(gpu, REG_A5XX_CP_ADDR_MODE_CNTL, 0x1); 893 gpu_write(gpu, REG_A5XX_VSC_ADDR_MODE_CNTL, 0x1); 894 gpu_write(gpu, REG_A5XX_GRAS_ADDR_MODE_CNTL, 0x1); 895 gpu_write(gpu, REG_A5XX_RB_ADDR_MODE_CNTL, 0x1); 896 gpu_write(gpu, REG_A5XX_PC_ADDR_MODE_CNTL, 0x1); 897 gpu_write(gpu, REG_A5XX_HLSQ_ADDR_MODE_CNTL, 0x1); 898 gpu_write(gpu, REG_A5XX_VFD_ADDR_MODE_CNTL, 0x1); 899 gpu_write(gpu, REG_A5XX_VPC_ADDR_MODE_CNTL, 0x1); 900 gpu_write(gpu, REG_A5XX_UCHE_ADDR_MODE_CNTL, 0x1); 901 gpu_write(gpu, REG_A5XX_SP_ADDR_MODE_CNTL, 0x1); 902 gpu_write(gpu, REG_A5XX_TPL1_ADDR_MODE_CNTL, 0x1); 903 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1); 904 905 /* 906 * VPC corner case with local memory load kill leads to corrupt 907 * internal state. Normal Disable does not work for all a5x chips. 908 * So do the following setting to disable it. 909 */ 910 if (adreno_gpu->info->quirks & ADRENO_QUIRK_LMLOADKILL_DISABLE) { 911 gpu_rmw(gpu, REG_A5XX_VPC_DBG_ECO_CNTL, 0, BIT(23)); 912 gpu_rmw(gpu, REG_A5XX_HLSQ_DBG_ECO_CNTL, BIT(18), 0); 913 } 914 915 ret = adreno_hw_init(gpu); 916 if (ret) 917 return ret; 918 919 if (adreno_is_a530(adreno_gpu) || adreno_is_a540(adreno_gpu)) 920 a5xx_gpmu_ucode_init(gpu); 921 922 gpu_write64(gpu, REG_A5XX_CP_ME_INSTR_BASE_LO, a5xx_gpu->pm4_iova); 923 gpu_write64(gpu, REG_A5XX_CP_PFP_INSTR_BASE_LO, a5xx_gpu->pfp_iova); 924 925 /* Set the ringbuffer address */ 926 gpu_write64(gpu, REG_A5XX_CP_RB_BASE, gpu->rb[0]->iova); 927 928 /* 929 * If the microcode supports the WHERE_AM_I opcode then we can use that 930 * in lieu of the RPTR shadow and enable preemption. Otherwise, we 931 * can't safely use the RPTR shadow or preemption. In either case, the 932 * RPTR shadow should be disabled in hardware. 933 */ 934 gpu_write(gpu, REG_A5XX_CP_RB_CNTL, 935 MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE); 936 937 /* Configure the RPTR shadow if needed: */ 938 if (a5xx_gpu->shadow_bo) { 939 gpu_write64(gpu, REG_A5XX_CP_RB_RPTR_ADDR, 940 shadowptr(a5xx_gpu, gpu->rb[0])); 941 } 942 943 a5xx_preempt_hw_init(gpu); 944 945 /* Disable the interrupts through the initial bringup stage */ 946 gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK, A5XX_INT_MASK); 947 948 /* Clear ME_HALT to start the micro engine */ 949 gpu_write(gpu, REG_A5XX_CP_PFP_ME_CNTL, 0); 950 ret = a5xx_me_init(gpu); 951 if (ret) 952 return ret; 953 954 ret = a5xx_power_init(gpu); 955 if (ret) 956 return ret; 957 958 /* 959 * Send a pipeline event stat to get misbehaving counters to start 960 * ticking correctly 961 */ 962 if (adreno_is_a530(adreno_gpu)) { 963 OUT_PKT7(gpu->rb[0], CP_EVENT_WRITE, 1); 964 OUT_RING(gpu->rb[0], CP_EVENT_WRITE_0_EVENT(STAT_EVENT)); 965 966 a5xx_flush(gpu, gpu->rb[0], true); 967 if (!a5xx_idle(gpu, gpu->rb[0])) 968 return -EINVAL; 969 } 970 971 /* 972 * If the chip that we are using does support loading one, then 973 * try to load a zap shader into the secure world. If successful 974 * we can use the CP to switch out of secure mode. If not then we 975 * have no resource but to try to switch ourselves out manually. If we 976 * guessed wrong then access to the RBBM_SECVID_TRUST_CNTL register will 977 * be blocked and a permissions violation will soon follow. 978 */ 979 ret = a5xx_zap_shader_init(gpu); 980 if (!ret) { 981 OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1); 982 OUT_RING(gpu->rb[0], 0x00000000); 983 984 a5xx_flush(gpu, gpu->rb[0], true); 985 if (!a5xx_idle(gpu, gpu->rb[0])) 986 return -EINVAL; 987 } else if (ret == -ENODEV) { 988 /* 989 * This device does not use zap shader (but print a warning 990 * just in case someone got their dt wrong.. hopefully they 991 * have a debug UART to realize the error of their ways... 992 * if you mess this up you are about to crash horribly) 993 */ 994 dev_warn_once(gpu->dev->dev, 995 "Zap shader not enabled - using SECVID_TRUST_CNTL instead\n"); 996 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TRUST_CNTL, 0x0); 997 } else { 998 return ret; 999 } 1000 1001 /* Last step - yield the ringbuffer */ 1002 a5xx_preempt_start(gpu); 1003 1004 return 0; 1005 } 1006 1007 static void a5xx_recover(struct msm_gpu *gpu) 1008 { 1009 int i; 1010 1011 adreno_dump_info(gpu); 1012 1013 for (i = 0; i < 8; i++) { 1014 printk("CP_SCRATCH_REG%d: %u\n", i, 1015 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(i))); 1016 } 1017 1018 if (hang_debug) 1019 a5xx_dump(gpu); 1020 1021 gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 1); 1022 gpu_read(gpu, REG_A5XX_RBBM_SW_RESET_CMD); 1023 gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 0); 1024 adreno_recover(gpu); 1025 } 1026 1027 static void a5xx_destroy(struct msm_gpu *gpu) 1028 { 1029 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1030 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 1031 1032 DBG("%s", gpu->name); 1033 1034 a5xx_preempt_fini(gpu); 1035 1036 if (a5xx_gpu->pm4_bo) { 1037 msm_gem_unpin_iova(a5xx_gpu->pm4_bo, gpu->aspace); 1038 drm_gem_object_put(a5xx_gpu->pm4_bo); 1039 } 1040 1041 if (a5xx_gpu->pfp_bo) { 1042 msm_gem_unpin_iova(a5xx_gpu->pfp_bo, gpu->aspace); 1043 drm_gem_object_put(a5xx_gpu->pfp_bo); 1044 } 1045 1046 if (a5xx_gpu->gpmu_bo) { 1047 msm_gem_unpin_iova(a5xx_gpu->gpmu_bo, gpu->aspace); 1048 drm_gem_object_put(a5xx_gpu->gpmu_bo); 1049 } 1050 1051 if (a5xx_gpu->shadow_bo) { 1052 msm_gem_unpin_iova(a5xx_gpu->shadow_bo, gpu->aspace); 1053 drm_gem_object_put(a5xx_gpu->shadow_bo); 1054 } 1055 1056 adreno_gpu_cleanup(adreno_gpu); 1057 kfree(a5xx_gpu); 1058 } 1059 1060 static inline bool _a5xx_check_idle(struct msm_gpu *gpu) 1061 { 1062 if (gpu_read(gpu, REG_A5XX_RBBM_STATUS) & ~A5XX_RBBM_STATUS_HI_BUSY) 1063 return false; 1064 1065 /* 1066 * Nearly every abnormality ends up pausing the GPU and triggering a 1067 * fault so we can safely just watch for this one interrupt to fire 1068 */ 1069 return !(gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS) & 1070 A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT); 1071 } 1072 1073 bool a5xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring) 1074 { 1075 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1076 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 1077 1078 if (ring != a5xx_gpu->cur_ring) { 1079 WARN(1, "Tried to idle a non-current ringbuffer\n"); 1080 return false; 1081 } 1082 1083 /* wait for CP to drain ringbuffer: */ 1084 if (!adreno_idle(gpu, ring)) 1085 return false; 1086 1087 if (spin_until(_a5xx_check_idle(gpu))) { 1088 DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n", 1089 gpu->name, __builtin_return_address(0), 1090 gpu_read(gpu, REG_A5XX_RBBM_STATUS), 1091 gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS), 1092 gpu_read(gpu, REG_A5XX_CP_RB_RPTR), 1093 gpu_read(gpu, REG_A5XX_CP_RB_WPTR)); 1094 return false; 1095 } 1096 1097 return true; 1098 } 1099 1100 static int a5xx_fault_handler(void *arg, unsigned long iova, int flags, void *data) 1101 { 1102 struct msm_gpu *gpu = arg; 1103 struct adreno_smmu_fault_info *info = data; 1104 char block[12] = "unknown"; 1105 u32 scratch[] = { 1106 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(4)), 1107 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(5)), 1108 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(6)), 1109 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(7)), 1110 }; 1111 1112 if (info) 1113 snprintf(block, sizeof(block), "%x", info->fsynr1); 1114 1115 return adreno_fault_handler(gpu, iova, flags, info, block, scratch); 1116 } 1117 1118 static void a5xx_cp_err_irq(struct msm_gpu *gpu) 1119 { 1120 u32 status = gpu_read(gpu, REG_A5XX_CP_INTERRUPT_STATUS); 1121 1122 if (status & A5XX_CP_INT_CP_OPCODE_ERROR) { 1123 u32 val; 1124 1125 gpu_write(gpu, REG_A5XX_CP_PFP_STAT_ADDR, 0); 1126 1127 /* 1128 * REG_A5XX_CP_PFP_STAT_DATA is indexed, and we want index 1 so 1129 * read it twice 1130 */ 1131 1132 gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA); 1133 val = gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA); 1134 1135 dev_err_ratelimited(gpu->dev->dev, "CP | opcode error | possible opcode=0x%8.8X\n", 1136 val); 1137 } 1138 1139 if (status & A5XX_CP_INT_CP_HW_FAULT_ERROR) 1140 dev_err_ratelimited(gpu->dev->dev, "CP | HW fault | status=0x%8.8X\n", 1141 gpu_read(gpu, REG_A5XX_CP_HW_FAULT)); 1142 1143 if (status & A5XX_CP_INT_CP_DMA_ERROR) 1144 dev_err_ratelimited(gpu->dev->dev, "CP | DMA error\n"); 1145 1146 if (status & A5XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) { 1147 u32 val = gpu_read(gpu, REG_A5XX_CP_PROTECT_STATUS); 1148 1149 dev_err_ratelimited(gpu->dev->dev, 1150 "CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n", 1151 val & (1 << 24) ? "WRITE" : "READ", 1152 (val & 0xFFFFF) >> 2, val); 1153 } 1154 1155 if (status & A5XX_CP_INT_CP_AHB_ERROR) { 1156 u32 status = gpu_read(gpu, REG_A5XX_CP_AHB_FAULT); 1157 const char *access[16] = { "reserved", "reserved", 1158 "timestamp lo", "timestamp hi", "pfp read", "pfp write", 1159 "", "", "me read", "me write", "", "", "crashdump read", 1160 "crashdump write" }; 1161 1162 dev_err_ratelimited(gpu->dev->dev, 1163 "CP | AHB error | addr=%X access=%s error=%d | status=0x%8.8X\n", 1164 status & 0xFFFFF, access[(status >> 24) & 0xF], 1165 (status & (1 << 31)), status); 1166 } 1167 } 1168 1169 static void a5xx_rbbm_err_irq(struct msm_gpu *gpu, u32 status) 1170 { 1171 if (status & A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR) { 1172 u32 val = gpu_read(gpu, REG_A5XX_RBBM_AHB_ERROR_STATUS); 1173 1174 dev_err_ratelimited(gpu->dev->dev, 1175 "RBBM | AHB bus error | %s | addr=0x%X | ports=0x%X:0x%X\n", 1176 val & (1 << 28) ? "WRITE" : "READ", 1177 (val & 0xFFFFF) >> 2, (val >> 20) & 0x3, 1178 (val >> 24) & 0xF); 1179 1180 /* Clear the error */ 1181 gpu_write(gpu, REG_A5XX_RBBM_AHB_CMD, (1 << 4)); 1182 1183 /* Clear the interrupt */ 1184 gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD, 1185 A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR); 1186 } 1187 1188 if (status & A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT) 1189 dev_err_ratelimited(gpu->dev->dev, "RBBM | AHB transfer timeout\n"); 1190 1191 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT) 1192 dev_err_ratelimited(gpu->dev->dev, "RBBM | ME master split | status=0x%X\n", 1193 gpu_read(gpu, REG_A5XX_RBBM_AHB_ME_SPLIT_STATUS)); 1194 1195 if (status & A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT) 1196 dev_err_ratelimited(gpu->dev->dev, "RBBM | PFP master split | status=0x%X\n", 1197 gpu_read(gpu, REG_A5XX_RBBM_AHB_PFP_SPLIT_STATUS)); 1198 1199 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT) 1200 dev_err_ratelimited(gpu->dev->dev, "RBBM | ETS master split | status=0x%X\n", 1201 gpu_read(gpu, REG_A5XX_RBBM_AHB_ETS_SPLIT_STATUS)); 1202 1203 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW) 1204 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB ASYNC overflow\n"); 1205 1206 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW) 1207 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB bus overflow\n"); 1208 } 1209 1210 static void a5xx_uche_err_irq(struct msm_gpu *gpu) 1211 { 1212 uint64_t addr = (uint64_t) gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_HI); 1213 1214 addr |= gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_LO); 1215 1216 dev_err_ratelimited(gpu->dev->dev, "UCHE | Out of bounds access | addr=0x%llX\n", 1217 addr); 1218 } 1219 1220 static void a5xx_gpmu_err_irq(struct msm_gpu *gpu) 1221 { 1222 dev_err_ratelimited(gpu->dev->dev, "GPMU | voltage droop\n"); 1223 } 1224 1225 static void a5xx_fault_detect_irq(struct msm_gpu *gpu) 1226 { 1227 struct drm_device *dev = gpu->dev; 1228 struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu); 1229 1230 /* 1231 * If stalled on SMMU fault, we could trip the GPU's hang detection, 1232 * but the fault handler will trigger the devcore dump, and we want 1233 * to otherwise resume normally rather than killing the submit, so 1234 * just bail. 1235 */ 1236 if (gpu_read(gpu, REG_A5XX_RBBM_STATUS3) & BIT(24)) 1237 return; 1238 1239 DRM_DEV_ERROR(dev->dev, "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n", 1240 ring ? ring->id : -1, ring ? ring->fctx->last_fence : 0, 1241 gpu_read(gpu, REG_A5XX_RBBM_STATUS), 1242 gpu_read(gpu, REG_A5XX_CP_RB_RPTR), 1243 gpu_read(gpu, REG_A5XX_CP_RB_WPTR), 1244 gpu_read64(gpu, REG_A5XX_CP_IB1_BASE), 1245 gpu_read(gpu, REG_A5XX_CP_IB1_BUFSZ), 1246 gpu_read64(gpu, REG_A5XX_CP_IB2_BASE), 1247 gpu_read(gpu, REG_A5XX_CP_IB2_BUFSZ)); 1248 1249 /* Turn off the hangcheck timer to keep it from bothering us */ 1250 del_timer(&gpu->hangcheck_timer); 1251 1252 kthread_queue_work(gpu->worker, &gpu->recover_work); 1253 } 1254 1255 #define RBBM_ERROR_MASK \ 1256 (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \ 1257 A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \ 1258 A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \ 1259 A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \ 1260 A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \ 1261 A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW) 1262 1263 static irqreturn_t a5xx_irq(struct msm_gpu *gpu) 1264 { 1265 struct msm_drm_private *priv = gpu->dev->dev_private; 1266 u32 status = gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS); 1267 1268 /* 1269 * Clear all the interrupts except RBBM_AHB_ERROR - if we clear it 1270 * before the source is cleared the interrupt will storm. 1271 */ 1272 gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD, 1273 status & ~A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR); 1274 1275 if (priv->disable_err_irq) { 1276 status &= A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | 1277 A5XX_RBBM_INT_0_MASK_CP_SW; 1278 } 1279 1280 /* Pass status to a5xx_rbbm_err_irq because we've already cleared it */ 1281 if (status & RBBM_ERROR_MASK) 1282 a5xx_rbbm_err_irq(gpu, status); 1283 1284 if (status & A5XX_RBBM_INT_0_MASK_CP_HW_ERROR) 1285 a5xx_cp_err_irq(gpu); 1286 1287 if (status & A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT) 1288 a5xx_fault_detect_irq(gpu); 1289 1290 if (status & A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS) 1291 a5xx_uche_err_irq(gpu); 1292 1293 if (status & A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP) 1294 a5xx_gpmu_err_irq(gpu); 1295 1296 if (status & A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS) { 1297 a5xx_preempt_trigger(gpu); 1298 msm_gpu_retire(gpu); 1299 } 1300 1301 if (status & A5XX_RBBM_INT_0_MASK_CP_SW) 1302 a5xx_preempt_irq(gpu); 1303 1304 return IRQ_HANDLED; 1305 } 1306 1307 static const u32 a5xx_registers[] = { 1308 0x0000, 0x0002, 0x0004, 0x0020, 0x0022, 0x0026, 0x0029, 0x002B, 1309 0x002E, 0x0035, 0x0038, 0x0042, 0x0044, 0x0044, 0x0047, 0x0095, 1310 0x0097, 0x00BB, 0x03A0, 0x0464, 0x0469, 0x046F, 0x04D2, 0x04D3, 1311 0x04E0, 0x0533, 0x0540, 0x0555, 0x0800, 0x081A, 0x081F, 0x0841, 1312 0x0860, 0x0860, 0x0880, 0x08A0, 0x0B00, 0x0B12, 0x0B15, 0x0B28, 1313 0x0B78, 0x0B7F, 0x0BB0, 0x0BBD, 0x0BC0, 0x0BC6, 0x0BD0, 0x0C53, 1314 0x0C60, 0x0C61, 0x0C80, 0x0C82, 0x0C84, 0x0C85, 0x0C90, 0x0C98, 1315 0x0CA0, 0x0CA0, 0x0CB0, 0x0CB2, 0x2180, 0x2185, 0x2580, 0x2585, 1316 0x0CC1, 0x0CC1, 0x0CC4, 0x0CC7, 0x0CCC, 0x0CCC, 0x0CD0, 0x0CD8, 1317 0x0CE0, 0x0CE5, 0x0CE8, 0x0CE8, 0x0CEC, 0x0CF1, 0x0CFB, 0x0D0E, 1318 0x2100, 0x211E, 0x2140, 0x2145, 0x2500, 0x251E, 0x2540, 0x2545, 1319 0x0D10, 0x0D17, 0x0D20, 0x0D23, 0x0D30, 0x0D30, 0x20C0, 0x20C0, 1320 0x24C0, 0x24C0, 0x0E40, 0x0E43, 0x0E4A, 0x0E4A, 0x0E50, 0x0E57, 1321 0x0E60, 0x0E7C, 0x0E80, 0x0E8E, 0x0E90, 0x0E96, 0x0EA0, 0x0EA8, 1322 0x0EB0, 0x0EB2, 0xE140, 0xE147, 0xE150, 0xE187, 0xE1A0, 0xE1A9, 1323 0xE1B0, 0xE1B6, 0xE1C0, 0xE1C7, 0xE1D0, 0xE1D1, 0xE200, 0xE201, 1324 0xE210, 0xE21C, 0xE240, 0xE268, 0xE000, 0xE006, 0xE010, 0xE09A, 1325 0xE0A0, 0xE0A4, 0xE0AA, 0xE0EB, 0xE100, 0xE105, 0xE380, 0xE38F, 1326 0xE3B0, 0xE3B0, 0xE400, 0xE405, 0xE408, 0xE4E9, 0xE4F0, 0xE4F0, 1327 0xE280, 0xE280, 0xE282, 0xE2A3, 0xE2A5, 0xE2C2, 0xE940, 0xE947, 1328 0xE950, 0xE987, 0xE9A0, 0xE9A9, 0xE9B0, 0xE9B6, 0xE9C0, 0xE9C7, 1329 0xE9D0, 0xE9D1, 0xEA00, 0xEA01, 0xEA10, 0xEA1C, 0xEA40, 0xEA68, 1330 0xE800, 0xE806, 0xE810, 0xE89A, 0xE8A0, 0xE8A4, 0xE8AA, 0xE8EB, 1331 0xE900, 0xE905, 0xEB80, 0xEB8F, 0xEBB0, 0xEBB0, 0xEC00, 0xEC05, 1332 0xEC08, 0xECE9, 0xECF0, 0xECF0, 0xEA80, 0xEA80, 0xEA82, 0xEAA3, 1333 0xEAA5, 0xEAC2, 0xA800, 0xA800, 0xA820, 0xA828, 0xA840, 0xA87D, 1334 0XA880, 0xA88D, 0xA890, 0xA8A3, 0xA8D0, 0xA8D8, 0xA8E0, 0xA8F5, 1335 0xAC60, 0xAC60, ~0, 1336 }; 1337 1338 static void a5xx_dump(struct msm_gpu *gpu) 1339 { 1340 DRM_DEV_INFO(gpu->dev->dev, "status: %08x\n", 1341 gpu_read(gpu, REG_A5XX_RBBM_STATUS)); 1342 adreno_dump(gpu); 1343 } 1344 1345 static int a5xx_pm_resume(struct msm_gpu *gpu) 1346 { 1347 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1348 int ret; 1349 1350 /* Turn on the core power */ 1351 ret = msm_gpu_pm_resume(gpu); 1352 if (ret) 1353 return ret; 1354 1355 /* Adreno 506, 508, 509, 510, 512 needs manual RBBM sus/res control */ 1356 if (!(adreno_is_a530(adreno_gpu) || adreno_is_a540(adreno_gpu))) { 1357 /* Halt the sp_input_clk at HM level */ 1358 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0x00000055); 1359 a5xx_set_hwcg(gpu, true); 1360 /* Turn on sp_input_clk at HM level */ 1361 gpu_rmw(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0xff, 0); 1362 return 0; 1363 } 1364 1365 /* Turn the RBCCU domain first to limit the chances of voltage droop */ 1366 gpu_write(gpu, REG_A5XX_GPMU_RBCCU_POWER_CNTL, 0x778000); 1367 1368 /* Wait 3 usecs before polling */ 1369 udelay(3); 1370 1371 ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS, 1372 (1 << 20), (1 << 20)); 1373 if (ret) { 1374 DRM_ERROR("%s: timeout waiting for RBCCU GDSC enable: %X\n", 1375 gpu->name, 1376 gpu_read(gpu, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS)); 1377 return ret; 1378 } 1379 1380 /* Turn on the SP domain */ 1381 gpu_write(gpu, REG_A5XX_GPMU_SP_POWER_CNTL, 0x778000); 1382 ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_SP_PWR_CLK_STATUS, 1383 (1 << 20), (1 << 20)); 1384 if (ret) 1385 DRM_ERROR("%s: timeout waiting for SP GDSC enable\n", 1386 gpu->name); 1387 1388 return ret; 1389 } 1390 1391 static int a5xx_pm_suspend(struct msm_gpu *gpu) 1392 { 1393 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1394 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 1395 u32 mask = 0xf; 1396 int i, ret; 1397 1398 /* A506, A508, A510 have 3 XIN ports in VBIF */ 1399 if (adreno_is_a506(adreno_gpu) || adreno_is_a508(adreno_gpu) || 1400 adreno_is_a510(adreno_gpu)) 1401 mask = 0x7; 1402 1403 /* Clear the VBIF pipe before shutting down */ 1404 gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, mask); 1405 spin_until((gpu_read(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL1) & 1406 mask) == mask); 1407 1408 gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0); 1409 1410 /* 1411 * Reset the VBIF before power collapse to avoid issue with FIFO 1412 * entries on Adreno A510 and A530 (the others will tend to lock up) 1413 */ 1414 if (adreno_is_a510(adreno_gpu) || adreno_is_a530(adreno_gpu)) { 1415 gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x003C0000); 1416 gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x00000000); 1417 } 1418 1419 ret = msm_gpu_pm_suspend(gpu); 1420 if (ret) 1421 return ret; 1422 1423 if (a5xx_gpu->has_whereami) 1424 for (i = 0; i < gpu->nr_rings; i++) 1425 a5xx_gpu->shadow[i] = 0; 1426 1427 return 0; 1428 } 1429 1430 static int a5xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value) 1431 { 1432 *value = gpu_read64(gpu, REG_A5XX_RBBM_ALWAYSON_COUNTER_LO); 1433 1434 return 0; 1435 } 1436 1437 struct a5xx_crashdumper { 1438 void *ptr; 1439 struct drm_gem_object *bo; 1440 u64 iova; 1441 }; 1442 1443 struct a5xx_gpu_state { 1444 struct msm_gpu_state base; 1445 u32 *hlsqregs; 1446 }; 1447 1448 static int a5xx_crashdumper_init(struct msm_gpu *gpu, 1449 struct a5xx_crashdumper *dumper) 1450 { 1451 dumper->ptr = msm_gem_kernel_new(gpu->dev, 1452 SZ_1M, MSM_BO_WC, gpu->aspace, 1453 &dumper->bo, &dumper->iova); 1454 1455 if (!IS_ERR(dumper->ptr)) 1456 msm_gem_object_set_name(dumper->bo, "crashdump"); 1457 1458 return PTR_ERR_OR_ZERO(dumper->ptr); 1459 } 1460 1461 static int a5xx_crashdumper_run(struct msm_gpu *gpu, 1462 struct a5xx_crashdumper *dumper) 1463 { 1464 u32 val; 1465 1466 if (IS_ERR_OR_NULL(dumper->ptr)) 1467 return -EINVAL; 1468 1469 gpu_write64(gpu, REG_A5XX_CP_CRASH_SCRIPT_BASE_LO, dumper->iova); 1470 1471 gpu_write(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, 1); 1472 1473 return gpu_poll_timeout(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, val, 1474 val & 0x04, 100, 10000); 1475 } 1476 1477 /* 1478 * These are a list of the registers that need to be read through the HLSQ 1479 * aperture through the crashdumper. These are not nominally accessible from 1480 * the CPU on a secure platform. 1481 */ 1482 static const struct { 1483 u32 type; 1484 u32 regoffset; 1485 u32 count; 1486 } a5xx_hlsq_aperture_regs[] = { 1487 { 0x35, 0xe00, 0x32 }, /* HSLQ non-context */ 1488 { 0x31, 0x2080, 0x1 }, /* HLSQ 2D context 0 */ 1489 { 0x33, 0x2480, 0x1 }, /* HLSQ 2D context 1 */ 1490 { 0x32, 0xe780, 0x62 }, /* HLSQ 3D context 0 */ 1491 { 0x34, 0xef80, 0x62 }, /* HLSQ 3D context 1 */ 1492 { 0x3f, 0x0ec0, 0x40 }, /* SP non-context */ 1493 { 0x3d, 0x2040, 0x1 }, /* SP 2D context 0 */ 1494 { 0x3b, 0x2440, 0x1 }, /* SP 2D context 1 */ 1495 { 0x3e, 0xe580, 0x170 }, /* SP 3D context 0 */ 1496 { 0x3c, 0xed80, 0x170 }, /* SP 3D context 1 */ 1497 { 0x3a, 0x0f00, 0x1c }, /* TP non-context */ 1498 { 0x38, 0x2000, 0xa }, /* TP 2D context 0 */ 1499 { 0x36, 0x2400, 0xa }, /* TP 2D context 1 */ 1500 { 0x39, 0xe700, 0x80 }, /* TP 3D context 0 */ 1501 { 0x37, 0xef00, 0x80 }, /* TP 3D context 1 */ 1502 }; 1503 1504 static void a5xx_gpu_state_get_hlsq_regs(struct msm_gpu *gpu, 1505 struct a5xx_gpu_state *a5xx_state) 1506 { 1507 struct a5xx_crashdumper dumper = { 0 }; 1508 u32 offset, count = 0; 1509 u64 *ptr; 1510 int i; 1511 1512 if (a5xx_crashdumper_init(gpu, &dumper)) 1513 return; 1514 1515 /* The script will be written at offset 0 */ 1516 ptr = dumper.ptr; 1517 1518 /* Start writing the data at offset 256k */ 1519 offset = dumper.iova + (256 * SZ_1K); 1520 1521 /* Count how many additional registers to get from the HLSQ aperture */ 1522 for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) 1523 count += a5xx_hlsq_aperture_regs[i].count; 1524 1525 a5xx_state->hlsqregs = kcalloc(count, sizeof(u32), GFP_KERNEL); 1526 if (!a5xx_state->hlsqregs) 1527 return; 1528 1529 /* Build the crashdump script */ 1530 for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) { 1531 u32 type = a5xx_hlsq_aperture_regs[i].type; 1532 u32 c = a5xx_hlsq_aperture_regs[i].count; 1533 1534 /* Write the register to select the desired bank */ 1535 *ptr++ = ((u64) type << 8); 1536 *ptr++ = (((u64) REG_A5XX_HLSQ_DBG_READ_SEL) << 44) | 1537 (1 << 21) | 1; 1538 1539 *ptr++ = offset; 1540 *ptr++ = (((u64) REG_A5XX_HLSQ_DBG_AHB_READ_APERTURE) << 44) 1541 | c; 1542 1543 offset += c * sizeof(u32); 1544 } 1545 1546 /* Write two zeros to close off the script */ 1547 *ptr++ = 0; 1548 *ptr++ = 0; 1549 1550 if (a5xx_crashdumper_run(gpu, &dumper)) { 1551 kfree(a5xx_state->hlsqregs); 1552 msm_gem_kernel_put(dumper.bo, gpu->aspace); 1553 return; 1554 } 1555 1556 /* Copy the data from the crashdumper to the state */ 1557 memcpy(a5xx_state->hlsqregs, dumper.ptr + (256 * SZ_1K), 1558 count * sizeof(u32)); 1559 1560 msm_gem_kernel_put(dumper.bo, gpu->aspace); 1561 } 1562 1563 static struct msm_gpu_state *a5xx_gpu_state_get(struct msm_gpu *gpu) 1564 { 1565 struct a5xx_gpu_state *a5xx_state = kzalloc(sizeof(*a5xx_state), 1566 GFP_KERNEL); 1567 bool stalled = !!(gpu_read(gpu, REG_A5XX_RBBM_STATUS3) & BIT(24)); 1568 1569 if (!a5xx_state) 1570 return ERR_PTR(-ENOMEM); 1571 1572 /* Temporarily disable hardware clock gating before reading the hw */ 1573 a5xx_set_hwcg(gpu, false); 1574 1575 /* First get the generic state from the adreno core */ 1576 adreno_gpu_state_get(gpu, &(a5xx_state->base)); 1577 1578 a5xx_state->base.rbbm_status = gpu_read(gpu, REG_A5XX_RBBM_STATUS); 1579 1580 /* 1581 * Get the HLSQ regs with the help of the crashdumper, but only if 1582 * we are not stalled in an iommu fault (in which case the crashdumper 1583 * would not have access to memory) 1584 */ 1585 if (!stalled) 1586 a5xx_gpu_state_get_hlsq_regs(gpu, a5xx_state); 1587 1588 a5xx_set_hwcg(gpu, true); 1589 1590 return &a5xx_state->base; 1591 } 1592 1593 static void a5xx_gpu_state_destroy(struct kref *kref) 1594 { 1595 struct msm_gpu_state *state = container_of(kref, 1596 struct msm_gpu_state, ref); 1597 struct a5xx_gpu_state *a5xx_state = container_of(state, 1598 struct a5xx_gpu_state, base); 1599 1600 kfree(a5xx_state->hlsqregs); 1601 1602 adreno_gpu_state_destroy(state); 1603 kfree(a5xx_state); 1604 } 1605 1606 static int a5xx_gpu_state_put(struct msm_gpu_state *state) 1607 { 1608 if (IS_ERR_OR_NULL(state)) 1609 return 1; 1610 1611 return kref_put(&state->ref, a5xx_gpu_state_destroy); 1612 } 1613 1614 1615 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP) 1616 static void a5xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state, 1617 struct drm_printer *p) 1618 { 1619 int i, j; 1620 u32 pos = 0; 1621 struct a5xx_gpu_state *a5xx_state = container_of(state, 1622 struct a5xx_gpu_state, base); 1623 1624 if (IS_ERR_OR_NULL(state)) 1625 return; 1626 1627 adreno_show(gpu, state, p); 1628 1629 /* Dump the additional a5xx HLSQ registers */ 1630 if (!a5xx_state->hlsqregs) 1631 return; 1632 1633 drm_printf(p, "registers-hlsq:\n"); 1634 1635 for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) { 1636 u32 o = a5xx_hlsq_aperture_regs[i].regoffset; 1637 u32 c = a5xx_hlsq_aperture_regs[i].count; 1638 1639 for (j = 0; j < c; j++, pos++, o++) { 1640 /* 1641 * To keep the crashdump simple we pull the entire range 1642 * for each register type but not all of the registers 1643 * in the range are valid. Fortunately invalid registers 1644 * stick out like a sore thumb with a value of 1645 * 0xdeadbeef 1646 */ 1647 if (a5xx_state->hlsqregs[pos] == 0xdeadbeef) 1648 continue; 1649 1650 drm_printf(p, " - { offset: 0x%04x, value: 0x%08x }\n", 1651 o << 2, a5xx_state->hlsqregs[pos]); 1652 } 1653 } 1654 } 1655 #endif 1656 1657 static struct msm_ringbuffer *a5xx_active_ring(struct msm_gpu *gpu) 1658 { 1659 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1660 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 1661 1662 return a5xx_gpu->cur_ring; 1663 } 1664 1665 static u64 a5xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate) 1666 { 1667 u64 busy_cycles; 1668 1669 busy_cycles = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_0_LO); 1670 *out_sample_rate = clk_get_rate(gpu->core_clk); 1671 1672 return busy_cycles; 1673 } 1674 1675 static uint32_t a5xx_get_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring) 1676 { 1677 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1678 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 1679 1680 if (a5xx_gpu->has_whereami) 1681 return a5xx_gpu->shadow[ring->id]; 1682 1683 return ring->memptrs->rptr = gpu_read(gpu, REG_A5XX_CP_RB_RPTR); 1684 } 1685 1686 static const struct adreno_gpu_funcs funcs = { 1687 .base = { 1688 .get_param = adreno_get_param, 1689 .set_param = adreno_set_param, 1690 .hw_init = a5xx_hw_init, 1691 .ucode_load = a5xx_ucode_load, 1692 .pm_suspend = a5xx_pm_suspend, 1693 .pm_resume = a5xx_pm_resume, 1694 .recover = a5xx_recover, 1695 .submit = a5xx_submit, 1696 .active_ring = a5xx_active_ring, 1697 .irq = a5xx_irq, 1698 .destroy = a5xx_destroy, 1699 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP) 1700 .show = a5xx_show, 1701 #endif 1702 #if defined(CONFIG_DEBUG_FS) 1703 .debugfs_init = a5xx_debugfs_init, 1704 #endif 1705 .gpu_busy = a5xx_gpu_busy, 1706 .gpu_state_get = a5xx_gpu_state_get, 1707 .gpu_state_put = a5xx_gpu_state_put, 1708 .create_address_space = adreno_create_address_space, 1709 .get_rptr = a5xx_get_rptr, 1710 }, 1711 .get_timestamp = a5xx_get_timestamp, 1712 }; 1713 1714 static void check_speed_bin(struct device *dev) 1715 { 1716 struct nvmem_cell *cell; 1717 u32 val; 1718 1719 /* 1720 * If the OPP table specifies a opp-supported-hw property then we have 1721 * to set something with dev_pm_opp_set_supported_hw() or the table 1722 * doesn't get populated so pick an arbitrary value that should 1723 * ensure the default frequencies are selected but not conflict with any 1724 * actual bins 1725 */ 1726 val = 0x80; 1727 1728 cell = nvmem_cell_get(dev, "speed_bin"); 1729 1730 if (!IS_ERR(cell)) { 1731 void *buf = nvmem_cell_read(cell, NULL); 1732 1733 if (!IS_ERR(buf)) { 1734 u8 bin = *((u8 *) buf); 1735 1736 val = (1 << bin); 1737 kfree(buf); 1738 } 1739 1740 nvmem_cell_put(cell); 1741 } 1742 1743 devm_pm_opp_set_supported_hw(dev, &val, 1); 1744 } 1745 1746 struct msm_gpu *a5xx_gpu_init(struct drm_device *dev) 1747 { 1748 struct msm_drm_private *priv = dev->dev_private; 1749 struct platform_device *pdev = priv->gpu_pdev; 1750 struct adreno_platform_config *config = pdev->dev.platform_data; 1751 struct a5xx_gpu *a5xx_gpu = NULL; 1752 struct adreno_gpu *adreno_gpu; 1753 struct msm_gpu *gpu; 1754 unsigned int nr_rings; 1755 int ret; 1756 1757 if (!pdev) { 1758 DRM_DEV_ERROR(dev->dev, "No A5XX device is defined\n"); 1759 return ERR_PTR(-ENXIO); 1760 } 1761 1762 a5xx_gpu = kzalloc(sizeof(*a5xx_gpu), GFP_KERNEL); 1763 if (!a5xx_gpu) 1764 return ERR_PTR(-ENOMEM); 1765 1766 adreno_gpu = &a5xx_gpu->base; 1767 gpu = &adreno_gpu->base; 1768 1769 adreno_gpu->registers = a5xx_registers; 1770 1771 a5xx_gpu->lm_leakage = 0x4E001A; 1772 1773 check_speed_bin(&pdev->dev); 1774 1775 nr_rings = 4; 1776 1777 if (config->info->revn == 510) 1778 nr_rings = 1; 1779 1780 ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, nr_rings); 1781 if (ret) { 1782 a5xx_destroy(&(a5xx_gpu->base.base)); 1783 return ERR_PTR(ret); 1784 } 1785 1786 if (gpu->aspace) 1787 msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu, a5xx_fault_handler); 1788 1789 /* Set up the preemption specific bits and pieces for each ringbuffer */ 1790 a5xx_preempt_init(gpu); 1791 1792 return gpu; 1793 } 1794