1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2016-2017 The Linux Foundation. All rights reserved. 3 */ 4 5 #include <linux/kernel.h> 6 #include <linux/types.h> 7 #include <linux/cpumask.h> 8 #include <linux/qcom_scm.h> 9 #include <linux/pm_opp.h> 10 #include <linux/nvmem-consumer.h> 11 #include <linux/slab.h> 12 #include "msm_gem.h" 13 #include "msm_mmu.h" 14 #include "a5xx_gpu.h" 15 16 extern bool hang_debug; 17 static void a5xx_dump(struct msm_gpu *gpu); 18 19 #define GPU_PAS_ID 13 20 21 void a5xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring, 22 bool sync) 23 { 24 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 25 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 26 uint32_t wptr; 27 unsigned long flags; 28 29 /* 30 * Most flush operations need to issue a WHERE_AM_I opcode to sync up 31 * the rptr shadow 32 */ 33 if (a5xx_gpu->has_whereami && sync) { 34 OUT_PKT7(ring, CP_WHERE_AM_I, 2); 35 OUT_RING(ring, lower_32_bits(shadowptr(a5xx_gpu, ring))); 36 OUT_RING(ring, upper_32_bits(shadowptr(a5xx_gpu, ring))); 37 } 38 39 spin_lock_irqsave(&ring->preempt_lock, flags); 40 41 /* Copy the shadow to the actual register */ 42 ring->cur = ring->next; 43 44 /* Make sure to wrap wptr if we need to */ 45 wptr = get_wptr(ring); 46 47 spin_unlock_irqrestore(&ring->preempt_lock, flags); 48 49 /* Make sure everything is posted before making a decision */ 50 mb(); 51 52 /* Update HW if this is the current ring and we are not in preempt */ 53 if (a5xx_gpu->cur_ring == ring && !a5xx_in_preempt(a5xx_gpu)) 54 gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr); 55 } 56 57 static void a5xx_submit_in_rb(struct msm_gpu *gpu, struct msm_gem_submit *submit) 58 { 59 struct msm_drm_private *priv = gpu->dev->dev_private; 60 struct msm_ringbuffer *ring = submit->ring; 61 struct msm_gem_object *obj; 62 uint32_t *ptr, dwords; 63 unsigned int i; 64 65 for (i = 0; i < submit->nr_cmds; i++) { 66 switch (submit->cmd[i].type) { 67 case MSM_SUBMIT_CMD_IB_TARGET_BUF: 68 break; 69 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF: 70 if (priv->lastctx == submit->queue->ctx) 71 break; 72 fallthrough; 73 case MSM_SUBMIT_CMD_BUF: 74 /* copy commands into RB: */ 75 obj = submit->bos[submit->cmd[i].idx].obj; 76 dwords = submit->cmd[i].size; 77 78 ptr = msm_gem_get_vaddr(&obj->base); 79 80 /* _get_vaddr() shouldn't fail at this point, 81 * since we've already mapped it once in 82 * submit_reloc() 83 */ 84 if (WARN_ON(!ptr)) 85 return; 86 87 for (i = 0; i < dwords; i++) { 88 /* normally the OUT_PKTn() would wait 89 * for space for the packet. But since 90 * we just OUT_RING() the whole thing, 91 * need to call adreno_wait_ring() 92 * ourself: 93 */ 94 adreno_wait_ring(ring, 1); 95 OUT_RING(ring, ptr[i]); 96 } 97 98 msm_gem_put_vaddr(&obj->base); 99 100 break; 101 } 102 } 103 104 a5xx_flush(gpu, ring, true); 105 a5xx_preempt_trigger(gpu); 106 107 /* we might not necessarily have a cmd from userspace to 108 * trigger an event to know that submit has completed, so 109 * do this manually: 110 */ 111 a5xx_idle(gpu, ring); 112 ring->memptrs->fence = submit->seqno; 113 msm_gpu_retire(gpu); 114 } 115 116 static void a5xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) 117 { 118 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 119 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 120 struct msm_drm_private *priv = gpu->dev->dev_private; 121 struct msm_ringbuffer *ring = submit->ring; 122 unsigned int i, ibs = 0; 123 124 if (IS_ENABLED(CONFIG_DRM_MSM_GPU_SUDO) && submit->in_rb) { 125 priv->lastctx = NULL; 126 a5xx_submit_in_rb(gpu, submit); 127 return; 128 } 129 130 OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1); 131 OUT_RING(ring, 0x02); 132 133 /* Turn off protected mode to write to special registers */ 134 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1); 135 OUT_RING(ring, 0); 136 137 /* Set the save preemption record for the ring/command */ 138 OUT_PKT4(ring, REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 2); 139 OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[submit->ring->id])); 140 OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[submit->ring->id])); 141 142 /* Turn back on protected mode */ 143 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1); 144 OUT_RING(ring, 1); 145 146 /* Enable local preemption for finegrain preemption */ 147 OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1); 148 OUT_RING(ring, 0x02); 149 150 /* Allow CP_CONTEXT_SWITCH_YIELD packets in the IB2 */ 151 OUT_PKT7(ring, CP_YIELD_ENABLE, 1); 152 OUT_RING(ring, 0x02); 153 154 /* Submit the commands */ 155 for (i = 0; i < submit->nr_cmds; i++) { 156 switch (submit->cmd[i].type) { 157 case MSM_SUBMIT_CMD_IB_TARGET_BUF: 158 break; 159 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF: 160 if (priv->lastctx == submit->queue->ctx) 161 break; 162 fallthrough; 163 case MSM_SUBMIT_CMD_BUF: 164 OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3); 165 OUT_RING(ring, lower_32_bits(submit->cmd[i].iova)); 166 OUT_RING(ring, upper_32_bits(submit->cmd[i].iova)); 167 OUT_RING(ring, submit->cmd[i].size); 168 ibs++; 169 break; 170 } 171 } 172 173 /* 174 * Write the render mode to NULL (0) to indicate to the CP that the IBs 175 * are done rendering - otherwise a lucky preemption would start 176 * replaying from the last checkpoint 177 */ 178 OUT_PKT7(ring, CP_SET_RENDER_MODE, 5); 179 OUT_RING(ring, 0); 180 OUT_RING(ring, 0); 181 OUT_RING(ring, 0); 182 OUT_RING(ring, 0); 183 OUT_RING(ring, 0); 184 185 /* Turn off IB level preemptions */ 186 OUT_PKT7(ring, CP_YIELD_ENABLE, 1); 187 OUT_RING(ring, 0x01); 188 189 /* Write the fence to the scratch register */ 190 OUT_PKT4(ring, REG_A5XX_CP_SCRATCH_REG(2), 1); 191 OUT_RING(ring, submit->seqno); 192 193 /* 194 * Execute a CACHE_FLUSH_TS event. This will ensure that the 195 * timestamp is written to the memory and then triggers the interrupt 196 */ 197 OUT_PKT7(ring, CP_EVENT_WRITE, 4); 198 OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(CACHE_FLUSH_TS) | 199 CP_EVENT_WRITE_0_IRQ); 200 OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence))); 201 OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence))); 202 OUT_RING(ring, submit->seqno); 203 204 /* Yield the floor on command completion */ 205 OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4); 206 /* 207 * If dword[2:1] are non zero, they specify an address for the CP to 208 * write the value of dword[3] to on preemption complete. Write 0 to 209 * skip the write 210 */ 211 OUT_RING(ring, 0x00); 212 OUT_RING(ring, 0x00); 213 /* Data value - not used if the address above is 0 */ 214 OUT_RING(ring, 0x01); 215 /* Set bit 0 to trigger an interrupt on preempt complete */ 216 OUT_RING(ring, 0x01); 217 218 /* A WHERE_AM_I packet is not needed after a YIELD */ 219 a5xx_flush(gpu, ring, false); 220 221 /* Check to see if we need to start preemption */ 222 a5xx_preempt_trigger(gpu); 223 } 224 225 static const struct adreno_five_hwcg_regs { 226 u32 offset; 227 u32 value; 228 } a5xx_hwcg[] = { 229 {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, 230 {REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222}, 231 {REG_A5XX_RBBM_CLOCK_CNTL_SP2, 0x02222222}, 232 {REG_A5XX_RBBM_CLOCK_CNTL_SP3, 0x02222222}, 233 {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, 234 {REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220}, 235 {REG_A5XX_RBBM_CLOCK_CNTL2_SP2, 0x02222220}, 236 {REG_A5XX_RBBM_CLOCK_CNTL2_SP3, 0x02222220}, 237 {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, 238 {REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF}, 239 {REG_A5XX_RBBM_CLOCK_HYST_SP2, 0x0000F3CF}, 240 {REG_A5XX_RBBM_CLOCK_HYST_SP3, 0x0000F3CF}, 241 {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, 242 {REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080}, 243 {REG_A5XX_RBBM_CLOCK_DELAY_SP2, 0x00000080}, 244 {REG_A5XX_RBBM_CLOCK_DELAY_SP3, 0x00000080}, 245 {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, 246 {REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222}, 247 {REG_A5XX_RBBM_CLOCK_CNTL_TP2, 0x22222222}, 248 {REG_A5XX_RBBM_CLOCK_CNTL_TP3, 0x22222222}, 249 {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, 250 {REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, 251 {REG_A5XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222}, 252 {REG_A5XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222}, 253 {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, 254 {REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222}, 255 {REG_A5XX_RBBM_CLOCK_CNTL3_TP2, 0x00002222}, 256 {REG_A5XX_RBBM_CLOCK_CNTL3_TP3, 0x00002222}, 257 {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, 258 {REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, 259 {REG_A5XX_RBBM_CLOCK_HYST_TP2, 0x77777777}, 260 {REG_A5XX_RBBM_CLOCK_HYST_TP3, 0x77777777}, 261 {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, 262 {REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, 263 {REG_A5XX_RBBM_CLOCK_HYST2_TP2, 0x77777777}, 264 {REG_A5XX_RBBM_CLOCK_HYST2_TP3, 0x77777777}, 265 {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, 266 {REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777}, 267 {REG_A5XX_RBBM_CLOCK_HYST3_TP2, 0x00007777}, 268 {REG_A5XX_RBBM_CLOCK_HYST3_TP3, 0x00007777}, 269 {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, 270 {REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, 271 {REG_A5XX_RBBM_CLOCK_DELAY_TP2, 0x11111111}, 272 {REG_A5XX_RBBM_CLOCK_DELAY_TP3, 0x11111111}, 273 {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, 274 {REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, 275 {REG_A5XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111}, 276 {REG_A5XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111}, 277 {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, 278 {REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111}, 279 {REG_A5XX_RBBM_CLOCK_DELAY3_TP2, 0x00001111}, 280 {REG_A5XX_RBBM_CLOCK_DELAY3_TP3, 0x00001111}, 281 {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, 282 {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, 283 {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, 284 {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, 285 {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444}, 286 {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, 287 {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, 288 {REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222}, 289 {REG_A5XX_RBBM_CLOCK_CNTL_RB2, 0x22222222}, 290 {REG_A5XX_RBBM_CLOCK_CNTL_RB3, 0x22222222}, 291 {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, 292 {REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222}, 293 {REG_A5XX_RBBM_CLOCK_CNTL2_RB2, 0x00222222}, 294 {REG_A5XX_RBBM_CLOCK_CNTL2_RB3, 0x00222222}, 295 {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, 296 {REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220}, 297 {REG_A5XX_RBBM_CLOCK_CNTL_CCU2, 0x00022220}, 298 {REG_A5XX_RBBM_CLOCK_CNTL_CCU3, 0x00022220}, 299 {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, 300 {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, 301 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, 302 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404}, 303 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU2, 0x04040404}, 304 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU3, 0x04040404}, 305 {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, 306 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, 307 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002}, 308 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2, 0x00000002}, 309 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3, 0x00000002}, 310 {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, 311 {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, 312 {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, 313 {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, 314 {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, 315 {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, 316 {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, 317 {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, 318 {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, 319 {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, 320 {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222} 321 }, a50x_hwcg[] = { 322 {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, 323 {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, 324 {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, 325 {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, 326 {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, 327 {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, 328 {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, 329 {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, 330 {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, 331 {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, 332 {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, 333 {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, 334 {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, 335 {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, 336 {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, 337 {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, 338 {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, 339 {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00FFFFF4}, 340 {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, 341 {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, 342 {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, 343 {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, 344 {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, 345 {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, 346 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, 347 {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, 348 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, 349 {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, 350 {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, 351 {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, 352 {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, 353 {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, 354 {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, 355 {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, 356 {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, 357 {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, 358 {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, 359 {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, 360 }, a512_hwcg[] = { 361 {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, 362 {REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222}, 363 {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, 364 {REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220}, 365 {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, 366 {REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF}, 367 {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, 368 {REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080}, 369 {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, 370 {REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222}, 371 {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, 372 {REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, 373 {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, 374 {REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222}, 375 {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, 376 {REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, 377 {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, 378 {REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, 379 {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, 380 {REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777}, 381 {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, 382 {REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, 383 {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, 384 {REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, 385 {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, 386 {REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111}, 387 {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, 388 {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, 389 {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, 390 {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, 391 {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444}, 392 {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, 393 {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, 394 {REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222}, 395 {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, 396 {REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222}, 397 {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, 398 {REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220}, 399 {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, 400 {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, 401 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, 402 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404}, 403 {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, 404 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, 405 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002}, 406 {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, 407 {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, 408 {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, 409 {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, 410 {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, 411 {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, 412 {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, 413 {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, 414 {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, 415 {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, 416 {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, 417 }; 418 419 void a5xx_set_hwcg(struct msm_gpu *gpu, bool state) 420 { 421 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 422 const struct adreno_five_hwcg_regs *regs; 423 unsigned int i, sz; 424 425 if (adreno_is_a508(adreno_gpu)) { 426 regs = a50x_hwcg; 427 sz = ARRAY_SIZE(a50x_hwcg); 428 } else if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu)) { 429 regs = a512_hwcg; 430 sz = ARRAY_SIZE(a512_hwcg); 431 } else { 432 regs = a5xx_hwcg; 433 sz = ARRAY_SIZE(a5xx_hwcg); 434 } 435 436 for (i = 0; i < sz; i++) 437 gpu_write(gpu, regs[i].offset, 438 state ? regs[i].value : 0); 439 440 if (adreno_is_a540(adreno_gpu)) { 441 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_DELAY_GPMU, state ? 0x00000770 : 0); 442 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_HYST_GPMU, state ? 0x00000004 : 0); 443 } 444 445 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, state ? 0xAAA8AA00 : 0); 446 gpu_write(gpu, REG_A5XX_RBBM_ISDB_CNT, state ? 0x182 : 0x180); 447 } 448 449 static int a5xx_me_init(struct msm_gpu *gpu) 450 { 451 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 452 struct msm_ringbuffer *ring = gpu->rb[0]; 453 454 OUT_PKT7(ring, CP_ME_INIT, 8); 455 456 OUT_RING(ring, 0x0000002F); 457 458 /* Enable multiple hardware contexts */ 459 OUT_RING(ring, 0x00000003); 460 461 /* Enable error detection */ 462 OUT_RING(ring, 0x20000000); 463 464 /* Don't enable header dump */ 465 OUT_RING(ring, 0x00000000); 466 OUT_RING(ring, 0x00000000); 467 468 /* Specify workarounds for various microcode issues */ 469 if (adreno_is_a530(adreno_gpu)) { 470 /* Workaround for token end syncs 471 * Force a WFI after every direct-render 3D mode draw and every 472 * 2D mode 3 draw 473 */ 474 OUT_RING(ring, 0x0000000B); 475 } else if (adreno_is_a510(adreno_gpu)) { 476 /* Workaround for token and syncs */ 477 OUT_RING(ring, 0x00000001); 478 } else { 479 /* No workarounds enabled */ 480 OUT_RING(ring, 0x00000000); 481 } 482 483 OUT_RING(ring, 0x00000000); 484 OUT_RING(ring, 0x00000000); 485 486 a5xx_flush(gpu, ring, true); 487 return a5xx_idle(gpu, ring) ? 0 : -EINVAL; 488 } 489 490 static int a5xx_preempt_start(struct msm_gpu *gpu) 491 { 492 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 493 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 494 struct msm_ringbuffer *ring = gpu->rb[0]; 495 496 if (gpu->nr_rings == 1) 497 return 0; 498 499 /* Turn off protected mode to write to special registers */ 500 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1); 501 OUT_RING(ring, 0); 502 503 /* Set the save preemption record for the ring/command */ 504 OUT_PKT4(ring, REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 2); 505 OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[ring->id])); 506 OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[ring->id])); 507 508 /* Turn back on protected mode */ 509 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1); 510 OUT_RING(ring, 1); 511 512 OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1); 513 OUT_RING(ring, 0x00); 514 515 OUT_PKT7(ring, CP_PREEMPT_ENABLE_LOCAL, 1); 516 OUT_RING(ring, 0x01); 517 518 OUT_PKT7(ring, CP_YIELD_ENABLE, 1); 519 OUT_RING(ring, 0x01); 520 521 /* Yield the floor on command completion */ 522 OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4); 523 OUT_RING(ring, 0x00); 524 OUT_RING(ring, 0x00); 525 OUT_RING(ring, 0x01); 526 OUT_RING(ring, 0x01); 527 528 /* The WHERE_AMI_I packet is not needed after a YIELD is issued */ 529 a5xx_flush(gpu, ring, false); 530 531 return a5xx_idle(gpu, ring) ? 0 : -EINVAL; 532 } 533 534 static void a5xx_ucode_check_version(struct a5xx_gpu *a5xx_gpu, 535 struct drm_gem_object *obj) 536 { 537 u32 *buf = msm_gem_get_vaddr(obj); 538 539 if (IS_ERR(buf)) 540 return; 541 542 /* 543 * If the lowest nibble is 0xa that is an indication that this microcode 544 * has been patched. The actual version is in dword [3] but we only care 545 * about the patchlevel which is the lowest nibble of dword [3] 546 */ 547 if (((buf[0] & 0xf) == 0xa) && (buf[2] & 0xf) >= 1) 548 a5xx_gpu->has_whereami = true; 549 550 msm_gem_put_vaddr(obj); 551 } 552 553 static int a5xx_ucode_init(struct msm_gpu *gpu) 554 { 555 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 556 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 557 int ret; 558 559 if (!a5xx_gpu->pm4_bo) { 560 a5xx_gpu->pm4_bo = adreno_fw_create_bo(gpu, 561 adreno_gpu->fw[ADRENO_FW_PM4], &a5xx_gpu->pm4_iova); 562 563 564 if (IS_ERR(a5xx_gpu->pm4_bo)) { 565 ret = PTR_ERR(a5xx_gpu->pm4_bo); 566 a5xx_gpu->pm4_bo = NULL; 567 DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PM4: %d\n", 568 ret); 569 return ret; 570 } 571 572 msm_gem_object_set_name(a5xx_gpu->pm4_bo, "pm4fw"); 573 } 574 575 if (!a5xx_gpu->pfp_bo) { 576 a5xx_gpu->pfp_bo = adreno_fw_create_bo(gpu, 577 adreno_gpu->fw[ADRENO_FW_PFP], &a5xx_gpu->pfp_iova); 578 579 if (IS_ERR(a5xx_gpu->pfp_bo)) { 580 ret = PTR_ERR(a5xx_gpu->pfp_bo); 581 a5xx_gpu->pfp_bo = NULL; 582 DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PFP: %d\n", 583 ret); 584 return ret; 585 } 586 587 msm_gem_object_set_name(a5xx_gpu->pfp_bo, "pfpfw"); 588 a5xx_ucode_check_version(a5xx_gpu, a5xx_gpu->pfp_bo); 589 } 590 591 gpu_write64(gpu, REG_A5XX_CP_ME_INSTR_BASE_LO, 592 REG_A5XX_CP_ME_INSTR_BASE_HI, a5xx_gpu->pm4_iova); 593 594 gpu_write64(gpu, REG_A5XX_CP_PFP_INSTR_BASE_LO, 595 REG_A5XX_CP_PFP_INSTR_BASE_HI, a5xx_gpu->pfp_iova); 596 597 return 0; 598 } 599 600 #define SCM_GPU_ZAP_SHADER_RESUME 0 601 602 static int a5xx_zap_shader_resume(struct msm_gpu *gpu) 603 { 604 int ret; 605 606 ret = qcom_scm_set_remote_state(SCM_GPU_ZAP_SHADER_RESUME, GPU_PAS_ID); 607 if (ret) 608 DRM_ERROR("%s: zap-shader resume failed: %d\n", 609 gpu->name, ret); 610 611 return ret; 612 } 613 614 static int a5xx_zap_shader_init(struct msm_gpu *gpu) 615 { 616 static bool loaded; 617 int ret; 618 619 /* 620 * If the zap shader is already loaded into memory we just need to kick 621 * the remote processor to reinitialize it 622 */ 623 if (loaded) 624 return a5xx_zap_shader_resume(gpu); 625 626 ret = adreno_zap_shader_load(gpu, GPU_PAS_ID); 627 628 loaded = !ret; 629 return ret; 630 } 631 632 #define A5XX_INT_MASK (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \ 633 A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \ 634 A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \ 635 A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \ 636 A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \ 637 A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW | \ 638 A5XX_RBBM_INT_0_MASK_CP_HW_ERROR | \ 639 A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT | \ 640 A5XX_RBBM_INT_0_MASK_CP_SW | \ 641 A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \ 642 A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \ 643 A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP) 644 645 static int a5xx_hw_init(struct msm_gpu *gpu) 646 { 647 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 648 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 649 u32 regbit; 650 int ret; 651 652 gpu_write(gpu, REG_A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003); 653 654 if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu) || 655 adreno_is_a540(adreno_gpu)) 656 gpu_write(gpu, REG_A5XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009); 657 658 /* Make all blocks contribute to the GPU BUSY perf counter */ 659 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF); 660 661 /* Enable RBBM error reporting bits */ 662 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL0, 0x00000001); 663 664 if (adreno_gpu->info->quirks & ADRENO_QUIRK_FAULT_DETECT_MASK) { 665 /* 666 * Mask out the activity signals from RB1-3 to avoid false 667 * positives 668 */ 669 670 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11, 671 0xF0000000); 672 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12, 673 0xFFFFFFFF); 674 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13, 675 0xFFFFFFFF); 676 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14, 677 0xFFFFFFFF); 678 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15, 679 0xFFFFFFFF); 680 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16, 681 0xFFFFFFFF); 682 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17, 683 0xFFFFFFFF); 684 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18, 685 0xFFFFFFFF); 686 } 687 688 /* Enable fault detection */ 689 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_INT_CNTL, 690 (1 << 30) | 0xFFFF); 691 692 /* Turn on performance counters */ 693 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_CNTL, 0x01); 694 695 /* Select CP0 to always count cycles */ 696 gpu_write(gpu, REG_A5XX_CP_PERFCTR_CP_SEL_0, PERF_CP_ALWAYS_COUNT); 697 698 /* Select RBBM0 to countable 6 to get the busy status for devfreq */ 699 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_SEL_0, 6); 700 701 /* Increase VFD cache access so LRZ and other data gets evicted less */ 702 gpu_write(gpu, REG_A5XX_UCHE_CACHE_WAYS, 0x02); 703 704 /* Disable L2 bypass in the UCHE */ 705 gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_LO, 0xFFFF0000); 706 gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_HI, 0x0001FFFF); 707 gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_LO, 0xFFFF0000); 708 gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_HI, 0x0001FFFF); 709 710 /* Set the GMEM VA range (0 to gpu->gmem) */ 711 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_LO, 0x00100000); 712 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_HI, 0x00000000); 713 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_LO, 714 0x00100000 + adreno_gpu->gmem - 1); 715 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_HI, 0x00000000); 716 717 if (adreno_is_a508(adreno_gpu) || adreno_is_a510(adreno_gpu)) { 718 gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x20); 719 if (adreno_is_a508(adreno_gpu)) 720 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400); 721 else 722 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x20); 723 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x40000030); 724 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x20100D0A); 725 } else { 726 gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x40); 727 if (adreno_is_a530(adreno_gpu)) 728 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x40); 729 else 730 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400); 731 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060); 732 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16); 733 } 734 735 if (adreno_is_a508(adreno_gpu)) 736 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 737 (0x100 << 11 | 0x100 << 22)); 738 else if (adreno_is_a509(adreno_gpu) || adreno_is_a510(adreno_gpu) || 739 adreno_is_a512(adreno_gpu)) 740 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 741 (0x200 << 11 | 0x200 << 22)); 742 else 743 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 744 (0x400 << 11 | 0x300 << 22)); 745 746 if (adreno_gpu->info->quirks & ADRENO_QUIRK_TWO_PASS_USE_WFI) 747 gpu_rmw(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8)); 748 749 /* 750 * Disable the RB sampler datapath DP2 clock gating optimization 751 * for 1-SP GPUs, as it is enabled by default. 752 */ 753 if (adreno_is_a508(adreno_gpu) || adreno_is_a509(adreno_gpu) || 754 adreno_is_a512(adreno_gpu)) 755 gpu_rmw(gpu, REG_A5XX_RB_DBG_ECO_CNTL, 0, (1 << 9)); 756 757 /* Disable UCHE global filter as SP can invalidate/flush independently */ 758 gpu_write(gpu, REG_A5XX_UCHE_MODE_CNTL, BIT(29)); 759 760 /* Enable USE_RETENTION_FLOPS */ 761 gpu_write(gpu, REG_A5XX_CP_CHICKEN_DBG, 0x02000000); 762 763 /* Enable ME/PFP split notification */ 764 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL1, 0xA6FFFFFF); 765 766 /* 767 * In A5x, CCU can send context_done event of a particular context to 768 * UCHE which ultimately reaches CP even when there is valid 769 * transaction of that context inside CCU. This can let CP to program 770 * config registers, which will make the "valid transaction" inside 771 * CCU to be interpreted differently. This can cause gpu fault. This 772 * bug is fixed in latest A510 revision. To enable this bug fix - 773 * bit[11] of RB_DBG_ECO_CNTL need to be set to 0, default is 1 774 * (disable). For older A510 version this bit is unused. 775 */ 776 if (adreno_is_a510(adreno_gpu)) 777 gpu_rmw(gpu, REG_A5XX_RB_DBG_ECO_CNTL, (1 << 11), 0); 778 779 /* Enable HWCG */ 780 a5xx_set_hwcg(gpu, true); 781 782 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL2, 0x0000003F); 783 784 /* Set the highest bank bit */ 785 if (adreno_is_a540(adreno_gpu)) 786 regbit = 2; 787 else 788 regbit = 1; 789 790 gpu_write(gpu, REG_A5XX_TPL1_MODE_CNTL, regbit << 7); 791 gpu_write(gpu, REG_A5XX_RB_MODE_CNTL, regbit << 1); 792 793 if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu) || 794 adreno_is_a540(adreno_gpu)) 795 gpu_write(gpu, REG_A5XX_UCHE_DBG_ECO_CNTL_2, regbit); 796 797 /* Disable All flat shading optimization (ALLFLATOPTDIS) */ 798 gpu_rmw(gpu, REG_A5XX_VPC_DBG_ECO_CNTL, 0, (1 << 10)); 799 800 /* Protect registers from the CP */ 801 gpu_write(gpu, REG_A5XX_CP_PROTECT_CNTL, 0x00000007); 802 803 /* RBBM */ 804 gpu_write(gpu, REG_A5XX_CP_PROTECT(0), ADRENO_PROTECT_RW(0x04, 4)); 805 gpu_write(gpu, REG_A5XX_CP_PROTECT(1), ADRENO_PROTECT_RW(0x08, 8)); 806 gpu_write(gpu, REG_A5XX_CP_PROTECT(2), ADRENO_PROTECT_RW(0x10, 16)); 807 gpu_write(gpu, REG_A5XX_CP_PROTECT(3), ADRENO_PROTECT_RW(0x20, 32)); 808 gpu_write(gpu, REG_A5XX_CP_PROTECT(4), ADRENO_PROTECT_RW(0x40, 64)); 809 gpu_write(gpu, REG_A5XX_CP_PROTECT(5), ADRENO_PROTECT_RW(0x80, 64)); 810 811 /* Content protect */ 812 gpu_write(gpu, REG_A5XX_CP_PROTECT(6), 813 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO, 814 16)); 815 gpu_write(gpu, REG_A5XX_CP_PROTECT(7), 816 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TRUST_CNTL, 2)); 817 818 /* CP */ 819 gpu_write(gpu, REG_A5XX_CP_PROTECT(8), ADRENO_PROTECT_RW(0x800, 64)); 820 gpu_write(gpu, REG_A5XX_CP_PROTECT(9), ADRENO_PROTECT_RW(0x840, 8)); 821 gpu_write(gpu, REG_A5XX_CP_PROTECT(10), ADRENO_PROTECT_RW(0x880, 32)); 822 gpu_write(gpu, REG_A5XX_CP_PROTECT(11), ADRENO_PROTECT_RW(0xAA0, 1)); 823 824 /* RB */ 825 gpu_write(gpu, REG_A5XX_CP_PROTECT(12), ADRENO_PROTECT_RW(0xCC0, 1)); 826 gpu_write(gpu, REG_A5XX_CP_PROTECT(13), ADRENO_PROTECT_RW(0xCF0, 2)); 827 828 /* VPC */ 829 gpu_write(gpu, REG_A5XX_CP_PROTECT(14), ADRENO_PROTECT_RW(0xE68, 8)); 830 gpu_write(gpu, REG_A5XX_CP_PROTECT(15), ADRENO_PROTECT_RW(0xE70, 16)); 831 832 /* UCHE */ 833 gpu_write(gpu, REG_A5XX_CP_PROTECT(16), ADRENO_PROTECT_RW(0xE80, 16)); 834 835 if (adreno_is_a508(adreno_gpu) || adreno_is_a509(adreno_gpu) || 836 adreno_is_a510(adreno_gpu) || adreno_is_a512(adreno_gpu) || 837 adreno_is_a530(adreno_gpu)) 838 gpu_write(gpu, REG_A5XX_CP_PROTECT(17), 839 ADRENO_PROTECT_RW(0x10000, 0x8000)); 840 841 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_CNTL, 0); 842 /* 843 * Disable the trusted memory range - we don't actually supported secure 844 * memory rendering at this point in time and we don't want to block off 845 * part of the virtual memory space. 846 */ 847 gpu_write64(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO, 848 REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI, 0x00000000); 849 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000); 850 851 /* Put the GPU into 64 bit by default */ 852 gpu_write(gpu, REG_A5XX_CP_ADDR_MODE_CNTL, 0x1); 853 gpu_write(gpu, REG_A5XX_VSC_ADDR_MODE_CNTL, 0x1); 854 gpu_write(gpu, REG_A5XX_GRAS_ADDR_MODE_CNTL, 0x1); 855 gpu_write(gpu, REG_A5XX_RB_ADDR_MODE_CNTL, 0x1); 856 gpu_write(gpu, REG_A5XX_PC_ADDR_MODE_CNTL, 0x1); 857 gpu_write(gpu, REG_A5XX_HLSQ_ADDR_MODE_CNTL, 0x1); 858 gpu_write(gpu, REG_A5XX_VFD_ADDR_MODE_CNTL, 0x1); 859 gpu_write(gpu, REG_A5XX_VPC_ADDR_MODE_CNTL, 0x1); 860 gpu_write(gpu, REG_A5XX_UCHE_ADDR_MODE_CNTL, 0x1); 861 gpu_write(gpu, REG_A5XX_SP_ADDR_MODE_CNTL, 0x1); 862 gpu_write(gpu, REG_A5XX_TPL1_ADDR_MODE_CNTL, 0x1); 863 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1); 864 865 /* 866 * VPC corner case with local memory load kill leads to corrupt 867 * internal state. Normal Disable does not work for all a5x chips. 868 * So do the following setting to disable it. 869 */ 870 if (adreno_gpu->info->quirks & ADRENO_QUIRK_LMLOADKILL_DISABLE) { 871 gpu_rmw(gpu, REG_A5XX_VPC_DBG_ECO_CNTL, 0, BIT(23)); 872 gpu_rmw(gpu, REG_A5XX_HLSQ_DBG_ECO_CNTL, BIT(18), 0); 873 } 874 875 ret = adreno_hw_init(gpu); 876 if (ret) 877 return ret; 878 879 if (!(adreno_is_a508(adreno_gpu) || adreno_is_a509(adreno_gpu) || 880 adreno_is_a510(adreno_gpu) || adreno_is_a512(adreno_gpu))) 881 a5xx_gpmu_ucode_init(gpu); 882 883 ret = a5xx_ucode_init(gpu); 884 if (ret) 885 return ret; 886 887 /* Set the ringbuffer address */ 888 gpu_write64(gpu, REG_A5XX_CP_RB_BASE, REG_A5XX_CP_RB_BASE_HI, 889 gpu->rb[0]->iova); 890 891 /* 892 * If the microcode supports the WHERE_AM_I opcode then we can use that 893 * in lieu of the RPTR shadow and enable preemption. Otherwise, we 894 * can't safely use the RPTR shadow or preemption. In either case, the 895 * RPTR shadow should be disabled in hardware. 896 */ 897 gpu_write(gpu, REG_A5XX_CP_RB_CNTL, 898 MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE); 899 900 /* Create a privileged buffer for the RPTR shadow */ 901 if (a5xx_gpu->has_whereami) { 902 if (!a5xx_gpu->shadow_bo) { 903 a5xx_gpu->shadow = msm_gem_kernel_new(gpu->dev, 904 sizeof(u32) * gpu->nr_rings, 905 MSM_BO_WC | MSM_BO_MAP_PRIV, 906 gpu->aspace, &a5xx_gpu->shadow_bo, 907 &a5xx_gpu->shadow_iova); 908 909 if (IS_ERR(a5xx_gpu->shadow)) 910 return PTR_ERR(a5xx_gpu->shadow); 911 } 912 913 gpu_write64(gpu, REG_A5XX_CP_RB_RPTR_ADDR, 914 REG_A5XX_CP_RB_RPTR_ADDR_HI, shadowptr(a5xx_gpu, gpu->rb[0])); 915 } else if (gpu->nr_rings > 1) { 916 /* Disable preemption if WHERE_AM_I isn't available */ 917 a5xx_preempt_fini(gpu); 918 gpu->nr_rings = 1; 919 } 920 921 a5xx_preempt_hw_init(gpu); 922 923 /* Disable the interrupts through the initial bringup stage */ 924 gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK, A5XX_INT_MASK); 925 926 /* Clear ME_HALT to start the micro engine */ 927 gpu_write(gpu, REG_A5XX_CP_PFP_ME_CNTL, 0); 928 ret = a5xx_me_init(gpu); 929 if (ret) 930 return ret; 931 932 ret = a5xx_power_init(gpu); 933 if (ret) 934 return ret; 935 936 /* 937 * Send a pipeline event stat to get misbehaving counters to start 938 * ticking correctly 939 */ 940 if (adreno_is_a530(adreno_gpu)) { 941 OUT_PKT7(gpu->rb[0], CP_EVENT_WRITE, 1); 942 OUT_RING(gpu->rb[0], CP_EVENT_WRITE_0_EVENT(STAT_EVENT)); 943 944 a5xx_flush(gpu, gpu->rb[0], true); 945 if (!a5xx_idle(gpu, gpu->rb[0])) 946 return -EINVAL; 947 } 948 949 /* 950 * If the chip that we are using does support loading one, then 951 * try to load a zap shader into the secure world. If successful 952 * we can use the CP to switch out of secure mode. If not then we 953 * have no resource but to try to switch ourselves out manually. If we 954 * guessed wrong then access to the RBBM_SECVID_TRUST_CNTL register will 955 * be blocked and a permissions violation will soon follow. 956 */ 957 ret = a5xx_zap_shader_init(gpu); 958 if (!ret) { 959 OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1); 960 OUT_RING(gpu->rb[0], 0x00000000); 961 962 a5xx_flush(gpu, gpu->rb[0], true); 963 if (!a5xx_idle(gpu, gpu->rb[0])) 964 return -EINVAL; 965 } else if (ret == -ENODEV) { 966 /* 967 * This device does not use zap shader (but print a warning 968 * just in case someone got their dt wrong.. hopefully they 969 * have a debug UART to realize the error of their ways... 970 * if you mess this up you are about to crash horribly) 971 */ 972 dev_warn_once(gpu->dev->dev, 973 "Zap shader not enabled - using SECVID_TRUST_CNTL instead\n"); 974 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TRUST_CNTL, 0x0); 975 } else { 976 return ret; 977 } 978 979 /* Last step - yield the ringbuffer */ 980 a5xx_preempt_start(gpu); 981 982 return 0; 983 } 984 985 static void a5xx_recover(struct msm_gpu *gpu) 986 { 987 int i; 988 989 adreno_dump_info(gpu); 990 991 for (i = 0; i < 8; i++) { 992 printk("CP_SCRATCH_REG%d: %u\n", i, 993 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(i))); 994 } 995 996 if (hang_debug) 997 a5xx_dump(gpu); 998 999 gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 1); 1000 gpu_read(gpu, REG_A5XX_RBBM_SW_RESET_CMD); 1001 gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 0); 1002 adreno_recover(gpu); 1003 } 1004 1005 static void a5xx_destroy(struct msm_gpu *gpu) 1006 { 1007 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1008 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 1009 1010 DBG("%s", gpu->name); 1011 1012 a5xx_preempt_fini(gpu); 1013 1014 if (a5xx_gpu->pm4_bo) { 1015 msm_gem_unpin_iova(a5xx_gpu->pm4_bo, gpu->aspace); 1016 drm_gem_object_put(a5xx_gpu->pm4_bo); 1017 } 1018 1019 if (a5xx_gpu->pfp_bo) { 1020 msm_gem_unpin_iova(a5xx_gpu->pfp_bo, gpu->aspace); 1021 drm_gem_object_put(a5xx_gpu->pfp_bo); 1022 } 1023 1024 if (a5xx_gpu->gpmu_bo) { 1025 msm_gem_unpin_iova(a5xx_gpu->gpmu_bo, gpu->aspace); 1026 drm_gem_object_put(a5xx_gpu->gpmu_bo); 1027 } 1028 1029 if (a5xx_gpu->shadow_bo) { 1030 msm_gem_unpin_iova(a5xx_gpu->shadow_bo, gpu->aspace); 1031 drm_gem_object_put(a5xx_gpu->shadow_bo); 1032 } 1033 1034 adreno_gpu_cleanup(adreno_gpu); 1035 kfree(a5xx_gpu); 1036 } 1037 1038 static inline bool _a5xx_check_idle(struct msm_gpu *gpu) 1039 { 1040 if (gpu_read(gpu, REG_A5XX_RBBM_STATUS) & ~A5XX_RBBM_STATUS_HI_BUSY) 1041 return false; 1042 1043 /* 1044 * Nearly every abnormality ends up pausing the GPU and triggering a 1045 * fault so we can safely just watch for this one interrupt to fire 1046 */ 1047 return !(gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS) & 1048 A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT); 1049 } 1050 1051 bool a5xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring) 1052 { 1053 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1054 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 1055 1056 if (ring != a5xx_gpu->cur_ring) { 1057 WARN(1, "Tried to idle a non-current ringbuffer\n"); 1058 return false; 1059 } 1060 1061 /* wait for CP to drain ringbuffer: */ 1062 if (!adreno_idle(gpu, ring)) 1063 return false; 1064 1065 if (spin_until(_a5xx_check_idle(gpu))) { 1066 DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n", 1067 gpu->name, __builtin_return_address(0), 1068 gpu_read(gpu, REG_A5XX_RBBM_STATUS), 1069 gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS), 1070 gpu_read(gpu, REG_A5XX_CP_RB_RPTR), 1071 gpu_read(gpu, REG_A5XX_CP_RB_WPTR)); 1072 return false; 1073 } 1074 1075 return true; 1076 } 1077 1078 static int a5xx_fault_handler(void *arg, unsigned long iova, int flags, void *data) 1079 { 1080 struct msm_gpu *gpu = arg; 1081 pr_warn_ratelimited("*** gpu fault: iova=%08lx, flags=%d (%u,%u,%u,%u)\n", 1082 iova, flags, 1083 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(4)), 1084 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(5)), 1085 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(6)), 1086 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(7))); 1087 1088 return 0; 1089 } 1090 1091 static void a5xx_cp_err_irq(struct msm_gpu *gpu) 1092 { 1093 u32 status = gpu_read(gpu, REG_A5XX_CP_INTERRUPT_STATUS); 1094 1095 if (status & A5XX_CP_INT_CP_OPCODE_ERROR) { 1096 u32 val; 1097 1098 gpu_write(gpu, REG_A5XX_CP_PFP_STAT_ADDR, 0); 1099 1100 /* 1101 * REG_A5XX_CP_PFP_STAT_DATA is indexed, and we want index 1 so 1102 * read it twice 1103 */ 1104 1105 gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA); 1106 val = gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA); 1107 1108 dev_err_ratelimited(gpu->dev->dev, "CP | opcode error | possible opcode=0x%8.8X\n", 1109 val); 1110 } 1111 1112 if (status & A5XX_CP_INT_CP_HW_FAULT_ERROR) 1113 dev_err_ratelimited(gpu->dev->dev, "CP | HW fault | status=0x%8.8X\n", 1114 gpu_read(gpu, REG_A5XX_CP_HW_FAULT)); 1115 1116 if (status & A5XX_CP_INT_CP_DMA_ERROR) 1117 dev_err_ratelimited(gpu->dev->dev, "CP | DMA error\n"); 1118 1119 if (status & A5XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) { 1120 u32 val = gpu_read(gpu, REG_A5XX_CP_PROTECT_STATUS); 1121 1122 dev_err_ratelimited(gpu->dev->dev, 1123 "CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n", 1124 val & (1 << 24) ? "WRITE" : "READ", 1125 (val & 0xFFFFF) >> 2, val); 1126 } 1127 1128 if (status & A5XX_CP_INT_CP_AHB_ERROR) { 1129 u32 status = gpu_read(gpu, REG_A5XX_CP_AHB_FAULT); 1130 const char *access[16] = { "reserved", "reserved", 1131 "timestamp lo", "timestamp hi", "pfp read", "pfp write", 1132 "", "", "me read", "me write", "", "", "crashdump read", 1133 "crashdump write" }; 1134 1135 dev_err_ratelimited(gpu->dev->dev, 1136 "CP | AHB error | addr=%X access=%s error=%d | status=0x%8.8X\n", 1137 status & 0xFFFFF, access[(status >> 24) & 0xF], 1138 (status & (1 << 31)), status); 1139 } 1140 } 1141 1142 static void a5xx_rbbm_err_irq(struct msm_gpu *gpu, u32 status) 1143 { 1144 if (status & A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR) { 1145 u32 val = gpu_read(gpu, REG_A5XX_RBBM_AHB_ERROR_STATUS); 1146 1147 dev_err_ratelimited(gpu->dev->dev, 1148 "RBBM | AHB bus error | %s | addr=0x%X | ports=0x%X:0x%X\n", 1149 val & (1 << 28) ? "WRITE" : "READ", 1150 (val & 0xFFFFF) >> 2, (val >> 20) & 0x3, 1151 (val >> 24) & 0xF); 1152 1153 /* Clear the error */ 1154 gpu_write(gpu, REG_A5XX_RBBM_AHB_CMD, (1 << 4)); 1155 1156 /* Clear the interrupt */ 1157 gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD, 1158 A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR); 1159 } 1160 1161 if (status & A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT) 1162 dev_err_ratelimited(gpu->dev->dev, "RBBM | AHB transfer timeout\n"); 1163 1164 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT) 1165 dev_err_ratelimited(gpu->dev->dev, "RBBM | ME master split | status=0x%X\n", 1166 gpu_read(gpu, REG_A5XX_RBBM_AHB_ME_SPLIT_STATUS)); 1167 1168 if (status & A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT) 1169 dev_err_ratelimited(gpu->dev->dev, "RBBM | PFP master split | status=0x%X\n", 1170 gpu_read(gpu, REG_A5XX_RBBM_AHB_PFP_SPLIT_STATUS)); 1171 1172 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT) 1173 dev_err_ratelimited(gpu->dev->dev, "RBBM | ETS master split | status=0x%X\n", 1174 gpu_read(gpu, REG_A5XX_RBBM_AHB_ETS_SPLIT_STATUS)); 1175 1176 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW) 1177 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB ASYNC overflow\n"); 1178 1179 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW) 1180 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB bus overflow\n"); 1181 } 1182 1183 static void a5xx_uche_err_irq(struct msm_gpu *gpu) 1184 { 1185 uint64_t addr = (uint64_t) gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_HI); 1186 1187 addr |= gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_LO); 1188 1189 dev_err_ratelimited(gpu->dev->dev, "UCHE | Out of bounds access | addr=0x%llX\n", 1190 addr); 1191 } 1192 1193 static void a5xx_gpmu_err_irq(struct msm_gpu *gpu) 1194 { 1195 dev_err_ratelimited(gpu->dev->dev, "GPMU | voltage droop\n"); 1196 } 1197 1198 static void a5xx_fault_detect_irq(struct msm_gpu *gpu) 1199 { 1200 struct drm_device *dev = gpu->dev; 1201 struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu); 1202 1203 /* 1204 * If stalled on SMMU fault, we could trip the GPU's hang detection, 1205 * but the fault handler will trigger the devcore dump, and we want 1206 * to otherwise resume normally rather than killing the submit, so 1207 * just bail. 1208 */ 1209 if (gpu_read(gpu, REG_A5XX_RBBM_STATUS3) & BIT(24)) 1210 return; 1211 1212 DRM_DEV_ERROR(dev->dev, "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n", 1213 ring ? ring->id : -1, ring ? ring->seqno : 0, 1214 gpu_read(gpu, REG_A5XX_RBBM_STATUS), 1215 gpu_read(gpu, REG_A5XX_CP_RB_RPTR), 1216 gpu_read(gpu, REG_A5XX_CP_RB_WPTR), 1217 gpu_read64(gpu, REG_A5XX_CP_IB1_BASE, REG_A5XX_CP_IB1_BASE_HI), 1218 gpu_read(gpu, REG_A5XX_CP_IB1_BUFSZ), 1219 gpu_read64(gpu, REG_A5XX_CP_IB2_BASE, REG_A5XX_CP_IB2_BASE_HI), 1220 gpu_read(gpu, REG_A5XX_CP_IB2_BUFSZ)); 1221 1222 /* Turn off the hangcheck timer to keep it from bothering us */ 1223 del_timer(&gpu->hangcheck_timer); 1224 1225 kthread_queue_work(gpu->worker, &gpu->recover_work); 1226 } 1227 1228 #define RBBM_ERROR_MASK \ 1229 (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \ 1230 A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \ 1231 A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \ 1232 A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \ 1233 A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \ 1234 A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW) 1235 1236 static irqreturn_t a5xx_irq(struct msm_gpu *gpu) 1237 { 1238 u32 status = gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS); 1239 1240 /* 1241 * Clear all the interrupts except RBBM_AHB_ERROR - if we clear it 1242 * before the source is cleared the interrupt will storm. 1243 */ 1244 gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD, 1245 status & ~A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR); 1246 1247 /* Pass status to a5xx_rbbm_err_irq because we've already cleared it */ 1248 if (status & RBBM_ERROR_MASK) 1249 a5xx_rbbm_err_irq(gpu, status); 1250 1251 if (status & A5XX_RBBM_INT_0_MASK_CP_HW_ERROR) 1252 a5xx_cp_err_irq(gpu); 1253 1254 if (status & A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT) 1255 a5xx_fault_detect_irq(gpu); 1256 1257 if (status & A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS) 1258 a5xx_uche_err_irq(gpu); 1259 1260 if (status & A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP) 1261 a5xx_gpmu_err_irq(gpu); 1262 1263 if (status & A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS) { 1264 a5xx_preempt_trigger(gpu); 1265 msm_gpu_retire(gpu); 1266 } 1267 1268 if (status & A5XX_RBBM_INT_0_MASK_CP_SW) 1269 a5xx_preempt_irq(gpu); 1270 1271 return IRQ_HANDLED; 1272 } 1273 1274 static const u32 a5xx_registers[] = { 1275 0x0000, 0x0002, 0x0004, 0x0020, 0x0022, 0x0026, 0x0029, 0x002B, 1276 0x002E, 0x0035, 0x0038, 0x0042, 0x0044, 0x0044, 0x0047, 0x0095, 1277 0x0097, 0x00BB, 0x03A0, 0x0464, 0x0469, 0x046F, 0x04D2, 0x04D3, 1278 0x04E0, 0x0533, 0x0540, 0x0555, 0x0800, 0x081A, 0x081F, 0x0841, 1279 0x0860, 0x0860, 0x0880, 0x08A0, 0x0B00, 0x0B12, 0x0B15, 0x0B28, 1280 0x0B78, 0x0B7F, 0x0BB0, 0x0BBD, 0x0BC0, 0x0BC6, 0x0BD0, 0x0C53, 1281 0x0C60, 0x0C61, 0x0C80, 0x0C82, 0x0C84, 0x0C85, 0x0C90, 0x0C98, 1282 0x0CA0, 0x0CA0, 0x0CB0, 0x0CB2, 0x2180, 0x2185, 0x2580, 0x2585, 1283 0x0CC1, 0x0CC1, 0x0CC4, 0x0CC7, 0x0CCC, 0x0CCC, 0x0CD0, 0x0CD8, 1284 0x0CE0, 0x0CE5, 0x0CE8, 0x0CE8, 0x0CEC, 0x0CF1, 0x0CFB, 0x0D0E, 1285 0x2100, 0x211E, 0x2140, 0x2145, 0x2500, 0x251E, 0x2540, 0x2545, 1286 0x0D10, 0x0D17, 0x0D20, 0x0D23, 0x0D30, 0x0D30, 0x20C0, 0x20C0, 1287 0x24C0, 0x24C0, 0x0E40, 0x0E43, 0x0E4A, 0x0E4A, 0x0E50, 0x0E57, 1288 0x0E60, 0x0E7C, 0x0E80, 0x0E8E, 0x0E90, 0x0E96, 0x0EA0, 0x0EA8, 1289 0x0EB0, 0x0EB2, 0xE140, 0xE147, 0xE150, 0xE187, 0xE1A0, 0xE1A9, 1290 0xE1B0, 0xE1B6, 0xE1C0, 0xE1C7, 0xE1D0, 0xE1D1, 0xE200, 0xE201, 1291 0xE210, 0xE21C, 0xE240, 0xE268, 0xE000, 0xE006, 0xE010, 0xE09A, 1292 0xE0A0, 0xE0A4, 0xE0AA, 0xE0EB, 0xE100, 0xE105, 0xE380, 0xE38F, 1293 0xE3B0, 0xE3B0, 0xE400, 0xE405, 0xE408, 0xE4E9, 0xE4F0, 0xE4F0, 1294 0xE280, 0xE280, 0xE282, 0xE2A3, 0xE2A5, 0xE2C2, 0xE940, 0xE947, 1295 0xE950, 0xE987, 0xE9A0, 0xE9A9, 0xE9B0, 0xE9B6, 0xE9C0, 0xE9C7, 1296 0xE9D0, 0xE9D1, 0xEA00, 0xEA01, 0xEA10, 0xEA1C, 0xEA40, 0xEA68, 1297 0xE800, 0xE806, 0xE810, 0xE89A, 0xE8A0, 0xE8A4, 0xE8AA, 0xE8EB, 1298 0xE900, 0xE905, 0xEB80, 0xEB8F, 0xEBB0, 0xEBB0, 0xEC00, 0xEC05, 1299 0xEC08, 0xECE9, 0xECF0, 0xECF0, 0xEA80, 0xEA80, 0xEA82, 0xEAA3, 1300 0xEAA5, 0xEAC2, 0xA800, 0xA800, 0xA820, 0xA828, 0xA840, 0xA87D, 1301 0XA880, 0xA88D, 0xA890, 0xA8A3, 0xA8D0, 0xA8D8, 0xA8E0, 0xA8F5, 1302 0xAC60, 0xAC60, ~0, 1303 }; 1304 1305 static void a5xx_dump(struct msm_gpu *gpu) 1306 { 1307 DRM_DEV_INFO(gpu->dev->dev, "status: %08x\n", 1308 gpu_read(gpu, REG_A5XX_RBBM_STATUS)); 1309 adreno_dump(gpu); 1310 } 1311 1312 static int a5xx_pm_resume(struct msm_gpu *gpu) 1313 { 1314 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1315 int ret; 1316 1317 /* Turn on the core power */ 1318 ret = msm_gpu_pm_resume(gpu); 1319 if (ret) 1320 return ret; 1321 1322 /* Adreno 508, 509, 510, 512 needs manual RBBM sus/res control */ 1323 if (!(adreno_is_a530(adreno_gpu) || adreno_is_a540(adreno_gpu))) { 1324 /* Halt the sp_input_clk at HM level */ 1325 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0x00000055); 1326 a5xx_set_hwcg(gpu, true); 1327 /* Turn on sp_input_clk at HM level */ 1328 gpu_rmw(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0xff, 0); 1329 return 0; 1330 } 1331 1332 /* Turn the RBCCU domain first to limit the chances of voltage droop */ 1333 gpu_write(gpu, REG_A5XX_GPMU_RBCCU_POWER_CNTL, 0x778000); 1334 1335 /* Wait 3 usecs before polling */ 1336 udelay(3); 1337 1338 ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS, 1339 (1 << 20), (1 << 20)); 1340 if (ret) { 1341 DRM_ERROR("%s: timeout waiting for RBCCU GDSC enable: %X\n", 1342 gpu->name, 1343 gpu_read(gpu, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS)); 1344 return ret; 1345 } 1346 1347 /* Turn on the SP domain */ 1348 gpu_write(gpu, REG_A5XX_GPMU_SP_POWER_CNTL, 0x778000); 1349 ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_SP_PWR_CLK_STATUS, 1350 (1 << 20), (1 << 20)); 1351 if (ret) 1352 DRM_ERROR("%s: timeout waiting for SP GDSC enable\n", 1353 gpu->name); 1354 1355 return ret; 1356 } 1357 1358 static int a5xx_pm_suspend(struct msm_gpu *gpu) 1359 { 1360 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1361 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 1362 u32 mask = 0xf; 1363 int i, ret; 1364 1365 /* A508, A510 have 3 XIN ports in VBIF */ 1366 if (adreno_is_a508(adreno_gpu) || adreno_is_a510(adreno_gpu)) 1367 mask = 0x7; 1368 1369 /* Clear the VBIF pipe before shutting down */ 1370 gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, mask); 1371 spin_until((gpu_read(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL1) & 1372 mask) == mask); 1373 1374 gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0); 1375 1376 /* 1377 * Reset the VBIF before power collapse to avoid issue with FIFO 1378 * entries on Adreno A510 and A530 (the others will tend to lock up) 1379 */ 1380 if (adreno_is_a510(adreno_gpu) || adreno_is_a530(adreno_gpu)) { 1381 gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x003C0000); 1382 gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x00000000); 1383 } 1384 1385 ret = msm_gpu_pm_suspend(gpu); 1386 if (ret) 1387 return ret; 1388 1389 if (a5xx_gpu->has_whereami) 1390 for (i = 0; i < gpu->nr_rings; i++) 1391 a5xx_gpu->shadow[i] = 0; 1392 1393 return 0; 1394 } 1395 1396 static int a5xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value) 1397 { 1398 *value = gpu_read64(gpu, REG_A5XX_RBBM_ALWAYSON_COUNTER_LO, 1399 REG_A5XX_RBBM_ALWAYSON_COUNTER_HI); 1400 1401 return 0; 1402 } 1403 1404 struct a5xx_crashdumper { 1405 void *ptr; 1406 struct drm_gem_object *bo; 1407 u64 iova; 1408 }; 1409 1410 struct a5xx_gpu_state { 1411 struct msm_gpu_state base; 1412 u32 *hlsqregs; 1413 }; 1414 1415 static int a5xx_crashdumper_init(struct msm_gpu *gpu, 1416 struct a5xx_crashdumper *dumper) 1417 { 1418 dumper->ptr = msm_gem_kernel_new_locked(gpu->dev, 1419 SZ_1M, MSM_BO_WC, gpu->aspace, 1420 &dumper->bo, &dumper->iova); 1421 1422 if (!IS_ERR(dumper->ptr)) 1423 msm_gem_object_set_name(dumper->bo, "crashdump"); 1424 1425 return PTR_ERR_OR_ZERO(dumper->ptr); 1426 } 1427 1428 static int a5xx_crashdumper_run(struct msm_gpu *gpu, 1429 struct a5xx_crashdumper *dumper) 1430 { 1431 u32 val; 1432 1433 if (IS_ERR_OR_NULL(dumper->ptr)) 1434 return -EINVAL; 1435 1436 gpu_write64(gpu, REG_A5XX_CP_CRASH_SCRIPT_BASE_LO, 1437 REG_A5XX_CP_CRASH_SCRIPT_BASE_HI, dumper->iova); 1438 1439 gpu_write(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, 1); 1440 1441 return gpu_poll_timeout(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, val, 1442 val & 0x04, 100, 10000); 1443 } 1444 1445 /* 1446 * These are a list of the registers that need to be read through the HLSQ 1447 * aperture through the crashdumper. These are not nominally accessible from 1448 * the CPU on a secure platform. 1449 */ 1450 static const struct { 1451 u32 type; 1452 u32 regoffset; 1453 u32 count; 1454 } a5xx_hlsq_aperture_regs[] = { 1455 { 0x35, 0xe00, 0x32 }, /* HSLQ non-context */ 1456 { 0x31, 0x2080, 0x1 }, /* HLSQ 2D context 0 */ 1457 { 0x33, 0x2480, 0x1 }, /* HLSQ 2D context 1 */ 1458 { 0x32, 0xe780, 0x62 }, /* HLSQ 3D context 0 */ 1459 { 0x34, 0xef80, 0x62 }, /* HLSQ 3D context 1 */ 1460 { 0x3f, 0x0ec0, 0x40 }, /* SP non-context */ 1461 { 0x3d, 0x2040, 0x1 }, /* SP 2D context 0 */ 1462 { 0x3b, 0x2440, 0x1 }, /* SP 2D context 1 */ 1463 { 0x3e, 0xe580, 0x170 }, /* SP 3D context 0 */ 1464 { 0x3c, 0xed80, 0x170 }, /* SP 3D context 1 */ 1465 { 0x3a, 0x0f00, 0x1c }, /* TP non-context */ 1466 { 0x38, 0x2000, 0xa }, /* TP 2D context 0 */ 1467 { 0x36, 0x2400, 0xa }, /* TP 2D context 1 */ 1468 { 0x39, 0xe700, 0x80 }, /* TP 3D context 0 */ 1469 { 0x37, 0xef00, 0x80 }, /* TP 3D context 1 */ 1470 }; 1471 1472 static void a5xx_gpu_state_get_hlsq_regs(struct msm_gpu *gpu, 1473 struct a5xx_gpu_state *a5xx_state) 1474 { 1475 struct a5xx_crashdumper dumper = { 0 }; 1476 u32 offset, count = 0; 1477 u64 *ptr; 1478 int i; 1479 1480 if (a5xx_crashdumper_init(gpu, &dumper)) 1481 return; 1482 1483 /* The script will be written at offset 0 */ 1484 ptr = dumper.ptr; 1485 1486 /* Start writing the data at offset 256k */ 1487 offset = dumper.iova + (256 * SZ_1K); 1488 1489 /* Count how many additional registers to get from the HLSQ aperture */ 1490 for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) 1491 count += a5xx_hlsq_aperture_regs[i].count; 1492 1493 a5xx_state->hlsqregs = kcalloc(count, sizeof(u32), GFP_KERNEL); 1494 if (!a5xx_state->hlsqregs) 1495 return; 1496 1497 /* Build the crashdump script */ 1498 for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) { 1499 u32 type = a5xx_hlsq_aperture_regs[i].type; 1500 u32 c = a5xx_hlsq_aperture_regs[i].count; 1501 1502 /* Write the register to select the desired bank */ 1503 *ptr++ = ((u64) type << 8); 1504 *ptr++ = (((u64) REG_A5XX_HLSQ_DBG_READ_SEL) << 44) | 1505 (1 << 21) | 1; 1506 1507 *ptr++ = offset; 1508 *ptr++ = (((u64) REG_A5XX_HLSQ_DBG_AHB_READ_APERTURE) << 44) 1509 | c; 1510 1511 offset += c * sizeof(u32); 1512 } 1513 1514 /* Write two zeros to close off the script */ 1515 *ptr++ = 0; 1516 *ptr++ = 0; 1517 1518 if (a5xx_crashdumper_run(gpu, &dumper)) { 1519 kfree(a5xx_state->hlsqregs); 1520 msm_gem_kernel_put(dumper.bo, gpu->aspace, true); 1521 return; 1522 } 1523 1524 /* Copy the data from the crashdumper to the state */ 1525 memcpy(a5xx_state->hlsqregs, dumper.ptr + (256 * SZ_1K), 1526 count * sizeof(u32)); 1527 1528 msm_gem_kernel_put(dumper.bo, gpu->aspace, true); 1529 } 1530 1531 static struct msm_gpu_state *a5xx_gpu_state_get(struct msm_gpu *gpu) 1532 { 1533 struct a5xx_gpu_state *a5xx_state = kzalloc(sizeof(*a5xx_state), 1534 GFP_KERNEL); 1535 bool stalled = !!(gpu_read(gpu, REG_A5XX_RBBM_STATUS3) & BIT(24)); 1536 1537 if (!a5xx_state) 1538 return ERR_PTR(-ENOMEM); 1539 1540 /* Temporarily disable hardware clock gating before reading the hw */ 1541 a5xx_set_hwcg(gpu, false); 1542 1543 /* First get the generic state from the adreno core */ 1544 adreno_gpu_state_get(gpu, &(a5xx_state->base)); 1545 1546 a5xx_state->base.rbbm_status = gpu_read(gpu, REG_A5XX_RBBM_STATUS); 1547 1548 /* 1549 * Get the HLSQ regs with the help of the crashdumper, but only if 1550 * we are not stalled in an iommu fault (in which case the crashdumper 1551 * would not have access to memory) 1552 */ 1553 if (!stalled) 1554 a5xx_gpu_state_get_hlsq_regs(gpu, a5xx_state); 1555 1556 a5xx_set_hwcg(gpu, true); 1557 1558 return &a5xx_state->base; 1559 } 1560 1561 static void a5xx_gpu_state_destroy(struct kref *kref) 1562 { 1563 struct msm_gpu_state *state = container_of(kref, 1564 struct msm_gpu_state, ref); 1565 struct a5xx_gpu_state *a5xx_state = container_of(state, 1566 struct a5xx_gpu_state, base); 1567 1568 kfree(a5xx_state->hlsqregs); 1569 1570 adreno_gpu_state_destroy(state); 1571 kfree(a5xx_state); 1572 } 1573 1574 static int a5xx_gpu_state_put(struct msm_gpu_state *state) 1575 { 1576 if (IS_ERR_OR_NULL(state)) 1577 return 1; 1578 1579 return kref_put(&state->ref, a5xx_gpu_state_destroy); 1580 } 1581 1582 1583 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP) 1584 static void a5xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state, 1585 struct drm_printer *p) 1586 { 1587 int i, j; 1588 u32 pos = 0; 1589 struct a5xx_gpu_state *a5xx_state = container_of(state, 1590 struct a5xx_gpu_state, base); 1591 1592 if (IS_ERR_OR_NULL(state)) 1593 return; 1594 1595 adreno_show(gpu, state, p); 1596 1597 /* Dump the additional a5xx HLSQ registers */ 1598 if (!a5xx_state->hlsqregs) 1599 return; 1600 1601 drm_printf(p, "registers-hlsq:\n"); 1602 1603 for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) { 1604 u32 o = a5xx_hlsq_aperture_regs[i].regoffset; 1605 u32 c = a5xx_hlsq_aperture_regs[i].count; 1606 1607 for (j = 0; j < c; j++, pos++, o++) { 1608 /* 1609 * To keep the crashdump simple we pull the entire range 1610 * for each register type but not all of the registers 1611 * in the range are valid. Fortunately invalid registers 1612 * stick out like a sore thumb with a value of 1613 * 0xdeadbeef 1614 */ 1615 if (a5xx_state->hlsqregs[pos] == 0xdeadbeef) 1616 continue; 1617 1618 drm_printf(p, " - { offset: 0x%04x, value: 0x%08x }\n", 1619 o << 2, a5xx_state->hlsqregs[pos]); 1620 } 1621 } 1622 } 1623 #endif 1624 1625 static struct msm_ringbuffer *a5xx_active_ring(struct msm_gpu *gpu) 1626 { 1627 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1628 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 1629 1630 return a5xx_gpu->cur_ring; 1631 } 1632 1633 static unsigned long a5xx_gpu_busy(struct msm_gpu *gpu) 1634 { 1635 u64 busy_cycles, busy_time; 1636 1637 /* Only read the gpu busy if the hardware is already active */ 1638 if (pm_runtime_get_if_in_use(&gpu->pdev->dev) == 0) 1639 return 0; 1640 1641 busy_cycles = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_0_LO, 1642 REG_A5XX_RBBM_PERFCTR_RBBM_0_HI); 1643 1644 busy_time = busy_cycles - gpu->devfreq.busy_cycles; 1645 do_div(busy_time, clk_get_rate(gpu->core_clk) / 1000000); 1646 1647 gpu->devfreq.busy_cycles = busy_cycles; 1648 1649 pm_runtime_put(&gpu->pdev->dev); 1650 1651 if (WARN_ON(busy_time > ~0LU)) 1652 return ~0LU; 1653 1654 return (unsigned long)busy_time; 1655 } 1656 1657 static uint32_t a5xx_get_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring) 1658 { 1659 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1660 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 1661 1662 if (a5xx_gpu->has_whereami) 1663 return a5xx_gpu->shadow[ring->id]; 1664 1665 return ring->memptrs->rptr = gpu_read(gpu, REG_A5XX_CP_RB_RPTR); 1666 } 1667 1668 static const struct adreno_gpu_funcs funcs = { 1669 .base = { 1670 .get_param = adreno_get_param, 1671 .hw_init = a5xx_hw_init, 1672 .pm_suspend = a5xx_pm_suspend, 1673 .pm_resume = a5xx_pm_resume, 1674 .recover = a5xx_recover, 1675 .submit = a5xx_submit, 1676 .active_ring = a5xx_active_ring, 1677 .irq = a5xx_irq, 1678 .destroy = a5xx_destroy, 1679 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP) 1680 .show = a5xx_show, 1681 #endif 1682 #if defined(CONFIG_DEBUG_FS) 1683 .debugfs_init = a5xx_debugfs_init, 1684 #endif 1685 .gpu_busy = a5xx_gpu_busy, 1686 .gpu_state_get = a5xx_gpu_state_get, 1687 .gpu_state_put = a5xx_gpu_state_put, 1688 .create_address_space = adreno_iommu_create_address_space, 1689 .get_rptr = a5xx_get_rptr, 1690 }, 1691 .get_timestamp = a5xx_get_timestamp, 1692 }; 1693 1694 static void check_speed_bin(struct device *dev) 1695 { 1696 struct nvmem_cell *cell; 1697 u32 val; 1698 1699 /* 1700 * If the OPP table specifies a opp-supported-hw property then we have 1701 * to set something with dev_pm_opp_set_supported_hw() or the table 1702 * doesn't get populated so pick an arbitrary value that should 1703 * ensure the default frequencies are selected but not conflict with any 1704 * actual bins 1705 */ 1706 val = 0x80; 1707 1708 cell = nvmem_cell_get(dev, "speed_bin"); 1709 1710 if (!IS_ERR(cell)) { 1711 void *buf = nvmem_cell_read(cell, NULL); 1712 1713 if (!IS_ERR(buf)) { 1714 u8 bin = *((u8 *) buf); 1715 1716 val = (1 << bin); 1717 kfree(buf); 1718 } 1719 1720 nvmem_cell_put(cell); 1721 } 1722 1723 devm_pm_opp_set_supported_hw(dev, &val, 1); 1724 } 1725 1726 struct msm_gpu *a5xx_gpu_init(struct drm_device *dev) 1727 { 1728 struct msm_drm_private *priv = dev->dev_private; 1729 struct platform_device *pdev = priv->gpu_pdev; 1730 struct a5xx_gpu *a5xx_gpu = NULL; 1731 struct adreno_gpu *adreno_gpu; 1732 struct msm_gpu *gpu; 1733 int ret; 1734 1735 if (!pdev) { 1736 DRM_DEV_ERROR(dev->dev, "No A5XX device is defined\n"); 1737 return ERR_PTR(-ENXIO); 1738 } 1739 1740 a5xx_gpu = kzalloc(sizeof(*a5xx_gpu), GFP_KERNEL); 1741 if (!a5xx_gpu) 1742 return ERR_PTR(-ENOMEM); 1743 1744 adreno_gpu = &a5xx_gpu->base; 1745 gpu = &adreno_gpu->base; 1746 1747 adreno_gpu->registers = a5xx_registers; 1748 1749 a5xx_gpu->lm_leakage = 0x4E001A; 1750 1751 check_speed_bin(&pdev->dev); 1752 1753 ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 4); 1754 if (ret) { 1755 a5xx_destroy(&(a5xx_gpu->base.base)); 1756 return ERR_PTR(ret); 1757 } 1758 1759 if (gpu->aspace) 1760 msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu, a5xx_fault_handler); 1761 1762 /* Set up the preemption specific bits and pieces for each ringbuffer */ 1763 a5xx_preempt_init(gpu); 1764 1765 return gpu; 1766 } 1767