xref: /openbmc/linux/drivers/gpu/drm/msm/adreno/a5xx_gpu.c (revision 5497b23e)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2016-2017 The Linux Foundation. All rights reserved.
3  */
4 
5 #include <linux/kernel.h>
6 #include <linux/types.h>
7 #include <linux/cpumask.h>
8 #include <linux/qcom_scm.h>
9 #include <linux/pm_opp.h>
10 #include <linux/nvmem-consumer.h>
11 #include <linux/slab.h>
12 #include "msm_gem.h"
13 #include "msm_mmu.h"
14 #include "a5xx_gpu.h"
15 
16 extern bool hang_debug;
17 static void a5xx_dump(struct msm_gpu *gpu);
18 
19 #define GPU_PAS_ID 13
20 
21 void a5xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring,
22 		bool sync)
23 {
24 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
25 	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
26 	uint32_t wptr;
27 	unsigned long flags;
28 
29 	/*
30 	 * Most flush operations need to issue a WHERE_AM_I opcode to sync up
31 	 * the rptr shadow
32 	 */
33 	if (a5xx_gpu->has_whereami && sync) {
34 		OUT_PKT7(ring, CP_WHERE_AM_I, 2);
35 		OUT_RING(ring, lower_32_bits(shadowptr(a5xx_gpu, ring)));
36 		OUT_RING(ring, upper_32_bits(shadowptr(a5xx_gpu, ring)));
37 	}
38 
39 	spin_lock_irqsave(&ring->preempt_lock, flags);
40 
41 	/* Copy the shadow to the actual register */
42 	ring->cur = ring->next;
43 
44 	/* Make sure to wrap wptr if we need to */
45 	wptr = get_wptr(ring);
46 
47 	spin_unlock_irqrestore(&ring->preempt_lock, flags);
48 
49 	/* Make sure everything is posted before making a decision */
50 	mb();
51 
52 	/* Update HW if this is the current ring and we are not in preempt */
53 	if (a5xx_gpu->cur_ring == ring && !a5xx_in_preempt(a5xx_gpu))
54 		gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);
55 }
56 
57 static void a5xx_submit_in_rb(struct msm_gpu *gpu, struct msm_gem_submit *submit)
58 {
59 	struct msm_drm_private *priv = gpu->dev->dev_private;
60 	struct msm_ringbuffer *ring = submit->ring;
61 	struct msm_gem_object *obj;
62 	uint32_t *ptr, dwords;
63 	unsigned int i;
64 
65 	for (i = 0; i < submit->nr_cmds; i++) {
66 		switch (submit->cmd[i].type) {
67 		case MSM_SUBMIT_CMD_IB_TARGET_BUF:
68 			break;
69 		case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
70 			if (priv->lastctx == submit->queue->ctx)
71 				break;
72 			fallthrough;
73 		case MSM_SUBMIT_CMD_BUF:
74 			/* copy commands into RB: */
75 			obj = submit->bos[submit->cmd[i].idx].obj;
76 			dwords = submit->cmd[i].size;
77 
78 			ptr = msm_gem_get_vaddr(&obj->base);
79 
80 			/* _get_vaddr() shouldn't fail at this point,
81 			 * since we've already mapped it once in
82 			 * submit_reloc()
83 			 */
84 			if (WARN_ON(!ptr))
85 				return;
86 
87 			for (i = 0; i < dwords; i++) {
88 				/* normally the OUT_PKTn() would wait
89 				 * for space for the packet.  But since
90 				 * we just OUT_RING() the whole thing,
91 				 * need to call adreno_wait_ring()
92 				 * ourself:
93 				 */
94 				adreno_wait_ring(ring, 1);
95 				OUT_RING(ring, ptr[i]);
96 			}
97 
98 			msm_gem_put_vaddr(&obj->base);
99 
100 			break;
101 		}
102 	}
103 
104 	a5xx_flush(gpu, ring, true);
105 	a5xx_preempt_trigger(gpu);
106 
107 	/* we might not necessarily have a cmd from userspace to
108 	 * trigger an event to know that submit has completed, so
109 	 * do this manually:
110 	 */
111 	a5xx_idle(gpu, ring);
112 	ring->memptrs->fence = submit->seqno;
113 	msm_gpu_retire(gpu);
114 }
115 
116 static void a5xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
117 {
118 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
119 	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
120 	struct msm_drm_private *priv = gpu->dev->dev_private;
121 	struct msm_ringbuffer *ring = submit->ring;
122 	unsigned int i, ibs = 0;
123 
124 	if (IS_ENABLED(CONFIG_DRM_MSM_GPU_SUDO) && submit->in_rb) {
125 		priv->lastctx = NULL;
126 		a5xx_submit_in_rb(gpu, submit);
127 		return;
128 	}
129 
130 	OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1);
131 	OUT_RING(ring, 0x02);
132 
133 	/* Turn off protected mode to write to special registers */
134 	OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
135 	OUT_RING(ring, 0);
136 
137 	/* Set the save preemption record for the ring/command */
138 	OUT_PKT4(ring, REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 2);
139 	OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[submit->ring->id]));
140 	OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[submit->ring->id]));
141 
142 	/* Turn back on protected mode */
143 	OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
144 	OUT_RING(ring, 1);
145 
146 	/* Enable local preemption for finegrain preemption */
147 	OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1);
148 	OUT_RING(ring, 0x02);
149 
150 	/* Allow CP_CONTEXT_SWITCH_YIELD packets in the IB2 */
151 	OUT_PKT7(ring, CP_YIELD_ENABLE, 1);
152 	OUT_RING(ring, 0x02);
153 
154 	/* Submit the commands */
155 	for (i = 0; i < submit->nr_cmds; i++) {
156 		switch (submit->cmd[i].type) {
157 		case MSM_SUBMIT_CMD_IB_TARGET_BUF:
158 			break;
159 		case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
160 			if (priv->lastctx == submit->queue->ctx)
161 				break;
162 			fallthrough;
163 		case MSM_SUBMIT_CMD_BUF:
164 			OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3);
165 			OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
166 			OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
167 			OUT_RING(ring, submit->cmd[i].size);
168 			ibs++;
169 			break;
170 		}
171 	}
172 
173 	/*
174 	 * Write the render mode to NULL (0) to indicate to the CP that the IBs
175 	 * are done rendering - otherwise a lucky preemption would start
176 	 * replaying from the last checkpoint
177 	 */
178 	OUT_PKT7(ring, CP_SET_RENDER_MODE, 5);
179 	OUT_RING(ring, 0);
180 	OUT_RING(ring, 0);
181 	OUT_RING(ring, 0);
182 	OUT_RING(ring, 0);
183 	OUT_RING(ring, 0);
184 
185 	/* Turn off IB level preemptions */
186 	OUT_PKT7(ring, CP_YIELD_ENABLE, 1);
187 	OUT_RING(ring, 0x01);
188 
189 	/* Write the fence to the scratch register */
190 	OUT_PKT4(ring, REG_A5XX_CP_SCRATCH_REG(2), 1);
191 	OUT_RING(ring, submit->seqno);
192 
193 	/*
194 	 * Execute a CACHE_FLUSH_TS event. This will ensure that the
195 	 * timestamp is written to the memory and then triggers the interrupt
196 	 */
197 	OUT_PKT7(ring, CP_EVENT_WRITE, 4);
198 	OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(CACHE_FLUSH_TS) |
199 		CP_EVENT_WRITE_0_IRQ);
200 	OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence)));
201 	OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence)));
202 	OUT_RING(ring, submit->seqno);
203 
204 	/* Yield the floor on command completion */
205 	OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4);
206 	/*
207 	 * If dword[2:1] are non zero, they specify an address for the CP to
208 	 * write the value of dword[3] to on preemption complete. Write 0 to
209 	 * skip the write
210 	 */
211 	OUT_RING(ring, 0x00);
212 	OUT_RING(ring, 0x00);
213 	/* Data value - not used if the address above is 0 */
214 	OUT_RING(ring, 0x01);
215 	/* Set bit 0 to trigger an interrupt on preempt complete */
216 	OUT_RING(ring, 0x01);
217 
218 	/* A WHERE_AM_I packet is not needed after a YIELD */
219 	a5xx_flush(gpu, ring, false);
220 
221 	/* Check to see if we need to start preemption */
222 	a5xx_preempt_trigger(gpu);
223 }
224 
225 static const struct adreno_five_hwcg_regs {
226 	u32 offset;
227 	u32 value;
228 } a5xx_hwcg[] = {
229 	{REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
230 	{REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222},
231 	{REG_A5XX_RBBM_CLOCK_CNTL_SP2, 0x02222222},
232 	{REG_A5XX_RBBM_CLOCK_CNTL_SP3, 0x02222222},
233 	{REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
234 	{REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220},
235 	{REG_A5XX_RBBM_CLOCK_CNTL2_SP2, 0x02222220},
236 	{REG_A5XX_RBBM_CLOCK_CNTL2_SP3, 0x02222220},
237 	{REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
238 	{REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF},
239 	{REG_A5XX_RBBM_CLOCK_HYST_SP2, 0x0000F3CF},
240 	{REG_A5XX_RBBM_CLOCK_HYST_SP3, 0x0000F3CF},
241 	{REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
242 	{REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
243 	{REG_A5XX_RBBM_CLOCK_DELAY_SP2, 0x00000080},
244 	{REG_A5XX_RBBM_CLOCK_DELAY_SP3, 0x00000080},
245 	{REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
246 	{REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222},
247 	{REG_A5XX_RBBM_CLOCK_CNTL_TP2, 0x22222222},
248 	{REG_A5XX_RBBM_CLOCK_CNTL_TP3, 0x22222222},
249 	{REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
250 	{REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
251 	{REG_A5XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222},
252 	{REG_A5XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222},
253 	{REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222},
254 	{REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222},
255 	{REG_A5XX_RBBM_CLOCK_CNTL3_TP2, 0x00002222},
256 	{REG_A5XX_RBBM_CLOCK_CNTL3_TP3, 0x00002222},
257 	{REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
258 	{REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
259 	{REG_A5XX_RBBM_CLOCK_HYST_TP2, 0x77777777},
260 	{REG_A5XX_RBBM_CLOCK_HYST_TP3, 0x77777777},
261 	{REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
262 	{REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
263 	{REG_A5XX_RBBM_CLOCK_HYST2_TP2, 0x77777777},
264 	{REG_A5XX_RBBM_CLOCK_HYST2_TP3, 0x77777777},
265 	{REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777},
266 	{REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777},
267 	{REG_A5XX_RBBM_CLOCK_HYST3_TP2, 0x00007777},
268 	{REG_A5XX_RBBM_CLOCK_HYST3_TP3, 0x00007777},
269 	{REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
270 	{REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
271 	{REG_A5XX_RBBM_CLOCK_DELAY_TP2, 0x11111111},
272 	{REG_A5XX_RBBM_CLOCK_DELAY_TP3, 0x11111111},
273 	{REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
274 	{REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
275 	{REG_A5XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111},
276 	{REG_A5XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111},
277 	{REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111},
278 	{REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111},
279 	{REG_A5XX_RBBM_CLOCK_DELAY3_TP2, 0x00001111},
280 	{REG_A5XX_RBBM_CLOCK_DELAY3_TP3, 0x00001111},
281 	{REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
282 	{REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
283 	{REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
284 	{REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
285 	{REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444},
286 	{REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
287 	{REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
288 	{REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
289 	{REG_A5XX_RBBM_CLOCK_CNTL_RB2, 0x22222222},
290 	{REG_A5XX_RBBM_CLOCK_CNTL_RB3, 0x22222222},
291 	{REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222},
292 	{REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222},
293 	{REG_A5XX_RBBM_CLOCK_CNTL2_RB2, 0x00222222},
294 	{REG_A5XX_RBBM_CLOCK_CNTL2_RB3, 0x00222222},
295 	{REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220},
296 	{REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220},
297 	{REG_A5XX_RBBM_CLOCK_CNTL_CCU2, 0x00022220},
298 	{REG_A5XX_RBBM_CLOCK_CNTL_CCU3, 0x00022220},
299 	{REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222},
300 	{REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555},
301 	{REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404},
302 	{REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404},
303 	{REG_A5XX_RBBM_CLOCK_HYST_RB_CCU2, 0x04040404},
304 	{REG_A5XX_RBBM_CLOCK_HYST_RB_CCU3, 0x04040404},
305 	{REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044},
306 	{REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002},
307 	{REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002},
308 	{REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2, 0x00000002},
309 	{REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3, 0x00000002},
310 	{REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011},
311 	{REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
312 	{REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
313 	{REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
314 	{REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
315 	{REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
316 	{REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
317 	{REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
318 	{REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
319 	{REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
320 	{REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}
321 }, a50x_hwcg[] = {
322 	{REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
323 	{REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
324 	{REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
325 	{REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
326 	{REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
327 	{REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
328 	{REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222},
329 	{REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
330 	{REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
331 	{REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777},
332 	{REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
333 	{REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
334 	{REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111},
335 	{REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
336 	{REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
337 	{REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
338 	{REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
339 	{REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00FFFFF4},
340 	{REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
341 	{REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
342 	{REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222},
343 	{REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220},
344 	{REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222},
345 	{REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555},
346 	{REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404},
347 	{REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044},
348 	{REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002},
349 	{REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011},
350 	{REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
351 	{REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
352 	{REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
353 	{REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
354 	{REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
355 	{REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
356 	{REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
357 	{REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
358 	{REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
359 	{REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
360 }, a512_hwcg[] = {
361 	{REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
362 	{REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222},
363 	{REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
364 	{REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220},
365 	{REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
366 	{REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF},
367 	{REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
368 	{REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
369 	{REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
370 	{REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222},
371 	{REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
372 	{REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
373 	{REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222},
374 	{REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222},
375 	{REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
376 	{REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
377 	{REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
378 	{REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
379 	{REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777},
380 	{REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777},
381 	{REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
382 	{REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
383 	{REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
384 	{REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
385 	{REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111},
386 	{REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111},
387 	{REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
388 	{REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
389 	{REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
390 	{REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
391 	{REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444},
392 	{REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
393 	{REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
394 	{REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
395 	{REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222},
396 	{REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222},
397 	{REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220},
398 	{REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220},
399 	{REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222},
400 	{REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555},
401 	{REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404},
402 	{REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404},
403 	{REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044},
404 	{REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002},
405 	{REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002},
406 	{REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011},
407 	{REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
408 	{REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
409 	{REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
410 	{REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
411 	{REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
412 	{REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
413 	{REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
414 	{REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
415 	{REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
416 	{REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
417 };
418 
419 void a5xx_set_hwcg(struct msm_gpu *gpu, bool state)
420 {
421 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
422 	const struct adreno_five_hwcg_regs *regs;
423 	unsigned int i, sz;
424 
425 	if (adreno_is_a508(adreno_gpu)) {
426 		regs = a50x_hwcg;
427 		sz = ARRAY_SIZE(a50x_hwcg);
428 	} else if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu)) {
429 		regs = a512_hwcg;
430 		sz = ARRAY_SIZE(a512_hwcg);
431 	} else {
432 		regs = a5xx_hwcg;
433 		sz = ARRAY_SIZE(a5xx_hwcg);
434 	}
435 
436 	for (i = 0; i < sz; i++)
437 		gpu_write(gpu, regs[i].offset,
438 			  state ? regs[i].value : 0);
439 
440 	if (adreno_is_a540(adreno_gpu)) {
441 		gpu_write(gpu, REG_A5XX_RBBM_CLOCK_DELAY_GPMU, state ? 0x00000770 : 0);
442 		gpu_write(gpu, REG_A5XX_RBBM_CLOCK_HYST_GPMU, state ? 0x00000004 : 0);
443 	}
444 
445 	gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, state ? 0xAAA8AA00 : 0);
446 	gpu_write(gpu, REG_A5XX_RBBM_ISDB_CNT, state ? 0x182 : 0x180);
447 }
448 
449 static int a5xx_me_init(struct msm_gpu *gpu)
450 {
451 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
452 	struct msm_ringbuffer *ring = gpu->rb[0];
453 
454 	OUT_PKT7(ring, CP_ME_INIT, 8);
455 
456 	OUT_RING(ring, 0x0000002F);
457 
458 	/* Enable multiple hardware contexts */
459 	OUT_RING(ring, 0x00000003);
460 
461 	/* Enable error detection */
462 	OUT_RING(ring, 0x20000000);
463 
464 	/* Don't enable header dump */
465 	OUT_RING(ring, 0x00000000);
466 	OUT_RING(ring, 0x00000000);
467 
468 	/* Specify workarounds for various microcode issues */
469 	if (adreno_is_a530(adreno_gpu)) {
470 		/* Workaround for token end syncs
471 		 * Force a WFI after every direct-render 3D mode draw and every
472 		 * 2D mode 3 draw
473 		 */
474 		OUT_RING(ring, 0x0000000B);
475 	} else if (adreno_is_a510(adreno_gpu)) {
476 		/* Workaround for token and syncs */
477 		OUT_RING(ring, 0x00000001);
478 	} else {
479 		/* No workarounds enabled */
480 		OUT_RING(ring, 0x00000000);
481 	}
482 
483 	OUT_RING(ring, 0x00000000);
484 	OUT_RING(ring, 0x00000000);
485 
486 	a5xx_flush(gpu, ring, true);
487 	return a5xx_idle(gpu, ring) ? 0 : -EINVAL;
488 }
489 
490 static int a5xx_preempt_start(struct msm_gpu *gpu)
491 {
492 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
493 	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
494 	struct msm_ringbuffer *ring = gpu->rb[0];
495 
496 	if (gpu->nr_rings == 1)
497 		return 0;
498 
499 	/* Turn off protected mode to write to special registers */
500 	OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
501 	OUT_RING(ring, 0);
502 
503 	/* Set the save preemption record for the ring/command */
504 	OUT_PKT4(ring, REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 2);
505 	OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[ring->id]));
506 	OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[ring->id]));
507 
508 	/* Turn back on protected mode */
509 	OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
510 	OUT_RING(ring, 1);
511 
512 	OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1);
513 	OUT_RING(ring, 0x00);
514 
515 	OUT_PKT7(ring, CP_PREEMPT_ENABLE_LOCAL, 1);
516 	OUT_RING(ring, 0x01);
517 
518 	OUT_PKT7(ring, CP_YIELD_ENABLE, 1);
519 	OUT_RING(ring, 0x01);
520 
521 	/* Yield the floor on command completion */
522 	OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4);
523 	OUT_RING(ring, 0x00);
524 	OUT_RING(ring, 0x00);
525 	OUT_RING(ring, 0x01);
526 	OUT_RING(ring, 0x01);
527 
528 	/* The WHERE_AMI_I packet is not needed after a YIELD is issued */
529 	a5xx_flush(gpu, ring, false);
530 
531 	return a5xx_idle(gpu, ring) ? 0 : -EINVAL;
532 }
533 
534 static void a5xx_ucode_check_version(struct a5xx_gpu *a5xx_gpu,
535 		struct drm_gem_object *obj)
536 {
537 	u32 *buf = msm_gem_get_vaddr(obj);
538 
539 	if (IS_ERR(buf))
540 		return;
541 
542 	/*
543 	 * If the lowest nibble is 0xa that is an indication that this microcode
544 	 * has been patched. The actual version is in dword [3] but we only care
545 	 * about the patchlevel which is the lowest nibble of dword [3]
546 	 */
547 	if (((buf[0] & 0xf) == 0xa) && (buf[2] & 0xf) >= 1)
548 		a5xx_gpu->has_whereami = true;
549 
550 	msm_gem_put_vaddr(obj);
551 }
552 
553 static int a5xx_ucode_init(struct msm_gpu *gpu)
554 {
555 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
556 	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
557 	int ret;
558 
559 	if (!a5xx_gpu->pm4_bo) {
560 		a5xx_gpu->pm4_bo = adreno_fw_create_bo(gpu,
561 			adreno_gpu->fw[ADRENO_FW_PM4], &a5xx_gpu->pm4_iova);
562 
563 
564 		if (IS_ERR(a5xx_gpu->pm4_bo)) {
565 			ret = PTR_ERR(a5xx_gpu->pm4_bo);
566 			a5xx_gpu->pm4_bo = NULL;
567 			DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PM4: %d\n",
568 				ret);
569 			return ret;
570 		}
571 
572 		msm_gem_object_set_name(a5xx_gpu->pm4_bo, "pm4fw");
573 	}
574 
575 	if (!a5xx_gpu->pfp_bo) {
576 		a5xx_gpu->pfp_bo = adreno_fw_create_bo(gpu,
577 			adreno_gpu->fw[ADRENO_FW_PFP], &a5xx_gpu->pfp_iova);
578 
579 		if (IS_ERR(a5xx_gpu->pfp_bo)) {
580 			ret = PTR_ERR(a5xx_gpu->pfp_bo);
581 			a5xx_gpu->pfp_bo = NULL;
582 			DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PFP: %d\n",
583 				ret);
584 			return ret;
585 		}
586 
587 		msm_gem_object_set_name(a5xx_gpu->pfp_bo, "pfpfw");
588 		a5xx_ucode_check_version(a5xx_gpu, a5xx_gpu->pfp_bo);
589 	}
590 
591 	gpu_write64(gpu, REG_A5XX_CP_ME_INSTR_BASE_LO,
592 		REG_A5XX_CP_ME_INSTR_BASE_HI, a5xx_gpu->pm4_iova);
593 
594 	gpu_write64(gpu, REG_A5XX_CP_PFP_INSTR_BASE_LO,
595 		REG_A5XX_CP_PFP_INSTR_BASE_HI, a5xx_gpu->pfp_iova);
596 
597 	return 0;
598 }
599 
600 #define SCM_GPU_ZAP_SHADER_RESUME 0
601 
602 static int a5xx_zap_shader_resume(struct msm_gpu *gpu)
603 {
604 	int ret;
605 
606 	ret = qcom_scm_set_remote_state(SCM_GPU_ZAP_SHADER_RESUME, GPU_PAS_ID);
607 	if (ret)
608 		DRM_ERROR("%s: zap-shader resume failed: %d\n",
609 			gpu->name, ret);
610 
611 	return ret;
612 }
613 
614 static int a5xx_zap_shader_init(struct msm_gpu *gpu)
615 {
616 	static bool loaded;
617 	int ret;
618 
619 	/*
620 	 * If the zap shader is already loaded into memory we just need to kick
621 	 * the remote processor to reinitialize it
622 	 */
623 	if (loaded)
624 		return a5xx_zap_shader_resume(gpu);
625 
626 	ret = adreno_zap_shader_load(gpu, GPU_PAS_ID);
627 
628 	loaded = !ret;
629 	return ret;
630 }
631 
632 #define A5XX_INT_MASK (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
633 	  A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
634 	  A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
635 	  A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
636 	  A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
637 	  A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW | \
638 	  A5XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
639 	  A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT | \
640 	  A5XX_RBBM_INT_0_MASK_CP_SW | \
641 	  A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
642 	  A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
643 	  A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
644 
645 static int a5xx_hw_init(struct msm_gpu *gpu)
646 {
647 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
648 	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
649 	u32 regbit;
650 	int ret;
651 
652 	gpu_write(gpu, REG_A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003);
653 
654 	if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu) ||
655 	    adreno_is_a540(adreno_gpu))
656 		gpu_write(gpu, REG_A5XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009);
657 
658 	/* Make all blocks contribute to the GPU BUSY perf counter */
659 	gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF);
660 
661 	/* Enable RBBM error reporting bits */
662 	gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL0, 0x00000001);
663 
664 	if (adreno_gpu->info->quirks & ADRENO_QUIRK_FAULT_DETECT_MASK) {
665 		/*
666 		 * Mask out the activity signals from RB1-3 to avoid false
667 		 * positives
668 		 */
669 
670 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11,
671 			0xF0000000);
672 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12,
673 			0xFFFFFFFF);
674 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13,
675 			0xFFFFFFFF);
676 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14,
677 			0xFFFFFFFF);
678 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15,
679 			0xFFFFFFFF);
680 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16,
681 			0xFFFFFFFF);
682 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17,
683 			0xFFFFFFFF);
684 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18,
685 			0xFFFFFFFF);
686 	}
687 
688 	/* Enable fault detection */
689 	gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_INT_CNTL,
690 		(1 << 30) | 0xFFFF);
691 
692 	/* Turn on performance counters */
693 	gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_CNTL, 0x01);
694 
695 	/* Select CP0 to always count cycles */
696 	gpu_write(gpu, REG_A5XX_CP_PERFCTR_CP_SEL_0, PERF_CP_ALWAYS_COUNT);
697 
698 	/* Select RBBM0 to countable 6 to get the busy status for devfreq */
699 	gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_SEL_0, 6);
700 
701 	/* Increase VFD cache access so LRZ and other data gets evicted less */
702 	gpu_write(gpu, REG_A5XX_UCHE_CACHE_WAYS, 0x02);
703 
704 	/* Disable L2 bypass in the UCHE */
705 	gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_LO, 0xFFFF0000);
706 	gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_HI, 0x0001FFFF);
707 	gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_LO, 0xFFFF0000);
708 	gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_HI, 0x0001FFFF);
709 
710 	/* Set the GMEM VA range (0 to gpu->gmem) */
711 	gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_LO, 0x00100000);
712 	gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_HI, 0x00000000);
713 	gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_LO,
714 		0x00100000 + adreno_gpu->gmem - 1);
715 	gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_HI, 0x00000000);
716 
717 	if (adreno_is_a508(adreno_gpu) || adreno_is_a510(adreno_gpu)) {
718 		gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x20);
719 		if (adreno_is_a508(adreno_gpu))
720 			gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400);
721 		else
722 			gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x20);
723 		gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x40000030);
724 		gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x20100D0A);
725 	} else {
726 		gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x40);
727 		if (adreno_is_a530(adreno_gpu))
728 			gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x40);
729 		else
730 			gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400);
731 		gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060);
732 		gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16);
733 	}
734 
735 	if (adreno_is_a508(adreno_gpu))
736 		gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL,
737 			  (0x100 << 11 | 0x100 << 22));
738 	else if (adreno_is_a509(adreno_gpu) || adreno_is_a510(adreno_gpu) ||
739 		 adreno_is_a512(adreno_gpu))
740 		gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL,
741 			  (0x200 << 11 | 0x200 << 22));
742 	else
743 		gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL,
744 			  (0x400 << 11 | 0x300 << 22));
745 
746 	if (adreno_gpu->info->quirks & ADRENO_QUIRK_TWO_PASS_USE_WFI)
747 		gpu_rmw(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8));
748 
749 	/*
750 	 * Disable the RB sampler datapath DP2 clock gating optimization
751 	 * for 1-SP GPUs, as it is enabled by default.
752 	 */
753 	if (adreno_is_a508(adreno_gpu) || adreno_is_a509(adreno_gpu) ||
754 	    adreno_is_a512(adreno_gpu))
755 		gpu_rmw(gpu, REG_A5XX_RB_DBG_ECO_CNTL, 0, (1 << 9));
756 
757 	/* Disable UCHE global filter as SP can invalidate/flush independently */
758 	gpu_write(gpu, REG_A5XX_UCHE_MODE_CNTL, BIT(29));
759 
760 	/* Enable USE_RETENTION_FLOPS */
761 	gpu_write(gpu, REG_A5XX_CP_CHICKEN_DBG, 0x02000000);
762 
763 	/* Enable ME/PFP split notification */
764 	gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL1, 0xA6FFFFFF);
765 
766 	/*
767 	 *  In A5x, CCU can send context_done event of a particular context to
768 	 *  UCHE which ultimately reaches CP even when there is valid
769 	 *  transaction of that context inside CCU. This can let CP to program
770 	 *  config registers, which will make the "valid transaction" inside
771 	 *  CCU to be interpreted differently. This can cause gpu fault. This
772 	 *  bug is fixed in latest A510 revision. To enable this bug fix -
773 	 *  bit[11] of RB_DBG_ECO_CNTL need to be set to 0, default is 1
774 	 *  (disable). For older A510 version this bit is unused.
775 	 */
776 	if (adreno_is_a510(adreno_gpu))
777 		gpu_rmw(gpu, REG_A5XX_RB_DBG_ECO_CNTL, (1 << 11), 0);
778 
779 	/* Enable HWCG */
780 	a5xx_set_hwcg(gpu, true);
781 
782 	gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL2, 0x0000003F);
783 
784 	/* Set the highest bank bit */
785 	if (adreno_is_a540(adreno_gpu))
786 		regbit = 2;
787 	else
788 		regbit = 1;
789 
790 	gpu_write(gpu, REG_A5XX_TPL1_MODE_CNTL, regbit << 7);
791 	gpu_write(gpu, REG_A5XX_RB_MODE_CNTL, regbit << 1);
792 
793 	if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu) ||
794 	    adreno_is_a540(adreno_gpu))
795 		gpu_write(gpu, REG_A5XX_UCHE_DBG_ECO_CNTL_2, regbit);
796 
797 	/* Disable All flat shading optimization (ALLFLATOPTDIS) */
798 	gpu_rmw(gpu, REG_A5XX_VPC_DBG_ECO_CNTL, 0, (1 << 10));
799 
800 	/* Protect registers from the CP */
801 	gpu_write(gpu, REG_A5XX_CP_PROTECT_CNTL, 0x00000007);
802 
803 	/* RBBM */
804 	gpu_write(gpu, REG_A5XX_CP_PROTECT(0), ADRENO_PROTECT_RW(0x04, 4));
805 	gpu_write(gpu, REG_A5XX_CP_PROTECT(1), ADRENO_PROTECT_RW(0x08, 8));
806 	gpu_write(gpu, REG_A5XX_CP_PROTECT(2), ADRENO_PROTECT_RW(0x10, 16));
807 	gpu_write(gpu, REG_A5XX_CP_PROTECT(3), ADRENO_PROTECT_RW(0x20, 32));
808 	gpu_write(gpu, REG_A5XX_CP_PROTECT(4), ADRENO_PROTECT_RW(0x40, 64));
809 	gpu_write(gpu, REG_A5XX_CP_PROTECT(5), ADRENO_PROTECT_RW(0x80, 64));
810 
811 	/* Content protect */
812 	gpu_write(gpu, REG_A5XX_CP_PROTECT(6),
813 		ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
814 			16));
815 	gpu_write(gpu, REG_A5XX_CP_PROTECT(7),
816 		ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TRUST_CNTL, 2));
817 
818 	/* CP */
819 	gpu_write(gpu, REG_A5XX_CP_PROTECT(8), ADRENO_PROTECT_RW(0x800, 64));
820 	gpu_write(gpu, REG_A5XX_CP_PROTECT(9), ADRENO_PROTECT_RW(0x840, 8));
821 	gpu_write(gpu, REG_A5XX_CP_PROTECT(10), ADRENO_PROTECT_RW(0x880, 32));
822 	gpu_write(gpu, REG_A5XX_CP_PROTECT(11), ADRENO_PROTECT_RW(0xAA0, 1));
823 
824 	/* RB */
825 	gpu_write(gpu, REG_A5XX_CP_PROTECT(12), ADRENO_PROTECT_RW(0xCC0, 1));
826 	gpu_write(gpu, REG_A5XX_CP_PROTECT(13), ADRENO_PROTECT_RW(0xCF0, 2));
827 
828 	/* VPC */
829 	gpu_write(gpu, REG_A5XX_CP_PROTECT(14), ADRENO_PROTECT_RW(0xE68, 8));
830 	gpu_write(gpu, REG_A5XX_CP_PROTECT(15), ADRENO_PROTECT_RW(0xE70, 16));
831 
832 	/* UCHE */
833 	gpu_write(gpu, REG_A5XX_CP_PROTECT(16), ADRENO_PROTECT_RW(0xE80, 16));
834 
835 	if (adreno_is_a508(adreno_gpu) || adreno_is_a509(adreno_gpu) ||
836 	    adreno_is_a510(adreno_gpu) || adreno_is_a512(adreno_gpu) ||
837 	    adreno_is_a530(adreno_gpu))
838 		gpu_write(gpu, REG_A5XX_CP_PROTECT(17),
839 			ADRENO_PROTECT_RW(0x10000, 0x8000));
840 
841 	gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_CNTL, 0);
842 	/*
843 	 * Disable the trusted memory range - we don't actually supported secure
844 	 * memory rendering at this point in time and we don't want to block off
845 	 * part of the virtual memory space.
846 	 */
847 	gpu_write64(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
848 		REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI, 0x00000000);
849 	gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
850 
851 	/* Put the GPU into 64 bit by default */
852 	gpu_write(gpu, REG_A5XX_CP_ADDR_MODE_CNTL, 0x1);
853 	gpu_write(gpu, REG_A5XX_VSC_ADDR_MODE_CNTL, 0x1);
854 	gpu_write(gpu, REG_A5XX_GRAS_ADDR_MODE_CNTL, 0x1);
855 	gpu_write(gpu, REG_A5XX_RB_ADDR_MODE_CNTL, 0x1);
856 	gpu_write(gpu, REG_A5XX_PC_ADDR_MODE_CNTL, 0x1);
857 	gpu_write(gpu, REG_A5XX_HLSQ_ADDR_MODE_CNTL, 0x1);
858 	gpu_write(gpu, REG_A5XX_VFD_ADDR_MODE_CNTL, 0x1);
859 	gpu_write(gpu, REG_A5XX_VPC_ADDR_MODE_CNTL, 0x1);
860 	gpu_write(gpu, REG_A5XX_UCHE_ADDR_MODE_CNTL, 0x1);
861 	gpu_write(gpu, REG_A5XX_SP_ADDR_MODE_CNTL, 0x1);
862 	gpu_write(gpu, REG_A5XX_TPL1_ADDR_MODE_CNTL, 0x1);
863 	gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1);
864 
865 	/*
866 	 * VPC corner case with local memory load kill leads to corrupt
867 	 * internal state. Normal Disable does not work for all a5x chips.
868 	 * So do the following setting to disable it.
869 	 */
870 	if (adreno_gpu->info->quirks & ADRENO_QUIRK_LMLOADKILL_DISABLE) {
871 		gpu_rmw(gpu, REG_A5XX_VPC_DBG_ECO_CNTL, 0, BIT(23));
872 		gpu_rmw(gpu, REG_A5XX_HLSQ_DBG_ECO_CNTL, BIT(18), 0);
873 	}
874 
875 	ret = adreno_hw_init(gpu);
876 	if (ret)
877 		return ret;
878 
879 	if (!(adreno_is_a508(adreno_gpu) || adreno_is_a509(adreno_gpu) ||
880 	      adreno_is_a510(adreno_gpu) || adreno_is_a512(adreno_gpu)))
881 		a5xx_gpmu_ucode_init(gpu);
882 
883 	ret = a5xx_ucode_init(gpu);
884 	if (ret)
885 		return ret;
886 
887 	/* Set the ringbuffer address */
888 	gpu_write64(gpu, REG_A5XX_CP_RB_BASE, REG_A5XX_CP_RB_BASE_HI,
889 		gpu->rb[0]->iova);
890 
891 	/*
892 	 * If the microcode supports the WHERE_AM_I opcode then we can use that
893 	 * in lieu of the RPTR shadow and enable preemption. Otherwise, we
894 	 * can't safely use the RPTR shadow or preemption. In either case, the
895 	 * RPTR shadow should be disabled in hardware.
896 	 */
897 	gpu_write(gpu, REG_A5XX_CP_RB_CNTL,
898 		MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE);
899 
900 	/* Create a privileged buffer for the RPTR shadow */
901 	if (a5xx_gpu->has_whereami) {
902 		if (!a5xx_gpu->shadow_bo) {
903 			a5xx_gpu->shadow = msm_gem_kernel_new(gpu->dev,
904 				sizeof(u32) * gpu->nr_rings,
905 				MSM_BO_UNCACHED | MSM_BO_MAP_PRIV,
906 				gpu->aspace, &a5xx_gpu->shadow_bo,
907 				&a5xx_gpu->shadow_iova);
908 
909 			if (IS_ERR(a5xx_gpu->shadow))
910 				return PTR_ERR(a5xx_gpu->shadow);
911 		}
912 
913 		gpu_write64(gpu, REG_A5XX_CP_RB_RPTR_ADDR,
914 			REG_A5XX_CP_RB_RPTR_ADDR_HI, shadowptr(a5xx_gpu, gpu->rb[0]));
915 	} else if (gpu->nr_rings > 1) {
916 		/* Disable preemption if WHERE_AM_I isn't available */
917 		a5xx_preempt_fini(gpu);
918 		gpu->nr_rings = 1;
919 	}
920 
921 	a5xx_preempt_hw_init(gpu);
922 
923 	/* Disable the interrupts through the initial bringup stage */
924 	gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK, A5XX_INT_MASK);
925 
926 	/* Clear ME_HALT to start the micro engine */
927 	gpu_write(gpu, REG_A5XX_CP_PFP_ME_CNTL, 0);
928 	ret = a5xx_me_init(gpu);
929 	if (ret)
930 		return ret;
931 
932 	ret = a5xx_power_init(gpu);
933 	if (ret)
934 		return ret;
935 
936 	/*
937 	 * Send a pipeline event stat to get misbehaving counters to start
938 	 * ticking correctly
939 	 */
940 	if (adreno_is_a530(adreno_gpu)) {
941 		OUT_PKT7(gpu->rb[0], CP_EVENT_WRITE, 1);
942 		OUT_RING(gpu->rb[0], CP_EVENT_WRITE_0_EVENT(STAT_EVENT));
943 
944 		a5xx_flush(gpu, gpu->rb[0], true);
945 		if (!a5xx_idle(gpu, gpu->rb[0]))
946 			return -EINVAL;
947 	}
948 
949 	/*
950 	 * If the chip that we are using does support loading one, then
951 	 * try to load a zap shader into the secure world. If successful
952 	 * we can use the CP to switch out of secure mode. If not then we
953 	 * have no resource but to try to switch ourselves out manually. If we
954 	 * guessed wrong then access to the RBBM_SECVID_TRUST_CNTL register will
955 	 * be blocked and a permissions violation will soon follow.
956 	 */
957 	ret = a5xx_zap_shader_init(gpu);
958 	if (!ret) {
959 		OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1);
960 		OUT_RING(gpu->rb[0], 0x00000000);
961 
962 		a5xx_flush(gpu, gpu->rb[0], true);
963 		if (!a5xx_idle(gpu, gpu->rb[0]))
964 			return -EINVAL;
965 	} else if (ret == -ENODEV) {
966 		/*
967 		 * This device does not use zap shader (but print a warning
968 		 * just in case someone got their dt wrong.. hopefully they
969 		 * have a debug UART to realize the error of their ways...
970 		 * if you mess this up you are about to crash horribly)
971 		 */
972 		dev_warn_once(gpu->dev->dev,
973 			"Zap shader not enabled - using SECVID_TRUST_CNTL instead\n");
974 		gpu_write(gpu, REG_A5XX_RBBM_SECVID_TRUST_CNTL, 0x0);
975 	} else {
976 		return ret;
977 	}
978 
979 	/* Last step - yield the ringbuffer */
980 	a5xx_preempt_start(gpu);
981 
982 	return 0;
983 }
984 
985 static void a5xx_recover(struct msm_gpu *gpu)
986 {
987 	int i;
988 
989 	adreno_dump_info(gpu);
990 
991 	for (i = 0; i < 8; i++) {
992 		printk("CP_SCRATCH_REG%d: %u\n", i,
993 			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(i)));
994 	}
995 
996 	if (hang_debug)
997 		a5xx_dump(gpu);
998 
999 	gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 1);
1000 	gpu_read(gpu, REG_A5XX_RBBM_SW_RESET_CMD);
1001 	gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 0);
1002 	adreno_recover(gpu);
1003 }
1004 
1005 static void a5xx_destroy(struct msm_gpu *gpu)
1006 {
1007 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1008 	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
1009 
1010 	DBG("%s", gpu->name);
1011 
1012 	a5xx_preempt_fini(gpu);
1013 
1014 	if (a5xx_gpu->pm4_bo) {
1015 		msm_gem_unpin_iova(a5xx_gpu->pm4_bo, gpu->aspace);
1016 		drm_gem_object_put(a5xx_gpu->pm4_bo);
1017 	}
1018 
1019 	if (a5xx_gpu->pfp_bo) {
1020 		msm_gem_unpin_iova(a5xx_gpu->pfp_bo, gpu->aspace);
1021 		drm_gem_object_put(a5xx_gpu->pfp_bo);
1022 	}
1023 
1024 	if (a5xx_gpu->gpmu_bo) {
1025 		msm_gem_unpin_iova(a5xx_gpu->gpmu_bo, gpu->aspace);
1026 		drm_gem_object_put(a5xx_gpu->gpmu_bo);
1027 	}
1028 
1029 	if (a5xx_gpu->shadow_bo) {
1030 		msm_gem_unpin_iova(a5xx_gpu->shadow_bo, gpu->aspace);
1031 		drm_gem_object_put(a5xx_gpu->shadow_bo);
1032 	}
1033 
1034 	adreno_gpu_cleanup(adreno_gpu);
1035 	kfree(a5xx_gpu);
1036 }
1037 
1038 static inline bool _a5xx_check_idle(struct msm_gpu *gpu)
1039 {
1040 	if (gpu_read(gpu, REG_A5XX_RBBM_STATUS) & ~A5XX_RBBM_STATUS_HI_BUSY)
1041 		return false;
1042 
1043 	/*
1044 	 * Nearly every abnormality ends up pausing the GPU and triggering a
1045 	 * fault so we can safely just watch for this one interrupt to fire
1046 	 */
1047 	return !(gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS) &
1048 		A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT);
1049 }
1050 
1051 bool a5xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
1052 {
1053 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1054 	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
1055 
1056 	if (ring != a5xx_gpu->cur_ring) {
1057 		WARN(1, "Tried to idle a non-current ringbuffer\n");
1058 		return false;
1059 	}
1060 
1061 	/* wait for CP to drain ringbuffer: */
1062 	if (!adreno_idle(gpu, ring))
1063 		return false;
1064 
1065 	if (spin_until(_a5xx_check_idle(gpu))) {
1066 		DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n",
1067 			gpu->name, __builtin_return_address(0),
1068 			gpu_read(gpu, REG_A5XX_RBBM_STATUS),
1069 			gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS),
1070 			gpu_read(gpu, REG_A5XX_CP_RB_RPTR),
1071 			gpu_read(gpu, REG_A5XX_CP_RB_WPTR));
1072 		return false;
1073 	}
1074 
1075 	return true;
1076 }
1077 
1078 static int a5xx_fault_handler(void *arg, unsigned long iova, int flags)
1079 {
1080 	struct msm_gpu *gpu = arg;
1081 	pr_warn_ratelimited("*** gpu fault: iova=%08lx, flags=%d (%u,%u,%u,%u)\n",
1082 			iova, flags,
1083 			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(4)),
1084 			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(5)),
1085 			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(6)),
1086 			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(7)));
1087 
1088 	return -EFAULT;
1089 }
1090 
1091 static void a5xx_cp_err_irq(struct msm_gpu *gpu)
1092 {
1093 	u32 status = gpu_read(gpu, REG_A5XX_CP_INTERRUPT_STATUS);
1094 
1095 	if (status & A5XX_CP_INT_CP_OPCODE_ERROR) {
1096 		u32 val;
1097 
1098 		gpu_write(gpu, REG_A5XX_CP_PFP_STAT_ADDR, 0);
1099 
1100 		/*
1101 		 * REG_A5XX_CP_PFP_STAT_DATA is indexed, and we want index 1 so
1102 		 * read it twice
1103 		 */
1104 
1105 		gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
1106 		val = gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
1107 
1108 		dev_err_ratelimited(gpu->dev->dev, "CP | opcode error | possible opcode=0x%8.8X\n",
1109 			val);
1110 	}
1111 
1112 	if (status & A5XX_CP_INT_CP_HW_FAULT_ERROR)
1113 		dev_err_ratelimited(gpu->dev->dev, "CP | HW fault | status=0x%8.8X\n",
1114 			gpu_read(gpu, REG_A5XX_CP_HW_FAULT));
1115 
1116 	if (status & A5XX_CP_INT_CP_DMA_ERROR)
1117 		dev_err_ratelimited(gpu->dev->dev, "CP | DMA error\n");
1118 
1119 	if (status & A5XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) {
1120 		u32 val = gpu_read(gpu, REG_A5XX_CP_PROTECT_STATUS);
1121 
1122 		dev_err_ratelimited(gpu->dev->dev,
1123 			"CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n",
1124 			val & (1 << 24) ? "WRITE" : "READ",
1125 			(val & 0xFFFFF) >> 2, val);
1126 	}
1127 
1128 	if (status & A5XX_CP_INT_CP_AHB_ERROR) {
1129 		u32 status = gpu_read(gpu, REG_A5XX_CP_AHB_FAULT);
1130 		const char *access[16] = { "reserved", "reserved",
1131 			"timestamp lo", "timestamp hi", "pfp read", "pfp write",
1132 			"", "", "me read", "me write", "", "", "crashdump read",
1133 			"crashdump write" };
1134 
1135 		dev_err_ratelimited(gpu->dev->dev,
1136 			"CP | AHB error | addr=%X access=%s error=%d | status=0x%8.8X\n",
1137 			status & 0xFFFFF, access[(status >> 24) & 0xF],
1138 			(status & (1 << 31)), status);
1139 	}
1140 }
1141 
1142 static void a5xx_rbbm_err_irq(struct msm_gpu *gpu, u32 status)
1143 {
1144 	if (status & A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR) {
1145 		u32 val = gpu_read(gpu, REG_A5XX_RBBM_AHB_ERROR_STATUS);
1146 
1147 		dev_err_ratelimited(gpu->dev->dev,
1148 			"RBBM | AHB bus error | %s | addr=0x%X | ports=0x%X:0x%X\n",
1149 			val & (1 << 28) ? "WRITE" : "READ",
1150 			(val & 0xFFFFF) >> 2, (val >> 20) & 0x3,
1151 			(val >> 24) & 0xF);
1152 
1153 		/* Clear the error */
1154 		gpu_write(gpu, REG_A5XX_RBBM_AHB_CMD, (1 << 4));
1155 
1156 		/* Clear the interrupt */
1157 		gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD,
1158 			A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR);
1159 	}
1160 
1161 	if (status & A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT)
1162 		dev_err_ratelimited(gpu->dev->dev, "RBBM | AHB transfer timeout\n");
1163 
1164 	if (status & A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT)
1165 		dev_err_ratelimited(gpu->dev->dev, "RBBM | ME master split | status=0x%X\n",
1166 			gpu_read(gpu, REG_A5XX_RBBM_AHB_ME_SPLIT_STATUS));
1167 
1168 	if (status & A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT)
1169 		dev_err_ratelimited(gpu->dev->dev, "RBBM | PFP master split | status=0x%X\n",
1170 			gpu_read(gpu, REG_A5XX_RBBM_AHB_PFP_SPLIT_STATUS));
1171 
1172 	if (status & A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT)
1173 		dev_err_ratelimited(gpu->dev->dev, "RBBM | ETS master split | status=0x%X\n",
1174 			gpu_read(gpu, REG_A5XX_RBBM_AHB_ETS_SPLIT_STATUS));
1175 
1176 	if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
1177 		dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB ASYNC overflow\n");
1178 
1179 	if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW)
1180 		dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB bus overflow\n");
1181 }
1182 
1183 static void a5xx_uche_err_irq(struct msm_gpu *gpu)
1184 {
1185 	uint64_t addr = (uint64_t) gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_HI);
1186 
1187 	addr |= gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_LO);
1188 
1189 	dev_err_ratelimited(gpu->dev->dev, "UCHE | Out of bounds access | addr=0x%llX\n",
1190 		addr);
1191 }
1192 
1193 static void a5xx_gpmu_err_irq(struct msm_gpu *gpu)
1194 {
1195 	dev_err_ratelimited(gpu->dev->dev, "GPMU | voltage droop\n");
1196 }
1197 
1198 static void a5xx_fault_detect_irq(struct msm_gpu *gpu)
1199 {
1200 	struct drm_device *dev = gpu->dev;
1201 	struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);
1202 
1203 	DRM_DEV_ERROR(dev->dev, "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n",
1204 		ring ? ring->id : -1, ring ? ring->seqno : 0,
1205 		gpu_read(gpu, REG_A5XX_RBBM_STATUS),
1206 		gpu_read(gpu, REG_A5XX_CP_RB_RPTR),
1207 		gpu_read(gpu, REG_A5XX_CP_RB_WPTR),
1208 		gpu_read64(gpu, REG_A5XX_CP_IB1_BASE, REG_A5XX_CP_IB1_BASE_HI),
1209 		gpu_read(gpu, REG_A5XX_CP_IB1_BUFSZ),
1210 		gpu_read64(gpu, REG_A5XX_CP_IB2_BASE, REG_A5XX_CP_IB2_BASE_HI),
1211 		gpu_read(gpu, REG_A5XX_CP_IB2_BUFSZ));
1212 
1213 	/* Turn off the hangcheck timer to keep it from bothering us */
1214 	del_timer(&gpu->hangcheck_timer);
1215 
1216 	kthread_queue_work(gpu->worker, &gpu->recover_work);
1217 }
1218 
1219 #define RBBM_ERROR_MASK \
1220 	(A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
1221 	A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
1222 	A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
1223 	A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
1224 	A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
1225 	A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
1226 
1227 static irqreturn_t a5xx_irq(struct msm_gpu *gpu)
1228 {
1229 	u32 status = gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS);
1230 
1231 	/*
1232 	 * Clear all the interrupts except RBBM_AHB_ERROR - if we clear it
1233 	 * before the source is cleared the interrupt will storm.
1234 	 */
1235 	gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD,
1236 		status & ~A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR);
1237 
1238 	/* Pass status to a5xx_rbbm_err_irq because we've already cleared it */
1239 	if (status & RBBM_ERROR_MASK)
1240 		a5xx_rbbm_err_irq(gpu, status);
1241 
1242 	if (status & A5XX_RBBM_INT_0_MASK_CP_HW_ERROR)
1243 		a5xx_cp_err_irq(gpu);
1244 
1245 	if (status & A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT)
1246 		a5xx_fault_detect_irq(gpu);
1247 
1248 	if (status & A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
1249 		a5xx_uche_err_irq(gpu);
1250 
1251 	if (status & A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
1252 		a5xx_gpmu_err_irq(gpu);
1253 
1254 	if (status & A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS) {
1255 		a5xx_preempt_trigger(gpu);
1256 		msm_gpu_retire(gpu);
1257 	}
1258 
1259 	if (status & A5XX_RBBM_INT_0_MASK_CP_SW)
1260 		a5xx_preempt_irq(gpu);
1261 
1262 	return IRQ_HANDLED;
1263 }
1264 
1265 static const u32 a5xx_registers[] = {
1266 	0x0000, 0x0002, 0x0004, 0x0020, 0x0022, 0x0026, 0x0029, 0x002B,
1267 	0x002E, 0x0035, 0x0038, 0x0042, 0x0044, 0x0044, 0x0047, 0x0095,
1268 	0x0097, 0x00BB, 0x03A0, 0x0464, 0x0469, 0x046F, 0x04D2, 0x04D3,
1269 	0x04E0, 0x0533, 0x0540, 0x0555, 0x0800, 0x081A, 0x081F, 0x0841,
1270 	0x0860, 0x0860, 0x0880, 0x08A0, 0x0B00, 0x0B12, 0x0B15, 0x0B28,
1271 	0x0B78, 0x0B7F, 0x0BB0, 0x0BBD, 0x0BC0, 0x0BC6, 0x0BD0, 0x0C53,
1272 	0x0C60, 0x0C61, 0x0C80, 0x0C82, 0x0C84, 0x0C85, 0x0C90, 0x0C98,
1273 	0x0CA0, 0x0CA0, 0x0CB0, 0x0CB2, 0x2180, 0x2185, 0x2580, 0x2585,
1274 	0x0CC1, 0x0CC1, 0x0CC4, 0x0CC7, 0x0CCC, 0x0CCC, 0x0CD0, 0x0CD8,
1275 	0x0CE0, 0x0CE5, 0x0CE8, 0x0CE8, 0x0CEC, 0x0CF1, 0x0CFB, 0x0D0E,
1276 	0x2100, 0x211E, 0x2140, 0x2145, 0x2500, 0x251E, 0x2540, 0x2545,
1277 	0x0D10, 0x0D17, 0x0D20, 0x0D23, 0x0D30, 0x0D30, 0x20C0, 0x20C0,
1278 	0x24C0, 0x24C0, 0x0E40, 0x0E43, 0x0E4A, 0x0E4A, 0x0E50, 0x0E57,
1279 	0x0E60, 0x0E7C, 0x0E80, 0x0E8E, 0x0E90, 0x0E96, 0x0EA0, 0x0EA8,
1280 	0x0EB0, 0x0EB2, 0xE140, 0xE147, 0xE150, 0xE187, 0xE1A0, 0xE1A9,
1281 	0xE1B0, 0xE1B6, 0xE1C0, 0xE1C7, 0xE1D0, 0xE1D1, 0xE200, 0xE201,
1282 	0xE210, 0xE21C, 0xE240, 0xE268, 0xE000, 0xE006, 0xE010, 0xE09A,
1283 	0xE0A0, 0xE0A4, 0xE0AA, 0xE0EB, 0xE100, 0xE105, 0xE380, 0xE38F,
1284 	0xE3B0, 0xE3B0, 0xE400, 0xE405, 0xE408, 0xE4E9, 0xE4F0, 0xE4F0,
1285 	0xE280, 0xE280, 0xE282, 0xE2A3, 0xE2A5, 0xE2C2, 0xE940, 0xE947,
1286 	0xE950, 0xE987, 0xE9A0, 0xE9A9, 0xE9B0, 0xE9B6, 0xE9C0, 0xE9C7,
1287 	0xE9D0, 0xE9D1, 0xEA00, 0xEA01, 0xEA10, 0xEA1C, 0xEA40, 0xEA68,
1288 	0xE800, 0xE806, 0xE810, 0xE89A, 0xE8A0, 0xE8A4, 0xE8AA, 0xE8EB,
1289 	0xE900, 0xE905, 0xEB80, 0xEB8F, 0xEBB0, 0xEBB0, 0xEC00, 0xEC05,
1290 	0xEC08, 0xECE9, 0xECF0, 0xECF0, 0xEA80, 0xEA80, 0xEA82, 0xEAA3,
1291 	0xEAA5, 0xEAC2, 0xA800, 0xA800, 0xA820, 0xA828, 0xA840, 0xA87D,
1292 	0XA880, 0xA88D, 0xA890, 0xA8A3, 0xA8D0, 0xA8D8, 0xA8E0, 0xA8F5,
1293 	0xAC60, 0xAC60, ~0,
1294 };
1295 
1296 static void a5xx_dump(struct msm_gpu *gpu)
1297 {
1298 	DRM_DEV_INFO(gpu->dev->dev, "status:   %08x\n",
1299 		gpu_read(gpu, REG_A5XX_RBBM_STATUS));
1300 	adreno_dump(gpu);
1301 }
1302 
1303 static int a5xx_pm_resume(struct msm_gpu *gpu)
1304 {
1305 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1306 	int ret;
1307 
1308 	/* Turn on the core power */
1309 	ret = msm_gpu_pm_resume(gpu);
1310 	if (ret)
1311 		return ret;
1312 
1313 	/* Adreno 508, 509, 510, 512 needs manual RBBM sus/res control */
1314 	if (!(adreno_is_a530(adreno_gpu) || adreno_is_a540(adreno_gpu))) {
1315 		/* Halt the sp_input_clk at HM level */
1316 		gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0x00000055);
1317 		a5xx_set_hwcg(gpu, true);
1318 		/* Turn on sp_input_clk at HM level */
1319 		gpu_rmw(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0xff, 0);
1320 		return 0;
1321 	}
1322 
1323 	/* Turn the RBCCU domain first to limit the chances of voltage droop */
1324 	gpu_write(gpu, REG_A5XX_GPMU_RBCCU_POWER_CNTL, 0x778000);
1325 
1326 	/* Wait 3 usecs before polling */
1327 	udelay(3);
1328 
1329 	ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS,
1330 		(1 << 20), (1 << 20));
1331 	if (ret) {
1332 		DRM_ERROR("%s: timeout waiting for RBCCU GDSC enable: %X\n",
1333 			gpu->name,
1334 			gpu_read(gpu, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS));
1335 		return ret;
1336 	}
1337 
1338 	/* Turn on the SP domain */
1339 	gpu_write(gpu, REG_A5XX_GPMU_SP_POWER_CNTL, 0x778000);
1340 	ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_SP_PWR_CLK_STATUS,
1341 		(1 << 20), (1 << 20));
1342 	if (ret)
1343 		DRM_ERROR("%s: timeout waiting for SP GDSC enable\n",
1344 			gpu->name);
1345 
1346 	return ret;
1347 }
1348 
1349 static int a5xx_pm_suspend(struct msm_gpu *gpu)
1350 {
1351 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1352 	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
1353 	u32 mask = 0xf;
1354 	int i, ret;
1355 
1356 	/* A508, A510 have 3 XIN ports in VBIF */
1357 	if (adreno_is_a508(adreno_gpu) || adreno_is_a510(adreno_gpu))
1358 		mask = 0x7;
1359 
1360 	/* Clear the VBIF pipe before shutting down */
1361 	gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, mask);
1362 	spin_until((gpu_read(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL1) &
1363 				mask) == mask);
1364 
1365 	gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0);
1366 
1367 	/*
1368 	 * Reset the VBIF before power collapse to avoid issue with FIFO
1369 	 * entries on Adreno A510 and A530 (the others will tend to lock up)
1370 	 */
1371 	if (adreno_is_a510(adreno_gpu) || adreno_is_a530(adreno_gpu)) {
1372 		gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x003C0000);
1373 		gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x00000000);
1374 	}
1375 
1376 	ret = msm_gpu_pm_suspend(gpu);
1377 	if (ret)
1378 		return ret;
1379 
1380 	if (a5xx_gpu->has_whereami)
1381 		for (i = 0; i < gpu->nr_rings; i++)
1382 			a5xx_gpu->shadow[i] = 0;
1383 
1384 	return 0;
1385 }
1386 
1387 static int a5xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
1388 {
1389 	*value = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_CP_0_LO,
1390 		REG_A5XX_RBBM_PERFCTR_CP_0_HI);
1391 
1392 	return 0;
1393 }
1394 
1395 struct a5xx_crashdumper {
1396 	void *ptr;
1397 	struct drm_gem_object *bo;
1398 	u64 iova;
1399 };
1400 
1401 struct a5xx_gpu_state {
1402 	struct msm_gpu_state base;
1403 	u32 *hlsqregs;
1404 };
1405 
1406 static int a5xx_crashdumper_init(struct msm_gpu *gpu,
1407 		struct a5xx_crashdumper *dumper)
1408 {
1409 	dumper->ptr = msm_gem_kernel_new_locked(gpu->dev,
1410 		SZ_1M, MSM_BO_UNCACHED, gpu->aspace,
1411 		&dumper->bo, &dumper->iova);
1412 
1413 	if (!IS_ERR(dumper->ptr))
1414 		msm_gem_object_set_name(dumper->bo, "crashdump");
1415 
1416 	return PTR_ERR_OR_ZERO(dumper->ptr);
1417 }
1418 
1419 static int a5xx_crashdumper_run(struct msm_gpu *gpu,
1420 		struct a5xx_crashdumper *dumper)
1421 {
1422 	u32 val;
1423 
1424 	if (IS_ERR_OR_NULL(dumper->ptr))
1425 		return -EINVAL;
1426 
1427 	gpu_write64(gpu, REG_A5XX_CP_CRASH_SCRIPT_BASE_LO,
1428 		REG_A5XX_CP_CRASH_SCRIPT_BASE_HI, dumper->iova);
1429 
1430 	gpu_write(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, 1);
1431 
1432 	return gpu_poll_timeout(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, val,
1433 		val & 0x04, 100, 10000);
1434 }
1435 
1436 /*
1437  * These are a list of the registers that need to be read through the HLSQ
1438  * aperture through the crashdumper.  These are not nominally accessible from
1439  * the CPU on a secure platform.
1440  */
1441 static const struct {
1442 	u32 type;
1443 	u32 regoffset;
1444 	u32 count;
1445 } a5xx_hlsq_aperture_regs[] = {
1446 	{ 0x35, 0xe00, 0x32 },   /* HSLQ non-context */
1447 	{ 0x31, 0x2080, 0x1 },   /* HLSQ 2D context 0 */
1448 	{ 0x33, 0x2480, 0x1 },   /* HLSQ 2D context 1 */
1449 	{ 0x32, 0xe780, 0x62 },  /* HLSQ 3D context 0 */
1450 	{ 0x34, 0xef80, 0x62 },  /* HLSQ 3D context 1 */
1451 	{ 0x3f, 0x0ec0, 0x40 },  /* SP non-context */
1452 	{ 0x3d, 0x2040, 0x1 },   /* SP 2D context 0 */
1453 	{ 0x3b, 0x2440, 0x1 },   /* SP 2D context 1 */
1454 	{ 0x3e, 0xe580, 0x170 }, /* SP 3D context 0 */
1455 	{ 0x3c, 0xed80, 0x170 }, /* SP 3D context 1 */
1456 	{ 0x3a, 0x0f00, 0x1c },  /* TP non-context */
1457 	{ 0x38, 0x2000, 0xa },   /* TP 2D context 0 */
1458 	{ 0x36, 0x2400, 0xa },   /* TP 2D context 1 */
1459 	{ 0x39, 0xe700, 0x80 },  /* TP 3D context 0 */
1460 	{ 0x37, 0xef00, 0x80 },  /* TP 3D context 1 */
1461 };
1462 
1463 static void a5xx_gpu_state_get_hlsq_regs(struct msm_gpu *gpu,
1464 		struct a5xx_gpu_state *a5xx_state)
1465 {
1466 	struct a5xx_crashdumper dumper = { 0 };
1467 	u32 offset, count = 0;
1468 	u64 *ptr;
1469 	int i;
1470 
1471 	if (a5xx_crashdumper_init(gpu, &dumper))
1472 		return;
1473 
1474 	/* The script will be written at offset 0 */
1475 	ptr = dumper.ptr;
1476 
1477 	/* Start writing the data at offset 256k */
1478 	offset = dumper.iova + (256 * SZ_1K);
1479 
1480 	/* Count how many additional registers to get from the HLSQ aperture */
1481 	for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++)
1482 		count += a5xx_hlsq_aperture_regs[i].count;
1483 
1484 	a5xx_state->hlsqregs = kcalloc(count, sizeof(u32), GFP_KERNEL);
1485 	if (!a5xx_state->hlsqregs)
1486 		return;
1487 
1488 	/* Build the crashdump script */
1489 	for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) {
1490 		u32 type = a5xx_hlsq_aperture_regs[i].type;
1491 		u32 c = a5xx_hlsq_aperture_regs[i].count;
1492 
1493 		/* Write the register to select the desired bank */
1494 		*ptr++ = ((u64) type << 8);
1495 		*ptr++ = (((u64) REG_A5XX_HLSQ_DBG_READ_SEL) << 44) |
1496 			(1 << 21) | 1;
1497 
1498 		*ptr++ = offset;
1499 		*ptr++ = (((u64) REG_A5XX_HLSQ_DBG_AHB_READ_APERTURE) << 44)
1500 			| c;
1501 
1502 		offset += c * sizeof(u32);
1503 	}
1504 
1505 	/* Write two zeros to close off the script */
1506 	*ptr++ = 0;
1507 	*ptr++ = 0;
1508 
1509 	if (a5xx_crashdumper_run(gpu, &dumper)) {
1510 		kfree(a5xx_state->hlsqregs);
1511 		msm_gem_kernel_put(dumper.bo, gpu->aspace, true);
1512 		return;
1513 	}
1514 
1515 	/* Copy the data from the crashdumper to the state */
1516 	memcpy(a5xx_state->hlsqregs, dumper.ptr + (256 * SZ_1K),
1517 		count * sizeof(u32));
1518 
1519 	msm_gem_kernel_put(dumper.bo, gpu->aspace, true);
1520 }
1521 
1522 static struct msm_gpu_state *a5xx_gpu_state_get(struct msm_gpu *gpu)
1523 {
1524 	struct a5xx_gpu_state *a5xx_state = kzalloc(sizeof(*a5xx_state),
1525 			GFP_KERNEL);
1526 
1527 	if (!a5xx_state)
1528 		return ERR_PTR(-ENOMEM);
1529 
1530 	/* Temporarily disable hardware clock gating before reading the hw */
1531 	a5xx_set_hwcg(gpu, false);
1532 
1533 	/* First get the generic state from the adreno core */
1534 	adreno_gpu_state_get(gpu, &(a5xx_state->base));
1535 
1536 	a5xx_state->base.rbbm_status = gpu_read(gpu, REG_A5XX_RBBM_STATUS);
1537 
1538 	/* Get the HLSQ regs with the help of the crashdumper */
1539 	a5xx_gpu_state_get_hlsq_regs(gpu, a5xx_state);
1540 
1541 	a5xx_set_hwcg(gpu, true);
1542 
1543 	return &a5xx_state->base;
1544 }
1545 
1546 static void a5xx_gpu_state_destroy(struct kref *kref)
1547 {
1548 	struct msm_gpu_state *state = container_of(kref,
1549 		struct msm_gpu_state, ref);
1550 	struct a5xx_gpu_state *a5xx_state = container_of(state,
1551 		struct a5xx_gpu_state, base);
1552 
1553 	kfree(a5xx_state->hlsqregs);
1554 
1555 	adreno_gpu_state_destroy(state);
1556 	kfree(a5xx_state);
1557 }
1558 
1559 static int a5xx_gpu_state_put(struct msm_gpu_state *state)
1560 {
1561 	if (IS_ERR_OR_NULL(state))
1562 		return 1;
1563 
1564 	return kref_put(&state->ref, a5xx_gpu_state_destroy);
1565 }
1566 
1567 
1568 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
1569 static void a5xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state,
1570 		      struct drm_printer *p)
1571 {
1572 	int i, j;
1573 	u32 pos = 0;
1574 	struct a5xx_gpu_state *a5xx_state = container_of(state,
1575 		struct a5xx_gpu_state, base);
1576 
1577 	if (IS_ERR_OR_NULL(state))
1578 		return;
1579 
1580 	adreno_show(gpu, state, p);
1581 
1582 	/* Dump the additional a5xx HLSQ registers */
1583 	if (!a5xx_state->hlsqregs)
1584 		return;
1585 
1586 	drm_printf(p, "registers-hlsq:\n");
1587 
1588 	for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) {
1589 		u32 o = a5xx_hlsq_aperture_regs[i].regoffset;
1590 		u32 c = a5xx_hlsq_aperture_regs[i].count;
1591 
1592 		for (j = 0; j < c; j++, pos++, o++) {
1593 			/*
1594 			 * To keep the crashdump simple we pull the entire range
1595 			 * for each register type but not all of the registers
1596 			 * in the range are valid. Fortunately invalid registers
1597 			 * stick out like a sore thumb with a value of
1598 			 * 0xdeadbeef
1599 			 */
1600 			if (a5xx_state->hlsqregs[pos] == 0xdeadbeef)
1601 				continue;
1602 
1603 			drm_printf(p, "  - { offset: 0x%04x, value: 0x%08x }\n",
1604 				o << 2, a5xx_state->hlsqregs[pos]);
1605 		}
1606 	}
1607 }
1608 #endif
1609 
1610 static struct msm_ringbuffer *a5xx_active_ring(struct msm_gpu *gpu)
1611 {
1612 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1613 	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
1614 
1615 	return a5xx_gpu->cur_ring;
1616 }
1617 
1618 static unsigned long a5xx_gpu_busy(struct msm_gpu *gpu)
1619 {
1620 	u64 busy_cycles, busy_time;
1621 
1622 	/* Only read the gpu busy if the hardware is already active */
1623 	if (pm_runtime_get_if_in_use(&gpu->pdev->dev) == 0)
1624 		return 0;
1625 
1626 	busy_cycles = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_0_LO,
1627 			REG_A5XX_RBBM_PERFCTR_RBBM_0_HI);
1628 
1629 	busy_time = busy_cycles - gpu->devfreq.busy_cycles;
1630 	do_div(busy_time, clk_get_rate(gpu->core_clk) / 1000000);
1631 
1632 	gpu->devfreq.busy_cycles = busy_cycles;
1633 
1634 	pm_runtime_put(&gpu->pdev->dev);
1635 
1636 	if (WARN_ON(busy_time > ~0LU))
1637 		return ~0LU;
1638 
1639 	return (unsigned long)busy_time;
1640 }
1641 
1642 static uint32_t a5xx_get_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
1643 {
1644 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1645 	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
1646 
1647 	if (a5xx_gpu->has_whereami)
1648 		return a5xx_gpu->shadow[ring->id];
1649 
1650 	return ring->memptrs->rptr = gpu_read(gpu, REG_A5XX_CP_RB_RPTR);
1651 }
1652 
1653 static const struct adreno_gpu_funcs funcs = {
1654 	.base = {
1655 		.get_param = adreno_get_param,
1656 		.hw_init = a5xx_hw_init,
1657 		.pm_suspend = a5xx_pm_suspend,
1658 		.pm_resume = a5xx_pm_resume,
1659 		.recover = a5xx_recover,
1660 		.submit = a5xx_submit,
1661 		.active_ring = a5xx_active_ring,
1662 		.irq = a5xx_irq,
1663 		.destroy = a5xx_destroy,
1664 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
1665 		.show = a5xx_show,
1666 #endif
1667 #if defined(CONFIG_DEBUG_FS)
1668 		.debugfs_init = a5xx_debugfs_init,
1669 #endif
1670 		.gpu_busy = a5xx_gpu_busy,
1671 		.gpu_state_get = a5xx_gpu_state_get,
1672 		.gpu_state_put = a5xx_gpu_state_put,
1673 		.create_address_space = adreno_iommu_create_address_space,
1674 		.get_rptr = a5xx_get_rptr,
1675 	},
1676 	.get_timestamp = a5xx_get_timestamp,
1677 };
1678 
1679 static void check_speed_bin(struct device *dev)
1680 {
1681 	struct nvmem_cell *cell;
1682 	u32 val;
1683 
1684 	/*
1685 	 * If the OPP table specifies a opp-supported-hw property then we have
1686 	 * to set something with dev_pm_opp_set_supported_hw() or the table
1687 	 * doesn't get populated so pick an arbitrary value that should
1688 	 * ensure the default frequencies are selected but not conflict with any
1689 	 * actual bins
1690 	 */
1691 	val = 0x80;
1692 
1693 	cell = nvmem_cell_get(dev, "speed_bin");
1694 
1695 	if (!IS_ERR(cell)) {
1696 		void *buf = nvmem_cell_read(cell, NULL);
1697 
1698 		if (!IS_ERR(buf)) {
1699 			u8 bin = *((u8 *) buf);
1700 
1701 			val = (1 << bin);
1702 			kfree(buf);
1703 		}
1704 
1705 		nvmem_cell_put(cell);
1706 	}
1707 
1708 	dev_pm_opp_set_supported_hw(dev, &val, 1);
1709 }
1710 
1711 struct msm_gpu *a5xx_gpu_init(struct drm_device *dev)
1712 {
1713 	struct msm_drm_private *priv = dev->dev_private;
1714 	struct platform_device *pdev = priv->gpu_pdev;
1715 	struct a5xx_gpu *a5xx_gpu = NULL;
1716 	struct adreno_gpu *adreno_gpu;
1717 	struct msm_gpu *gpu;
1718 	int ret;
1719 
1720 	if (!pdev) {
1721 		DRM_DEV_ERROR(dev->dev, "No A5XX device is defined\n");
1722 		return ERR_PTR(-ENXIO);
1723 	}
1724 
1725 	a5xx_gpu = kzalloc(sizeof(*a5xx_gpu), GFP_KERNEL);
1726 	if (!a5xx_gpu)
1727 		return ERR_PTR(-ENOMEM);
1728 
1729 	adreno_gpu = &a5xx_gpu->base;
1730 	gpu = &adreno_gpu->base;
1731 
1732 	adreno_gpu->registers = a5xx_registers;
1733 
1734 	a5xx_gpu->lm_leakage = 0x4E001A;
1735 
1736 	check_speed_bin(&pdev->dev);
1737 
1738 	ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 4);
1739 	if (ret) {
1740 		a5xx_destroy(&(a5xx_gpu->base.base));
1741 		return ERR_PTR(ret);
1742 	}
1743 
1744 	if (gpu->aspace)
1745 		msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu, a5xx_fault_handler);
1746 
1747 	/* Set up the preemption specific bits and pieces for each ringbuffer */
1748 	a5xx_preempt_init(gpu);
1749 
1750 	return gpu;
1751 }
1752