xref: /openbmc/linux/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c (revision ae40e94f)
1 /*
2  * Copyright 2014 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 #include <linux/kernel.h>
24 #include <linux/firmware.h>
25 #include <drm/drmP.h>
26 #include "amdgpu.h"
27 #include "amdgpu_gfx.h"
28 #include "vi.h"
29 #include "vi_structs.h"
30 #include "vid.h"
31 #include "amdgpu_ucode.h"
32 #include "amdgpu_atombios.h"
33 #include "atombios_i2c.h"
34 #include "clearstate_vi.h"
35 
36 #include "gmc/gmc_8_2_d.h"
37 #include "gmc/gmc_8_2_sh_mask.h"
38 
39 #include "oss/oss_3_0_d.h"
40 #include "oss/oss_3_0_sh_mask.h"
41 
42 #include "bif/bif_5_0_d.h"
43 #include "bif/bif_5_0_sh_mask.h"
44 #include "gca/gfx_8_0_d.h"
45 #include "gca/gfx_8_0_enum.h"
46 #include "gca/gfx_8_0_sh_mask.h"
47 
48 #include "dce/dce_10_0_d.h"
49 #include "dce/dce_10_0_sh_mask.h"
50 
51 #include "smu/smu_7_1_3_d.h"
52 
53 #include "ivsrcid/ivsrcid_vislands30.h"
54 
55 #define GFX8_NUM_GFX_RINGS     1
56 #define GFX8_MEC_HPD_SIZE 4096
57 
58 #define TOPAZ_GB_ADDR_CONFIG_GOLDEN 0x22010001
59 #define CARRIZO_GB_ADDR_CONFIG_GOLDEN 0x22010001
60 #define POLARIS11_GB_ADDR_CONFIG_GOLDEN 0x22011002
61 #define TONGA_GB_ADDR_CONFIG_GOLDEN 0x22011003
62 
63 #define ARRAY_MODE(x)					((x) << GB_TILE_MODE0__ARRAY_MODE__SHIFT)
64 #define PIPE_CONFIG(x)					((x) << GB_TILE_MODE0__PIPE_CONFIG__SHIFT)
65 #define TILE_SPLIT(x)					((x) << GB_TILE_MODE0__TILE_SPLIT__SHIFT)
66 #define MICRO_TILE_MODE_NEW(x)				((x) << GB_TILE_MODE0__MICRO_TILE_MODE_NEW__SHIFT)
67 #define SAMPLE_SPLIT(x)					((x) << GB_TILE_MODE0__SAMPLE_SPLIT__SHIFT)
68 #define BANK_WIDTH(x)					((x) << GB_MACROTILE_MODE0__BANK_WIDTH__SHIFT)
69 #define BANK_HEIGHT(x)					((x) << GB_MACROTILE_MODE0__BANK_HEIGHT__SHIFT)
70 #define MACRO_TILE_ASPECT(x)				((x) << GB_MACROTILE_MODE0__MACRO_TILE_ASPECT__SHIFT)
71 #define NUM_BANKS(x)					((x) << GB_MACROTILE_MODE0__NUM_BANKS__SHIFT)
72 
73 #define RLC_CGTT_MGCG_OVERRIDE__CPF_MASK            0x00000001L
74 #define RLC_CGTT_MGCG_OVERRIDE__RLC_MASK            0x00000002L
75 #define RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK           0x00000004L
76 #define RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK           0x00000008L
77 #define RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK           0x00000010L
78 #define RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK           0x00000020L
79 
80 /* BPM SERDES CMD */
81 #define SET_BPM_SERDES_CMD    1
82 #define CLE_BPM_SERDES_CMD    0
83 
84 /* BPM Register Address*/
85 enum {
86 	BPM_REG_CGLS_EN = 0,        /* Enable/Disable CGLS */
87 	BPM_REG_CGLS_ON,            /* ON/OFF CGLS: shall be controlled by RLC FW */
88 	BPM_REG_CGCG_OVERRIDE,      /* Set/Clear CGCG Override */
89 	BPM_REG_MGCG_OVERRIDE,      /* Set/Clear MGCG Override */
90 	BPM_REG_FGCG_OVERRIDE,      /* Set/Clear FGCG Override */
91 	BPM_REG_FGCG_MAX
92 };
93 
94 #define RLC_FormatDirectRegListLength        14
95 
96 MODULE_FIRMWARE("amdgpu/carrizo_ce.bin");
97 MODULE_FIRMWARE("amdgpu/carrizo_pfp.bin");
98 MODULE_FIRMWARE("amdgpu/carrizo_me.bin");
99 MODULE_FIRMWARE("amdgpu/carrizo_mec.bin");
100 MODULE_FIRMWARE("amdgpu/carrizo_mec2.bin");
101 MODULE_FIRMWARE("amdgpu/carrizo_rlc.bin");
102 
103 MODULE_FIRMWARE("amdgpu/stoney_ce.bin");
104 MODULE_FIRMWARE("amdgpu/stoney_pfp.bin");
105 MODULE_FIRMWARE("amdgpu/stoney_me.bin");
106 MODULE_FIRMWARE("amdgpu/stoney_mec.bin");
107 MODULE_FIRMWARE("amdgpu/stoney_rlc.bin");
108 
109 MODULE_FIRMWARE("amdgpu/tonga_ce.bin");
110 MODULE_FIRMWARE("amdgpu/tonga_pfp.bin");
111 MODULE_FIRMWARE("amdgpu/tonga_me.bin");
112 MODULE_FIRMWARE("amdgpu/tonga_mec.bin");
113 MODULE_FIRMWARE("amdgpu/tonga_mec2.bin");
114 MODULE_FIRMWARE("amdgpu/tonga_rlc.bin");
115 
116 MODULE_FIRMWARE("amdgpu/topaz_ce.bin");
117 MODULE_FIRMWARE("amdgpu/topaz_pfp.bin");
118 MODULE_FIRMWARE("amdgpu/topaz_me.bin");
119 MODULE_FIRMWARE("amdgpu/topaz_mec.bin");
120 MODULE_FIRMWARE("amdgpu/topaz_rlc.bin");
121 
122 MODULE_FIRMWARE("amdgpu/fiji_ce.bin");
123 MODULE_FIRMWARE("amdgpu/fiji_pfp.bin");
124 MODULE_FIRMWARE("amdgpu/fiji_me.bin");
125 MODULE_FIRMWARE("amdgpu/fiji_mec.bin");
126 MODULE_FIRMWARE("amdgpu/fiji_mec2.bin");
127 MODULE_FIRMWARE("amdgpu/fiji_rlc.bin");
128 
129 MODULE_FIRMWARE("amdgpu/polaris10_ce.bin");
130 MODULE_FIRMWARE("amdgpu/polaris10_ce_2.bin");
131 MODULE_FIRMWARE("amdgpu/polaris10_pfp.bin");
132 MODULE_FIRMWARE("amdgpu/polaris10_pfp_2.bin");
133 MODULE_FIRMWARE("amdgpu/polaris10_me.bin");
134 MODULE_FIRMWARE("amdgpu/polaris10_me_2.bin");
135 MODULE_FIRMWARE("amdgpu/polaris10_mec.bin");
136 MODULE_FIRMWARE("amdgpu/polaris10_mec_2.bin");
137 MODULE_FIRMWARE("amdgpu/polaris10_mec2.bin");
138 MODULE_FIRMWARE("amdgpu/polaris10_mec2_2.bin");
139 MODULE_FIRMWARE("amdgpu/polaris10_rlc.bin");
140 
141 MODULE_FIRMWARE("amdgpu/polaris11_ce.bin");
142 MODULE_FIRMWARE("amdgpu/polaris11_ce_2.bin");
143 MODULE_FIRMWARE("amdgpu/polaris11_pfp.bin");
144 MODULE_FIRMWARE("amdgpu/polaris11_pfp_2.bin");
145 MODULE_FIRMWARE("amdgpu/polaris11_me.bin");
146 MODULE_FIRMWARE("amdgpu/polaris11_me_2.bin");
147 MODULE_FIRMWARE("amdgpu/polaris11_mec.bin");
148 MODULE_FIRMWARE("amdgpu/polaris11_mec_2.bin");
149 MODULE_FIRMWARE("amdgpu/polaris11_mec2.bin");
150 MODULE_FIRMWARE("amdgpu/polaris11_mec2_2.bin");
151 MODULE_FIRMWARE("amdgpu/polaris11_rlc.bin");
152 
153 MODULE_FIRMWARE("amdgpu/polaris12_ce.bin");
154 MODULE_FIRMWARE("amdgpu/polaris12_ce_2.bin");
155 MODULE_FIRMWARE("amdgpu/polaris12_pfp.bin");
156 MODULE_FIRMWARE("amdgpu/polaris12_pfp_2.bin");
157 MODULE_FIRMWARE("amdgpu/polaris12_me.bin");
158 MODULE_FIRMWARE("amdgpu/polaris12_me_2.bin");
159 MODULE_FIRMWARE("amdgpu/polaris12_mec.bin");
160 MODULE_FIRMWARE("amdgpu/polaris12_mec_2.bin");
161 MODULE_FIRMWARE("amdgpu/polaris12_mec2.bin");
162 MODULE_FIRMWARE("amdgpu/polaris12_mec2_2.bin");
163 MODULE_FIRMWARE("amdgpu/polaris12_rlc.bin");
164 
165 MODULE_FIRMWARE("amdgpu/vegam_ce.bin");
166 MODULE_FIRMWARE("amdgpu/vegam_pfp.bin");
167 MODULE_FIRMWARE("amdgpu/vegam_me.bin");
168 MODULE_FIRMWARE("amdgpu/vegam_mec.bin");
169 MODULE_FIRMWARE("amdgpu/vegam_mec2.bin");
170 MODULE_FIRMWARE("amdgpu/vegam_rlc.bin");
171 
172 static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] =
173 {
174 	{mmGDS_VMID0_BASE, mmGDS_VMID0_SIZE, mmGDS_GWS_VMID0, mmGDS_OA_VMID0},
175 	{mmGDS_VMID1_BASE, mmGDS_VMID1_SIZE, mmGDS_GWS_VMID1, mmGDS_OA_VMID1},
176 	{mmGDS_VMID2_BASE, mmGDS_VMID2_SIZE, mmGDS_GWS_VMID2, mmGDS_OA_VMID2},
177 	{mmGDS_VMID3_BASE, mmGDS_VMID3_SIZE, mmGDS_GWS_VMID3, mmGDS_OA_VMID3},
178 	{mmGDS_VMID4_BASE, mmGDS_VMID4_SIZE, mmGDS_GWS_VMID4, mmGDS_OA_VMID4},
179 	{mmGDS_VMID5_BASE, mmGDS_VMID5_SIZE, mmGDS_GWS_VMID5, mmGDS_OA_VMID5},
180 	{mmGDS_VMID6_BASE, mmGDS_VMID6_SIZE, mmGDS_GWS_VMID6, mmGDS_OA_VMID6},
181 	{mmGDS_VMID7_BASE, mmGDS_VMID7_SIZE, mmGDS_GWS_VMID7, mmGDS_OA_VMID7},
182 	{mmGDS_VMID8_BASE, mmGDS_VMID8_SIZE, mmGDS_GWS_VMID8, mmGDS_OA_VMID8},
183 	{mmGDS_VMID9_BASE, mmGDS_VMID9_SIZE, mmGDS_GWS_VMID9, mmGDS_OA_VMID9},
184 	{mmGDS_VMID10_BASE, mmGDS_VMID10_SIZE, mmGDS_GWS_VMID10, mmGDS_OA_VMID10},
185 	{mmGDS_VMID11_BASE, mmGDS_VMID11_SIZE, mmGDS_GWS_VMID11, mmGDS_OA_VMID11},
186 	{mmGDS_VMID12_BASE, mmGDS_VMID12_SIZE, mmGDS_GWS_VMID12, mmGDS_OA_VMID12},
187 	{mmGDS_VMID13_BASE, mmGDS_VMID13_SIZE, mmGDS_GWS_VMID13, mmGDS_OA_VMID13},
188 	{mmGDS_VMID14_BASE, mmGDS_VMID14_SIZE, mmGDS_GWS_VMID14, mmGDS_OA_VMID14},
189 	{mmGDS_VMID15_BASE, mmGDS_VMID15_SIZE, mmGDS_GWS_VMID15, mmGDS_OA_VMID15}
190 };
191 
192 static const u32 golden_settings_tonga_a11[] =
193 {
194 	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
195 	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
196 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
197 	mmGB_GPU_ID, 0x0000000f, 0x00000000,
198 	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
199 	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
200 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
201 	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
202 	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
203 	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
204 	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
205 	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
206 	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
207 	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
208 	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
209 	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
210 };
211 
212 static const u32 tonga_golden_common_all[] =
213 {
214 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
215 	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012,
216 	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A,
217 	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
218 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
219 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
220 	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
221 	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF
222 };
223 
224 static const u32 tonga_mgcg_cgcg_init[] =
225 {
226 	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
227 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
228 	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
229 	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
230 	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
231 	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
232 	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
233 	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
234 	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
235 	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
236 	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
237 	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
238 	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
239 	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
240 	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
241 	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
242 	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
243 	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
244 	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
245 	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
246 	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
247 	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
248 	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
249 	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
250 	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
251 	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
252 	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
253 	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
254 	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
255 	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
256 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
257 	mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000,
258 	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
259 	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
260 	mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005,
261 	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
262 	mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000,
263 	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
264 	mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007,
265 	mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005,
266 	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
267 	mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000,
268 	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
269 	mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007,
270 	mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005,
271 	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
272 	mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000,
273 	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
274 	mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007,
275 	mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005,
276 	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
277 	mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000,
278 	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
279 	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
280 	mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005,
281 	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
282 	mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000,
283 	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
284 	mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007,
285 	mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005,
286 	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
287 	mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000,
288 	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
289 	mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007,
290 	mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005,
291 	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
292 	mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000,
293 	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
294 	mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007,
295 	mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005,
296 	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
297 	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
298 	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
299 	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
300 	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
301 };
302 
303 static const u32 golden_settings_vegam_a11[] =
304 {
305 	mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208,
306 	mmCB_HW_CONTROL_2, 0x0f000000, 0x0d000000,
307 	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
308 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
309 	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
310 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
311 	mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x3a00161a,
312 	mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002e,
313 	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
314 	mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,
315 	mmSQ_CONFIG, 0x07f80000, 0x01180000,
316 	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
317 	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
318 	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
319 	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
320 	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054,
321 	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
322 };
323 
324 static const u32 vegam_golden_common_all[] =
325 {
326 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
327 	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
328 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
329 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
330 	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
331 	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
332 };
333 
334 static const u32 golden_settings_polaris11_a11[] =
335 {
336 	mmCB_HW_CONTROL, 0x0000f3cf, 0x00007208,
337 	mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000,
338 	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
339 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
340 	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
341 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
342 	mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012,
343 	mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000,
344 	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
345 	mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,
346 	mmSQ_CONFIG, 0x07f80000, 0x01180000,
347 	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
348 	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
349 	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3,
350 	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
351 	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
352 	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
353 };
354 
355 static const u32 polaris11_golden_common_all[] =
356 {
357 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
358 	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011002,
359 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
360 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
361 	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
362 	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
363 };
364 
365 static const u32 golden_settings_polaris10_a11[] =
366 {
367 	mmATC_MISC_CG, 0x000c0fc0, 0x000c0200,
368 	mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208,
369 	mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000,
370 	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
371 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
372 	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
373 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
374 	mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012,
375 	mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002a,
376 	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
377 	mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,
378 	mmSQ_CONFIG, 0x07f80000, 0x07180000,
379 	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
380 	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
381 	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
382 	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
383 	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
384 };
385 
386 static const u32 polaris10_golden_common_all[] =
387 {
388 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
389 	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012,
390 	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A,
391 	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
392 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
393 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
394 	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
395 	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
396 };
397 
398 static const u32 fiji_golden_common_all[] =
399 {
400 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
401 	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
402 	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
403 	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
404 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
405 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
406 	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
407 	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
408 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
409 	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009,
410 };
411 
412 static const u32 golden_settings_fiji_a10[] =
413 {
414 	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
415 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
416 	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
417 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
418 	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
419 	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
420 	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
421 	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
422 	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
423 	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
424 	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
425 };
426 
427 static const u32 fiji_mgcg_cgcg_init[] =
428 {
429 	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
430 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
431 	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
432 	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
433 	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
434 	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
435 	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
436 	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
437 	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
438 	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
439 	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
440 	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
441 	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
442 	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
443 	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
444 	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
445 	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
446 	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
447 	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
448 	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
449 	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
450 	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
451 	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
452 	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
453 	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
454 	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
455 	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
456 	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
457 	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
458 	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
459 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
460 	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
461 	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
462 	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
463 	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
464 };
465 
466 static const u32 golden_settings_iceland_a11[] =
467 {
468 	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
469 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
470 	mmDB_DEBUG3, 0xc0000000, 0xc0000000,
471 	mmGB_GPU_ID, 0x0000000f, 0x00000000,
472 	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
473 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
474 	mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x00000002,
475 	mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000,
476 	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
477 	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
478 	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
479 	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
480 	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
481 	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1,
482 	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
483 	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010,
484 };
485 
486 static const u32 iceland_golden_common_all[] =
487 {
488 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
489 	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002,
490 	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,
491 	mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001,
492 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
493 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
494 	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
495 	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF
496 };
497 
498 static const u32 iceland_mgcg_cgcg_init[] =
499 {
500 	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
501 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
502 	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
503 	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
504 	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0xc0000100,
505 	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0xc0000100,
506 	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0xc0000100,
507 	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
508 	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
509 	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
510 	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
511 	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
512 	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
513 	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
514 	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
515 	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
516 	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
517 	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
518 	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
519 	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
520 	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
521 	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
522 	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0xff000100,
523 	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
524 	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
525 	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
526 	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
527 	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
528 	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
529 	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
530 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
531 	mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000,
532 	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
533 	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87,
534 	mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005,
535 	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
536 	mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000,
537 	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
538 	mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007,
539 	mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005,
540 	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
541 	mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000,
542 	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
543 	mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007,
544 	mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005,
545 	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
546 	mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000,
547 	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
548 	mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007,
549 	mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005,
550 	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
551 	mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000,
552 	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
553 	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87,
554 	mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005,
555 	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
556 	mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000,
557 	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
558 	mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007,
559 	mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005,
560 	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
561 	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
562 	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
563 	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
564 };
565 
566 static const u32 cz_golden_settings_a11[] =
567 {
568 	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
569 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
570 	mmGB_GPU_ID, 0x0000000f, 0x00000000,
571 	mmPA_SC_ENHANCE, 0xffffffff, 0x00000001,
572 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
573 	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
574 	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
575 	mmTA_CNTL_AUX, 0x000f000f, 0x00010000,
576 	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
577 	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
578 	mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f3,
579 	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001302
580 };
581 
582 static const u32 cz_golden_common_all[] =
583 {
584 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
585 	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002,
586 	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,
587 	mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001,
588 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
589 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
590 	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
591 	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF
592 };
593 
594 static const u32 cz_mgcg_cgcg_init[] =
595 {
596 	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
597 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
598 	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
599 	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
600 	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
601 	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
602 	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x00000100,
603 	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
604 	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
605 	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
606 	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
607 	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
608 	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
609 	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
610 	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
611 	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
612 	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
613 	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
614 	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
615 	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
616 	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
617 	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
618 	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
619 	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
620 	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
621 	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
622 	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
623 	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
624 	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
625 	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
626 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
627 	mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000,
628 	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
629 	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
630 	mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005,
631 	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
632 	mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000,
633 	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
634 	mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007,
635 	mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005,
636 	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
637 	mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000,
638 	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
639 	mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007,
640 	mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005,
641 	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
642 	mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000,
643 	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
644 	mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007,
645 	mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005,
646 	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
647 	mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000,
648 	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
649 	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
650 	mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005,
651 	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
652 	mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000,
653 	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
654 	mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007,
655 	mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005,
656 	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
657 	mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000,
658 	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
659 	mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007,
660 	mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005,
661 	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
662 	mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000,
663 	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
664 	mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007,
665 	mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005,
666 	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
667 	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
668 	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
669 	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
670 	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
671 };
672 
673 static const u32 stoney_golden_settings_a11[] =
674 {
675 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
676 	mmGB_GPU_ID, 0x0000000f, 0x00000000,
677 	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
678 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
679 	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
680 	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
681 	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
682 	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
683 	mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f1,
684 	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x10101010,
685 };
686 
687 static const u32 stoney_golden_common_all[] =
688 {
689 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
690 	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000000,
691 	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,
692 	mmGB_ADDR_CONFIG, 0xffffffff, 0x12010001,
693 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
694 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
695 	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
696 	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
697 };
698 
699 static const u32 stoney_mgcg_cgcg_init[] =
700 {
701 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
702 	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
703 	mmCP_MEM_SLP_CNTL, 0xffffffff, 0x00020201,
704 	mmRLC_MEM_SLP_CNTL, 0xffffffff, 0x00020201,
705 	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
706 };
707 
708 
709 static const char * const sq_edc_source_names[] = {
710 	"SQ_EDC_INFO_SOURCE_INVALID: No EDC error has occurred",
711 	"SQ_EDC_INFO_SOURCE_INST: EDC source is Instruction Fetch",
712 	"SQ_EDC_INFO_SOURCE_SGPR: EDC source is SGPR or SQC data return",
713 	"SQ_EDC_INFO_SOURCE_VGPR: EDC source is VGPR",
714 	"SQ_EDC_INFO_SOURCE_LDS: EDC source is LDS",
715 	"SQ_EDC_INFO_SOURCE_GDS: EDC source is GDS",
716 	"SQ_EDC_INFO_SOURCE_TA: EDC source is TA",
717 };
718 
719 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev);
720 static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev);
721 static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev);
722 static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev);
723 static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev);
724 static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev);
725 static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring);
726 static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring);
727 
728 static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev)
729 {
730 	switch (adev->asic_type) {
731 	case CHIP_TOPAZ:
732 		amdgpu_device_program_register_sequence(adev,
733 							iceland_mgcg_cgcg_init,
734 							ARRAY_SIZE(iceland_mgcg_cgcg_init));
735 		amdgpu_device_program_register_sequence(adev,
736 							golden_settings_iceland_a11,
737 							ARRAY_SIZE(golden_settings_iceland_a11));
738 		amdgpu_device_program_register_sequence(adev,
739 							iceland_golden_common_all,
740 							ARRAY_SIZE(iceland_golden_common_all));
741 		break;
742 	case CHIP_FIJI:
743 		amdgpu_device_program_register_sequence(adev,
744 							fiji_mgcg_cgcg_init,
745 							ARRAY_SIZE(fiji_mgcg_cgcg_init));
746 		amdgpu_device_program_register_sequence(adev,
747 							golden_settings_fiji_a10,
748 							ARRAY_SIZE(golden_settings_fiji_a10));
749 		amdgpu_device_program_register_sequence(adev,
750 							fiji_golden_common_all,
751 							ARRAY_SIZE(fiji_golden_common_all));
752 		break;
753 
754 	case CHIP_TONGA:
755 		amdgpu_device_program_register_sequence(adev,
756 							tonga_mgcg_cgcg_init,
757 							ARRAY_SIZE(tonga_mgcg_cgcg_init));
758 		amdgpu_device_program_register_sequence(adev,
759 							golden_settings_tonga_a11,
760 							ARRAY_SIZE(golden_settings_tonga_a11));
761 		amdgpu_device_program_register_sequence(adev,
762 							tonga_golden_common_all,
763 							ARRAY_SIZE(tonga_golden_common_all));
764 		break;
765 	case CHIP_VEGAM:
766 		amdgpu_device_program_register_sequence(adev,
767 							golden_settings_vegam_a11,
768 							ARRAY_SIZE(golden_settings_vegam_a11));
769 		amdgpu_device_program_register_sequence(adev,
770 							vegam_golden_common_all,
771 							ARRAY_SIZE(vegam_golden_common_all));
772 		break;
773 	case CHIP_POLARIS11:
774 	case CHIP_POLARIS12:
775 		amdgpu_device_program_register_sequence(adev,
776 							golden_settings_polaris11_a11,
777 							ARRAY_SIZE(golden_settings_polaris11_a11));
778 		amdgpu_device_program_register_sequence(adev,
779 							polaris11_golden_common_all,
780 							ARRAY_SIZE(polaris11_golden_common_all));
781 		break;
782 	case CHIP_POLARIS10:
783 		amdgpu_device_program_register_sequence(adev,
784 							golden_settings_polaris10_a11,
785 							ARRAY_SIZE(golden_settings_polaris10_a11));
786 		amdgpu_device_program_register_sequence(adev,
787 							polaris10_golden_common_all,
788 							ARRAY_SIZE(polaris10_golden_common_all));
789 		WREG32_SMC(ixCG_ACLK_CNTL, 0x0000001C);
790 		if (adev->pdev->revision == 0xc7 &&
791 		    ((adev->pdev->subsystem_device == 0xb37 && adev->pdev->subsystem_vendor == 0x1002) ||
792 		     (adev->pdev->subsystem_device == 0x4a8 && adev->pdev->subsystem_vendor == 0x1043) ||
793 		     (adev->pdev->subsystem_device == 0x9480 && adev->pdev->subsystem_vendor == 0x1682))) {
794 			amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1E, 0xDD);
795 			amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1F, 0xD0);
796 		}
797 		break;
798 	case CHIP_CARRIZO:
799 		amdgpu_device_program_register_sequence(adev,
800 							cz_mgcg_cgcg_init,
801 							ARRAY_SIZE(cz_mgcg_cgcg_init));
802 		amdgpu_device_program_register_sequence(adev,
803 							cz_golden_settings_a11,
804 							ARRAY_SIZE(cz_golden_settings_a11));
805 		amdgpu_device_program_register_sequence(adev,
806 							cz_golden_common_all,
807 							ARRAY_SIZE(cz_golden_common_all));
808 		break;
809 	case CHIP_STONEY:
810 		amdgpu_device_program_register_sequence(adev,
811 							stoney_mgcg_cgcg_init,
812 							ARRAY_SIZE(stoney_mgcg_cgcg_init));
813 		amdgpu_device_program_register_sequence(adev,
814 							stoney_golden_settings_a11,
815 							ARRAY_SIZE(stoney_golden_settings_a11));
816 		amdgpu_device_program_register_sequence(adev,
817 							stoney_golden_common_all,
818 							ARRAY_SIZE(stoney_golden_common_all));
819 		break;
820 	default:
821 		break;
822 	}
823 }
824 
825 static void gfx_v8_0_scratch_init(struct amdgpu_device *adev)
826 {
827 	adev->gfx.scratch.num_reg = 8;
828 	adev->gfx.scratch.reg_base = mmSCRATCH_REG0;
829 	adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
830 }
831 
832 static int gfx_v8_0_ring_test_ring(struct amdgpu_ring *ring)
833 {
834 	struct amdgpu_device *adev = ring->adev;
835 	uint32_t scratch;
836 	uint32_t tmp = 0;
837 	unsigned i;
838 	int r;
839 
840 	r = amdgpu_gfx_scratch_get(adev, &scratch);
841 	if (r)
842 		return r;
843 
844 	WREG32(scratch, 0xCAFEDEAD);
845 	r = amdgpu_ring_alloc(ring, 3);
846 	if (r)
847 		goto error_free_scratch;
848 
849 	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
850 	amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
851 	amdgpu_ring_write(ring, 0xDEADBEEF);
852 	amdgpu_ring_commit(ring);
853 
854 	for (i = 0; i < adev->usec_timeout; i++) {
855 		tmp = RREG32(scratch);
856 		if (tmp == 0xDEADBEEF)
857 			break;
858 		DRM_UDELAY(1);
859 	}
860 
861 	if (i >= adev->usec_timeout)
862 		r = -ETIMEDOUT;
863 
864 error_free_scratch:
865 	amdgpu_gfx_scratch_free(adev, scratch);
866 	return r;
867 }
868 
869 static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
870 {
871 	struct amdgpu_device *adev = ring->adev;
872 	struct amdgpu_ib ib;
873 	struct dma_fence *f = NULL;
874 
875 	unsigned int index;
876 	uint64_t gpu_addr;
877 	uint32_t tmp;
878 	long r;
879 
880 	r = amdgpu_device_wb_get(adev, &index);
881 	if (r)
882 		return r;
883 
884 	gpu_addr = adev->wb.gpu_addr + (index * 4);
885 	adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
886 	memset(&ib, 0, sizeof(ib));
887 	r = amdgpu_ib_get(adev, NULL, 16, &ib);
888 	if (r)
889 		goto err1;
890 
891 	ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
892 	ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
893 	ib.ptr[2] = lower_32_bits(gpu_addr);
894 	ib.ptr[3] = upper_32_bits(gpu_addr);
895 	ib.ptr[4] = 0xDEADBEEF;
896 	ib.length_dw = 5;
897 
898 	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
899 	if (r)
900 		goto err2;
901 
902 	r = dma_fence_wait_timeout(f, false, timeout);
903 	if (r == 0) {
904 		r = -ETIMEDOUT;
905 		goto err2;
906 	} else if (r < 0) {
907 		goto err2;
908 	}
909 
910 	tmp = adev->wb.wb[index];
911 	if (tmp == 0xDEADBEEF)
912 		r = 0;
913 	else
914 		r = -EINVAL;
915 
916 err2:
917 	amdgpu_ib_free(adev, &ib, NULL);
918 	dma_fence_put(f);
919 err1:
920 	amdgpu_device_wb_free(adev, index);
921 	return r;
922 }
923 
924 
925 static void gfx_v8_0_free_microcode(struct amdgpu_device *adev)
926 {
927 	release_firmware(adev->gfx.pfp_fw);
928 	adev->gfx.pfp_fw = NULL;
929 	release_firmware(adev->gfx.me_fw);
930 	adev->gfx.me_fw = NULL;
931 	release_firmware(adev->gfx.ce_fw);
932 	adev->gfx.ce_fw = NULL;
933 	release_firmware(adev->gfx.rlc_fw);
934 	adev->gfx.rlc_fw = NULL;
935 	release_firmware(adev->gfx.mec_fw);
936 	adev->gfx.mec_fw = NULL;
937 	if ((adev->asic_type != CHIP_STONEY) &&
938 	    (adev->asic_type != CHIP_TOPAZ))
939 		release_firmware(adev->gfx.mec2_fw);
940 	adev->gfx.mec2_fw = NULL;
941 
942 	kfree(adev->gfx.rlc.register_list_format);
943 }
944 
945 static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
946 {
947 	const char *chip_name;
948 	char fw_name[30];
949 	int err;
950 	struct amdgpu_firmware_info *info = NULL;
951 	const struct common_firmware_header *header = NULL;
952 	const struct gfx_firmware_header_v1_0 *cp_hdr;
953 	const struct rlc_firmware_header_v2_0 *rlc_hdr;
954 	unsigned int *tmp = NULL, i;
955 
956 	DRM_DEBUG("\n");
957 
958 	switch (adev->asic_type) {
959 	case CHIP_TOPAZ:
960 		chip_name = "topaz";
961 		break;
962 	case CHIP_TONGA:
963 		chip_name = "tonga";
964 		break;
965 	case CHIP_CARRIZO:
966 		chip_name = "carrizo";
967 		break;
968 	case CHIP_FIJI:
969 		chip_name = "fiji";
970 		break;
971 	case CHIP_STONEY:
972 		chip_name = "stoney";
973 		break;
974 	case CHIP_POLARIS10:
975 		chip_name = "polaris10";
976 		break;
977 	case CHIP_POLARIS11:
978 		chip_name = "polaris11";
979 		break;
980 	case CHIP_POLARIS12:
981 		chip_name = "polaris12";
982 		break;
983 	case CHIP_VEGAM:
984 		chip_name = "vegam";
985 		break;
986 	default:
987 		BUG();
988 	}
989 
990 	if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
991 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp_2.bin", chip_name);
992 		err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
993 		if (err == -ENOENT) {
994 			snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
995 			err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
996 		}
997 	} else {
998 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
999 		err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
1000 	}
1001 	if (err)
1002 		goto out;
1003 	err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
1004 	if (err)
1005 		goto out;
1006 	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
1007 	adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1008 	adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1009 
1010 	if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
1011 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me_2.bin", chip_name);
1012 		err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
1013 		if (err == -ENOENT) {
1014 			snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
1015 			err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
1016 		}
1017 	} else {
1018 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
1019 		err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
1020 	}
1021 	if (err)
1022 		goto out;
1023 	err = amdgpu_ucode_validate(adev->gfx.me_fw);
1024 	if (err)
1025 		goto out;
1026 	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
1027 	adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1028 
1029 	adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1030 
1031 	if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
1032 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce_2.bin", chip_name);
1033 		err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
1034 		if (err == -ENOENT) {
1035 			snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
1036 			err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
1037 		}
1038 	} else {
1039 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
1040 		err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
1041 	}
1042 	if (err)
1043 		goto out;
1044 	err = amdgpu_ucode_validate(adev->gfx.ce_fw);
1045 	if (err)
1046 		goto out;
1047 	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
1048 	adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1049 	adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1050 
1051 	/*
1052 	 * Support for MCBP/Virtualization in combination with chained IBs is
1053 	 * formal released on feature version #46
1054 	 */
1055 	if (adev->gfx.ce_feature_version >= 46 &&
1056 	    adev->gfx.pfp_feature_version >= 46) {
1057 		adev->virt.chained_ib_support = true;
1058 		DRM_INFO("Chained IB support enabled!\n");
1059 	} else
1060 		adev->virt.chained_ib_support = false;
1061 
1062 	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
1063 	err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
1064 	if (err)
1065 		goto out;
1066 	err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
1067 	rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1068 	adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
1069 	adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
1070 
1071 	adev->gfx.rlc.save_and_restore_offset =
1072 			le32_to_cpu(rlc_hdr->save_and_restore_offset);
1073 	adev->gfx.rlc.clear_state_descriptor_offset =
1074 			le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
1075 	adev->gfx.rlc.avail_scratch_ram_locations =
1076 			le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
1077 	adev->gfx.rlc.reg_restore_list_size =
1078 			le32_to_cpu(rlc_hdr->reg_restore_list_size);
1079 	adev->gfx.rlc.reg_list_format_start =
1080 			le32_to_cpu(rlc_hdr->reg_list_format_start);
1081 	adev->gfx.rlc.reg_list_format_separate_start =
1082 			le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
1083 	adev->gfx.rlc.starting_offsets_start =
1084 			le32_to_cpu(rlc_hdr->starting_offsets_start);
1085 	adev->gfx.rlc.reg_list_format_size_bytes =
1086 			le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
1087 	adev->gfx.rlc.reg_list_size_bytes =
1088 			le32_to_cpu(rlc_hdr->reg_list_size_bytes);
1089 
1090 	adev->gfx.rlc.register_list_format =
1091 			kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
1092 					adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
1093 
1094 	if (!adev->gfx.rlc.register_list_format) {
1095 		err = -ENOMEM;
1096 		goto out;
1097 	}
1098 
1099 	tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1100 			le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
1101 	for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++)
1102 		adev->gfx.rlc.register_list_format[i] =	le32_to_cpu(tmp[i]);
1103 
1104 	adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
1105 
1106 	tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1107 			le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
1108 	for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++)
1109 		adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
1110 
1111 	if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
1112 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec_2.bin", chip_name);
1113 		err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
1114 		if (err == -ENOENT) {
1115 			snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
1116 			err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
1117 		}
1118 	} else {
1119 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
1120 		err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
1121 	}
1122 	if (err)
1123 		goto out;
1124 	err = amdgpu_ucode_validate(adev->gfx.mec_fw);
1125 	if (err)
1126 		goto out;
1127 	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1128 	adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1129 	adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1130 
1131 	if ((adev->asic_type != CHIP_STONEY) &&
1132 	    (adev->asic_type != CHIP_TOPAZ)) {
1133 		if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
1134 			snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2_2.bin", chip_name);
1135 			err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
1136 			if (err == -ENOENT) {
1137 				snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
1138 				err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
1139 			}
1140 		} else {
1141 			snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
1142 			err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
1143 		}
1144 		if (!err) {
1145 			err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
1146 			if (err)
1147 				goto out;
1148 			cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1149 				adev->gfx.mec2_fw->data;
1150 			adev->gfx.mec2_fw_version =
1151 				le32_to_cpu(cp_hdr->header.ucode_version);
1152 			adev->gfx.mec2_feature_version =
1153 				le32_to_cpu(cp_hdr->ucode_feature_version);
1154 		} else {
1155 			err = 0;
1156 			adev->gfx.mec2_fw = NULL;
1157 		}
1158 	}
1159 
1160 	info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
1161 	info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
1162 	info->fw = adev->gfx.pfp_fw;
1163 	header = (const struct common_firmware_header *)info->fw->data;
1164 	adev->firmware.fw_size +=
1165 		ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1166 
1167 	info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
1168 	info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
1169 	info->fw = adev->gfx.me_fw;
1170 	header = (const struct common_firmware_header *)info->fw->data;
1171 	adev->firmware.fw_size +=
1172 		ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1173 
1174 	info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
1175 	info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
1176 	info->fw = adev->gfx.ce_fw;
1177 	header = (const struct common_firmware_header *)info->fw->data;
1178 	adev->firmware.fw_size +=
1179 		ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1180 
1181 	info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
1182 	info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
1183 	info->fw = adev->gfx.rlc_fw;
1184 	header = (const struct common_firmware_header *)info->fw->data;
1185 	adev->firmware.fw_size +=
1186 		ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1187 
1188 	info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
1189 	info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
1190 	info->fw = adev->gfx.mec_fw;
1191 	header = (const struct common_firmware_header *)info->fw->data;
1192 	adev->firmware.fw_size +=
1193 		ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1194 
1195 	/* we need account JT in */
1196 	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1197 	adev->firmware.fw_size +=
1198 		ALIGN(le32_to_cpu(cp_hdr->jt_size) << 2, PAGE_SIZE);
1199 
1200 	if (amdgpu_sriov_vf(adev)) {
1201 		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_STORAGE];
1202 		info->ucode_id = AMDGPU_UCODE_ID_STORAGE;
1203 		info->fw = adev->gfx.mec_fw;
1204 		adev->firmware.fw_size +=
1205 			ALIGN(le32_to_cpu(64 * PAGE_SIZE), PAGE_SIZE);
1206 	}
1207 
1208 	if (adev->gfx.mec2_fw) {
1209 		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
1210 		info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
1211 		info->fw = adev->gfx.mec2_fw;
1212 		header = (const struct common_firmware_header *)info->fw->data;
1213 		adev->firmware.fw_size +=
1214 			ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1215 	}
1216 
1217 out:
1218 	if (err) {
1219 		dev_err(adev->dev,
1220 			"gfx8: Failed to load firmware \"%s\"\n",
1221 			fw_name);
1222 		release_firmware(adev->gfx.pfp_fw);
1223 		adev->gfx.pfp_fw = NULL;
1224 		release_firmware(adev->gfx.me_fw);
1225 		adev->gfx.me_fw = NULL;
1226 		release_firmware(adev->gfx.ce_fw);
1227 		adev->gfx.ce_fw = NULL;
1228 		release_firmware(adev->gfx.rlc_fw);
1229 		adev->gfx.rlc_fw = NULL;
1230 		release_firmware(adev->gfx.mec_fw);
1231 		adev->gfx.mec_fw = NULL;
1232 		release_firmware(adev->gfx.mec2_fw);
1233 		adev->gfx.mec2_fw = NULL;
1234 	}
1235 	return err;
1236 }
1237 
1238 static void gfx_v8_0_get_csb_buffer(struct amdgpu_device *adev,
1239 				    volatile u32 *buffer)
1240 {
1241 	u32 count = 0, i;
1242 	const struct cs_section_def *sect = NULL;
1243 	const struct cs_extent_def *ext = NULL;
1244 
1245 	if (adev->gfx.rlc.cs_data == NULL)
1246 		return;
1247 	if (buffer == NULL)
1248 		return;
1249 
1250 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1251 	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1252 
1253 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1254 	buffer[count++] = cpu_to_le32(0x80000000);
1255 	buffer[count++] = cpu_to_le32(0x80000000);
1256 
1257 	for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
1258 		for (ext = sect->section; ext->extent != NULL; ++ext) {
1259 			if (sect->id == SECT_CONTEXT) {
1260 				buffer[count++] =
1261 					cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
1262 				buffer[count++] = cpu_to_le32(ext->reg_index -
1263 						PACKET3_SET_CONTEXT_REG_START);
1264 				for (i = 0; i < ext->reg_count; i++)
1265 					buffer[count++] = cpu_to_le32(ext->extent[i]);
1266 			} else {
1267 				return;
1268 			}
1269 		}
1270 	}
1271 
1272 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
1273 	buffer[count++] = cpu_to_le32(mmPA_SC_RASTER_CONFIG -
1274 			PACKET3_SET_CONTEXT_REG_START);
1275 	buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config);
1276 	buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config_1);
1277 
1278 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1279 	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
1280 
1281 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
1282 	buffer[count++] = cpu_to_le32(0);
1283 }
1284 
1285 static int gfx_v8_0_cp_jump_table_num(struct amdgpu_device *adev)
1286 {
1287 	if (adev->asic_type == CHIP_CARRIZO)
1288 		return 5;
1289 	else
1290 		return 4;
1291 }
1292 
1293 static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
1294 {
1295 	const struct cs_section_def *cs_data;
1296 	int r;
1297 
1298 	adev->gfx.rlc.cs_data = vi_cs_data;
1299 
1300 	cs_data = adev->gfx.rlc.cs_data;
1301 
1302 	if (cs_data) {
1303 		/* init clear state block */
1304 		r = amdgpu_gfx_rlc_init_csb(adev);
1305 		if (r)
1306 			return r;
1307 	}
1308 
1309 	if ((adev->asic_type == CHIP_CARRIZO) ||
1310 	    (adev->asic_type == CHIP_STONEY)) {
1311 		adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1312 		r = amdgpu_gfx_rlc_init_cpt(adev);
1313 		if (r)
1314 			return r;
1315 	}
1316 
1317 	return 0;
1318 }
1319 
1320 static void gfx_v8_0_mec_fini(struct amdgpu_device *adev)
1321 {
1322 	amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1323 }
1324 
1325 static int gfx_v8_0_mec_init(struct amdgpu_device *adev)
1326 {
1327 	int r;
1328 	u32 *hpd;
1329 	size_t mec_hpd_size;
1330 
1331 	bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1332 
1333 	/* take ownership of the relevant compute queues */
1334 	amdgpu_gfx_compute_queue_acquire(adev);
1335 
1336 	mec_hpd_size = adev->gfx.num_compute_rings * GFX8_MEC_HPD_SIZE;
1337 
1338 	r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1339 				      AMDGPU_GEM_DOMAIN_VRAM,
1340 				      &adev->gfx.mec.hpd_eop_obj,
1341 				      &adev->gfx.mec.hpd_eop_gpu_addr,
1342 				      (void **)&hpd);
1343 	if (r) {
1344 		dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1345 		return r;
1346 	}
1347 
1348 	memset(hpd, 0, mec_hpd_size);
1349 
1350 	amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
1351 	amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
1352 
1353 	return 0;
1354 }
1355 
1356 static const u32 vgpr_init_compute_shader[] =
1357 {
1358 	0x7e000209, 0x7e020208,
1359 	0x7e040207, 0x7e060206,
1360 	0x7e080205, 0x7e0a0204,
1361 	0x7e0c0203, 0x7e0e0202,
1362 	0x7e100201, 0x7e120200,
1363 	0x7e140209, 0x7e160208,
1364 	0x7e180207, 0x7e1a0206,
1365 	0x7e1c0205, 0x7e1e0204,
1366 	0x7e200203, 0x7e220202,
1367 	0x7e240201, 0x7e260200,
1368 	0x7e280209, 0x7e2a0208,
1369 	0x7e2c0207, 0x7e2e0206,
1370 	0x7e300205, 0x7e320204,
1371 	0x7e340203, 0x7e360202,
1372 	0x7e380201, 0x7e3a0200,
1373 	0x7e3c0209, 0x7e3e0208,
1374 	0x7e400207, 0x7e420206,
1375 	0x7e440205, 0x7e460204,
1376 	0x7e480203, 0x7e4a0202,
1377 	0x7e4c0201, 0x7e4e0200,
1378 	0x7e500209, 0x7e520208,
1379 	0x7e540207, 0x7e560206,
1380 	0x7e580205, 0x7e5a0204,
1381 	0x7e5c0203, 0x7e5e0202,
1382 	0x7e600201, 0x7e620200,
1383 	0x7e640209, 0x7e660208,
1384 	0x7e680207, 0x7e6a0206,
1385 	0x7e6c0205, 0x7e6e0204,
1386 	0x7e700203, 0x7e720202,
1387 	0x7e740201, 0x7e760200,
1388 	0x7e780209, 0x7e7a0208,
1389 	0x7e7c0207, 0x7e7e0206,
1390 	0xbf8a0000, 0xbf810000,
1391 };
1392 
1393 static const u32 sgpr_init_compute_shader[] =
1394 {
1395 	0xbe8a0100, 0xbe8c0102,
1396 	0xbe8e0104, 0xbe900106,
1397 	0xbe920108, 0xbe940100,
1398 	0xbe960102, 0xbe980104,
1399 	0xbe9a0106, 0xbe9c0108,
1400 	0xbe9e0100, 0xbea00102,
1401 	0xbea20104, 0xbea40106,
1402 	0xbea60108, 0xbea80100,
1403 	0xbeaa0102, 0xbeac0104,
1404 	0xbeae0106, 0xbeb00108,
1405 	0xbeb20100, 0xbeb40102,
1406 	0xbeb60104, 0xbeb80106,
1407 	0xbeba0108, 0xbebc0100,
1408 	0xbebe0102, 0xbec00104,
1409 	0xbec20106, 0xbec40108,
1410 	0xbec60100, 0xbec80102,
1411 	0xbee60004, 0xbee70005,
1412 	0xbeea0006, 0xbeeb0007,
1413 	0xbee80008, 0xbee90009,
1414 	0xbefc0000, 0xbf8a0000,
1415 	0xbf810000, 0x00000000,
1416 };
1417 
1418 static const u32 vgpr_init_regs[] =
1419 {
1420 	mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xffffffff,
1421 	mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, /* CU_GROUP_COUNT=1 */
1422 	mmCOMPUTE_NUM_THREAD_X, 256*4,
1423 	mmCOMPUTE_NUM_THREAD_Y, 1,
1424 	mmCOMPUTE_NUM_THREAD_Z, 1,
1425 	mmCOMPUTE_PGM_RSRC1, 0x100004f, /* VGPRS=15 (64 logical VGPRs), SGPRS=1 (16 SGPRs), BULKY=1 */
1426 	mmCOMPUTE_PGM_RSRC2, 20,
1427 	mmCOMPUTE_USER_DATA_0, 0xedcedc00,
1428 	mmCOMPUTE_USER_DATA_1, 0xedcedc01,
1429 	mmCOMPUTE_USER_DATA_2, 0xedcedc02,
1430 	mmCOMPUTE_USER_DATA_3, 0xedcedc03,
1431 	mmCOMPUTE_USER_DATA_4, 0xedcedc04,
1432 	mmCOMPUTE_USER_DATA_5, 0xedcedc05,
1433 	mmCOMPUTE_USER_DATA_6, 0xedcedc06,
1434 	mmCOMPUTE_USER_DATA_7, 0xedcedc07,
1435 	mmCOMPUTE_USER_DATA_8, 0xedcedc08,
1436 	mmCOMPUTE_USER_DATA_9, 0xedcedc09,
1437 };
1438 
1439 static const u32 sgpr1_init_regs[] =
1440 {
1441 	mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0x0f,
1442 	mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, /* CU_GROUP_COUNT=1 */
1443 	mmCOMPUTE_NUM_THREAD_X, 256*5,
1444 	mmCOMPUTE_NUM_THREAD_Y, 1,
1445 	mmCOMPUTE_NUM_THREAD_Z, 1,
1446 	mmCOMPUTE_PGM_RSRC1, 0x240, /* SGPRS=9 (80 GPRS) */
1447 	mmCOMPUTE_PGM_RSRC2, 20,
1448 	mmCOMPUTE_USER_DATA_0, 0xedcedc00,
1449 	mmCOMPUTE_USER_DATA_1, 0xedcedc01,
1450 	mmCOMPUTE_USER_DATA_2, 0xedcedc02,
1451 	mmCOMPUTE_USER_DATA_3, 0xedcedc03,
1452 	mmCOMPUTE_USER_DATA_4, 0xedcedc04,
1453 	mmCOMPUTE_USER_DATA_5, 0xedcedc05,
1454 	mmCOMPUTE_USER_DATA_6, 0xedcedc06,
1455 	mmCOMPUTE_USER_DATA_7, 0xedcedc07,
1456 	mmCOMPUTE_USER_DATA_8, 0xedcedc08,
1457 	mmCOMPUTE_USER_DATA_9, 0xedcedc09,
1458 };
1459 
1460 static const u32 sgpr2_init_regs[] =
1461 {
1462 	mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xf0,
1463 	mmCOMPUTE_RESOURCE_LIMITS, 0x1000000,
1464 	mmCOMPUTE_NUM_THREAD_X, 256*5,
1465 	mmCOMPUTE_NUM_THREAD_Y, 1,
1466 	mmCOMPUTE_NUM_THREAD_Z, 1,
1467 	mmCOMPUTE_PGM_RSRC1, 0x240, /* SGPRS=9 (80 GPRS) */
1468 	mmCOMPUTE_PGM_RSRC2, 20,
1469 	mmCOMPUTE_USER_DATA_0, 0xedcedc00,
1470 	mmCOMPUTE_USER_DATA_1, 0xedcedc01,
1471 	mmCOMPUTE_USER_DATA_2, 0xedcedc02,
1472 	mmCOMPUTE_USER_DATA_3, 0xedcedc03,
1473 	mmCOMPUTE_USER_DATA_4, 0xedcedc04,
1474 	mmCOMPUTE_USER_DATA_5, 0xedcedc05,
1475 	mmCOMPUTE_USER_DATA_6, 0xedcedc06,
1476 	mmCOMPUTE_USER_DATA_7, 0xedcedc07,
1477 	mmCOMPUTE_USER_DATA_8, 0xedcedc08,
1478 	mmCOMPUTE_USER_DATA_9, 0xedcedc09,
1479 };
1480 
1481 static const u32 sec_ded_counter_registers[] =
1482 {
1483 	mmCPC_EDC_ATC_CNT,
1484 	mmCPC_EDC_SCRATCH_CNT,
1485 	mmCPC_EDC_UCODE_CNT,
1486 	mmCPF_EDC_ATC_CNT,
1487 	mmCPF_EDC_ROQ_CNT,
1488 	mmCPF_EDC_TAG_CNT,
1489 	mmCPG_EDC_ATC_CNT,
1490 	mmCPG_EDC_DMA_CNT,
1491 	mmCPG_EDC_TAG_CNT,
1492 	mmDC_EDC_CSINVOC_CNT,
1493 	mmDC_EDC_RESTORE_CNT,
1494 	mmDC_EDC_STATE_CNT,
1495 	mmGDS_EDC_CNT,
1496 	mmGDS_EDC_GRBM_CNT,
1497 	mmGDS_EDC_OA_DED,
1498 	mmSPI_EDC_CNT,
1499 	mmSQC_ATC_EDC_GATCL1_CNT,
1500 	mmSQC_EDC_CNT,
1501 	mmSQ_EDC_DED_CNT,
1502 	mmSQ_EDC_INFO,
1503 	mmSQ_EDC_SEC_CNT,
1504 	mmTCC_EDC_CNT,
1505 	mmTCP_ATC_EDC_GATCL1_CNT,
1506 	mmTCP_EDC_CNT,
1507 	mmTD_EDC_CNT
1508 };
1509 
1510 static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
1511 {
1512 	struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
1513 	struct amdgpu_ib ib;
1514 	struct dma_fence *f = NULL;
1515 	int r, i;
1516 	u32 tmp;
1517 	unsigned total_size, vgpr_offset, sgpr_offset;
1518 	u64 gpu_addr;
1519 
1520 	/* only supported on CZ */
1521 	if (adev->asic_type != CHIP_CARRIZO)
1522 		return 0;
1523 
1524 	/* bail if the compute ring is not ready */
1525 	if (!ring->sched.ready)
1526 		return 0;
1527 
1528 	tmp = RREG32(mmGB_EDC_MODE);
1529 	WREG32(mmGB_EDC_MODE, 0);
1530 
1531 	total_size =
1532 		(((ARRAY_SIZE(vgpr_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;
1533 	total_size +=
1534 		(((ARRAY_SIZE(sgpr1_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;
1535 	total_size +=
1536 		(((ARRAY_SIZE(sgpr2_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;
1537 	total_size = ALIGN(total_size, 256);
1538 	vgpr_offset = total_size;
1539 	total_size += ALIGN(sizeof(vgpr_init_compute_shader), 256);
1540 	sgpr_offset = total_size;
1541 	total_size += sizeof(sgpr_init_compute_shader);
1542 
1543 	/* allocate an indirect buffer to put the commands in */
1544 	memset(&ib, 0, sizeof(ib));
1545 	r = amdgpu_ib_get(adev, NULL, total_size, &ib);
1546 	if (r) {
1547 		DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
1548 		return r;
1549 	}
1550 
1551 	/* load the compute shaders */
1552 	for (i = 0; i < ARRAY_SIZE(vgpr_init_compute_shader); i++)
1553 		ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_compute_shader[i];
1554 
1555 	for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
1556 		ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
1557 
1558 	/* init the ib length to 0 */
1559 	ib.length_dw = 0;
1560 
1561 	/* VGPR */
1562 	/* write the register state for the compute dispatch */
1563 	for (i = 0; i < ARRAY_SIZE(vgpr_init_regs); i += 2) {
1564 		ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
1565 		ib.ptr[ib.length_dw++] = vgpr_init_regs[i] - PACKET3_SET_SH_REG_START;
1566 		ib.ptr[ib.length_dw++] = vgpr_init_regs[i + 1];
1567 	}
1568 	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
1569 	gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
1570 	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
1571 	ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
1572 	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
1573 	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
1574 
1575 	/* write dispatch packet */
1576 	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
1577 	ib.ptr[ib.length_dw++] = 8; /* x */
1578 	ib.ptr[ib.length_dw++] = 1; /* y */
1579 	ib.ptr[ib.length_dw++] = 1; /* z */
1580 	ib.ptr[ib.length_dw++] =
1581 		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
1582 
1583 	/* write CS partial flush packet */
1584 	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
1585 	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
1586 
1587 	/* SGPR1 */
1588 	/* write the register state for the compute dispatch */
1589 	for (i = 0; i < ARRAY_SIZE(sgpr1_init_regs); i += 2) {
1590 		ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
1591 		ib.ptr[ib.length_dw++] = sgpr1_init_regs[i] - PACKET3_SET_SH_REG_START;
1592 		ib.ptr[ib.length_dw++] = sgpr1_init_regs[i + 1];
1593 	}
1594 	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
1595 	gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
1596 	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
1597 	ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
1598 	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
1599 	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
1600 
1601 	/* write dispatch packet */
1602 	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
1603 	ib.ptr[ib.length_dw++] = 8; /* x */
1604 	ib.ptr[ib.length_dw++] = 1; /* y */
1605 	ib.ptr[ib.length_dw++] = 1; /* z */
1606 	ib.ptr[ib.length_dw++] =
1607 		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
1608 
1609 	/* write CS partial flush packet */
1610 	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
1611 	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
1612 
1613 	/* SGPR2 */
1614 	/* write the register state for the compute dispatch */
1615 	for (i = 0; i < ARRAY_SIZE(sgpr2_init_regs); i += 2) {
1616 		ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
1617 		ib.ptr[ib.length_dw++] = sgpr2_init_regs[i] - PACKET3_SET_SH_REG_START;
1618 		ib.ptr[ib.length_dw++] = sgpr2_init_regs[i + 1];
1619 	}
1620 	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
1621 	gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
1622 	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
1623 	ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
1624 	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
1625 	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
1626 
1627 	/* write dispatch packet */
1628 	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
1629 	ib.ptr[ib.length_dw++] = 8; /* x */
1630 	ib.ptr[ib.length_dw++] = 1; /* y */
1631 	ib.ptr[ib.length_dw++] = 1; /* z */
1632 	ib.ptr[ib.length_dw++] =
1633 		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
1634 
1635 	/* write CS partial flush packet */
1636 	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
1637 	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
1638 
1639 	/* shedule the ib on the ring */
1640 	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1641 	if (r) {
1642 		DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
1643 		goto fail;
1644 	}
1645 
1646 	/* wait for the GPU to finish processing the IB */
1647 	r = dma_fence_wait(f, false);
1648 	if (r) {
1649 		DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
1650 		goto fail;
1651 	}
1652 
1653 	tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, DED_MODE, 2);
1654 	tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, PROP_FED, 1);
1655 	WREG32(mmGB_EDC_MODE, tmp);
1656 
1657 	tmp = RREG32(mmCC_GC_EDC_CONFIG);
1658 	tmp = REG_SET_FIELD(tmp, CC_GC_EDC_CONFIG, DIS_EDC, 0) | 1;
1659 	WREG32(mmCC_GC_EDC_CONFIG, tmp);
1660 
1661 
1662 	/* read back registers to clear the counters */
1663 	for (i = 0; i < ARRAY_SIZE(sec_ded_counter_registers); i++)
1664 		RREG32(sec_ded_counter_registers[i]);
1665 
1666 fail:
1667 	amdgpu_ib_free(adev, &ib, NULL);
1668 	dma_fence_put(f);
1669 
1670 	return r;
1671 }
1672 
1673 static int gfx_v8_0_gpu_early_init(struct amdgpu_device *adev)
1674 {
1675 	u32 gb_addr_config;
1676 	u32 mc_shared_chmap, mc_arb_ramcfg;
1677 	u32 dimm00_addr_map, dimm01_addr_map, dimm10_addr_map, dimm11_addr_map;
1678 	u32 tmp;
1679 	int ret;
1680 
1681 	switch (adev->asic_type) {
1682 	case CHIP_TOPAZ:
1683 		adev->gfx.config.max_shader_engines = 1;
1684 		adev->gfx.config.max_tile_pipes = 2;
1685 		adev->gfx.config.max_cu_per_sh = 6;
1686 		adev->gfx.config.max_sh_per_se = 1;
1687 		adev->gfx.config.max_backends_per_se = 2;
1688 		adev->gfx.config.max_texture_channel_caches = 2;
1689 		adev->gfx.config.max_gprs = 256;
1690 		adev->gfx.config.max_gs_threads = 32;
1691 		adev->gfx.config.max_hw_contexts = 8;
1692 
1693 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1694 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1695 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1696 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1697 		gb_addr_config = TOPAZ_GB_ADDR_CONFIG_GOLDEN;
1698 		break;
1699 	case CHIP_FIJI:
1700 		adev->gfx.config.max_shader_engines = 4;
1701 		adev->gfx.config.max_tile_pipes = 16;
1702 		adev->gfx.config.max_cu_per_sh = 16;
1703 		adev->gfx.config.max_sh_per_se = 1;
1704 		adev->gfx.config.max_backends_per_se = 4;
1705 		adev->gfx.config.max_texture_channel_caches = 16;
1706 		adev->gfx.config.max_gprs = 256;
1707 		adev->gfx.config.max_gs_threads = 32;
1708 		adev->gfx.config.max_hw_contexts = 8;
1709 
1710 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1711 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1712 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1713 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1714 		gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1715 		break;
1716 	case CHIP_POLARIS11:
1717 	case CHIP_POLARIS12:
1718 		ret = amdgpu_atombios_get_gfx_info(adev);
1719 		if (ret)
1720 			return ret;
1721 		adev->gfx.config.max_gprs = 256;
1722 		adev->gfx.config.max_gs_threads = 32;
1723 		adev->gfx.config.max_hw_contexts = 8;
1724 
1725 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1726 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1727 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1728 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1729 		gb_addr_config = POLARIS11_GB_ADDR_CONFIG_GOLDEN;
1730 		break;
1731 	case CHIP_POLARIS10:
1732 	case CHIP_VEGAM:
1733 		ret = amdgpu_atombios_get_gfx_info(adev);
1734 		if (ret)
1735 			return ret;
1736 		adev->gfx.config.max_gprs = 256;
1737 		adev->gfx.config.max_gs_threads = 32;
1738 		adev->gfx.config.max_hw_contexts = 8;
1739 
1740 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1741 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1742 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1743 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1744 		gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1745 		break;
1746 	case CHIP_TONGA:
1747 		adev->gfx.config.max_shader_engines = 4;
1748 		adev->gfx.config.max_tile_pipes = 8;
1749 		adev->gfx.config.max_cu_per_sh = 8;
1750 		adev->gfx.config.max_sh_per_se = 1;
1751 		adev->gfx.config.max_backends_per_se = 2;
1752 		adev->gfx.config.max_texture_channel_caches = 8;
1753 		adev->gfx.config.max_gprs = 256;
1754 		adev->gfx.config.max_gs_threads = 32;
1755 		adev->gfx.config.max_hw_contexts = 8;
1756 
1757 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1758 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1759 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1760 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1761 		gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1762 		break;
1763 	case CHIP_CARRIZO:
1764 		adev->gfx.config.max_shader_engines = 1;
1765 		adev->gfx.config.max_tile_pipes = 2;
1766 		adev->gfx.config.max_sh_per_se = 1;
1767 		adev->gfx.config.max_backends_per_se = 2;
1768 		adev->gfx.config.max_cu_per_sh = 8;
1769 		adev->gfx.config.max_texture_channel_caches = 2;
1770 		adev->gfx.config.max_gprs = 256;
1771 		adev->gfx.config.max_gs_threads = 32;
1772 		adev->gfx.config.max_hw_contexts = 8;
1773 
1774 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1775 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1776 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1777 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1778 		gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN;
1779 		break;
1780 	case CHIP_STONEY:
1781 		adev->gfx.config.max_shader_engines = 1;
1782 		adev->gfx.config.max_tile_pipes = 2;
1783 		adev->gfx.config.max_sh_per_se = 1;
1784 		adev->gfx.config.max_backends_per_se = 1;
1785 		adev->gfx.config.max_cu_per_sh = 3;
1786 		adev->gfx.config.max_texture_channel_caches = 2;
1787 		adev->gfx.config.max_gprs = 256;
1788 		adev->gfx.config.max_gs_threads = 16;
1789 		adev->gfx.config.max_hw_contexts = 8;
1790 
1791 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1792 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1793 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1794 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1795 		gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN;
1796 		break;
1797 	default:
1798 		adev->gfx.config.max_shader_engines = 2;
1799 		adev->gfx.config.max_tile_pipes = 4;
1800 		adev->gfx.config.max_cu_per_sh = 2;
1801 		adev->gfx.config.max_sh_per_se = 1;
1802 		adev->gfx.config.max_backends_per_se = 2;
1803 		adev->gfx.config.max_texture_channel_caches = 4;
1804 		adev->gfx.config.max_gprs = 256;
1805 		adev->gfx.config.max_gs_threads = 32;
1806 		adev->gfx.config.max_hw_contexts = 8;
1807 
1808 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1809 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1810 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1811 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1812 		gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1813 		break;
1814 	}
1815 
1816 	mc_shared_chmap = RREG32(mmMC_SHARED_CHMAP);
1817 	adev->gfx.config.mc_arb_ramcfg = RREG32(mmMC_ARB_RAMCFG);
1818 	mc_arb_ramcfg = adev->gfx.config.mc_arb_ramcfg;
1819 
1820 	adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes;
1821 	adev->gfx.config.mem_max_burst_length_bytes = 256;
1822 	if (adev->flags & AMD_IS_APU) {
1823 		/* Get memory bank mapping mode. */
1824 		tmp = RREG32(mmMC_FUS_DRAM0_BANK_ADDR_MAPPING);
1825 		dimm00_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
1826 		dimm01_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM1ADDRMAP);
1827 
1828 		tmp = RREG32(mmMC_FUS_DRAM1_BANK_ADDR_MAPPING);
1829 		dimm10_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
1830 		dimm11_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM1ADDRMAP);
1831 
1832 		/* Validate settings in case only one DIMM installed. */
1833 		if ((dimm00_addr_map == 0) || (dimm00_addr_map == 3) || (dimm00_addr_map == 4) || (dimm00_addr_map > 12))
1834 			dimm00_addr_map = 0;
1835 		if ((dimm01_addr_map == 0) || (dimm01_addr_map == 3) || (dimm01_addr_map == 4) || (dimm01_addr_map > 12))
1836 			dimm01_addr_map = 0;
1837 		if ((dimm10_addr_map == 0) || (dimm10_addr_map == 3) || (dimm10_addr_map == 4) || (dimm10_addr_map > 12))
1838 			dimm10_addr_map = 0;
1839 		if ((dimm11_addr_map == 0) || (dimm11_addr_map == 3) || (dimm11_addr_map == 4) || (dimm11_addr_map > 12))
1840 			dimm11_addr_map = 0;
1841 
1842 		/* If DIMM Addr map is 8GB, ROW size should be 2KB. Otherwise 1KB. */
1843 		/* If ROW size(DIMM1) != ROW size(DMIMM0), ROW size should be larger one. */
1844 		if ((dimm00_addr_map == 11) || (dimm01_addr_map == 11) || (dimm10_addr_map == 11) || (dimm11_addr_map == 11))
1845 			adev->gfx.config.mem_row_size_in_kb = 2;
1846 		else
1847 			adev->gfx.config.mem_row_size_in_kb = 1;
1848 	} else {
1849 		tmp = REG_GET_FIELD(mc_arb_ramcfg, MC_ARB_RAMCFG, NOOFCOLS);
1850 		adev->gfx.config.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1851 		if (adev->gfx.config.mem_row_size_in_kb > 4)
1852 			adev->gfx.config.mem_row_size_in_kb = 4;
1853 	}
1854 
1855 	adev->gfx.config.shader_engine_tile_size = 32;
1856 	adev->gfx.config.num_gpus = 1;
1857 	adev->gfx.config.multi_gpu_tile_size = 64;
1858 
1859 	/* fix up row size */
1860 	switch (adev->gfx.config.mem_row_size_in_kb) {
1861 	case 1:
1862 	default:
1863 		gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 0);
1864 		break;
1865 	case 2:
1866 		gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 1);
1867 		break;
1868 	case 4:
1869 		gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 2);
1870 		break;
1871 	}
1872 	adev->gfx.config.gb_addr_config = gb_addr_config;
1873 
1874 	return 0;
1875 }
1876 
1877 static int gfx_v8_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
1878 					int mec, int pipe, int queue)
1879 {
1880 	int r;
1881 	unsigned irq_type;
1882 	struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
1883 
1884 	ring = &adev->gfx.compute_ring[ring_id];
1885 
1886 	/* mec0 is me1 */
1887 	ring->me = mec + 1;
1888 	ring->pipe = pipe;
1889 	ring->queue = queue;
1890 
1891 	ring->ring_obj = NULL;
1892 	ring->use_doorbell = true;
1893 	ring->doorbell_index = adev->doorbell_index.mec_ring0 + ring_id;
1894 	ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
1895 				+ (ring_id * GFX8_MEC_HPD_SIZE);
1896 	sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1897 
1898 	irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
1899 		+ ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
1900 		+ ring->pipe;
1901 
1902 	/* type-2 packets are deprecated on MEC, use type-3 instead */
1903 	r = amdgpu_ring_init(adev, ring, 1024,
1904 			&adev->gfx.eop_irq, irq_type);
1905 	if (r)
1906 		return r;
1907 
1908 
1909 	return 0;
1910 }
1911 
1912 static void gfx_v8_0_sq_irq_work_func(struct work_struct *work);
1913 
1914 static int gfx_v8_0_sw_init(void *handle)
1915 {
1916 	int i, j, k, r, ring_id;
1917 	struct amdgpu_ring *ring;
1918 	struct amdgpu_kiq *kiq;
1919 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1920 
1921 	switch (adev->asic_type) {
1922 	case CHIP_TONGA:
1923 	case CHIP_CARRIZO:
1924 	case CHIP_FIJI:
1925 	case CHIP_POLARIS10:
1926 	case CHIP_POLARIS11:
1927 	case CHIP_POLARIS12:
1928 	case CHIP_VEGAM:
1929 		adev->gfx.mec.num_mec = 2;
1930 		break;
1931 	case CHIP_TOPAZ:
1932 	case CHIP_STONEY:
1933 	default:
1934 		adev->gfx.mec.num_mec = 1;
1935 		break;
1936 	}
1937 
1938 	adev->gfx.mec.num_pipe_per_mec = 4;
1939 	adev->gfx.mec.num_queue_per_pipe = 8;
1940 
1941 	/* EOP Event */
1942 	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_END_OF_PIPE, &adev->gfx.eop_irq);
1943 	if (r)
1944 		return r;
1945 
1946 	/* Privileged reg */
1947 	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_REG_FAULT,
1948 			      &adev->gfx.priv_reg_irq);
1949 	if (r)
1950 		return r;
1951 
1952 	/* Privileged inst */
1953 	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_INSTR_FAULT,
1954 			      &adev->gfx.priv_inst_irq);
1955 	if (r)
1956 		return r;
1957 
1958 	/* Add CP EDC/ECC irq  */
1959 	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_ECC_ERROR,
1960 			      &adev->gfx.cp_ecc_error_irq);
1961 	if (r)
1962 		return r;
1963 
1964 	/* SQ interrupts. */
1965 	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_SQ_INTERRUPT_MSG,
1966 			      &adev->gfx.sq_irq);
1967 	if (r) {
1968 		DRM_ERROR("amdgpu_irq_add() for SQ failed: %d\n", r);
1969 		return r;
1970 	}
1971 
1972 	INIT_WORK(&adev->gfx.sq_work.work, gfx_v8_0_sq_irq_work_func);
1973 
1974 	adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1975 
1976 	gfx_v8_0_scratch_init(adev);
1977 
1978 	r = gfx_v8_0_init_microcode(adev);
1979 	if (r) {
1980 		DRM_ERROR("Failed to load gfx firmware!\n");
1981 		return r;
1982 	}
1983 
1984 	r = adev->gfx.rlc.funcs->init(adev);
1985 	if (r) {
1986 		DRM_ERROR("Failed to init rlc BOs!\n");
1987 		return r;
1988 	}
1989 
1990 	r = gfx_v8_0_mec_init(adev);
1991 	if (r) {
1992 		DRM_ERROR("Failed to init MEC BOs!\n");
1993 		return r;
1994 	}
1995 
1996 	/* set up the gfx ring */
1997 	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
1998 		ring = &adev->gfx.gfx_ring[i];
1999 		ring->ring_obj = NULL;
2000 		sprintf(ring->name, "gfx");
2001 		/* no gfx doorbells on iceland */
2002 		if (adev->asic_type != CHIP_TOPAZ) {
2003 			ring->use_doorbell = true;
2004 			ring->doorbell_index = adev->doorbell_index.gfx_ring0;
2005 		}
2006 
2007 		r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2008 				     AMDGPU_CP_IRQ_GFX_EOP);
2009 		if (r)
2010 			return r;
2011 	}
2012 
2013 
2014 	/* set up the compute queues - allocate horizontally across pipes */
2015 	ring_id = 0;
2016 	for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2017 		for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2018 			for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2019 				if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j))
2020 					continue;
2021 
2022 				r = gfx_v8_0_compute_ring_init(adev,
2023 								ring_id,
2024 								i, k, j);
2025 				if (r)
2026 					return r;
2027 
2028 				ring_id++;
2029 			}
2030 		}
2031 	}
2032 
2033 	r = amdgpu_gfx_kiq_init(adev, GFX8_MEC_HPD_SIZE);
2034 	if (r) {
2035 		DRM_ERROR("Failed to init KIQ BOs!\n");
2036 		return r;
2037 	}
2038 
2039 	kiq = &adev->gfx.kiq;
2040 	r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
2041 	if (r)
2042 		return r;
2043 
2044 	/* create MQD for all compute queues as well as KIQ for SRIOV case */
2045 	r = amdgpu_gfx_compute_mqd_sw_init(adev, sizeof(struct vi_mqd_allocation));
2046 	if (r)
2047 		return r;
2048 
2049 	adev->gfx.ce_ram_size = 0x8000;
2050 
2051 	r = gfx_v8_0_gpu_early_init(adev);
2052 	if (r)
2053 		return r;
2054 
2055 	return 0;
2056 }
2057 
2058 static int gfx_v8_0_sw_fini(void *handle)
2059 {
2060 	int i;
2061 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2062 
2063 	amdgpu_bo_free_kernel(&adev->gds.oa_gfx_bo, NULL, NULL);
2064 	amdgpu_bo_free_kernel(&adev->gds.gws_gfx_bo, NULL, NULL);
2065 	amdgpu_bo_free_kernel(&adev->gds.gds_gfx_bo, NULL, NULL);
2066 
2067 	for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2068 		amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2069 	for (i = 0; i < adev->gfx.num_compute_rings; i++)
2070 		amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2071 
2072 	amdgpu_gfx_compute_mqd_sw_fini(adev);
2073 	amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
2074 	amdgpu_gfx_kiq_fini(adev);
2075 
2076 	gfx_v8_0_mec_fini(adev);
2077 	amdgpu_gfx_rlc_fini(adev);
2078 	amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
2079 				&adev->gfx.rlc.clear_state_gpu_addr,
2080 				(void **)&adev->gfx.rlc.cs_ptr);
2081 	if ((adev->asic_type == CHIP_CARRIZO) ||
2082 	    (adev->asic_type == CHIP_STONEY)) {
2083 		amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2084 				&adev->gfx.rlc.cp_table_gpu_addr,
2085 				(void **)&adev->gfx.rlc.cp_table_ptr);
2086 	}
2087 	gfx_v8_0_free_microcode(adev);
2088 
2089 	return 0;
2090 }
2091 
2092 static void gfx_v8_0_tiling_mode_table_init(struct amdgpu_device *adev)
2093 {
2094 	uint32_t *modearray, *mod2array;
2095 	const u32 num_tile_mode_states = ARRAY_SIZE(adev->gfx.config.tile_mode_array);
2096 	const u32 num_secondary_tile_mode_states = ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
2097 	u32 reg_offset;
2098 
2099 	modearray = adev->gfx.config.tile_mode_array;
2100 	mod2array = adev->gfx.config.macrotile_mode_array;
2101 
2102 	for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2103 		modearray[reg_offset] = 0;
2104 
2105 	for (reg_offset = 0; reg_offset <  num_secondary_tile_mode_states; reg_offset++)
2106 		mod2array[reg_offset] = 0;
2107 
2108 	switch (adev->asic_type) {
2109 	case CHIP_TOPAZ:
2110 		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2111 				PIPE_CONFIG(ADDR_SURF_P2) |
2112 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2113 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2114 		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2115 				PIPE_CONFIG(ADDR_SURF_P2) |
2116 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2117 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2118 		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2119 				PIPE_CONFIG(ADDR_SURF_P2) |
2120 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2121 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2122 		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2123 				PIPE_CONFIG(ADDR_SURF_P2) |
2124 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2125 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2126 		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2127 				PIPE_CONFIG(ADDR_SURF_P2) |
2128 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2129 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2130 		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2131 				PIPE_CONFIG(ADDR_SURF_P2) |
2132 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2133 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2134 		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2135 				PIPE_CONFIG(ADDR_SURF_P2) |
2136 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2137 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2138 		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2139 				PIPE_CONFIG(ADDR_SURF_P2));
2140 		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2141 				PIPE_CONFIG(ADDR_SURF_P2) |
2142 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2143 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2144 		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2145 				 PIPE_CONFIG(ADDR_SURF_P2) |
2146 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2147 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2148 		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2149 				 PIPE_CONFIG(ADDR_SURF_P2) |
2150 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2151 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2152 		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2153 				 PIPE_CONFIG(ADDR_SURF_P2) |
2154 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2155 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2156 		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2157 				 PIPE_CONFIG(ADDR_SURF_P2) |
2158 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2159 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2160 		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2161 				 PIPE_CONFIG(ADDR_SURF_P2) |
2162 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2163 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2164 		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2165 				 PIPE_CONFIG(ADDR_SURF_P2) |
2166 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2167 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2168 		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2169 				 PIPE_CONFIG(ADDR_SURF_P2) |
2170 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2171 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2172 		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2173 				 PIPE_CONFIG(ADDR_SURF_P2) |
2174 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2175 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2176 		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2177 				 PIPE_CONFIG(ADDR_SURF_P2) |
2178 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2179 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2180 		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2181 				 PIPE_CONFIG(ADDR_SURF_P2) |
2182 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2183 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2184 		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2185 				 PIPE_CONFIG(ADDR_SURF_P2) |
2186 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2187 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2188 		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2189 				 PIPE_CONFIG(ADDR_SURF_P2) |
2190 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2191 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2192 		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2193 				 PIPE_CONFIG(ADDR_SURF_P2) |
2194 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2195 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2196 		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2197 				 PIPE_CONFIG(ADDR_SURF_P2) |
2198 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2199 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2200 		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2201 				 PIPE_CONFIG(ADDR_SURF_P2) |
2202 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2203 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2204 		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2205 				 PIPE_CONFIG(ADDR_SURF_P2) |
2206 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2207 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2208 		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2209 				 PIPE_CONFIG(ADDR_SURF_P2) |
2210 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2211 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2212 
2213 		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2214 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2215 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2216 				NUM_BANKS(ADDR_SURF_8_BANK));
2217 		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2218 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2219 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2220 				NUM_BANKS(ADDR_SURF_8_BANK));
2221 		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2222 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2223 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2224 				NUM_BANKS(ADDR_SURF_8_BANK));
2225 		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2226 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2227 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2228 				NUM_BANKS(ADDR_SURF_8_BANK));
2229 		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2230 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2231 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2232 				NUM_BANKS(ADDR_SURF_8_BANK));
2233 		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2234 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2235 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2236 				NUM_BANKS(ADDR_SURF_8_BANK));
2237 		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2238 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2239 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2240 				NUM_BANKS(ADDR_SURF_8_BANK));
2241 		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2242 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2243 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2244 				NUM_BANKS(ADDR_SURF_16_BANK));
2245 		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2246 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2247 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2248 				NUM_BANKS(ADDR_SURF_16_BANK));
2249 		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2250 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2251 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2252 				 NUM_BANKS(ADDR_SURF_16_BANK));
2253 		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2254 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2255 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2256 				 NUM_BANKS(ADDR_SURF_16_BANK));
2257 		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2258 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2259 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2260 				 NUM_BANKS(ADDR_SURF_16_BANK));
2261 		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2262 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2263 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2264 				 NUM_BANKS(ADDR_SURF_16_BANK));
2265 		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2266 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2267 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2268 				 NUM_BANKS(ADDR_SURF_8_BANK));
2269 
2270 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2271 			if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&
2272 			    reg_offset != 23)
2273 				WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2274 
2275 		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2276 			if (reg_offset != 7)
2277 				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2278 
2279 		break;
2280 	case CHIP_FIJI:
2281 	case CHIP_VEGAM:
2282 		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2283 				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2284 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2285 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2286 		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2287 				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2288 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2289 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2290 		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2291 				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2292 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2293 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2294 		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2295 				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2296 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2297 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2298 		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2299 				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2300 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2301 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2302 		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2303 				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2304 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2305 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2306 		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2307 				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2308 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2309 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2310 		modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2311 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2312 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2313 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2314 		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2315 				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2316 		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2317 				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2318 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2319 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2320 		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2321 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2322 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2323 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2324 		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2325 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2326 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2327 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2328 		modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2329 				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2330 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2331 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2332 		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2333 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2334 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2335 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2336 		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2337 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2338 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2339 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2340 		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2341 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2342 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2343 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2344 		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2345 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2346 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2347 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2348 		modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2349 				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2350 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2351 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2352 		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2353 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2354 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2355 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2356 		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2357 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2358 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2359 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2360 		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2361 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2362 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2363 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2364 		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2365 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2366 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2367 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2368 		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2369 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2370 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2371 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2372 		modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2373 				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2374 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2375 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2376 		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2377 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2378 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2379 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2380 		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2381 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2382 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2383 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2384 		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2385 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2386 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2387 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2388 		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2389 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2390 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2391 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2392 		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2393 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2394 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2395 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2396 		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2397 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2398 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2399 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2400 		modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2401 				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2402 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2403 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2404 
2405 		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2406 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2407 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2408 				NUM_BANKS(ADDR_SURF_8_BANK));
2409 		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2410 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2411 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2412 				NUM_BANKS(ADDR_SURF_8_BANK));
2413 		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2414 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2415 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2416 				NUM_BANKS(ADDR_SURF_8_BANK));
2417 		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2418 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2419 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2420 				NUM_BANKS(ADDR_SURF_8_BANK));
2421 		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2422 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2423 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2424 				NUM_BANKS(ADDR_SURF_8_BANK));
2425 		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2426 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2427 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2428 				NUM_BANKS(ADDR_SURF_8_BANK));
2429 		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2430 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2431 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2432 				NUM_BANKS(ADDR_SURF_8_BANK));
2433 		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2434 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2435 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2436 				NUM_BANKS(ADDR_SURF_8_BANK));
2437 		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2438 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2439 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2440 				NUM_BANKS(ADDR_SURF_8_BANK));
2441 		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2442 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2443 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2444 				 NUM_BANKS(ADDR_SURF_8_BANK));
2445 		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2446 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2447 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2448 				 NUM_BANKS(ADDR_SURF_8_BANK));
2449 		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2450 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2451 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2452 				 NUM_BANKS(ADDR_SURF_8_BANK));
2453 		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2454 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2455 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2456 				 NUM_BANKS(ADDR_SURF_8_BANK));
2457 		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2458 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2459 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2460 				 NUM_BANKS(ADDR_SURF_4_BANK));
2461 
2462 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2463 			WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2464 
2465 		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2466 			if (reg_offset != 7)
2467 				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2468 
2469 		break;
2470 	case CHIP_TONGA:
2471 		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2472 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2473 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2474 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2475 		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2476 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2477 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2478 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2479 		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2480 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2481 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2482 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2483 		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2484 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2485 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2486 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2487 		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2488 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2489 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2490 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2491 		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2492 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2493 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2494 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2495 		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2496 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2497 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2498 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2499 		modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2500 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2501 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2502 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2503 		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2504 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2505 		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2506 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2507 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2508 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2509 		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2510 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2511 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2512 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2513 		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2514 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2515 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2516 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2517 		modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2518 				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2519 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2520 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2521 		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2522 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2523 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2524 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2525 		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2526 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2527 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2528 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2529 		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2530 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2531 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2532 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2533 		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2534 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2535 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2536 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2537 		modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2538 				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2539 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2540 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2541 		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2542 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2543 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2544 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2545 		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2546 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2547 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2548 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2549 		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2550 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2551 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2552 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2553 		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2554 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2555 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2556 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2557 		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2558 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2559 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2560 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2561 		modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2562 				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2563 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2564 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2565 		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2566 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2567 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2568 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2569 		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2570 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2571 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2572 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2573 		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2574 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2575 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2576 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2577 		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2578 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2579 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2580 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2581 		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2582 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2583 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2584 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2585 		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2586 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2587 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2588 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2589 		modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2590 				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2591 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2592 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2593 
2594 		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2595 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2596 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2597 				NUM_BANKS(ADDR_SURF_16_BANK));
2598 		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2599 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2600 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2601 				NUM_BANKS(ADDR_SURF_16_BANK));
2602 		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2603 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2604 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2605 				NUM_BANKS(ADDR_SURF_16_BANK));
2606 		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2607 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2608 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2609 				NUM_BANKS(ADDR_SURF_16_BANK));
2610 		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2611 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2612 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2613 				NUM_BANKS(ADDR_SURF_16_BANK));
2614 		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2615 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2616 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2617 				NUM_BANKS(ADDR_SURF_16_BANK));
2618 		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2619 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2620 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2621 				NUM_BANKS(ADDR_SURF_16_BANK));
2622 		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2623 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2624 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2625 				NUM_BANKS(ADDR_SURF_16_BANK));
2626 		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2627 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2628 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2629 				NUM_BANKS(ADDR_SURF_16_BANK));
2630 		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2631 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2632 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2633 				 NUM_BANKS(ADDR_SURF_16_BANK));
2634 		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2635 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2636 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2637 				 NUM_BANKS(ADDR_SURF_16_BANK));
2638 		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2639 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2640 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2641 				 NUM_BANKS(ADDR_SURF_8_BANK));
2642 		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2643 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2644 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2645 				 NUM_BANKS(ADDR_SURF_4_BANK));
2646 		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2647 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2648 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2649 				 NUM_BANKS(ADDR_SURF_4_BANK));
2650 
2651 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2652 			WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2653 
2654 		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2655 			if (reg_offset != 7)
2656 				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2657 
2658 		break;
2659 	case CHIP_POLARIS11:
2660 	case CHIP_POLARIS12:
2661 		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2662 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2663 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2664 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2665 		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2666 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2667 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2668 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2669 		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2670 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2671 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2672 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2673 		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2674 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2675 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2676 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2677 		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2678 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2679 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2680 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2681 		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2682 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2683 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2684 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2685 		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2686 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2687 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2688 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2689 		modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2690 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2691 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2692 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2693 		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2694 				PIPE_CONFIG(ADDR_SURF_P4_16x16));
2695 		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2696 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2697 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2698 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2699 		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2700 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2701 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2702 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2703 		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2704 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2705 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2706 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2707 		modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2708 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2709 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2710 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2711 		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2712 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2713 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2714 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2715 		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2716 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2717 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2718 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2719 		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2720 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2721 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2722 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2723 		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2724 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2725 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2726 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2727 		modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2728 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2729 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2730 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2731 		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2732 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2733 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2734 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2735 		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2736 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2737 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2738 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2739 		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2740 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2741 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2742 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2743 		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2744 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2745 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2746 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2747 		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2748 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2749 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2750 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2751 		modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2752 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2753 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2754 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2755 		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2756 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2757 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2758 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2759 		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2760 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2761 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2762 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2763 		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2764 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2765 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2766 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2767 		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2768 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2769 				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2770 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2771 		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2772 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2773 				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2774 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2775 		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2776 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2777 				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2778 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2779 		modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2780 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2781 				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2782 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2783 
2784 		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2785 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2786 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2787 				NUM_BANKS(ADDR_SURF_16_BANK));
2788 
2789 		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2790 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2791 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2792 				NUM_BANKS(ADDR_SURF_16_BANK));
2793 
2794 		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2795 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2796 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2797 				NUM_BANKS(ADDR_SURF_16_BANK));
2798 
2799 		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2800 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2801 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2802 				NUM_BANKS(ADDR_SURF_16_BANK));
2803 
2804 		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2805 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2806 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2807 				NUM_BANKS(ADDR_SURF_16_BANK));
2808 
2809 		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2810 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2811 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2812 				NUM_BANKS(ADDR_SURF_16_BANK));
2813 
2814 		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2815 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2816 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2817 				NUM_BANKS(ADDR_SURF_16_BANK));
2818 
2819 		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2820 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2821 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2822 				NUM_BANKS(ADDR_SURF_16_BANK));
2823 
2824 		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2825 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2826 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2827 				NUM_BANKS(ADDR_SURF_16_BANK));
2828 
2829 		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2830 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2831 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2832 				NUM_BANKS(ADDR_SURF_16_BANK));
2833 
2834 		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2835 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2836 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2837 				NUM_BANKS(ADDR_SURF_16_BANK));
2838 
2839 		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2840 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2841 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2842 				NUM_BANKS(ADDR_SURF_16_BANK));
2843 
2844 		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2845 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2846 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2847 				NUM_BANKS(ADDR_SURF_8_BANK));
2848 
2849 		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2850 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2851 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2852 				NUM_BANKS(ADDR_SURF_4_BANK));
2853 
2854 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2855 			WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2856 
2857 		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2858 			if (reg_offset != 7)
2859 				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2860 
2861 		break;
2862 	case CHIP_POLARIS10:
2863 		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2864 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2865 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2866 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2867 		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2868 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2869 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2870 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2871 		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2872 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2873 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2874 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2875 		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2876 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2877 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2878 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2879 		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2880 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2881 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2882 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2883 		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2884 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2885 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2886 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2887 		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2888 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2889 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2890 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2891 		modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2892 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2893 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2894 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2895 		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2896 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2897 		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2898 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2899 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2900 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2901 		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2902 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2903 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2904 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2905 		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2906 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2907 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2908 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2909 		modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2910 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2911 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2912 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2913 		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2914 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2915 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2916 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2917 		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2918 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2919 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2920 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2921 		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2922 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2923 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2924 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2925 		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2926 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2927 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2928 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2929 		modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2930 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2931 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2932 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2933 		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2934 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2935 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2936 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2937 		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2938 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2939 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2940 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2941 		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2942 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2943 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2944 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2945 		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2946 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2947 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2948 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2949 		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2950 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2951 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2952 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2953 		modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2954 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2955 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2956 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2957 		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2958 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2959 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2960 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2961 		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2962 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2963 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2964 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2965 		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2966 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2967 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2968 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2969 		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2970 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2971 				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2972 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2973 		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2974 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2975 				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2976 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2977 		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2978 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2979 				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2980 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2981 		modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2982 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2983 				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2984 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2985 
2986 		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2987 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2988 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2989 				NUM_BANKS(ADDR_SURF_16_BANK));
2990 
2991 		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2992 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2993 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2994 				NUM_BANKS(ADDR_SURF_16_BANK));
2995 
2996 		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2997 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2998 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2999 				NUM_BANKS(ADDR_SURF_16_BANK));
3000 
3001 		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3002 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3003 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3004 				NUM_BANKS(ADDR_SURF_16_BANK));
3005 
3006 		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3007 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3008 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3009 				NUM_BANKS(ADDR_SURF_16_BANK));
3010 
3011 		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3012 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3013 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3014 				NUM_BANKS(ADDR_SURF_16_BANK));
3015 
3016 		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3017 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3018 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3019 				NUM_BANKS(ADDR_SURF_16_BANK));
3020 
3021 		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3022 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3023 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3024 				NUM_BANKS(ADDR_SURF_16_BANK));
3025 
3026 		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3027 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3028 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3029 				NUM_BANKS(ADDR_SURF_16_BANK));
3030 
3031 		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3032 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3033 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3034 				NUM_BANKS(ADDR_SURF_16_BANK));
3035 
3036 		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3037 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3038 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3039 				NUM_BANKS(ADDR_SURF_16_BANK));
3040 
3041 		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3042 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3043 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3044 				NUM_BANKS(ADDR_SURF_8_BANK));
3045 
3046 		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3047 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3048 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3049 				NUM_BANKS(ADDR_SURF_4_BANK));
3050 
3051 		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3052 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3053 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3054 				NUM_BANKS(ADDR_SURF_4_BANK));
3055 
3056 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3057 			WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
3058 
3059 		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3060 			if (reg_offset != 7)
3061 				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
3062 
3063 		break;
3064 	case CHIP_STONEY:
3065 		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3066 				PIPE_CONFIG(ADDR_SURF_P2) |
3067 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
3068 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3069 		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3070 				PIPE_CONFIG(ADDR_SURF_P2) |
3071 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
3072 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3073 		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3074 				PIPE_CONFIG(ADDR_SURF_P2) |
3075 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
3076 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3077 		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3078 				PIPE_CONFIG(ADDR_SURF_P2) |
3079 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
3080 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3081 		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3082 				PIPE_CONFIG(ADDR_SURF_P2) |
3083 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3084 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3085 		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3086 				PIPE_CONFIG(ADDR_SURF_P2) |
3087 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3088 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3089 		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3090 				PIPE_CONFIG(ADDR_SURF_P2) |
3091 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3092 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3093 		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3094 				PIPE_CONFIG(ADDR_SURF_P2));
3095 		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3096 				PIPE_CONFIG(ADDR_SURF_P2) |
3097 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3098 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3099 		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3100 				 PIPE_CONFIG(ADDR_SURF_P2) |
3101 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3102 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3103 		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3104 				 PIPE_CONFIG(ADDR_SURF_P2) |
3105 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3106 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3107 		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3108 				 PIPE_CONFIG(ADDR_SURF_P2) |
3109 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3110 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3111 		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3112 				 PIPE_CONFIG(ADDR_SURF_P2) |
3113 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3114 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3115 		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
3116 				 PIPE_CONFIG(ADDR_SURF_P2) |
3117 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3118 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3119 		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3120 				 PIPE_CONFIG(ADDR_SURF_P2) |
3121 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3122 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3123 		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3124 				 PIPE_CONFIG(ADDR_SURF_P2) |
3125 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3126 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3127 		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3128 				 PIPE_CONFIG(ADDR_SURF_P2) |
3129 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3130 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3131 		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3132 				 PIPE_CONFIG(ADDR_SURF_P2) |
3133 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3134 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3135 		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
3136 				 PIPE_CONFIG(ADDR_SURF_P2) |
3137 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3138 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3139 		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
3140 				 PIPE_CONFIG(ADDR_SURF_P2) |
3141 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3142 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3143 		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3144 				 PIPE_CONFIG(ADDR_SURF_P2) |
3145 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3146 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3147 		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
3148 				 PIPE_CONFIG(ADDR_SURF_P2) |
3149 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3150 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3151 		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
3152 				 PIPE_CONFIG(ADDR_SURF_P2) |
3153 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3154 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3155 		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3156 				 PIPE_CONFIG(ADDR_SURF_P2) |
3157 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3158 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3159 		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3160 				 PIPE_CONFIG(ADDR_SURF_P2) |
3161 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3162 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3163 		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3164 				 PIPE_CONFIG(ADDR_SURF_P2) |
3165 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3166 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3167 
3168 		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3169 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3170 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3171 				NUM_BANKS(ADDR_SURF_8_BANK));
3172 		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3173 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3174 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3175 				NUM_BANKS(ADDR_SURF_8_BANK));
3176 		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3177 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3178 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3179 				NUM_BANKS(ADDR_SURF_8_BANK));
3180 		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3181 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3182 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3183 				NUM_BANKS(ADDR_SURF_8_BANK));
3184 		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3185 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3186 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3187 				NUM_BANKS(ADDR_SURF_8_BANK));
3188 		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3189 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3190 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3191 				NUM_BANKS(ADDR_SURF_8_BANK));
3192 		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3193 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3194 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3195 				NUM_BANKS(ADDR_SURF_8_BANK));
3196 		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3197 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3198 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3199 				NUM_BANKS(ADDR_SURF_16_BANK));
3200 		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3201 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3202 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3203 				NUM_BANKS(ADDR_SURF_16_BANK));
3204 		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3205 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3206 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3207 				 NUM_BANKS(ADDR_SURF_16_BANK));
3208 		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3209 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3210 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3211 				 NUM_BANKS(ADDR_SURF_16_BANK));
3212 		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3213 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3214 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3215 				 NUM_BANKS(ADDR_SURF_16_BANK));
3216 		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3217 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3218 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3219 				 NUM_BANKS(ADDR_SURF_16_BANK));
3220 		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3221 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3222 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3223 				 NUM_BANKS(ADDR_SURF_8_BANK));
3224 
3225 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3226 			if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&
3227 			    reg_offset != 23)
3228 				WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
3229 
3230 		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3231 			if (reg_offset != 7)
3232 				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
3233 
3234 		break;
3235 	default:
3236 		dev_warn(adev->dev,
3237 			 "Unknown chip type (%d) in function gfx_v8_0_tiling_mode_table_init() falling through to CHIP_CARRIZO\n",
3238 			 adev->asic_type);
3239 
3240 	case CHIP_CARRIZO:
3241 		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3242 				PIPE_CONFIG(ADDR_SURF_P2) |
3243 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
3244 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3245 		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3246 				PIPE_CONFIG(ADDR_SURF_P2) |
3247 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
3248 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3249 		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3250 				PIPE_CONFIG(ADDR_SURF_P2) |
3251 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
3252 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3253 		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3254 				PIPE_CONFIG(ADDR_SURF_P2) |
3255 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
3256 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3257 		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3258 				PIPE_CONFIG(ADDR_SURF_P2) |
3259 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3260 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3261 		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3262 				PIPE_CONFIG(ADDR_SURF_P2) |
3263 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3264 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3265 		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3266 				PIPE_CONFIG(ADDR_SURF_P2) |
3267 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3268 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3269 		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3270 				PIPE_CONFIG(ADDR_SURF_P2));
3271 		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3272 				PIPE_CONFIG(ADDR_SURF_P2) |
3273 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3274 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3275 		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3276 				 PIPE_CONFIG(ADDR_SURF_P2) |
3277 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3278 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3279 		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3280 				 PIPE_CONFIG(ADDR_SURF_P2) |
3281 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3282 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3283 		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3284 				 PIPE_CONFIG(ADDR_SURF_P2) |
3285 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3286 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3287 		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3288 				 PIPE_CONFIG(ADDR_SURF_P2) |
3289 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3290 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3291 		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
3292 				 PIPE_CONFIG(ADDR_SURF_P2) |
3293 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3294 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3295 		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3296 				 PIPE_CONFIG(ADDR_SURF_P2) |
3297 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3298 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3299 		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3300 				 PIPE_CONFIG(ADDR_SURF_P2) |
3301 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3302 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3303 		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3304 				 PIPE_CONFIG(ADDR_SURF_P2) |
3305 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3306 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3307 		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3308 				 PIPE_CONFIG(ADDR_SURF_P2) |
3309 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3310 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3311 		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
3312 				 PIPE_CONFIG(ADDR_SURF_P2) |
3313 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3314 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3315 		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
3316 				 PIPE_CONFIG(ADDR_SURF_P2) |
3317 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3318 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3319 		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3320 				 PIPE_CONFIG(ADDR_SURF_P2) |
3321 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3322 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3323 		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
3324 				 PIPE_CONFIG(ADDR_SURF_P2) |
3325 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3326 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3327 		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
3328 				 PIPE_CONFIG(ADDR_SURF_P2) |
3329 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3330 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3331 		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3332 				 PIPE_CONFIG(ADDR_SURF_P2) |
3333 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3334 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3335 		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3336 				 PIPE_CONFIG(ADDR_SURF_P2) |
3337 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3338 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3339 		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3340 				 PIPE_CONFIG(ADDR_SURF_P2) |
3341 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3342 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3343 
3344 		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3345 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3346 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3347 				NUM_BANKS(ADDR_SURF_8_BANK));
3348 		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3349 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3350 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3351 				NUM_BANKS(ADDR_SURF_8_BANK));
3352 		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3353 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3354 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3355 				NUM_BANKS(ADDR_SURF_8_BANK));
3356 		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3357 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3358 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3359 				NUM_BANKS(ADDR_SURF_8_BANK));
3360 		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3361 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3362 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3363 				NUM_BANKS(ADDR_SURF_8_BANK));
3364 		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3365 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3366 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3367 				NUM_BANKS(ADDR_SURF_8_BANK));
3368 		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3369 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3370 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3371 				NUM_BANKS(ADDR_SURF_8_BANK));
3372 		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3373 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3374 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3375 				NUM_BANKS(ADDR_SURF_16_BANK));
3376 		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3377 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3378 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3379 				NUM_BANKS(ADDR_SURF_16_BANK));
3380 		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3381 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3382 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3383 				 NUM_BANKS(ADDR_SURF_16_BANK));
3384 		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3385 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3386 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3387 				 NUM_BANKS(ADDR_SURF_16_BANK));
3388 		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3389 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3390 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3391 				 NUM_BANKS(ADDR_SURF_16_BANK));
3392 		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3393 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3394 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3395 				 NUM_BANKS(ADDR_SURF_16_BANK));
3396 		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3397 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3398 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3399 				 NUM_BANKS(ADDR_SURF_8_BANK));
3400 
3401 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3402 			if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&
3403 			    reg_offset != 23)
3404 				WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
3405 
3406 		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3407 			if (reg_offset != 7)
3408 				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
3409 
3410 		break;
3411 	}
3412 }
3413 
3414 static void gfx_v8_0_select_se_sh(struct amdgpu_device *adev,
3415 				  u32 se_num, u32 sh_num, u32 instance)
3416 {
3417 	u32 data;
3418 
3419 	if (instance == 0xffffffff)
3420 		data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
3421 	else
3422 		data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
3423 
3424 	if (se_num == 0xffffffff)
3425 		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
3426 	else
3427 		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
3428 
3429 	if (sh_num == 0xffffffff)
3430 		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
3431 	else
3432 		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
3433 
3434 	WREG32(mmGRBM_GFX_INDEX, data);
3435 }
3436 
3437 static void gfx_v8_0_select_me_pipe_q(struct amdgpu_device *adev,
3438 				  u32 me, u32 pipe, u32 q)
3439 {
3440 	vi_srbm_select(adev, me, pipe, q, 0);
3441 }
3442 
3443 static u32 gfx_v8_0_get_rb_active_bitmap(struct amdgpu_device *adev)
3444 {
3445 	u32 data, mask;
3446 
3447 	data =  RREG32(mmCC_RB_BACKEND_DISABLE) |
3448 		RREG32(mmGC_USER_RB_BACKEND_DISABLE);
3449 
3450 	data = REG_GET_FIELD(data, GC_USER_RB_BACKEND_DISABLE, BACKEND_DISABLE);
3451 
3452 	mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
3453 					 adev->gfx.config.max_sh_per_se);
3454 
3455 	return (~data) & mask;
3456 }
3457 
3458 static void
3459 gfx_v8_0_raster_config(struct amdgpu_device *adev, u32 *rconf, u32 *rconf1)
3460 {
3461 	switch (adev->asic_type) {
3462 	case CHIP_FIJI:
3463 	case CHIP_VEGAM:
3464 		*rconf |= RB_MAP_PKR0(2) | RB_MAP_PKR1(2) |
3465 			  RB_XSEL2(1) | PKR_MAP(2) |
3466 			  PKR_XSEL(1) | PKR_YSEL(1) |
3467 			  SE_MAP(2) | SE_XSEL(2) | SE_YSEL(3);
3468 		*rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(3) |
3469 			   SE_PAIR_YSEL(2);
3470 		break;
3471 	case CHIP_TONGA:
3472 	case CHIP_POLARIS10:
3473 		*rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) |
3474 			  SE_XSEL(1) | SE_YSEL(1);
3475 		*rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(2) |
3476 			   SE_PAIR_YSEL(2);
3477 		break;
3478 	case CHIP_TOPAZ:
3479 	case CHIP_CARRIZO:
3480 		*rconf |= RB_MAP_PKR0(2);
3481 		*rconf1 |= 0x0;
3482 		break;
3483 	case CHIP_POLARIS11:
3484 	case CHIP_POLARIS12:
3485 		*rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) |
3486 			  SE_XSEL(1) | SE_YSEL(1);
3487 		*rconf1 |= 0x0;
3488 		break;
3489 	case CHIP_STONEY:
3490 		*rconf |= 0x0;
3491 		*rconf1 |= 0x0;
3492 		break;
3493 	default:
3494 		DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type);
3495 		break;
3496 	}
3497 }
3498 
3499 static void
3500 gfx_v8_0_write_harvested_raster_configs(struct amdgpu_device *adev,
3501 					u32 raster_config, u32 raster_config_1,
3502 					unsigned rb_mask, unsigned num_rb)
3503 {
3504 	unsigned sh_per_se = max_t(unsigned, adev->gfx.config.max_sh_per_se, 1);
3505 	unsigned num_se = max_t(unsigned, adev->gfx.config.max_shader_engines, 1);
3506 	unsigned rb_per_pkr = min_t(unsigned, num_rb / num_se / sh_per_se, 2);
3507 	unsigned rb_per_se = num_rb / num_se;
3508 	unsigned se_mask[4];
3509 	unsigned se;
3510 
3511 	se_mask[0] = ((1 << rb_per_se) - 1) & rb_mask;
3512 	se_mask[1] = (se_mask[0] << rb_per_se) & rb_mask;
3513 	se_mask[2] = (se_mask[1] << rb_per_se) & rb_mask;
3514 	se_mask[3] = (se_mask[2] << rb_per_se) & rb_mask;
3515 
3516 	WARN_ON(!(num_se == 1 || num_se == 2 || num_se == 4));
3517 	WARN_ON(!(sh_per_se == 1 || sh_per_se == 2));
3518 	WARN_ON(!(rb_per_pkr == 1 || rb_per_pkr == 2));
3519 
3520 	if ((num_se > 2) && ((!se_mask[0] && !se_mask[1]) ||
3521 			     (!se_mask[2] && !se_mask[3]))) {
3522 		raster_config_1 &= ~SE_PAIR_MAP_MASK;
3523 
3524 		if (!se_mask[0] && !se_mask[1]) {
3525 			raster_config_1 |=
3526 				SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_3);
3527 		} else {
3528 			raster_config_1 |=
3529 				SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_0);
3530 		}
3531 	}
3532 
3533 	for (se = 0; se < num_se; se++) {
3534 		unsigned raster_config_se = raster_config;
3535 		unsigned pkr0_mask = ((1 << rb_per_pkr) - 1) << (se * rb_per_se);
3536 		unsigned pkr1_mask = pkr0_mask << rb_per_pkr;
3537 		int idx = (se / 2) * 2;
3538 
3539 		if ((num_se > 1) && (!se_mask[idx] || !se_mask[idx + 1])) {
3540 			raster_config_se &= ~SE_MAP_MASK;
3541 
3542 			if (!se_mask[idx]) {
3543 				raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_3);
3544 			} else {
3545 				raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_0);
3546 			}
3547 		}
3548 
3549 		pkr0_mask &= rb_mask;
3550 		pkr1_mask &= rb_mask;
3551 		if (rb_per_se > 2 && (!pkr0_mask || !pkr1_mask)) {
3552 			raster_config_se &= ~PKR_MAP_MASK;
3553 
3554 			if (!pkr0_mask) {
3555 				raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_3);
3556 			} else {
3557 				raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_0);
3558 			}
3559 		}
3560 
3561 		if (rb_per_se >= 2) {
3562 			unsigned rb0_mask = 1 << (se * rb_per_se);
3563 			unsigned rb1_mask = rb0_mask << 1;
3564 
3565 			rb0_mask &= rb_mask;
3566 			rb1_mask &= rb_mask;
3567 			if (!rb0_mask || !rb1_mask) {
3568 				raster_config_se &= ~RB_MAP_PKR0_MASK;
3569 
3570 				if (!rb0_mask) {
3571 					raster_config_se |=
3572 						RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_3);
3573 				} else {
3574 					raster_config_se |=
3575 						RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_0);
3576 				}
3577 			}
3578 
3579 			if (rb_per_se > 2) {
3580 				rb0_mask = 1 << (se * rb_per_se + rb_per_pkr);
3581 				rb1_mask = rb0_mask << 1;
3582 				rb0_mask &= rb_mask;
3583 				rb1_mask &= rb_mask;
3584 				if (!rb0_mask || !rb1_mask) {
3585 					raster_config_se &= ~RB_MAP_PKR1_MASK;
3586 
3587 					if (!rb0_mask) {
3588 						raster_config_se |=
3589 							RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_3);
3590 					} else {
3591 						raster_config_se |=
3592 							RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_0);
3593 					}
3594 				}
3595 			}
3596 		}
3597 
3598 		/* GRBM_GFX_INDEX has a different offset on VI */
3599 		gfx_v8_0_select_se_sh(adev, se, 0xffffffff, 0xffffffff);
3600 		WREG32(mmPA_SC_RASTER_CONFIG, raster_config_se);
3601 		WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
3602 	}
3603 
3604 	/* GRBM_GFX_INDEX has a different offset on VI */
3605 	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3606 }
3607 
3608 static void gfx_v8_0_setup_rb(struct amdgpu_device *adev)
3609 {
3610 	int i, j;
3611 	u32 data;
3612 	u32 raster_config = 0, raster_config_1 = 0;
3613 	u32 active_rbs = 0;
3614 	u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
3615 					adev->gfx.config.max_sh_per_se;
3616 	unsigned num_rb_pipes;
3617 
3618 	mutex_lock(&adev->grbm_idx_mutex);
3619 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3620 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
3621 			gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
3622 			data = gfx_v8_0_get_rb_active_bitmap(adev);
3623 			active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
3624 					       rb_bitmap_width_per_sh);
3625 		}
3626 	}
3627 	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3628 
3629 	adev->gfx.config.backend_enable_mask = active_rbs;
3630 	adev->gfx.config.num_rbs = hweight32(active_rbs);
3631 
3632 	num_rb_pipes = min_t(unsigned, adev->gfx.config.max_backends_per_se *
3633 			     adev->gfx.config.max_shader_engines, 16);
3634 
3635 	gfx_v8_0_raster_config(adev, &raster_config, &raster_config_1);
3636 
3637 	if (!adev->gfx.config.backend_enable_mask ||
3638 			adev->gfx.config.num_rbs >= num_rb_pipes) {
3639 		WREG32(mmPA_SC_RASTER_CONFIG, raster_config);
3640 		WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
3641 	} else {
3642 		gfx_v8_0_write_harvested_raster_configs(adev, raster_config, raster_config_1,
3643 							adev->gfx.config.backend_enable_mask,
3644 							num_rb_pipes);
3645 	}
3646 
3647 	/* cache the values for userspace */
3648 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3649 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
3650 			gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
3651 			adev->gfx.config.rb_config[i][j].rb_backend_disable =
3652 				RREG32(mmCC_RB_BACKEND_DISABLE);
3653 			adev->gfx.config.rb_config[i][j].user_rb_backend_disable =
3654 				RREG32(mmGC_USER_RB_BACKEND_DISABLE);
3655 			adev->gfx.config.rb_config[i][j].raster_config =
3656 				RREG32(mmPA_SC_RASTER_CONFIG);
3657 			adev->gfx.config.rb_config[i][j].raster_config_1 =
3658 				RREG32(mmPA_SC_RASTER_CONFIG_1);
3659 		}
3660 	}
3661 	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3662 	mutex_unlock(&adev->grbm_idx_mutex);
3663 }
3664 
3665 /**
3666  * gfx_v8_0_init_compute_vmid - gart enable
3667  *
3668  * @adev: amdgpu_device pointer
3669  *
3670  * Initialize compute vmid sh_mem registers
3671  *
3672  */
3673 #define DEFAULT_SH_MEM_BASES	(0x6000)
3674 #define FIRST_COMPUTE_VMID	(8)
3675 #define LAST_COMPUTE_VMID	(16)
3676 static void gfx_v8_0_init_compute_vmid(struct amdgpu_device *adev)
3677 {
3678 	int i;
3679 	uint32_t sh_mem_config;
3680 	uint32_t sh_mem_bases;
3681 
3682 	/*
3683 	 * Configure apertures:
3684 	 * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
3685 	 * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
3686 	 * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
3687 	 */
3688 	sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
3689 
3690 	sh_mem_config = SH_MEM_ADDRESS_MODE_HSA64 <<
3691 			SH_MEM_CONFIG__ADDRESS_MODE__SHIFT |
3692 			SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
3693 			SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT |
3694 			MTYPE_CC << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT |
3695 			SH_MEM_CONFIG__PRIVATE_ATC_MASK;
3696 
3697 	mutex_lock(&adev->srbm_mutex);
3698 	for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) {
3699 		vi_srbm_select(adev, 0, 0, 0, i);
3700 		/* CP and shaders */
3701 		WREG32(mmSH_MEM_CONFIG, sh_mem_config);
3702 		WREG32(mmSH_MEM_APE1_BASE, 1);
3703 		WREG32(mmSH_MEM_APE1_LIMIT, 0);
3704 		WREG32(mmSH_MEM_BASES, sh_mem_bases);
3705 	}
3706 	vi_srbm_select(adev, 0, 0, 0, 0);
3707 	mutex_unlock(&adev->srbm_mutex);
3708 }
3709 
3710 static void gfx_v8_0_config_init(struct amdgpu_device *adev)
3711 {
3712 	switch (adev->asic_type) {
3713 	default:
3714 		adev->gfx.config.double_offchip_lds_buf = 1;
3715 		break;
3716 	case CHIP_CARRIZO:
3717 	case CHIP_STONEY:
3718 		adev->gfx.config.double_offchip_lds_buf = 0;
3719 		break;
3720 	}
3721 }
3722 
3723 static void gfx_v8_0_constants_init(struct amdgpu_device *adev)
3724 {
3725 	u32 tmp, sh_static_mem_cfg;
3726 	int i;
3727 
3728 	WREG32_FIELD(GRBM_CNTL, READ_TIMEOUT, 0xFF);
3729 	WREG32(mmGB_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
3730 	WREG32(mmHDP_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
3731 	WREG32(mmDMIF_ADDR_CALC, adev->gfx.config.gb_addr_config);
3732 
3733 	gfx_v8_0_tiling_mode_table_init(adev);
3734 	gfx_v8_0_setup_rb(adev);
3735 	gfx_v8_0_get_cu_info(adev);
3736 	gfx_v8_0_config_init(adev);
3737 
3738 	/* XXX SH_MEM regs */
3739 	/* where to put LDS, scratch, GPUVM in FSA64 space */
3740 	sh_static_mem_cfg = REG_SET_FIELD(0, SH_STATIC_MEM_CONFIG,
3741 				   SWIZZLE_ENABLE, 1);
3742 	sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,
3743 				   ELEMENT_SIZE, 1);
3744 	sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,
3745 				   INDEX_STRIDE, 3);
3746 	WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg);
3747 
3748 	mutex_lock(&adev->srbm_mutex);
3749 	for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) {
3750 		vi_srbm_select(adev, 0, 0, 0, i);
3751 		/* CP and shaders */
3752 		if (i == 0) {
3753 			tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_UC);
3754 			tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC);
3755 			tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE,
3756 					    SH_MEM_ALIGNMENT_MODE_UNALIGNED);
3757 			WREG32(mmSH_MEM_CONFIG, tmp);
3758 			WREG32(mmSH_MEM_BASES, 0);
3759 		} else {
3760 			tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_NC);
3761 			tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC);
3762 			tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE,
3763 					    SH_MEM_ALIGNMENT_MODE_UNALIGNED);
3764 			WREG32(mmSH_MEM_CONFIG, tmp);
3765 			tmp = adev->gmc.shared_aperture_start >> 48;
3766 			WREG32(mmSH_MEM_BASES, tmp);
3767 		}
3768 
3769 		WREG32(mmSH_MEM_APE1_BASE, 1);
3770 		WREG32(mmSH_MEM_APE1_LIMIT, 0);
3771 	}
3772 	vi_srbm_select(adev, 0, 0, 0, 0);
3773 	mutex_unlock(&adev->srbm_mutex);
3774 
3775 	gfx_v8_0_init_compute_vmid(adev);
3776 
3777 	mutex_lock(&adev->grbm_idx_mutex);
3778 	/*
3779 	 * making sure that the following register writes will be broadcasted
3780 	 * to all the shaders
3781 	 */
3782 	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3783 
3784 	WREG32(mmPA_SC_FIFO_SIZE,
3785 		   (adev->gfx.config.sc_prim_fifo_size_frontend <<
3786 			PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT) |
3787 		   (adev->gfx.config.sc_prim_fifo_size_backend <<
3788 			PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT) |
3789 		   (adev->gfx.config.sc_hiz_tile_fifo_size <<
3790 			PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT) |
3791 		   (adev->gfx.config.sc_earlyz_tile_fifo_size <<
3792 			PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT));
3793 
3794 	tmp = RREG32(mmSPI_ARB_PRIORITY);
3795 	tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS0, 2);
3796 	tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS1, 2);
3797 	tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS2, 2);
3798 	tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS3, 2);
3799 	WREG32(mmSPI_ARB_PRIORITY, tmp);
3800 
3801 	mutex_unlock(&adev->grbm_idx_mutex);
3802 
3803 }
3804 
3805 static void gfx_v8_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
3806 {
3807 	u32 i, j, k;
3808 	u32 mask;
3809 
3810 	mutex_lock(&adev->grbm_idx_mutex);
3811 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3812 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
3813 			gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
3814 			for (k = 0; k < adev->usec_timeout; k++) {
3815 				if (RREG32(mmRLC_SERDES_CU_MASTER_BUSY) == 0)
3816 					break;
3817 				udelay(1);
3818 			}
3819 			if (k == adev->usec_timeout) {
3820 				gfx_v8_0_select_se_sh(adev, 0xffffffff,
3821 						      0xffffffff, 0xffffffff);
3822 				mutex_unlock(&adev->grbm_idx_mutex);
3823 				DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
3824 					 i, j);
3825 				return;
3826 			}
3827 		}
3828 	}
3829 	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3830 	mutex_unlock(&adev->grbm_idx_mutex);
3831 
3832 	mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
3833 		RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
3834 		RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
3835 		RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
3836 	for (k = 0; k < adev->usec_timeout; k++) {
3837 		if ((RREG32(mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
3838 			break;
3839 		udelay(1);
3840 	}
3841 }
3842 
3843 static void gfx_v8_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
3844 					       bool enable)
3845 {
3846 	u32 tmp = RREG32(mmCP_INT_CNTL_RING0);
3847 
3848 	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
3849 	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
3850 	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
3851 	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
3852 
3853 	WREG32(mmCP_INT_CNTL_RING0, tmp);
3854 }
3855 
3856 static void gfx_v8_0_init_csb(struct amdgpu_device *adev)
3857 {
3858 	/* csib */
3859 	WREG32(mmRLC_CSIB_ADDR_HI,
3860 			adev->gfx.rlc.clear_state_gpu_addr >> 32);
3861 	WREG32(mmRLC_CSIB_ADDR_LO,
3862 			adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
3863 	WREG32(mmRLC_CSIB_LENGTH,
3864 			adev->gfx.rlc.clear_state_size);
3865 }
3866 
3867 static void gfx_v8_0_parse_ind_reg_list(int *register_list_format,
3868 				int ind_offset,
3869 				int list_size,
3870 				int *unique_indices,
3871 				int *indices_count,
3872 				int max_indices,
3873 				int *ind_start_offsets,
3874 				int *offset_count,
3875 				int max_offset)
3876 {
3877 	int indices;
3878 	bool new_entry = true;
3879 
3880 	for (; ind_offset < list_size; ind_offset++) {
3881 
3882 		if (new_entry) {
3883 			new_entry = false;
3884 			ind_start_offsets[*offset_count] = ind_offset;
3885 			*offset_count = *offset_count + 1;
3886 			BUG_ON(*offset_count >= max_offset);
3887 		}
3888 
3889 		if (register_list_format[ind_offset] == 0xFFFFFFFF) {
3890 			new_entry = true;
3891 			continue;
3892 		}
3893 
3894 		ind_offset += 2;
3895 
3896 		/* look for the matching indice */
3897 		for (indices = 0;
3898 			indices < *indices_count;
3899 			indices++) {
3900 			if (unique_indices[indices] ==
3901 				register_list_format[ind_offset])
3902 				break;
3903 		}
3904 
3905 		if (indices >= *indices_count) {
3906 			unique_indices[*indices_count] =
3907 				register_list_format[ind_offset];
3908 			indices = *indices_count;
3909 			*indices_count = *indices_count + 1;
3910 			BUG_ON(*indices_count >= max_indices);
3911 		}
3912 
3913 		register_list_format[ind_offset] = indices;
3914 	}
3915 }
3916 
3917 static int gfx_v8_0_init_save_restore_list(struct amdgpu_device *adev)
3918 {
3919 	int i, temp, data;
3920 	int unique_indices[] = {0, 0, 0, 0, 0, 0, 0, 0};
3921 	int indices_count = 0;
3922 	int indirect_start_offsets[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
3923 	int offset_count = 0;
3924 
3925 	int list_size;
3926 	unsigned int *register_list_format =
3927 		kmalloc(adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
3928 	if (!register_list_format)
3929 		return -ENOMEM;
3930 	memcpy(register_list_format, adev->gfx.rlc.register_list_format,
3931 			adev->gfx.rlc.reg_list_format_size_bytes);
3932 
3933 	gfx_v8_0_parse_ind_reg_list(register_list_format,
3934 				RLC_FormatDirectRegListLength,
3935 				adev->gfx.rlc.reg_list_format_size_bytes >> 2,
3936 				unique_indices,
3937 				&indices_count,
3938 				ARRAY_SIZE(unique_indices),
3939 				indirect_start_offsets,
3940 				&offset_count,
3941 				ARRAY_SIZE(indirect_start_offsets));
3942 
3943 	/* save and restore list */
3944 	WREG32_FIELD(RLC_SRM_CNTL, AUTO_INCR_ADDR, 1);
3945 
3946 	WREG32(mmRLC_SRM_ARAM_ADDR, 0);
3947 	for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
3948 		WREG32(mmRLC_SRM_ARAM_DATA, adev->gfx.rlc.register_restore[i]);
3949 
3950 	/* indirect list */
3951 	WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_list_format_start);
3952 	for (i = 0; i < adev->gfx.rlc.reg_list_format_size_bytes >> 2; i++)
3953 		WREG32(mmRLC_GPM_SCRATCH_DATA, register_list_format[i]);
3954 
3955 	list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
3956 	list_size = list_size >> 1;
3957 	WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_restore_list_size);
3958 	WREG32(mmRLC_GPM_SCRATCH_DATA, list_size);
3959 
3960 	/* starting offsets starts */
3961 	WREG32(mmRLC_GPM_SCRATCH_ADDR,
3962 		adev->gfx.rlc.starting_offsets_start);
3963 	for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
3964 		WREG32(mmRLC_GPM_SCRATCH_DATA,
3965 				indirect_start_offsets[i]);
3966 
3967 	/* unique indices */
3968 	temp = mmRLC_SRM_INDEX_CNTL_ADDR_0;
3969 	data = mmRLC_SRM_INDEX_CNTL_DATA_0;
3970 	for (i = 0; i < ARRAY_SIZE(unique_indices); i++) {
3971 		if (unique_indices[i] != 0) {
3972 			WREG32(temp + i, unique_indices[i] & 0x3FFFF);
3973 			WREG32(data + i, unique_indices[i] >> 20);
3974 		}
3975 	}
3976 	kfree(register_list_format);
3977 
3978 	return 0;
3979 }
3980 
3981 static void gfx_v8_0_enable_save_restore_machine(struct amdgpu_device *adev)
3982 {
3983 	WREG32_FIELD(RLC_SRM_CNTL, SRM_ENABLE, 1);
3984 }
3985 
3986 static void gfx_v8_0_init_power_gating(struct amdgpu_device *adev)
3987 {
3988 	uint32_t data;
3989 
3990 	WREG32_FIELD(CP_RB_WPTR_POLL_CNTL, IDLE_POLL_COUNT, 0x60);
3991 
3992 	data = REG_SET_FIELD(0, RLC_PG_DELAY, POWER_UP_DELAY, 0x10);
3993 	data = REG_SET_FIELD(data, RLC_PG_DELAY, POWER_DOWN_DELAY, 0x10);
3994 	data = REG_SET_FIELD(data, RLC_PG_DELAY, CMD_PROPAGATE_DELAY, 0x10);
3995 	data = REG_SET_FIELD(data, RLC_PG_DELAY, MEM_SLEEP_DELAY, 0x10);
3996 	WREG32(mmRLC_PG_DELAY, data);
3997 
3998 	WREG32_FIELD(RLC_PG_DELAY_2, SERDES_CMD_DELAY, 0x3);
3999 	WREG32_FIELD(RLC_AUTO_PG_CTRL, GRBM_REG_SAVE_GFX_IDLE_THRESHOLD, 0x55f0);
4000 
4001 }
4002 
4003 static void cz_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
4004 						bool enable)
4005 {
4006 	WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PU_ENABLE, enable ? 1 : 0);
4007 }
4008 
4009 static void cz_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
4010 						  bool enable)
4011 {
4012 	WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PD_ENABLE, enable ? 1 : 0);
4013 }
4014 
4015 static void cz_enable_cp_power_gating(struct amdgpu_device *adev, bool enable)
4016 {
4017 	WREG32_FIELD(RLC_PG_CNTL, CP_PG_DISABLE, enable ? 0 : 1);
4018 }
4019 
4020 static void gfx_v8_0_init_pg(struct amdgpu_device *adev)
4021 {
4022 	if ((adev->asic_type == CHIP_CARRIZO) ||
4023 	    (adev->asic_type == CHIP_STONEY)) {
4024 		gfx_v8_0_init_csb(adev);
4025 		gfx_v8_0_init_save_restore_list(adev);
4026 		gfx_v8_0_enable_save_restore_machine(adev);
4027 		WREG32(mmRLC_JUMP_TABLE_RESTORE, adev->gfx.rlc.cp_table_gpu_addr >> 8);
4028 		gfx_v8_0_init_power_gating(adev);
4029 		WREG32(mmRLC_PG_ALWAYS_ON_CU_MASK, adev->gfx.cu_info.ao_cu_mask);
4030 	} else if ((adev->asic_type == CHIP_POLARIS11) ||
4031 		   (adev->asic_type == CHIP_POLARIS12) ||
4032 		   (adev->asic_type == CHIP_VEGAM)) {
4033 		gfx_v8_0_init_csb(adev);
4034 		gfx_v8_0_init_save_restore_list(adev);
4035 		gfx_v8_0_enable_save_restore_machine(adev);
4036 		gfx_v8_0_init_power_gating(adev);
4037 	}
4038 
4039 }
4040 
4041 static void gfx_v8_0_rlc_stop(struct amdgpu_device *adev)
4042 {
4043 	WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 0);
4044 
4045 	gfx_v8_0_enable_gui_idle_interrupt(adev, false);
4046 	gfx_v8_0_wait_for_rlc_serdes(adev);
4047 }
4048 
4049 static void gfx_v8_0_rlc_reset(struct amdgpu_device *adev)
4050 {
4051 	WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4052 	udelay(50);
4053 
4054 	WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
4055 	udelay(50);
4056 }
4057 
4058 static void gfx_v8_0_rlc_start(struct amdgpu_device *adev)
4059 {
4060 	WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 1);
4061 
4062 	/* carrizo do enable cp interrupt after cp inited */
4063 	if (!(adev->flags & AMD_IS_APU))
4064 		gfx_v8_0_enable_gui_idle_interrupt(adev, true);
4065 
4066 	udelay(50);
4067 }
4068 
4069 static int gfx_v8_0_rlc_resume(struct amdgpu_device *adev)
4070 {
4071 	if (amdgpu_sriov_vf(adev)) {
4072 		gfx_v8_0_init_csb(adev);
4073 		return 0;
4074 	}
4075 
4076 	adev->gfx.rlc.funcs->stop(adev);
4077 	adev->gfx.rlc.funcs->reset(adev);
4078 	gfx_v8_0_init_pg(adev);
4079 	adev->gfx.rlc.funcs->start(adev);
4080 
4081 	return 0;
4082 }
4083 
4084 static void gfx_v8_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
4085 {
4086 	int i;
4087 	u32 tmp = RREG32(mmCP_ME_CNTL);
4088 
4089 	if (enable) {
4090 		tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 0);
4091 		tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 0);
4092 		tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 0);
4093 	} else {
4094 		tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 1);
4095 		tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 1);
4096 		tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 1);
4097 		for (i = 0; i < adev->gfx.num_gfx_rings; i++)
4098 			adev->gfx.gfx_ring[i].sched.ready = false;
4099 	}
4100 	WREG32(mmCP_ME_CNTL, tmp);
4101 	udelay(50);
4102 }
4103 
4104 static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev)
4105 {
4106 	u32 count = 0;
4107 	const struct cs_section_def *sect = NULL;
4108 	const struct cs_extent_def *ext = NULL;
4109 
4110 	/* begin clear state */
4111 	count += 2;
4112 	/* context control state */
4113 	count += 3;
4114 
4115 	for (sect = vi_cs_data; sect->section != NULL; ++sect) {
4116 		for (ext = sect->section; ext->extent != NULL; ++ext) {
4117 			if (sect->id == SECT_CONTEXT)
4118 				count += 2 + ext->reg_count;
4119 			else
4120 				return 0;
4121 		}
4122 	}
4123 	/* pa_sc_raster_config/pa_sc_raster_config1 */
4124 	count += 4;
4125 	/* end clear state */
4126 	count += 2;
4127 	/* clear state */
4128 	count += 2;
4129 
4130 	return count;
4131 }
4132 
4133 static int gfx_v8_0_cp_gfx_start(struct amdgpu_device *adev)
4134 {
4135 	struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
4136 	const struct cs_section_def *sect = NULL;
4137 	const struct cs_extent_def *ext = NULL;
4138 	int r, i;
4139 
4140 	/* init the CP */
4141 	WREG32(mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
4142 	WREG32(mmCP_ENDIAN_SWAP, 0);
4143 	WREG32(mmCP_DEVICE_ID, 1);
4144 
4145 	gfx_v8_0_cp_gfx_enable(adev, true);
4146 
4147 	r = amdgpu_ring_alloc(ring, gfx_v8_0_get_csb_size(adev) + 4);
4148 	if (r) {
4149 		DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
4150 		return r;
4151 	}
4152 
4153 	/* clear state buffer */
4154 	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4155 	amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4156 
4157 	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4158 	amdgpu_ring_write(ring, 0x80000000);
4159 	amdgpu_ring_write(ring, 0x80000000);
4160 
4161 	for (sect = vi_cs_data; sect->section != NULL; ++sect) {
4162 		for (ext = sect->section; ext->extent != NULL; ++ext) {
4163 			if (sect->id == SECT_CONTEXT) {
4164 				amdgpu_ring_write(ring,
4165 				       PACKET3(PACKET3_SET_CONTEXT_REG,
4166 					       ext->reg_count));
4167 				amdgpu_ring_write(ring,
4168 				       ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
4169 				for (i = 0; i < ext->reg_count; i++)
4170 					amdgpu_ring_write(ring, ext->extent[i]);
4171 			}
4172 		}
4173 	}
4174 
4175 	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4176 	amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
4177 	amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config);
4178 	amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config_1);
4179 
4180 	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4181 	amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4182 
4183 	amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4184 	amdgpu_ring_write(ring, 0);
4185 
4186 	/* init the CE partitions */
4187 	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4188 	amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4189 	amdgpu_ring_write(ring, 0x8000);
4190 	amdgpu_ring_write(ring, 0x8000);
4191 
4192 	amdgpu_ring_commit(ring);
4193 
4194 	return 0;
4195 }
4196 static void gfx_v8_0_set_cpg_door_bell(struct amdgpu_device *adev, struct amdgpu_ring *ring)
4197 {
4198 	u32 tmp;
4199 	/* no gfx doorbells on iceland */
4200 	if (adev->asic_type == CHIP_TOPAZ)
4201 		return;
4202 
4203 	tmp = RREG32(mmCP_RB_DOORBELL_CONTROL);
4204 
4205 	if (ring->use_doorbell) {
4206 		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4207 				DOORBELL_OFFSET, ring->doorbell_index);
4208 		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4209 						DOORBELL_HIT, 0);
4210 		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4211 					    DOORBELL_EN, 1);
4212 	} else {
4213 		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
4214 	}
4215 
4216 	WREG32(mmCP_RB_DOORBELL_CONTROL, tmp);
4217 
4218 	if (adev->flags & AMD_IS_APU)
4219 		return;
4220 
4221 	tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
4222 					DOORBELL_RANGE_LOWER,
4223 					adev->doorbell_index.gfx_ring0);
4224 	WREG32(mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
4225 
4226 	WREG32(mmCP_RB_DOORBELL_RANGE_UPPER,
4227 		CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
4228 }
4229 
4230 static int gfx_v8_0_cp_gfx_resume(struct amdgpu_device *adev)
4231 {
4232 	struct amdgpu_ring *ring;
4233 	u32 tmp;
4234 	u32 rb_bufsz;
4235 	u64 rb_addr, rptr_addr, wptr_gpu_addr;
4236 
4237 	/* Set the write pointer delay */
4238 	WREG32(mmCP_RB_WPTR_DELAY, 0);
4239 
4240 	/* set the RB to use vmid 0 */
4241 	WREG32(mmCP_RB_VMID, 0);
4242 
4243 	/* Set ring buffer size */
4244 	ring = &adev->gfx.gfx_ring[0];
4245 	rb_bufsz = order_base_2(ring->ring_size / 8);
4246 	tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
4247 	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
4248 	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MTYPE, 3);
4249 	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MIN_IB_AVAILSZ, 1);
4250 #ifdef __BIG_ENDIAN
4251 	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
4252 #endif
4253 	WREG32(mmCP_RB0_CNTL, tmp);
4254 
4255 	/* Initialize the ring buffer's read and write pointers */
4256 	WREG32(mmCP_RB0_CNTL, tmp | CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK);
4257 	ring->wptr = 0;
4258 	WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
4259 
4260 	/* set the wb address wether it's enabled or not */
4261 	rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
4262 	WREG32(mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
4263 	WREG32(mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
4264 
4265 	wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
4266 	WREG32(mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
4267 	WREG32(mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
4268 	mdelay(1);
4269 	WREG32(mmCP_RB0_CNTL, tmp);
4270 
4271 	rb_addr = ring->gpu_addr >> 8;
4272 	WREG32(mmCP_RB0_BASE, rb_addr);
4273 	WREG32(mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
4274 
4275 	gfx_v8_0_set_cpg_door_bell(adev, ring);
4276 	/* start the ring */
4277 	amdgpu_ring_clear_ring(ring);
4278 	gfx_v8_0_cp_gfx_start(adev);
4279 	ring->sched.ready = true;
4280 
4281 	return 0;
4282 }
4283 
4284 static void gfx_v8_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
4285 {
4286 	int i;
4287 
4288 	if (enable) {
4289 		WREG32(mmCP_MEC_CNTL, 0);
4290 	} else {
4291 		WREG32(mmCP_MEC_CNTL, (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
4292 		for (i = 0; i < adev->gfx.num_compute_rings; i++)
4293 			adev->gfx.compute_ring[i].sched.ready = false;
4294 		adev->gfx.kiq.ring.sched.ready = false;
4295 	}
4296 	udelay(50);
4297 }
4298 
4299 /* KIQ functions */
4300 static void gfx_v8_0_kiq_setting(struct amdgpu_ring *ring)
4301 {
4302 	uint32_t tmp;
4303 	struct amdgpu_device *adev = ring->adev;
4304 
4305 	/* tell RLC which is KIQ queue */
4306 	tmp = RREG32(mmRLC_CP_SCHEDULERS);
4307 	tmp &= 0xffffff00;
4308 	tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
4309 	WREG32(mmRLC_CP_SCHEDULERS, tmp);
4310 	tmp |= 0x80;
4311 	WREG32(mmRLC_CP_SCHEDULERS, tmp);
4312 }
4313 
4314 static int gfx_v8_0_kiq_kcq_enable(struct amdgpu_device *adev)
4315 {
4316 	struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
4317 	uint64_t queue_mask = 0;
4318 	int r, i;
4319 
4320 	for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) {
4321 		if (!test_bit(i, adev->gfx.mec.queue_bitmap))
4322 			continue;
4323 
4324 		/* This situation may be hit in the future if a new HW
4325 		 * generation exposes more than 64 queues. If so, the
4326 		 * definition of queue_mask needs updating */
4327 		if (WARN_ON(i >= (sizeof(queue_mask)*8))) {
4328 			DRM_ERROR("Invalid KCQ enabled: %d\n", i);
4329 			break;
4330 		}
4331 
4332 		queue_mask |= (1ull << i);
4333 	}
4334 
4335 	r = amdgpu_ring_alloc(kiq_ring, (8 * adev->gfx.num_compute_rings) + 8);
4336 	if (r) {
4337 		DRM_ERROR("Failed to lock KIQ (%d).\n", r);
4338 		return r;
4339 	}
4340 	/* set resources */
4341 	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
4342 	amdgpu_ring_write(kiq_ring, 0);	/* vmid_mask:0 queue_type:0 (KIQ) */
4343 	amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask));	/* queue mask lo */
4344 	amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask));	/* queue mask hi */
4345 	amdgpu_ring_write(kiq_ring, 0);	/* gws mask lo */
4346 	amdgpu_ring_write(kiq_ring, 0);	/* gws mask hi */
4347 	amdgpu_ring_write(kiq_ring, 0);	/* oac mask */
4348 	amdgpu_ring_write(kiq_ring, 0);	/* gds heap base:0, gds heap size:0 */
4349 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4350 		struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
4351 		uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
4352 		uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
4353 
4354 		/* map queues */
4355 		amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
4356 		/* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
4357 		amdgpu_ring_write(kiq_ring,
4358 				  PACKET3_MAP_QUEUES_NUM_QUEUES(1));
4359 		amdgpu_ring_write(kiq_ring,
4360 				  PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index) |
4361 				  PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
4362 				  PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
4363 				  PACKET3_MAP_QUEUES_ME(ring->me == 1 ? 0 : 1)); /* doorbell */
4364 		amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
4365 		amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
4366 		amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
4367 		amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
4368 	}
4369 
4370 	amdgpu_ring_commit(kiq_ring);
4371 
4372 	return 0;
4373 }
4374 
4375 static int gfx_v8_0_deactivate_hqd(struct amdgpu_device *adev, u32 req)
4376 {
4377 	int i, r = 0;
4378 
4379 	if (RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK) {
4380 		WREG32_FIELD(CP_HQD_DEQUEUE_REQUEST, DEQUEUE_REQ, req);
4381 		for (i = 0; i < adev->usec_timeout; i++) {
4382 			if (!(RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK))
4383 				break;
4384 			udelay(1);
4385 		}
4386 		if (i == adev->usec_timeout)
4387 			r = -ETIMEDOUT;
4388 	}
4389 	WREG32(mmCP_HQD_DEQUEUE_REQUEST, 0);
4390 	WREG32(mmCP_HQD_PQ_RPTR, 0);
4391 	WREG32(mmCP_HQD_PQ_WPTR, 0);
4392 
4393 	return r;
4394 }
4395 
4396 static int gfx_v8_0_mqd_init(struct amdgpu_ring *ring)
4397 {
4398 	struct amdgpu_device *adev = ring->adev;
4399 	struct vi_mqd *mqd = ring->mqd_ptr;
4400 	uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
4401 	uint32_t tmp;
4402 
4403 	mqd->header = 0xC0310800;
4404 	mqd->compute_pipelinestat_enable = 0x00000001;
4405 	mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
4406 	mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
4407 	mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
4408 	mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
4409 	mqd->compute_misc_reserved = 0x00000003;
4410 	mqd->dynamic_cu_mask_addr_lo = lower_32_bits(ring->mqd_gpu_addr
4411 						     + offsetof(struct vi_mqd_allocation, dynamic_cu_mask));
4412 	mqd->dynamic_cu_mask_addr_hi = upper_32_bits(ring->mqd_gpu_addr
4413 						     + offsetof(struct vi_mqd_allocation, dynamic_cu_mask));
4414 	eop_base_addr = ring->eop_gpu_addr >> 8;
4415 	mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
4416 	mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
4417 
4418 	/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4419 	tmp = RREG32(mmCP_HQD_EOP_CONTROL);
4420 	tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
4421 			(order_base_2(GFX8_MEC_HPD_SIZE / 4) - 1));
4422 
4423 	mqd->cp_hqd_eop_control = tmp;
4424 
4425 	/* enable doorbell? */
4426 	tmp = REG_SET_FIELD(RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL),
4427 			    CP_HQD_PQ_DOORBELL_CONTROL,
4428 			    DOORBELL_EN,
4429 			    ring->use_doorbell ? 1 : 0);
4430 
4431 	mqd->cp_hqd_pq_doorbell_control = tmp;
4432 
4433 	/* set the pointer to the MQD */
4434 	mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
4435 	mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
4436 
4437 	/* set MQD vmid to 0 */
4438 	tmp = RREG32(mmCP_MQD_CONTROL);
4439 	tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
4440 	mqd->cp_mqd_control = tmp;
4441 
4442 	/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4443 	hqd_gpu_addr = ring->gpu_addr >> 8;
4444 	mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
4445 	mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4446 
4447 	/* set up the HQD, this is similar to CP_RB0_CNTL */
4448 	tmp = RREG32(mmCP_HQD_PQ_CONTROL);
4449 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
4450 			    (order_base_2(ring->ring_size / 4) - 1));
4451 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
4452 			((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
4453 #ifdef __BIG_ENDIAN
4454 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
4455 #endif
4456 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
4457 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
4458 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
4459 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
4460 	mqd->cp_hqd_pq_control = tmp;
4461 
4462 	/* set the wb address whether it's enabled or not */
4463 	wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
4464 	mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
4465 	mqd->cp_hqd_pq_rptr_report_addr_hi =
4466 		upper_32_bits(wb_gpu_addr) & 0xffff;
4467 
4468 	/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4469 	wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
4470 	mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
4471 	mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4472 
4473 	tmp = 0;
4474 	/* enable the doorbell if requested */
4475 	if (ring->use_doorbell) {
4476 		tmp = RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL);
4477 		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4478 				DOORBELL_OFFSET, ring->doorbell_index);
4479 
4480 		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4481 					 DOORBELL_EN, 1);
4482 		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4483 					 DOORBELL_SOURCE, 0);
4484 		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4485 					 DOORBELL_HIT, 0);
4486 	}
4487 
4488 	mqd->cp_hqd_pq_doorbell_control = tmp;
4489 
4490 	/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4491 	ring->wptr = 0;
4492 	mqd->cp_hqd_pq_wptr = ring->wptr;
4493 	mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR);
4494 
4495 	/* set the vmid for the queue */
4496 	mqd->cp_hqd_vmid = 0;
4497 
4498 	tmp = RREG32(mmCP_HQD_PERSISTENT_STATE);
4499 	tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
4500 	mqd->cp_hqd_persistent_state = tmp;
4501 
4502 	/* set MTYPE */
4503 	tmp = RREG32(mmCP_HQD_IB_CONTROL);
4504 	tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
4505 	tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MTYPE, 3);
4506 	mqd->cp_hqd_ib_control = tmp;
4507 
4508 	tmp = RREG32(mmCP_HQD_IQ_TIMER);
4509 	tmp = REG_SET_FIELD(tmp, CP_HQD_IQ_TIMER, MTYPE, 3);
4510 	mqd->cp_hqd_iq_timer = tmp;
4511 
4512 	tmp = RREG32(mmCP_HQD_CTX_SAVE_CONTROL);
4513 	tmp = REG_SET_FIELD(tmp, CP_HQD_CTX_SAVE_CONTROL, MTYPE, 3);
4514 	mqd->cp_hqd_ctx_save_control = tmp;
4515 
4516 	/* defaults */
4517 	mqd->cp_hqd_eop_rptr = RREG32(mmCP_HQD_EOP_RPTR);
4518 	mqd->cp_hqd_eop_wptr = RREG32(mmCP_HQD_EOP_WPTR);
4519 	mqd->cp_hqd_pipe_priority = RREG32(mmCP_HQD_PIPE_PRIORITY);
4520 	mqd->cp_hqd_queue_priority = RREG32(mmCP_HQD_QUEUE_PRIORITY);
4521 	mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM);
4522 	mqd->cp_hqd_ctx_save_base_addr_lo = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_LO);
4523 	mqd->cp_hqd_ctx_save_base_addr_hi = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_HI);
4524 	mqd->cp_hqd_cntl_stack_offset = RREG32(mmCP_HQD_CNTL_STACK_OFFSET);
4525 	mqd->cp_hqd_cntl_stack_size = RREG32(mmCP_HQD_CNTL_STACK_SIZE);
4526 	mqd->cp_hqd_wg_state_offset = RREG32(mmCP_HQD_WG_STATE_OFFSET);
4527 	mqd->cp_hqd_ctx_save_size = RREG32(mmCP_HQD_CTX_SAVE_SIZE);
4528 	mqd->cp_hqd_eop_done_events = RREG32(mmCP_HQD_EOP_EVENTS);
4529 	mqd->cp_hqd_error = RREG32(mmCP_HQD_ERROR);
4530 	mqd->cp_hqd_eop_wptr_mem = RREG32(mmCP_HQD_EOP_WPTR_MEM);
4531 	mqd->cp_hqd_eop_dones = RREG32(mmCP_HQD_EOP_DONES);
4532 
4533 	/* activate the queue */
4534 	mqd->cp_hqd_active = 1;
4535 
4536 	return 0;
4537 }
4538 
4539 int gfx_v8_0_mqd_commit(struct amdgpu_device *adev,
4540 			struct vi_mqd *mqd)
4541 {
4542 	uint32_t mqd_reg;
4543 	uint32_t *mqd_data;
4544 
4545 	/* HQD registers extend from mmCP_MQD_BASE_ADDR to mmCP_HQD_ERROR */
4546 	mqd_data = &mqd->cp_mqd_base_addr_lo;
4547 
4548 	/* disable wptr polling */
4549 	WREG32_FIELD(CP_PQ_WPTR_POLL_CNTL, EN, 0);
4550 
4551 	/* program all HQD registers */
4552 	for (mqd_reg = mmCP_HQD_VMID; mqd_reg <= mmCP_HQD_EOP_CONTROL; mqd_reg++)
4553 		WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);
4554 
4555 	/* Tonga errata: EOP RPTR/WPTR should be left unmodified.
4556 	 * This is safe since EOP RPTR==WPTR for any inactive HQD
4557 	 * on ASICs that do not support context-save.
4558 	 * EOP writes/reads can start anywhere in the ring.
4559 	 */
4560 	if (adev->asic_type != CHIP_TONGA) {
4561 		WREG32(mmCP_HQD_EOP_RPTR, mqd->cp_hqd_eop_rptr);
4562 		WREG32(mmCP_HQD_EOP_WPTR, mqd->cp_hqd_eop_wptr);
4563 		WREG32(mmCP_HQD_EOP_WPTR_MEM, mqd->cp_hqd_eop_wptr_mem);
4564 	}
4565 
4566 	for (mqd_reg = mmCP_HQD_EOP_EVENTS; mqd_reg <= mmCP_HQD_ERROR; mqd_reg++)
4567 		WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);
4568 
4569 	/* activate the HQD */
4570 	for (mqd_reg = mmCP_MQD_BASE_ADDR; mqd_reg <= mmCP_HQD_ACTIVE; mqd_reg++)
4571 		WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);
4572 
4573 	return 0;
4574 }
4575 
4576 static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring)
4577 {
4578 	struct amdgpu_device *adev = ring->adev;
4579 	struct vi_mqd *mqd = ring->mqd_ptr;
4580 	int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
4581 
4582 	gfx_v8_0_kiq_setting(ring);
4583 
4584 	if (adev->in_gpu_reset) { /* for GPU_RESET case */
4585 		/* reset MQD to a clean status */
4586 		if (adev->gfx.mec.mqd_backup[mqd_idx])
4587 			memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation));
4588 
4589 		/* reset ring buffer */
4590 		ring->wptr = 0;
4591 		amdgpu_ring_clear_ring(ring);
4592 		mutex_lock(&adev->srbm_mutex);
4593 		vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4594 		gfx_v8_0_mqd_commit(adev, mqd);
4595 		vi_srbm_select(adev, 0, 0, 0, 0);
4596 		mutex_unlock(&adev->srbm_mutex);
4597 	} else {
4598 		memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation));
4599 		((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
4600 		((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
4601 		mutex_lock(&adev->srbm_mutex);
4602 		vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4603 		gfx_v8_0_mqd_init(ring);
4604 		gfx_v8_0_mqd_commit(adev, mqd);
4605 		vi_srbm_select(adev, 0, 0, 0, 0);
4606 		mutex_unlock(&adev->srbm_mutex);
4607 
4608 		if (adev->gfx.mec.mqd_backup[mqd_idx])
4609 			memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation));
4610 	}
4611 
4612 	return 0;
4613 }
4614 
4615 static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring)
4616 {
4617 	struct amdgpu_device *adev = ring->adev;
4618 	struct vi_mqd *mqd = ring->mqd_ptr;
4619 	int mqd_idx = ring - &adev->gfx.compute_ring[0];
4620 
4621 	if (!adev->in_gpu_reset && !adev->in_suspend) {
4622 		memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation));
4623 		((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
4624 		((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
4625 		mutex_lock(&adev->srbm_mutex);
4626 		vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4627 		gfx_v8_0_mqd_init(ring);
4628 		vi_srbm_select(adev, 0, 0, 0, 0);
4629 		mutex_unlock(&adev->srbm_mutex);
4630 
4631 		if (adev->gfx.mec.mqd_backup[mqd_idx])
4632 			memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation));
4633 	} else if (adev->in_gpu_reset) { /* for GPU_RESET case */
4634 		/* reset MQD to a clean status */
4635 		if (adev->gfx.mec.mqd_backup[mqd_idx])
4636 			memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation));
4637 		/* reset ring buffer */
4638 		ring->wptr = 0;
4639 		amdgpu_ring_clear_ring(ring);
4640 	} else {
4641 		amdgpu_ring_clear_ring(ring);
4642 	}
4643 	return 0;
4644 }
4645 
4646 static void gfx_v8_0_set_mec_doorbell_range(struct amdgpu_device *adev)
4647 {
4648 	if (adev->asic_type > CHIP_TONGA) {
4649 		WREG32(mmCP_MEC_DOORBELL_RANGE_LOWER, adev->doorbell_index.kiq << 2);
4650 		WREG32(mmCP_MEC_DOORBELL_RANGE_UPPER, adev->doorbell_index.mec_ring7 << 2);
4651 	}
4652 	/* enable doorbells */
4653 	WREG32_FIELD(CP_PQ_STATUS, DOORBELL_ENABLE, 1);
4654 }
4655 
4656 static int gfx_v8_0_kiq_resume(struct amdgpu_device *adev)
4657 {
4658 	struct amdgpu_ring *ring;
4659 	int r;
4660 
4661 	ring = &adev->gfx.kiq.ring;
4662 
4663 	r = amdgpu_bo_reserve(ring->mqd_obj, false);
4664 	if (unlikely(r != 0))
4665 		return r;
4666 
4667 	r = amdgpu_bo_kmap(ring->mqd_obj, &ring->mqd_ptr);
4668 	if (unlikely(r != 0))
4669 		return r;
4670 
4671 	gfx_v8_0_kiq_init_queue(ring);
4672 	amdgpu_bo_kunmap(ring->mqd_obj);
4673 	ring->mqd_ptr = NULL;
4674 	amdgpu_bo_unreserve(ring->mqd_obj);
4675 	ring->sched.ready = true;
4676 	return 0;
4677 }
4678 
4679 static int gfx_v8_0_kcq_resume(struct amdgpu_device *adev)
4680 {
4681 	struct amdgpu_ring *ring = NULL;
4682 	int r = 0, i;
4683 
4684 	gfx_v8_0_cp_compute_enable(adev, true);
4685 
4686 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4687 		ring = &adev->gfx.compute_ring[i];
4688 
4689 		r = amdgpu_bo_reserve(ring->mqd_obj, false);
4690 		if (unlikely(r != 0))
4691 			goto done;
4692 		r = amdgpu_bo_kmap(ring->mqd_obj, &ring->mqd_ptr);
4693 		if (!r) {
4694 			r = gfx_v8_0_kcq_init_queue(ring);
4695 			amdgpu_bo_kunmap(ring->mqd_obj);
4696 			ring->mqd_ptr = NULL;
4697 		}
4698 		amdgpu_bo_unreserve(ring->mqd_obj);
4699 		if (r)
4700 			goto done;
4701 	}
4702 
4703 	gfx_v8_0_set_mec_doorbell_range(adev);
4704 
4705 	r = gfx_v8_0_kiq_kcq_enable(adev);
4706 	if (r)
4707 		goto done;
4708 
4709 done:
4710 	return r;
4711 }
4712 
4713 static int gfx_v8_0_cp_test_all_rings(struct amdgpu_device *adev)
4714 {
4715 	int r, i;
4716 	struct amdgpu_ring *ring;
4717 
4718 	/* collect all the ring_tests here, gfx, kiq, compute */
4719 	ring = &adev->gfx.gfx_ring[0];
4720 	r = amdgpu_ring_test_helper(ring);
4721 	if (r)
4722 		return r;
4723 
4724 	ring = &adev->gfx.kiq.ring;
4725 	r = amdgpu_ring_test_helper(ring);
4726 	if (r)
4727 		return r;
4728 
4729 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4730 		ring = &adev->gfx.compute_ring[i];
4731 		amdgpu_ring_test_helper(ring);
4732 	}
4733 
4734 	return 0;
4735 }
4736 
4737 static int gfx_v8_0_cp_resume(struct amdgpu_device *adev)
4738 {
4739 	int r;
4740 
4741 	if (!(adev->flags & AMD_IS_APU))
4742 		gfx_v8_0_enable_gui_idle_interrupt(adev, false);
4743 
4744 	r = gfx_v8_0_kiq_resume(adev);
4745 	if (r)
4746 		return r;
4747 
4748 	r = gfx_v8_0_cp_gfx_resume(adev);
4749 	if (r)
4750 		return r;
4751 
4752 	r = gfx_v8_0_kcq_resume(adev);
4753 	if (r)
4754 		return r;
4755 
4756 	r = gfx_v8_0_cp_test_all_rings(adev);
4757 	if (r)
4758 		return r;
4759 
4760 	gfx_v8_0_enable_gui_idle_interrupt(adev, true);
4761 
4762 	return 0;
4763 }
4764 
4765 static void gfx_v8_0_cp_enable(struct amdgpu_device *adev, bool enable)
4766 {
4767 	gfx_v8_0_cp_gfx_enable(adev, enable);
4768 	gfx_v8_0_cp_compute_enable(adev, enable);
4769 }
4770 
4771 static int gfx_v8_0_hw_init(void *handle)
4772 {
4773 	int r;
4774 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4775 
4776 	gfx_v8_0_init_golden_registers(adev);
4777 	gfx_v8_0_constants_init(adev);
4778 
4779 	r = adev->gfx.rlc.funcs->resume(adev);
4780 	if (r)
4781 		return r;
4782 
4783 	r = gfx_v8_0_cp_resume(adev);
4784 
4785 	return r;
4786 }
4787 
4788 static int gfx_v8_0_kcq_disable(struct amdgpu_device *adev)
4789 {
4790 	int r, i;
4791 	struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
4792 
4793 	r = amdgpu_ring_alloc(kiq_ring, 6 * adev->gfx.num_compute_rings);
4794 	if (r)
4795 		DRM_ERROR("Failed to lock KIQ (%d).\n", r);
4796 
4797 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4798 		struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
4799 
4800 		amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
4801 		amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
4802 						PACKET3_UNMAP_QUEUES_ACTION(1) | /* RESET_QUEUES */
4803 						PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
4804 						PACKET3_UNMAP_QUEUES_ENGINE_SEL(0) |
4805 						PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
4806 		amdgpu_ring_write(kiq_ring, PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
4807 		amdgpu_ring_write(kiq_ring, 0);
4808 		amdgpu_ring_write(kiq_ring, 0);
4809 		amdgpu_ring_write(kiq_ring, 0);
4810 	}
4811 	r = amdgpu_ring_test_helper(kiq_ring);
4812 	if (r)
4813 		DRM_ERROR("KCQ disable failed\n");
4814 
4815 	return r;
4816 }
4817 
4818 static bool gfx_v8_0_is_idle(void *handle)
4819 {
4820 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4821 
4822 	if (REG_GET_FIELD(RREG32(mmGRBM_STATUS), GRBM_STATUS, GUI_ACTIVE)
4823 		|| RREG32(mmGRBM_STATUS2) != 0x8)
4824 		return false;
4825 	else
4826 		return true;
4827 }
4828 
4829 static bool gfx_v8_0_rlc_is_idle(void *handle)
4830 {
4831 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4832 
4833 	if (RREG32(mmGRBM_STATUS2) != 0x8)
4834 		return false;
4835 	else
4836 		return true;
4837 }
4838 
4839 static int gfx_v8_0_wait_for_rlc_idle(void *handle)
4840 {
4841 	unsigned int i;
4842 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4843 
4844 	for (i = 0; i < adev->usec_timeout; i++) {
4845 		if (gfx_v8_0_rlc_is_idle(handle))
4846 			return 0;
4847 
4848 		udelay(1);
4849 	}
4850 	return -ETIMEDOUT;
4851 }
4852 
4853 static int gfx_v8_0_wait_for_idle(void *handle)
4854 {
4855 	unsigned int i;
4856 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4857 
4858 	for (i = 0; i < adev->usec_timeout; i++) {
4859 		if (gfx_v8_0_is_idle(handle))
4860 			return 0;
4861 
4862 		udelay(1);
4863 	}
4864 	return -ETIMEDOUT;
4865 }
4866 
4867 static int gfx_v8_0_hw_fini(void *handle)
4868 {
4869 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4870 
4871 	amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4872 	amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4873 
4874 	amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
4875 
4876 	amdgpu_irq_put(adev, &adev->gfx.sq_irq, 0);
4877 
4878 	/* disable KCQ to avoid CPC touch memory not valid anymore */
4879 	gfx_v8_0_kcq_disable(adev);
4880 
4881 	if (amdgpu_sriov_vf(adev)) {
4882 		pr_debug("For SRIOV client, shouldn't do anything.\n");
4883 		return 0;
4884 	}
4885 	amdgpu_gfx_rlc_enter_safe_mode(adev);
4886 	if (!gfx_v8_0_wait_for_idle(adev))
4887 		gfx_v8_0_cp_enable(adev, false);
4888 	else
4889 		pr_err("cp is busy, skip halt cp\n");
4890 	if (!gfx_v8_0_wait_for_rlc_idle(adev))
4891 		adev->gfx.rlc.funcs->stop(adev);
4892 	else
4893 		pr_err("rlc is busy, skip halt rlc\n");
4894 	amdgpu_gfx_rlc_exit_safe_mode(adev);
4895 	return 0;
4896 }
4897 
4898 static int gfx_v8_0_suspend(void *handle)
4899 {
4900 	return gfx_v8_0_hw_fini(handle);
4901 }
4902 
4903 static int gfx_v8_0_resume(void *handle)
4904 {
4905 	return gfx_v8_0_hw_init(handle);
4906 }
4907 
4908 static bool gfx_v8_0_check_soft_reset(void *handle)
4909 {
4910 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4911 	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4912 	u32 tmp;
4913 
4914 	/* GRBM_STATUS */
4915 	tmp = RREG32(mmGRBM_STATUS);
4916 	if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
4917 		   GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
4918 		   GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
4919 		   GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
4920 		   GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
4921 		   GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK |
4922 		   GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4923 		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4924 						GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4925 		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4926 						GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
4927 		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
4928 						SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1);
4929 	}
4930 
4931 	/* GRBM_STATUS2 */
4932 	tmp = RREG32(mmGRBM_STATUS2);
4933 	if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
4934 		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4935 						GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4936 
4937 	if (REG_GET_FIELD(tmp, GRBM_STATUS2, CPF_BUSY) ||
4938 	    REG_GET_FIELD(tmp, GRBM_STATUS2, CPC_BUSY) ||
4939 	    REG_GET_FIELD(tmp, GRBM_STATUS2, CPG_BUSY)) {
4940 		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4941 						SOFT_RESET_CPF, 1);
4942 		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4943 						SOFT_RESET_CPC, 1);
4944 		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4945 						SOFT_RESET_CPG, 1);
4946 		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET,
4947 						SOFT_RESET_GRBM, 1);
4948 	}
4949 
4950 	/* SRBM_STATUS */
4951 	tmp = RREG32(mmSRBM_STATUS);
4952 	if (REG_GET_FIELD(tmp, SRBM_STATUS, GRBM_RQ_PENDING))
4953 		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
4954 						SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1);
4955 	if (REG_GET_FIELD(tmp, SRBM_STATUS, SEM_BUSY))
4956 		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
4957 						SRBM_SOFT_RESET, SOFT_RESET_SEM, 1);
4958 
4959 	if (grbm_soft_reset || srbm_soft_reset) {
4960 		adev->gfx.grbm_soft_reset = grbm_soft_reset;
4961 		adev->gfx.srbm_soft_reset = srbm_soft_reset;
4962 		return true;
4963 	} else {
4964 		adev->gfx.grbm_soft_reset = 0;
4965 		adev->gfx.srbm_soft_reset = 0;
4966 		return false;
4967 	}
4968 }
4969 
4970 static int gfx_v8_0_pre_soft_reset(void *handle)
4971 {
4972 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4973 	u32 grbm_soft_reset = 0;
4974 
4975 	if ((!adev->gfx.grbm_soft_reset) &&
4976 	    (!adev->gfx.srbm_soft_reset))
4977 		return 0;
4978 
4979 	grbm_soft_reset = adev->gfx.grbm_soft_reset;
4980 
4981 	/* stop the rlc */
4982 	adev->gfx.rlc.funcs->stop(adev);
4983 
4984 	if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
4985 	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX))
4986 		/* Disable GFX parsing/prefetching */
4987 		gfx_v8_0_cp_gfx_enable(adev, false);
4988 
4989 	if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
4990 	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) ||
4991 	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) ||
4992 	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) {
4993 		int i;
4994 
4995 		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4996 			struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
4997 
4998 			mutex_lock(&adev->srbm_mutex);
4999 			vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
5000 			gfx_v8_0_deactivate_hqd(adev, 2);
5001 			vi_srbm_select(adev, 0, 0, 0, 0);
5002 			mutex_unlock(&adev->srbm_mutex);
5003 		}
5004 		/* Disable MEC parsing/prefetching */
5005 		gfx_v8_0_cp_compute_enable(adev, false);
5006 	}
5007 
5008        return 0;
5009 }
5010 
5011 static int gfx_v8_0_soft_reset(void *handle)
5012 {
5013 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5014 	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5015 	u32 tmp;
5016 
5017 	if ((!adev->gfx.grbm_soft_reset) &&
5018 	    (!adev->gfx.srbm_soft_reset))
5019 		return 0;
5020 
5021 	grbm_soft_reset = adev->gfx.grbm_soft_reset;
5022 	srbm_soft_reset = adev->gfx.srbm_soft_reset;
5023 
5024 	if (grbm_soft_reset || srbm_soft_reset) {
5025 		tmp = RREG32(mmGMCON_DEBUG);
5026 		tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 1);
5027 		tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 1);
5028 		WREG32(mmGMCON_DEBUG, tmp);
5029 		udelay(50);
5030 	}
5031 
5032 	if (grbm_soft_reset) {
5033 		tmp = RREG32(mmGRBM_SOFT_RESET);
5034 		tmp |= grbm_soft_reset;
5035 		dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5036 		WREG32(mmGRBM_SOFT_RESET, tmp);
5037 		tmp = RREG32(mmGRBM_SOFT_RESET);
5038 
5039 		udelay(50);
5040 
5041 		tmp &= ~grbm_soft_reset;
5042 		WREG32(mmGRBM_SOFT_RESET, tmp);
5043 		tmp = RREG32(mmGRBM_SOFT_RESET);
5044 	}
5045 
5046 	if (srbm_soft_reset) {
5047 		tmp = RREG32(mmSRBM_SOFT_RESET);
5048 		tmp |= srbm_soft_reset;
5049 		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5050 		WREG32(mmSRBM_SOFT_RESET, tmp);
5051 		tmp = RREG32(mmSRBM_SOFT_RESET);
5052 
5053 		udelay(50);
5054 
5055 		tmp &= ~srbm_soft_reset;
5056 		WREG32(mmSRBM_SOFT_RESET, tmp);
5057 		tmp = RREG32(mmSRBM_SOFT_RESET);
5058 	}
5059 
5060 	if (grbm_soft_reset || srbm_soft_reset) {
5061 		tmp = RREG32(mmGMCON_DEBUG);
5062 		tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 0);
5063 		tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 0);
5064 		WREG32(mmGMCON_DEBUG, tmp);
5065 	}
5066 
5067 	/* Wait a little for things to settle down */
5068 	udelay(50);
5069 
5070 	return 0;
5071 }
5072 
5073 static int gfx_v8_0_post_soft_reset(void *handle)
5074 {
5075 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5076 	u32 grbm_soft_reset = 0;
5077 
5078 	if ((!adev->gfx.grbm_soft_reset) &&
5079 	    (!adev->gfx.srbm_soft_reset))
5080 		return 0;
5081 
5082 	grbm_soft_reset = adev->gfx.grbm_soft_reset;
5083 
5084 	if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
5085 	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) ||
5086 	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) ||
5087 	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) {
5088 		int i;
5089 
5090 		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5091 			struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
5092 
5093 			mutex_lock(&adev->srbm_mutex);
5094 			vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
5095 			gfx_v8_0_deactivate_hqd(adev, 2);
5096 			vi_srbm_select(adev, 0, 0, 0, 0);
5097 			mutex_unlock(&adev->srbm_mutex);
5098 		}
5099 		gfx_v8_0_kiq_resume(adev);
5100 		gfx_v8_0_kcq_resume(adev);
5101 	}
5102 
5103 	if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
5104 	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX))
5105 		gfx_v8_0_cp_gfx_resume(adev);
5106 
5107 	gfx_v8_0_cp_test_all_rings(adev);
5108 
5109 	adev->gfx.rlc.funcs->start(adev);
5110 
5111 	return 0;
5112 }
5113 
5114 /**
5115  * gfx_v8_0_get_gpu_clock_counter - return GPU clock counter snapshot
5116  *
5117  * @adev: amdgpu_device pointer
5118  *
5119  * Fetches a GPU clock counter snapshot.
5120  * Returns the 64 bit clock counter snapshot.
5121  */
5122 static uint64_t gfx_v8_0_get_gpu_clock_counter(struct amdgpu_device *adev)
5123 {
5124 	uint64_t clock;
5125 
5126 	mutex_lock(&adev->gfx.gpu_clock_mutex);
5127 	WREG32(mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
5128 	clock = (uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_LSB) |
5129 		((uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
5130 	mutex_unlock(&adev->gfx.gpu_clock_mutex);
5131 	return clock;
5132 }
5133 
5134 static void gfx_v8_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
5135 					  uint32_t vmid,
5136 					  uint32_t gds_base, uint32_t gds_size,
5137 					  uint32_t gws_base, uint32_t gws_size,
5138 					  uint32_t oa_base, uint32_t oa_size)
5139 {
5140 	/* GDS Base */
5141 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5142 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5143 				WRITE_DATA_DST_SEL(0)));
5144 	amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_base);
5145 	amdgpu_ring_write(ring, 0);
5146 	amdgpu_ring_write(ring, gds_base);
5147 
5148 	/* GDS Size */
5149 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5150 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5151 				WRITE_DATA_DST_SEL(0)));
5152 	amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_size);
5153 	amdgpu_ring_write(ring, 0);
5154 	amdgpu_ring_write(ring, gds_size);
5155 
5156 	/* GWS */
5157 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5158 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5159 				WRITE_DATA_DST_SEL(0)));
5160 	amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].gws);
5161 	amdgpu_ring_write(ring, 0);
5162 	amdgpu_ring_write(ring, gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
5163 
5164 	/* OA */
5165 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5166 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5167 				WRITE_DATA_DST_SEL(0)));
5168 	amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].oa);
5169 	amdgpu_ring_write(ring, 0);
5170 	amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base));
5171 }
5172 
5173 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
5174 {
5175 	WREG32(mmSQ_IND_INDEX,
5176 		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
5177 		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
5178 		(address << SQ_IND_INDEX__INDEX__SHIFT) |
5179 		(SQ_IND_INDEX__FORCE_READ_MASK));
5180 	return RREG32(mmSQ_IND_DATA);
5181 }
5182 
5183 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
5184 			   uint32_t wave, uint32_t thread,
5185 			   uint32_t regno, uint32_t num, uint32_t *out)
5186 {
5187 	WREG32(mmSQ_IND_INDEX,
5188 		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
5189 		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
5190 		(regno << SQ_IND_INDEX__INDEX__SHIFT) |
5191 		(thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
5192 		(SQ_IND_INDEX__FORCE_READ_MASK) |
5193 		(SQ_IND_INDEX__AUTO_INCR_MASK));
5194 	while (num--)
5195 		*(out++) = RREG32(mmSQ_IND_DATA);
5196 }
5197 
5198 static void gfx_v8_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
5199 {
5200 	/* type 0 wave data */
5201 	dst[(*no_fields)++] = 0;
5202 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
5203 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
5204 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
5205 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
5206 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
5207 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
5208 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
5209 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
5210 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
5211 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
5212 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
5213 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
5214 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_LO);
5215 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_HI);
5216 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_LO);
5217 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_HI);
5218 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
5219 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
5220 }
5221 
5222 static void gfx_v8_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
5223 				     uint32_t wave, uint32_t start,
5224 				     uint32_t size, uint32_t *dst)
5225 {
5226 	wave_read_regs(
5227 		adev, simd, wave, 0,
5228 		start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
5229 }
5230 
5231 
5232 static const struct amdgpu_gfx_funcs gfx_v8_0_gfx_funcs = {
5233 	.get_gpu_clock_counter = &gfx_v8_0_get_gpu_clock_counter,
5234 	.select_se_sh = &gfx_v8_0_select_se_sh,
5235 	.read_wave_data = &gfx_v8_0_read_wave_data,
5236 	.read_wave_sgprs = &gfx_v8_0_read_wave_sgprs,
5237 	.select_me_pipe_q = &gfx_v8_0_select_me_pipe_q
5238 };
5239 
5240 static int gfx_v8_0_early_init(void *handle)
5241 {
5242 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5243 
5244 	adev->gfx.num_gfx_rings = GFX8_NUM_GFX_RINGS;
5245 	adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS;
5246 	adev->gfx.funcs = &gfx_v8_0_gfx_funcs;
5247 	gfx_v8_0_set_ring_funcs(adev);
5248 	gfx_v8_0_set_irq_funcs(adev);
5249 	gfx_v8_0_set_gds_init(adev);
5250 	gfx_v8_0_set_rlc_funcs(adev);
5251 
5252 	return 0;
5253 }
5254 
5255 static int gfx_v8_0_late_init(void *handle)
5256 {
5257 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5258 	int r;
5259 
5260 	r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
5261 	if (r)
5262 		return r;
5263 
5264 	r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
5265 	if (r)
5266 		return r;
5267 
5268 	/* requires IBs so do in late init after IB pool is initialized */
5269 	r = gfx_v8_0_do_edc_gpr_workarounds(adev);
5270 	if (r)
5271 		return r;
5272 
5273 	r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0);
5274 	if (r) {
5275 		DRM_ERROR("amdgpu_irq_get() failed to get IRQ for EDC, r: %d.\n", r);
5276 		return r;
5277 	}
5278 
5279 	r = amdgpu_irq_get(adev, &adev->gfx.sq_irq, 0);
5280 	if (r) {
5281 		DRM_ERROR(
5282 			"amdgpu_irq_get() failed to get IRQ for SQ, r: %d.\n",
5283 			r);
5284 		return r;
5285 	}
5286 
5287 	return 0;
5288 }
5289 
5290 static void gfx_v8_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
5291 						       bool enable)
5292 {
5293 	if (((adev->asic_type == CHIP_POLARIS11) ||
5294 	    (adev->asic_type == CHIP_POLARIS12) ||
5295 	    (adev->asic_type == CHIP_VEGAM)) &&
5296 	    adev->powerplay.pp_funcs->set_powergating_by_smu)
5297 		/* Send msg to SMU via Powerplay */
5298 		amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, enable);
5299 
5300 	WREG32_FIELD(RLC_PG_CNTL, STATIC_PER_CU_PG_ENABLE, enable ? 1 : 0);
5301 }
5302 
5303 static void gfx_v8_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
5304 							bool enable)
5305 {
5306 	WREG32_FIELD(RLC_PG_CNTL, DYN_PER_CU_PG_ENABLE, enable ? 1 : 0);
5307 }
5308 
5309 static void polaris11_enable_gfx_quick_mg_power_gating(struct amdgpu_device *adev,
5310 		bool enable)
5311 {
5312 	WREG32_FIELD(RLC_PG_CNTL, QUICK_PG_ENABLE, enable ? 1 : 0);
5313 }
5314 
5315 static void cz_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
5316 					  bool enable)
5317 {
5318 	WREG32_FIELD(RLC_PG_CNTL, GFX_POWER_GATING_ENABLE, enable ? 1 : 0);
5319 }
5320 
5321 static void cz_enable_gfx_pipeline_power_gating(struct amdgpu_device *adev,
5322 						bool enable)
5323 {
5324 	WREG32_FIELD(RLC_PG_CNTL, GFX_PIPELINE_PG_ENABLE, enable ? 1 : 0);
5325 
5326 	/* Read any GFX register to wake up GFX. */
5327 	if (!enable)
5328 		RREG32(mmDB_RENDER_CONTROL);
5329 }
5330 
5331 static void cz_update_gfx_cg_power_gating(struct amdgpu_device *adev,
5332 					  bool enable)
5333 {
5334 	if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
5335 		cz_enable_gfx_cg_power_gating(adev, true);
5336 		if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
5337 			cz_enable_gfx_pipeline_power_gating(adev, true);
5338 	} else {
5339 		cz_enable_gfx_cg_power_gating(adev, false);
5340 		cz_enable_gfx_pipeline_power_gating(adev, false);
5341 	}
5342 }
5343 
5344 static int gfx_v8_0_set_powergating_state(void *handle,
5345 					  enum amd_powergating_state state)
5346 {
5347 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5348 	bool enable = (state == AMD_PG_STATE_GATE);
5349 
5350 	if (amdgpu_sriov_vf(adev))
5351 		return 0;
5352 
5353 	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG |
5354 				AMD_PG_SUPPORT_RLC_SMU_HS |
5355 				AMD_PG_SUPPORT_CP |
5356 				AMD_PG_SUPPORT_GFX_DMG))
5357 		amdgpu_gfx_rlc_enter_safe_mode(adev);
5358 	switch (adev->asic_type) {
5359 	case CHIP_CARRIZO:
5360 	case CHIP_STONEY:
5361 
5362 		if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
5363 			cz_enable_sck_slow_down_on_power_up(adev, true);
5364 			cz_enable_sck_slow_down_on_power_down(adev, true);
5365 		} else {
5366 			cz_enable_sck_slow_down_on_power_up(adev, false);
5367 			cz_enable_sck_slow_down_on_power_down(adev, false);
5368 		}
5369 		if (adev->pg_flags & AMD_PG_SUPPORT_CP)
5370 			cz_enable_cp_power_gating(adev, true);
5371 		else
5372 			cz_enable_cp_power_gating(adev, false);
5373 
5374 		cz_update_gfx_cg_power_gating(adev, enable);
5375 
5376 		if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
5377 			gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true);
5378 		else
5379 			gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false);
5380 
5381 		if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
5382 			gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true);
5383 		else
5384 			gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false);
5385 		break;
5386 	case CHIP_POLARIS11:
5387 	case CHIP_POLARIS12:
5388 	case CHIP_VEGAM:
5389 		if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
5390 			gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true);
5391 		else
5392 			gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false);
5393 
5394 		if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
5395 			gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true);
5396 		else
5397 			gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false);
5398 
5399 		if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_QUICK_MG) && enable)
5400 			polaris11_enable_gfx_quick_mg_power_gating(adev, true);
5401 		else
5402 			polaris11_enable_gfx_quick_mg_power_gating(adev, false);
5403 		break;
5404 	default:
5405 		break;
5406 	}
5407 	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG |
5408 				AMD_PG_SUPPORT_RLC_SMU_HS |
5409 				AMD_PG_SUPPORT_CP |
5410 				AMD_PG_SUPPORT_GFX_DMG))
5411 		amdgpu_gfx_rlc_exit_safe_mode(adev);
5412 	return 0;
5413 }
5414 
5415 static void gfx_v8_0_get_clockgating_state(void *handle, u32 *flags)
5416 {
5417 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5418 	int data;
5419 
5420 	if (amdgpu_sriov_vf(adev))
5421 		*flags = 0;
5422 
5423 	/* AMD_CG_SUPPORT_GFX_MGCG */
5424 	data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5425 	if (!(data & RLC_CGTT_MGCG_OVERRIDE__CPF_MASK))
5426 		*flags |= AMD_CG_SUPPORT_GFX_MGCG;
5427 
5428 	/* AMD_CG_SUPPORT_GFX_CGLG */
5429 	data = RREG32(mmRLC_CGCG_CGLS_CTRL);
5430 	if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5431 		*flags |= AMD_CG_SUPPORT_GFX_CGCG;
5432 
5433 	/* AMD_CG_SUPPORT_GFX_CGLS */
5434 	if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5435 		*flags |= AMD_CG_SUPPORT_GFX_CGLS;
5436 
5437 	/* AMD_CG_SUPPORT_GFX_CGTS */
5438 	data = RREG32(mmCGTS_SM_CTRL_REG);
5439 	if (!(data & CGTS_SM_CTRL_REG__OVERRIDE_MASK))
5440 		*flags |= AMD_CG_SUPPORT_GFX_CGTS;
5441 
5442 	/* AMD_CG_SUPPORT_GFX_CGTS_LS */
5443 	if (!(data & CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK))
5444 		*flags |= AMD_CG_SUPPORT_GFX_CGTS_LS;
5445 
5446 	/* AMD_CG_SUPPORT_GFX_RLC_LS */
5447 	data = RREG32(mmRLC_MEM_SLP_CNTL);
5448 	if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5449 		*flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5450 
5451 	/* AMD_CG_SUPPORT_GFX_CP_LS */
5452 	data = RREG32(mmCP_MEM_SLP_CNTL);
5453 	if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5454 		*flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5455 }
5456 
5457 static void gfx_v8_0_send_serdes_cmd(struct amdgpu_device *adev,
5458 				     uint32_t reg_addr, uint32_t cmd)
5459 {
5460 	uint32_t data;
5461 
5462 	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
5463 
5464 	WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5465 	WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5466 
5467 	data = RREG32(mmRLC_SERDES_WR_CTRL);
5468 	if (adev->asic_type == CHIP_STONEY)
5469 		data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK |
5470 			  RLC_SERDES_WR_CTRL__READ_COMMAND_MASK |
5471 			  RLC_SERDES_WR_CTRL__P1_SELECT_MASK |
5472 			  RLC_SERDES_WR_CTRL__P2_SELECT_MASK |
5473 			  RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK |
5474 			  RLC_SERDES_WR_CTRL__POWER_DOWN_MASK |
5475 			  RLC_SERDES_WR_CTRL__POWER_UP_MASK |
5476 			  RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK |
5477 			  RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK);
5478 	else
5479 		data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK |
5480 			  RLC_SERDES_WR_CTRL__READ_COMMAND_MASK |
5481 			  RLC_SERDES_WR_CTRL__P1_SELECT_MASK |
5482 			  RLC_SERDES_WR_CTRL__P2_SELECT_MASK |
5483 			  RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK |
5484 			  RLC_SERDES_WR_CTRL__POWER_DOWN_MASK |
5485 			  RLC_SERDES_WR_CTRL__POWER_UP_MASK |
5486 			  RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK |
5487 			  RLC_SERDES_WR_CTRL__BPM_DATA_MASK |
5488 			  RLC_SERDES_WR_CTRL__REG_ADDR_MASK |
5489 			  RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK);
5490 	data |= (RLC_SERDES_WR_CTRL__RSVD_BPM_ADDR_MASK |
5491 		 (cmd << RLC_SERDES_WR_CTRL__BPM_DATA__SHIFT) |
5492 		 (reg_addr << RLC_SERDES_WR_CTRL__REG_ADDR__SHIFT) |
5493 		 (0xff << RLC_SERDES_WR_CTRL__BPM_ADDR__SHIFT));
5494 
5495 	WREG32(mmRLC_SERDES_WR_CTRL, data);
5496 }
5497 
5498 #define MSG_ENTER_RLC_SAFE_MODE     1
5499 #define MSG_EXIT_RLC_SAFE_MODE      0
5500 #define RLC_GPR_REG2__REQ_MASK 0x00000001
5501 #define RLC_GPR_REG2__REQ__SHIFT 0
5502 #define RLC_GPR_REG2__MESSAGE__SHIFT 0x00000001
5503 #define RLC_GPR_REG2__MESSAGE_MASK 0x0000001e
5504 
5505 static bool gfx_v8_0_is_rlc_enabled(struct amdgpu_device *adev)
5506 {
5507 	uint32_t rlc_setting;
5508 
5509 	rlc_setting = RREG32(mmRLC_CNTL);
5510 	if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
5511 		return false;
5512 
5513 	return true;
5514 }
5515 
5516 static void gfx_v8_0_set_safe_mode(struct amdgpu_device *adev)
5517 {
5518 	uint32_t data;
5519 	unsigned i;
5520 	data = RREG32(mmRLC_CNTL);
5521 	data |= RLC_SAFE_MODE__CMD_MASK;
5522 	data &= ~RLC_SAFE_MODE__MESSAGE_MASK;
5523 	data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
5524 	WREG32(mmRLC_SAFE_MODE, data);
5525 
5526 	/* wait for RLC_SAFE_MODE */
5527 	for (i = 0; i < adev->usec_timeout; i++) {
5528 		if ((RREG32(mmRLC_GPM_STAT) &
5529 		     (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK |
5530 		      RLC_GPM_STAT__GFX_POWER_STATUS_MASK)) ==
5531 		    (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK |
5532 		     RLC_GPM_STAT__GFX_POWER_STATUS_MASK))
5533 			break;
5534 		udelay(1);
5535 	}
5536 	for (i = 0; i < adev->usec_timeout; i++) {
5537 		if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
5538 			break;
5539 		udelay(1);
5540 	}
5541 }
5542 
5543 static void gfx_v8_0_unset_safe_mode(struct amdgpu_device *adev)
5544 {
5545 	uint32_t data;
5546 	unsigned i;
5547 
5548 	data = RREG32(mmRLC_CNTL);
5549 	data |= RLC_SAFE_MODE__CMD_MASK;
5550 	data &= ~RLC_SAFE_MODE__MESSAGE_MASK;
5551 	WREG32(mmRLC_SAFE_MODE, data);
5552 
5553 	for (i = 0; i < adev->usec_timeout; i++) {
5554 		if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
5555 			break;
5556 		udelay(1);
5557 	}
5558 }
5559 
5560 static const struct amdgpu_rlc_funcs iceland_rlc_funcs = {
5561 	.is_rlc_enabled = gfx_v8_0_is_rlc_enabled,
5562 	.set_safe_mode = gfx_v8_0_set_safe_mode,
5563 	.unset_safe_mode = gfx_v8_0_unset_safe_mode,
5564 	.init = gfx_v8_0_rlc_init,
5565 	.get_csb_size = gfx_v8_0_get_csb_size,
5566 	.get_csb_buffer = gfx_v8_0_get_csb_buffer,
5567 	.get_cp_table_num = gfx_v8_0_cp_jump_table_num,
5568 	.resume = gfx_v8_0_rlc_resume,
5569 	.stop = gfx_v8_0_rlc_stop,
5570 	.reset = gfx_v8_0_rlc_reset,
5571 	.start = gfx_v8_0_rlc_start
5572 };
5573 
5574 static void gfx_v8_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
5575 						      bool enable)
5576 {
5577 	uint32_t temp, data;
5578 
5579 	amdgpu_gfx_rlc_enter_safe_mode(adev);
5580 
5581 	/* It is disabled by HW by default */
5582 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
5583 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
5584 			if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS)
5585 				/* 1 - RLC memory Light sleep */
5586 				WREG32_FIELD(RLC_MEM_SLP_CNTL, RLC_MEM_LS_EN, 1);
5587 
5588 			if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS)
5589 				WREG32_FIELD(CP_MEM_SLP_CNTL, CP_MEM_LS_EN, 1);
5590 		}
5591 
5592 		/* 3 - RLC_CGTT_MGCG_OVERRIDE */
5593 		temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5594 		if (adev->flags & AMD_IS_APU)
5595 			data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |
5596 				  RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |
5597 				  RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK);
5598 		else
5599 			data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |
5600 				  RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |
5601 				  RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK |
5602 				  RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK);
5603 
5604 		if (temp != data)
5605 			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
5606 
5607 		/* 4 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5608 		gfx_v8_0_wait_for_rlc_serdes(adev);
5609 
5610 		/* 5 - clear mgcg override */
5611 		gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, CLE_BPM_SERDES_CMD);
5612 
5613 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS) {
5614 			/* 6 - Enable CGTS(Tree Shade) MGCG /MGLS */
5615 			temp = data = RREG32(mmCGTS_SM_CTRL_REG);
5616 			data &= ~(CGTS_SM_CTRL_REG__SM_MODE_MASK);
5617 			data |= (0x2 << CGTS_SM_CTRL_REG__SM_MODE__SHIFT);
5618 			data |= CGTS_SM_CTRL_REG__SM_MODE_ENABLE_MASK;
5619 			data &= ~CGTS_SM_CTRL_REG__OVERRIDE_MASK;
5620 			if ((adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) &&
5621 			    (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS_LS))
5622 				data &= ~CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK;
5623 			data |= CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN_MASK;
5624 			data |= (0x96 << CGTS_SM_CTRL_REG__ON_MONITOR_ADD__SHIFT);
5625 			if (temp != data)
5626 				WREG32(mmCGTS_SM_CTRL_REG, data);
5627 		}
5628 		udelay(50);
5629 
5630 		/* 7 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5631 		gfx_v8_0_wait_for_rlc_serdes(adev);
5632 	} else {
5633 		/* 1 - MGCG_OVERRIDE[0] for CP and MGCG_OVERRIDE[1] for RLC */
5634 		temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5635 		data |= (RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |
5636 				RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |
5637 				RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK |
5638 				RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK);
5639 		if (temp != data)
5640 			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
5641 
5642 		/* 2 - disable MGLS in RLC */
5643 		data = RREG32(mmRLC_MEM_SLP_CNTL);
5644 		if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
5645 			data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
5646 			WREG32(mmRLC_MEM_SLP_CNTL, data);
5647 		}
5648 
5649 		/* 3 - disable MGLS in CP */
5650 		data = RREG32(mmCP_MEM_SLP_CNTL);
5651 		if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
5652 			data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
5653 			WREG32(mmCP_MEM_SLP_CNTL, data);
5654 		}
5655 
5656 		/* 4 - Disable CGTS(Tree Shade) MGCG and MGLS */
5657 		temp = data = RREG32(mmCGTS_SM_CTRL_REG);
5658 		data |= (CGTS_SM_CTRL_REG__OVERRIDE_MASK |
5659 				CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK);
5660 		if (temp != data)
5661 			WREG32(mmCGTS_SM_CTRL_REG, data);
5662 
5663 		/* 5 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5664 		gfx_v8_0_wait_for_rlc_serdes(adev);
5665 
5666 		/* 6 - set mgcg override */
5667 		gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, SET_BPM_SERDES_CMD);
5668 
5669 		udelay(50);
5670 
5671 		/* 7- wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5672 		gfx_v8_0_wait_for_rlc_serdes(adev);
5673 	}
5674 
5675 	amdgpu_gfx_rlc_exit_safe_mode(adev);
5676 }
5677 
5678 static void gfx_v8_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
5679 						      bool enable)
5680 {
5681 	uint32_t temp, temp1, data, data1;
5682 
5683 	temp = data = RREG32(mmRLC_CGCG_CGLS_CTRL);
5684 
5685 	amdgpu_gfx_rlc_enter_safe_mode(adev);
5686 
5687 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
5688 		temp1 = data1 =	RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5689 		data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK;
5690 		if (temp1 != data1)
5691 			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
5692 
5693 		/* : wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5694 		gfx_v8_0_wait_for_rlc_serdes(adev);
5695 
5696 		/* 2 - clear cgcg override */
5697 		gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, CLE_BPM_SERDES_CMD);
5698 
5699 		/* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5700 		gfx_v8_0_wait_for_rlc_serdes(adev);
5701 
5702 		/* 3 - write cmd to set CGLS */
5703 		gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, SET_BPM_SERDES_CMD);
5704 
5705 		/* 4 - enable cgcg */
5706 		data |= RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5707 
5708 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5709 			/* enable cgls*/
5710 			data |= RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5711 
5712 			temp1 = data1 =	RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5713 			data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK;
5714 
5715 			if (temp1 != data1)
5716 				WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
5717 		} else {
5718 			data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5719 		}
5720 
5721 		if (temp != data)
5722 			WREG32(mmRLC_CGCG_CGLS_CTRL, data);
5723 
5724 		/* 5 enable cntx_empty_int_enable/cntx_busy_int_enable/
5725 		 * Cmp_busy/GFX_Idle interrupts
5726 		 */
5727 		gfx_v8_0_enable_gui_idle_interrupt(adev, true);
5728 	} else {
5729 		/* disable cntx_empty_int_enable & GFX Idle interrupt */
5730 		gfx_v8_0_enable_gui_idle_interrupt(adev, false);
5731 
5732 		/* TEST CGCG */
5733 		temp1 = data1 =	RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5734 		data1 |= (RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK |
5735 				RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK);
5736 		if (temp1 != data1)
5737 			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
5738 
5739 		/* read gfx register to wake up cgcg */
5740 		RREG32(mmCB_CGTT_SCLK_CTRL);
5741 		RREG32(mmCB_CGTT_SCLK_CTRL);
5742 		RREG32(mmCB_CGTT_SCLK_CTRL);
5743 		RREG32(mmCB_CGTT_SCLK_CTRL);
5744 
5745 		/* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5746 		gfx_v8_0_wait_for_rlc_serdes(adev);
5747 
5748 		/* write cmd to Set CGCG Overrride */
5749 		gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, SET_BPM_SERDES_CMD);
5750 
5751 		/* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5752 		gfx_v8_0_wait_for_rlc_serdes(adev);
5753 
5754 		/* write cmd to Clear CGLS */
5755 		gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, CLE_BPM_SERDES_CMD);
5756 
5757 		/* disable cgcg, cgls should be disabled too. */
5758 		data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK |
5759 			  RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
5760 		if (temp != data)
5761 			WREG32(mmRLC_CGCG_CGLS_CTRL, data);
5762 		/* enable interrupts again for PG */
5763 		gfx_v8_0_enable_gui_idle_interrupt(adev, true);
5764 	}
5765 
5766 	gfx_v8_0_wait_for_rlc_serdes(adev);
5767 
5768 	amdgpu_gfx_rlc_exit_safe_mode(adev);
5769 }
5770 static int gfx_v8_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5771 					    bool enable)
5772 {
5773 	if (enable) {
5774 		/* CGCG/CGLS should be enabled after MGCG/MGLS/TS(CG/LS)
5775 		 * ===  MGCG + MGLS + TS(CG/LS) ===
5776 		 */
5777 		gfx_v8_0_update_medium_grain_clock_gating(adev, enable);
5778 		gfx_v8_0_update_coarse_grain_clock_gating(adev, enable);
5779 	} else {
5780 		/* CGCG/CGLS should be disabled before MGCG/MGLS/TS(CG/LS)
5781 		 * ===  CGCG + CGLS ===
5782 		 */
5783 		gfx_v8_0_update_coarse_grain_clock_gating(adev, enable);
5784 		gfx_v8_0_update_medium_grain_clock_gating(adev, enable);
5785 	}
5786 	return 0;
5787 }
5788 
5789 static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev,
5790 					  enum amd_clockgating_state state)
5791 {
5792 	uint32_t msg_id, pp_state = 0;
5793 	uint32_t pp_support_state = 0;
5794 
5795 	if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) {
5796 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5797 			pp_support_state = PP_STATE_SUPPORT_LS;
5798 			pp_state = PP_STATE_LS;
5799 		}
5800 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
5801 			pp_support_state |= PP_STATE_SUPPORT_CG;
5802 			pp_state |= PP_STATE_CG;
5803 		}
5804 		if (state == AMD_CG_STATE_UNGATE)
5805 			pp_state = 0;
5806 
5807 		msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5808 				PP_BLOCK_GFX_CG,
5809 				pp_support_state,
5810 				pp_state);
5811 		if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
5812 			amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5813 	}
5814 
5815 	if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) {
5816 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
5817 			pp_support_state = PP_STATE_SUPPORT_LS;
5818 			pp_state = PP_STATE_LS;
5819 		}
5820 
5821 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5822 			pp_support_state |= PP_STATE_SUPPORT_CG;
5823 			pp_state |= PP_STATE_CG;
5824 		}
5825 
5826 		if (state == AMD_CG_STATE_UNGATE)
5827 			pp_state = 0;
5828 
5829 		msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5830 				PP_BLOCK_GFX_MG,
5831 				pp_support_state,
5832 				pp_state);
5833 		if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
5834 			amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5835 	}
5836 
5837 	return 0;
5838 }
5839 
5840 static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
5841 					  enum amd_clockgating_state state)
5842 {
5843 
5844 	uint32_t msg_id, pp_state = 0;
5845 	uint32_t pp_support_state = 0;
5846 
5847 	if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) {
5848 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5849 			pp_support_state = PP_STATE_SUPPORT_LS;
5850 			pp_state = PP_STATE_LS;
5851 		}
5852 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
5853 			pp_support_state |= PP_STATE_SUPPORT_CG;
5854 			pp_state |= PP_STATE_CG;
5855 		}
5856 		if (state == AMD_CG_STATE_UNGATE)
5857 			pp_state = 0;
5858 
5859 		msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5860 				PP_BLOCK_GFX_CG,
5861 				pp_support_state,
5862 				pp_state);
5863 		if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
5864 			amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5865 	}
5866 
5867 	if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_3D_CGCG | AMD_CG_SUPPORT_GFX_3D_CGLS)) {
5868 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) {
5869 			pp_support_state = PP_STATE_SUPPORT_LS;
5870 			pp_state = PP_STATE_LS;
5871 		}
5872 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) {
5873 			pp_support_state |= PP_STATE_SUPPORT_CG;
5874 			pp_state |= PP_STATE_CG;
5875 		}
5876 		if (state == AMD_CG_STATE_UNGATE)
5877 			pp_state = 0;
5878 
5879 		msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5880 				PP_BLOCK_GFX_3D,
5881 				pp_support_state,
5882 				pp_state);
5883 		if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
5884 			amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5885 	}
5886 
5887 	if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) {
5888 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
5889 			pp_support_state = PP_STATE_SUPPORT_LS;
5890 			pp_state = PP_STATE_LS;
5891 		}
5892 
5893 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5894 			pp_support_state |= PP_STATE_SUPPORT_CG;
5895 			pp_state |= PP_STATE_CG;
5896 		}
5897 
5898 		if (state == AMD_CG_STATE_UNGATE)
5899 			pp_state = 0;
5900 
5901 		msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5902 				PP_BLOCK_GFX_MG,
5903 				pp_support_state,
5904 				pp_state);
5905 		if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
5906 			amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5907 	}
5908 
5909 	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
5910 		pp_support_state = PP_STATE_SUPPORT_LS;
5911 
5912 		if (state == AMD_CG_STATE_UNGATE)
5913 			pp_state = 0;
5914 		else
5915 			pp_state = PP_STATE_LS;
5916 
5917 		msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5918 				PP_BLOCK_GFX_RLC,
5919 				pp_support_state,
5920 				pp_state);
5921 		if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
5922 			amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5923 	}
5924 
5925 	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
5926 		pp_support_state = PP_STATE_SUPPORT_LS;
5927 
5928 		if (state == AMD_CG_STATE_UNGATE)
5929 			pp_state = 0;
5930 		else
5931 			pp_state = PP_STATE_LS;
5932 		msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5933 			PP_BLOCK_GFX_CP,
5934 			pp_support_state,
5935 			pp_state);
5936 		if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
5937 			amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5938 	}
5939 
5940 	return 0;
5941 }
5942 
5943 static int gfx_v8_0_set_clockgating_state(void *handle,
5944 					  enum amd_clockgating_state state)
5945 {
5946 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5947 
5948 	if (amdgpu_sriov_vf(adev))
5949 		return 0;
5950 
5951 	switch (adev->asic_type) {
5952 	case CHIP_FIJI:
5953 	case CHIP_CARRIZO:
5954 	case CHIP_STONEY:
5955 		gfx_v8_0_update_gfx_clock_gating(adev,
5956 						 state == AMD_CG_STATE_GATE);
5957 		break;
5958 	case CHIP_TONGA:
5959 		gfx_v8_0_tonga_update_gfx_clock_gating(adev, state);
5960 		break;
5961 	case CHIP_POLARIS10:
5962 	case CHIP_POLARIS11:
5963 	case CHIP_POLARIS12:
5964 	case CHIP_VEGAM:
5965 		gfx_v8_0_polaris_update_gfx_clock_gating(adev, state);
5966 		break;
5967 	default:
5968 		break;
5969 	}
5970 	return 0;
5971 }
5972 
5973 static u64 gfx_v8_0_ring_get_rptr(struct amdgpu_ring *ring)
5974 {
5975 	return ring->adev->wb.wb[ring->rptr_offs];
5976 }
5977 
5978 static u64 gfx_v8_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5979 {
5980 	struct amdgpu_device *adev = ring->adev;
5981 
5982 	if (ring->use_doorbell)
5983 		/* XXX check if swapping is necessary on BE */
5984 		return ring->adev->wb.wb[ring->wptr_offs];
5985 	else
5986 		return RREG32(mmCP_RB0_WPTR);
5987 }
5988 
5989 static void gfx_v8_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5990 {
5991 	struct amdgpu_device *adev = ring->adev;
5992 
5993 	if (ring->use_doorbell) {
5994 		/* XXX check if swapping is necessary on BE */
5995 		adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
5996 		WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
5997 	} else {
5998 		WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
5999 		(void)RREG32(mmCP_RB0_WPTR);
6000 	}
6001 }
6002 
6003 static void gfx_v8_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
6004 {
6005 	u32 ref_and_mask, reg_mem_engine;
6006 
6007 	if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) ||
6008 	    (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)) {
6009 		switch (ring->me) {
6010 		case 1:
6011 			ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe;
6012 			break;
6013 		case 2:
6014 			ref_and_mask = GPU_HDP_FLUSH_DONE__CP6_MASK << ring->pipe;
6015 			break;
6016 		default:
6017 			return;
6018 		}
6019 		reg_mem_engine = 0;
6020 	} else {
6021 		ref_and_mask = GPU_HDP_FLUSH_DONE__CP0_MASK;
6022 		reg_mem_engine = WAIT_REG_MEM_ENGINE(1); /* pfp */
6023 	}
6024 
6025 	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6026 	amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
6027 				 WAIT_REG_MEM_FUNCTION(3) |  /* == */
6028 				 reg_mem_engine));
6029 	amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ);
6030 	amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE);
6031 	amdgpu_ring_write(ring, ref_and_mask);
6032 	amdgpu_ring_write(ring, ref_and_mask);
6033 	amdgpu_ring_write(ring, 0x20); /* poll interval */
6034 }
6035 
6036 static void gfx_v8_0_ring_emit_vgt_flush(struct amdgpu_ring *ring)
6037 {
6038 	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
6039 	amdgpu_ring_write(ring, EVENT_TYPE(VS_PARTIAL_FLUSH) |
6040 		EVENT_INDEX(4));
6041 
6042 	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
6043 	amdgpu_ring_write(ring, EVENT_TYPE(VGT_FLUSH) |
6044 		EVENT_INDEX(0));
6045 }
6046 
6047 static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
6048 					struct amdgpu_job *job,
6049 					struct amdgpu_ib *ib,
6050 					uint32_t flags)
6051 {
6052 	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
6053 	u32 header, control = 0;
6054 
6055 	if (ib->flags & AMDGPU_IB_FLAG_CE)
6056 		header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
6057 	else
6058 		header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
6059 
6060 	control |= ib->length_dw | (vmid << 24);
6061 
6062 	if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
6063 		control |= INDIRECT_BUFFER_PRE_ENB(1);
6064 
6065 		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
6066 			gfx_v8_0_ring_emit_de_meta(ring);
6067 	}
6068 
6069 	amdgpu_ring_write(ring, header);
6070 	amdgpu_ring_write(ring,
6071 #ifdef __BIG_ENDIAN
6072 			  (2 << 0) |
6073 #endif
6074 			  (ib->gpu_addr & 0xFFFFFFFC));
6075 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
6076 	amdgpu_ring_write(ring, control);
6077 }
6078 
6079 static void gfx_v8_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
6080 					  struct amdgpu_job *job,
6081 					  struct amdgpu_ib *ib,
6082 					  uint32_t flags)
6083 {
6084 	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
6085 	u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
6086 
6087 	/* Currently, there is a high possibility to get wave ID mismatch
6088 	 * between ME and GDS, leading to a hw deadlock, because ME generates
6089 	 * different wave IDs than the GDS expects. This situation happens
6090 	 * randomly when at least 5 compute pipes use GDS ordered append.
6091 	 * The wave IDs generated by ME are also wrong after suspend/resume.
6092 	 * Those are probably bugs somewhere else in the kernel driver.
6093 	 *
6094 	 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
6095 	 * GDS to 0 for this ring (me/pipe).
6096 	 */
6097 	if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
6098 		amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
6099 		amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID - PACKET3_SET_CONFIG_REG_START);
6100 		amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
6101 	}
6102 
6103 	amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
6104 	amdgpu_ring_write(ring,
6105 #ifdef __BIG_ENDIAN
6106 				(2 << 0) |
6107 #endif
6108 				(ib->gpu_addr & 0xFFFFFFFC));
6109 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
6110 	amdgpu_ring_write(ring, control);
6111 }
6112 
6113 static void gfx_v8_0_ring_emit_fence_gfx(struct amdgpu_ring *ring, u64 addr,
6114 					 u64 seq, unsigned flags)
6115 {
6116 	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
6117 	bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
6118 
6119 	/* EVENT_WRITE_EOP - flush caches, send int */
6120 	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
6121 	amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
6122 				 EOP_TC_ACTION_EN |
6123 				 EOP_TC_WB_ACTION_EN |
6124 				 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
6125 				 EVENT_INDEX(5)));
6126 	amdgpu_ring_write(ring, addr & 0xfffffffc);
6127 	amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
6128 			  DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
6129 	amdgpu_ring_write(ring, lower_32_bits(seq));
6130 	amdgpu_ring_write(ring, upper_32_bits(seq));
6131 
6132 }
6133 
6134 static void gfx_v8_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
6135 {
6136 	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
6137 	uint32_t seq = ring->fence_drv.sync_seq;
6138 	uint64_t addr = ring->fence_drv.gpu_addr;
6139 
6140 	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6141 	amdgpu_ring_write(ring, (WAIT_REG_MEM_MEM_SPACE(1) | /* memory */
6142 				 WAIT_REG_MEM_FUNCTION(3) | /* equal */
6143 				 WAIT_REG_MEM_ENGINE(usepfp))); /* pfp or me */
6144 	amdgpu_ring_write(ring, addr & 0xfffffffc);
6145 	amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
6146 	amdgpu_ring_write(ring, seq);
6147 	amdgpu_ring_write(ring, 0xffffffff);
6148 	amdgpu_ring_write(ring, 4); /* poll interval */
6149 }
6150 
6151 static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
6152 					unsigned vmid, uint64_t pd_addr)
6153 {
6154 	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
6155 
6156 	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
6157 
6158 	/* wait for the invalidate to complete */
6159 	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6160 	amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
6161 				 WAIT_REG_MEM_FUNCTION(0) |  /* always */
6162 				 WAIT_REG_MEM_ENGINE(0))); /* me */
6163 	amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST);
6164 	amdgpu_ring_write(ring, 0);
6165 	amdgpu_ring_write(ring, 0); /* ref */
6166 	amdgpu_ring_write(ring, 0); /* mask */
6167 	amdgpu_ring_write(ring, 0x20); /* poll interval */
6168 
6169 	/* compute doesn't have PFP */
6170 	if (usepfp) {
6171 		/* sync PFP to ME, otherwise we might get invalid PFP reads */
6172 		amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6173 		amdgpu_ring_write(ring, 0x0);
6174 	}
6175 }
6176 
6177 static u64 gfx_v8_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
6178 {
6179 	return ring->adev->wb.wb[ring->wptr_offs];
6180 }
6181 
6182 static void gfx_v8_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
6183 {
6184 	struct amdgpu_device *adev = ring->adev;
6185 
6186 	/* XXX check if swapping is necessary on BE */
6187 	adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
6188 	WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
6189 }
6190 
6191 static void gfx_v8_0_ring_set_pipe_percent(struct amdgpu_ring *ring,
6192 					   bool acquire)
6193 {
6194 	struct amdgpu_device *adev = ring->adev;
6195 	int pipe_num, tmp, reg;
6196 	int pipe_percent = acquire ? SPI_WCL_PIPE_PERCENT_GFX__VALUE_MASK : 0x1;
6197 
6198 	pipe_num = ring->me * adev->gfx.mec.num_pipe_per_mec + ring->pipe;
6199 
6200 	/* first me only has 2 entries, GFX and HP3D */
6201 	if (ring->me > 0)
6202 		pipe_num -= 2;
6203 
6204 	reg = mmSPI_WCL_PIPE_PERCENT_GFX + pipe_num;
6205 	tmp = RREG32(reg);
6206 	tmp = REG_SET_FIELD(tmp, SPI_WCL_PIPE_PERCENT_GFX, VALUE, pipe_percent);
6207 	WREG32(reg, tmp);
6208 }
6209 
6210 static void gfx_v8_0_pipe_reserve_resources(struct amdgpu_device *adev,
6211 					    struct amdgpu_ring *ring,
6212 					    bool acquire)
6213 {
6214 	int i, pipe;
6215 	bool reserve;
6216 	struct amdgpu_ring *iring;
6217 
6218 	mutex_lock(&adev->gfx.pipe_reserve_mutex);
6219 	pipe = amdgpu_gfx_queue_to_bit(adev, ring->me, ring->pipe, 0);
6220 	if (acquire)
6221 		set_bit(pipe, adev->gfx.pipe_reserve_bitmap);
6222 	else
6223 		clear_bit(pipe, adev->gfx.pipe_reserve_bitmap);
6224 
6225 	if (!bitmap_weight(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES)) {
6226 		/* Clear all reservations - everyone reacquires all resources */
6227 		for (i = 0; i < adev->gfx.num_gfx_rings; ++i)
6228 			gfx_v8_0_ring_set_pipe_percent(&adev->gfx.gfx_ring[i],
6229 						       true);
6230 
6231 		for (i = 0; i < adev->gfx.num_compute_rings; ++i)
6232 			gfx_v8_0_ring_set_pipe_percent(&adev->gfx.compute_ring[i],
6233 						       true);
6234 	} else {
6235 		/* Lower all pipes without a current reservation */
6236 		for (i = 0; i < adev->gfx.num_gfx_rings; ++i) {
6237 			iring = &adev->gfx.gfx_ring[i];
6238 			pipe = amdgpu_gfx_queue_to_bit(adev,
6239 						       iring->me,
6240 						       iring->pipe,
6241 						       0);
6242 			reserve = test_bit(pipe, adev->gfx.pipe_reserve_bitmap);
6243 			gfx_v8_0_ring_set_pipe_percent(iring, reserve);
6244 		}
6245 
6246 		for (i = 0; i < adev->gfx.num_compute_rings; ++i) {
6247 			iring = &adev->gfx.compute_ring[i];
6248 			pipe = amdgpu_gfx_queue_to_bit(adev,
6249 						       iring->me,
6250 						       iring->pipe,
6251 						       0);
6252 			reserve = test_bit(pipe, adev->gfx.pipe_reserve_bitmap);
6253 			gfx_v8_0_ring_set_pipe_percent(iring, reserve);
6254 		}
6255 	}
6256 
6257 	mutex_unlock(&adev->gfx.pipe_reserve_mutex);
6258 }
6259 
6260 static void gfx_v8_0_hqd_set_priority(struct amdgpu_device *adev,
6261 				      struct amdgpu_ring *ring,
6262 				      bool acquire)
6263 {
6264 	uint32_t pipe_priority = acquire ? 0x2 : 0x0;
6265 	uint32_t queue_priority = acquire ? 0xf : 0x0;
6266 
6267 	mutex_lock(&adev->srbm_mutex);
6268 	vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
6269 
6270 	WREG32(mmCP_HQD_PIPE_PRIORITY, pipe_priority);
6271 	WREG32(mmCP_HQD_QUEUE_PRIORITY, queue_priority);
6272 
6273 	vi_srbm_select(adev, 0, 0, 0, 0);
6274 	mutex_unlock(&adev->srbm_mutex);
6275 }
6276 static void gfx_v8_0_ring_set_priority_compute(struct amdgpu_ring *ring,
6277 					       enum drm_sched_priority priority)
6278 {
6279 	struct amdgpu_device *adev = ring->adev;
6280 	bool acquire = priority == DRM_SCHED_PRIORITY_HIGH_HW;
6281 
6282 	if (ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE)
6283 		return;
6284 
6285 	gfx_v8_0_hqd_set_priority(adev, ring, acquire);
6286 	gfx_v8_0_pipe_reserve_resources(adev, ring, acquire);
6287 }
6288 
6289 static void gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring *ring,
6290 					     u64 addr, u64 seq,
6291 					     unsigned flags)
6292 {
6293 	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
6294 	bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
6295 
6296 	/* RELEASE_MEM - flush caches, send int */
6297 	amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
6298 	amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
6299 				 EOP_TC_ACTION_EN |
6300 				 EOP_TC_WB_ACTION_EN |
6301 				 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
6302 				 EVENT_INDEX(5)));
6303 	amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
6304 	amdgpu_ring_write(ring, addr & 0xfffffffc);
6305 	amdgpu_ring_write(ring, upper_32_bits(addr));
6306 	amdgpu_ring_write(ring, lower_32_bits(seq));
6307 	amdgpu_ring_write(ring, upper_32_bits(seq));
6308 }
6309 
6310 static void gfx_v8_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
6311 					 u64 seq, unsigned int flags)
6312 {
6313 	/* we only allocate 32bit for each seq wb address */
6314 	BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
6315 
6316 	/* write fence seq to the "addr" */
6317 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6318 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
6319 				 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
6320 	amdgpu_ring_write(ring, lower_32_bits(addr));
6321 	amdgpu_ring_write(ring, upper_32_bits(addr));
6322 	amdgpu_ring_write(ring, lower_32_bits(seq));
6323 
6324 	if (flags & AMDGPU_FENCE_FLAG_INT) {
6325 		/* set register to trigger INT */
6326 		amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6327 		amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
6328 					 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
6329 		amdgpu_ring_write(ring, mmCPC_INT_STATUS);
6330 		amdgpu_ring_write(ring, 0);
6331 		amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
6332 	}
6333 }
6334 
6335 static void gfx_v8_ring_emit_sb(struct amdgpu_ring *ring)
6336 {
6337 	amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
6338 	amdgpu_ring_write(ring, 0);
6339 }
6340 
6341 static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
6342 {
6343 	uint32_t dw2 = 0;
6344 
6345 	if (amdgpu_sriov_vf(ring->adev))
6346 		gfx_v8_0_ring_emit_ce_meta(ring);
6347 
6348 	dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
6349 	if (flags & AMDGPU_HAVE_CTX_SWITCH) {
6350 		gfx_v8_0_ring_emit_vgt_flush(ring);
6351 		/* set load_global_config & load_global_uconfig */
6352 		dw2 |= 0x8001;
6353 		/* set load_cs_sh_regs */
6354 		dw2 |= 0x01000000;
6355 		/* set load_per_context_state & load_gfx_sh_regs for GFX */
6356 		dw2 |= 0x10002;
6357 
6358 		/* set load_ce_ram if preamble presented */
6359 		if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
6360 			dw2 |= 0x10000000;
6361 	} else {
6362 		/* still load_ce_ram if this is the first time preamble presented
6363 		 * although there is no context switch happens.
6364 		 */
6365 		if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
6366 			dw2 |= 0x10000000;
6367 	}
6368 
6369 	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6370 	amdgpu_ring_write(ring, dw2);
6371 	amdgpu_ring_write(ring, 0);
6372 }
6373 
6374 static unsigned gfx_v8_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
6375 {
6376 	unsigned ret;
6377 
6378 	amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
6379 	amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
6380 	amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
6381 	amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */
6382 	ret = ring->wptr & ring->buf_mask;
6383 	amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */
6384 	return ret;
6385 }
6386 
6387 static void gfx_v8_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
6388 {
6389 	unsigned cur;
6390 
6391 	BUG_ON(offset > ring->buf_mask);
6392 	BUG_ON(ring->ring[offset] != 0x55aa55aa);
6393 
6394 	cur = (ring->wptr & ring->buf_mask) - 1;
6395 	if (likely(cur > offset))
6396 		ring->ring[offset] = cur - offset;
6397 	else
6398 		ring->ring[offset] = (ring->ring_size >> 2) - offset + cur;
6399 }
6400 
6401 static void gfx_v8_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg)
6402 {
6403 	struct amdgpu_device *adev = ring->adev;
6404 
6405 	amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
6406 	amdgpu_ring_write(ring, 0 |	/* src: register*/
6407 				(5 << 8) |	/* dst: memory */
6408 				(1 << 20));	/* write confirm */
6409 	amdgpu_ring_write(ring, reg);
6410 	amdgpu_ring_write(ring, 0);
6411 	amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
6412 				adev->virt.reg_val_offs * 4));
6413 	amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
6414 				adev->virt.reg_val_offs * 4));
6415 }
6416 
6417 static void gfx_v8_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
6418 				  uint32_t val)
6419 {
6420 	uint32_t cmd;
6421 
6422 	switch (ring->funcs->type) {
6423 	case AMDGPU_RING_TYPE_GFX:
6424 		cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
6425 		break;
6426 	case AMDGPU_RING_TYPE_KIQ:
6427 		cmd = 1 << 16; /* no inc addr */
6428 		break;
6429 	default:
6430 		cmd = WR_CONFIRM;
6431 		break;
6432 	}
6433 
6434 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6435 	amdgpu_ring_write(ring, cmd);
6436 	amdgpu_ring_write(ring, reg);
6437 	amdgpu_ring_write(ring, 0);
6438 	amdgpu_ring_write(ring, val);
6439 }
6440 
6441 static void gfx_v8_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
6442 {
6443 	struct amdgpu_device *adev = ring->adev;
6444 	uint32_t value = 0;
6445 
6446 	value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
6447 	value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
6448 	value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
6449 	value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
6450 	WREG32(mmSQ_CMD, value);
6451 }
6452 
6453 static void gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
6454 						 enum amdgpu_interrupt_state state)
6455 {
6456 	WREG32_FIELD(CP_INT_CNTL_RING0, TIME_STAMP_INT_ENABLE,
6457 		     state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
6458 }
6459 
6460 static void gfx_v8_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
6461 						     int me, int pipe,
6462 						     enum amdgpu_interrupt_state state)
6463 {
6464 	u32 mec_int_cntl, mec_int_cntl_reg;
6465 
6466 	/*
6467 	 * amdgpu controls only the first MEC. That's why this function only
6468 	 * handles the setting of interrupts for this specific MEC. All other
6469 	 * pipes' interrupts are set by amdkfd.
6470 	 */
6471 
6472 	if (me == 1) {
6473 		switch (pipe) {
6474 		case 0:
6475 			mec_int_cntl_reg = mmCP_ME1_PIPE0_INT_CNTL;
6476 			break;
6477 		case 1:
6478 			mec_int_cntl_reg = mmCP_ME1_PIPE1_INT_CNTL;
6479 			break;
6480 		case 2:
6481 			mec_int_cntl_reg = mmCP_ME1_PIPE2_INT_CNTL;
6482 			break;
6483 		case 3:
6484 			mec_int_cntl_reg = mmCP_ME1_PIPE3_INT_CNTL;
6485 			break;
6486 		default:
6487 			DRM_DEBUG("invalid pipe %d\n", pipe);
6488 			return;
6489 		}
6490 	} else {
6491 		DRM_DEBUG("invalid me %d\n", me);
6492 		return;
6493 	}
6494 
6495 	switch (state) {
6496 	case AMDGPU_IRQ_STATE_DISABLE:
6497 		mec_int_cntl = RREG32(mec_int_cntl_reg);
6498 		mec_int_cntl &= ~CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
6499 		WREG32(mec_int_cntl_reg, mec_int_cntl);
6500 		break;
6501 	case AMDGPU_IRQ_STATE_ENABLE:
6502 		mec_int_cntl = RREG32(mec_int_cntl_reg);
6503 		mec_int_cntl |= CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
6504 		WREG32(mec_int_cntl_reg, mec_int_cntl);
6505 		break;
6506 	default:
6507 		break;
6508 	}
6509 }
6510 
6511 static int gfx_v8_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
6512 					     struct amdgpu_irq_src *source,
6513 					     unsigned type,
6514 					     enum amdgpu_interrupt_state state)
6515 {
6516 	WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_REG_INT_ENABLE,
6517 		     state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
6518 
6519 	return 0;
6520 }
6521 
6522 static int gfx_v8_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
6523 					      struct amdgpu_irq_src *source,
6524 					      unsigned type,
6525 					      enum amdgpu_interrupt_state state)
6526 {
6527 	WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_INSTR_INT_ENABLE,
6528 		     state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
6529 
6530 	return 0;
6531 }
6532 
6533 static int gfx_v8_0_set_eop_interrupt_state(struct amdgpu_device *adev,
6534 					    struct amdgpu_irq_src *src,
6535 					    unsigned type,
6536 					    enum amdgpu_interrupt_state state)
6537 {
6538 	switch (type) {
6539 	case AMDGPU_CP_IRQ_GFX_EOP:
6540 		gfx_v8_0_set_gfx_eop_interrupt_state(adev, state);
6541 		break;
6542 	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
6543 		gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
6544 		break;
6545 	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
6546 		gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
6547 		break;
6548 	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
6549 		gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
6550 		break;
6551 	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
6552 		gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
6553 		break;
6554 	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
6555 		gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
6556 		break;
6557 	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
6558 		gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
6559 		break;
6560 	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
6561 		gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
6562 		break;
6563 	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
6564 		gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
6565 		break;
6566 	default:
6567 		break;
6568 	}
6569 	return 0;
6570 }
6571 
6572 static int gfx_v8_0_set_cp_ecc_int_state(struct amdgpu_device *adev,
6573 					 struct amdgpu_irq_src *source,
6574 					 unsigned int type,
6575 					 enum amdgpu_interrupt_state state)
6576 {
6577 	int enable_flag;
6578 
6579 	switch (state) {
6580 	case AMDGPU_IRQ_STATE_DISABLE:
6581 		enable_flag = 0;
6582 		break;
6583 
6584 	case AMDGPU_IRQ_STATE_ENABLE:
6585 		enable_flag = 1;
6586 		break;
6587 
6588 	default:
6589 		return -EINVAL;
6590 	}
6591 
6592 	WREG32_FIELD(CP_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6593 	WREG32_FIELD(CP_INT_CNTL_RING0, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6594 	WREG32_FIELD(CP_INT_CNTL_RING1, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6595 	WREG32_FIELD(CP_INT_CNTL_RING2, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6596 	WREG32_FIELD(CPC_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6597 	WREG32_FIELD(CP_ME1_PIPE0_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6598 		     enable_flag);
6599 	WREG32_FIELD(CP_ME1_PIPE1_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6600 		     enable_flag);
6601 	WREG32_FIELD(CP_ME1_PIPE2_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6602 		     enable_flag);
6603 	WREG32_FIELD(CP_ME1_PIPE3_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6604 		     enable_flag);
6605 	WREG32_FIELD(CP_ME2_PIPE0_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6606 		     enable_flag);
6607 	WREG32_FIELD(CP_ME2_PIPE1_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6608 		     enable_flag);
6609 	WREG32_FIELD(CP_ME2_PIPE2_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6610 		     enable_flag);
6611 	WREG32_FIELD(CP_ME2_PIPE3_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6612 		     enable_flag);
6613 
6614 	return 0;
6615 }
6616 
6617 static int gfx_v8_0_set_sq_int_state(struct amdgpu_device *adev,
6618 				     struct amdgpu_irq_src *source,
6619 				     unsigned int type,
6620 				     enum amdgpu_interrupt_state state)
6621 {
6622 	int enable_flag;
6623 
6624 	switch (state) {
6625 	case AMDGPU_IRQ_STATE_DISABLE:
6626 		enable_flag = 1;
6627 		break;
6628 
6629 	case AMDGPU_IRQ_STATE_ENABLE:
6630 		enable_flag = 0;
6631 		break;
6632 
6633 	default:
6634 		return -EINVAL;
6635 	}
6636 
6637 	WREG32_FIELD(SQ_INTERRUPT_MSG_CTRL, STALL,
6638 		     enable_flag);
6639 
6640 	return 0;
6641 }
6642 
6643 static int gfx_v8_0_eop_irq(struct amdgpu_device *adev,
6644 			    struct amdgpu_irq_src *source,
6645 			    struct amdgpu_iv_entry *entry)
6646 {
6647 	int i;
6648 	u8 me_id, pipe_id, queue_id;
6649 	struct amdgpu_ring *ring;
6650 
6651 	DRM_DEBUG("IH: CP EOP\n");
6652 	me_id = (entry->ring_id & 0x0c) >> 2;
6653 	pipe_id = (entry->ring_id & 0x03) >> 0;
6654 	queue_id = (entry->ring_id & 0x70) >> 4;
6655 
6656 	switch (me_id) {
6657 	case 0:
6658 		amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
6659 		break;
6660 	case 1:
6661 	case 2:
6662 		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6663 			ring = &adev->gfx.compute_ring[i];
6664 			/* Per-queue interrupt is supported for MEC starting from VI.
6665 			  * The interrupt can only be enabled/disabled per pipe instead of per queue.
6666 			  */
6667 			if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
6668 				amdgpu_fence_process(ring);
6669 		}
6670 		break;
6671 	}
6672 	return 0;
6673 }
6674 
6675 static void gfx_v8_0_fault(struct amdgpu_device *adev,
6676 			   struct amdgpu_iv_entry *entry)
6677 {
6678 	u8 me_id, pipe_id, queue_id;
6679 	struct amdgpu_ring *ring;
6680 	int i;
6681 
6682 	me_id = (entry->ring_id & 0x0c) >> 2;
6683 	pipe_id = (entry->ring_id & 0x03) >> 0;
6684 	queue_id = (entry->ring_id & 0x70) >> 4;
6685 
6686 	switch (me_id) {
6687 	case 0:
6688 		drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
6689 		break;
6690 	case 1:
6691 	case 2:
6692 		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6693 			ring = &adev->gfx.compute_ring[i];
6694 			if (ring->me == me_id && ring->pipe == pipe_id &&
6695 			    ring->queue == queue_id)
6696 				drm_sched_fault(&ring->sched);
6697 		}
6698 		break;
6699 	}
6700 }
6701 
6702 static int gfx_v8_0_priv_reg_irq(struct amdgpu_device *adev,
6703 				 struct amdgpu_irq_src *source,
6704 				 struct amdgpu_iv_entry *entry)
6705 {
6706 	DRM_ERROR("Illegal register access in command stream\n");
6707 	gfx_v8_0_fault(adev, entry);
6708 	return 0;
6709 }
6710 
6711 static int gfx_v8_0_priv_inst_irq(struct amdgpu_device *adev,
6712 				  struct amdgpu_irq_src *source,
6713 				  struct amdgpu_iv_entry *entry)
6714 {
6715 	DRM_ERROR("Illegal instruction in command stream\n");
6716 	gfx_v8_0_fault(adev, entry);
6717 	return 0;
6718 }
6719 
6720 static int gfx_v8_0_cp_ecc_error_irq(struct amdgpu_device *adev,
6721 				     struct amdgpu_irq_src *source,
6722 				     struct amdgpu_iv_entry *entry)
6723 {
6724 	DRM_ERROR("CP EDC/ECC error detected.");
6725 	return 0;
6726 }
6727 
6728 static void gfx_v8_0_parse_sq_irq(struct amdgpu_device *adev, unsigned ih_data)
6729 {
6730 	u32 enc, se_id, sh_id, cu_id;
6731 	char type[20];
6732 	int sq_edc_source = -1;
6733 
6734 	enc = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, ENCODING);
6735 	se_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, SE_ID);
6736 
6737 	switch (enc) {
6738 		case 0:
6739 			DRM_INFO("SQ general purpose intr detected:"
6740 					"se_id %d, immed_overflow %d, host_reg_overflow %d,"
6741 					"host_cmd_overflow %d, cmd_timestamp %d,"
6742 					"reg_timestamp %d, thread_trace_buff_full %d,"
6743 					"wlt %d, thread_trace %d.\n",
6744 					se_id,
6745 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, IMMED_OVERFLOW),
6746 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_REG_OVERFLOW),
6747 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_CMD_OVERFLOW),
6748 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, CMD_TIMESTAMP),
6749 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, REG_TIMESTAMP),
6750 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE_BUF_FULL),
6751 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, WLT),
6752 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE)
6753 					);
6754 			break;
6755 		case 1:
6756 		case 2:
6757 
6758 			cu_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, CU_ID);
6759 			sh_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SH_ID);
6760 
6761 			/*
6762 			 * This function can be called either directly from ISR
6763 			 * or from BH in which case we can access SQ_EDC_INFO
6764 			 * instance
6765 			 */
6766 			if (in_task()) {
6767 				mutex_lock(&adev->grbm_idx_mutex);
6768 				gfx_v8_0_select_se_sh(adev, se_id, sh_id, cu_id);
6769 
6770 				sq_edc_source = REG_GET_FIELD(RREG32(mmSQ_EDC_INFO), SQ_EDC_INFO, SOURCE);
6771 
6772 				gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
6773 				mutex_unlock(&adev->grbm_idx_mutex);
6774 			}
6775 
6776 			if (enc == 1)
6777 				sprintf(type, "instruction intr");
6778 			else
6779 				sprintf(type, "EDC/ECC error");
6780 
6781 			DRM_INFO(
6782 				"SQ %s detected: "
6783 					"se_id %d, sh_id %d, cu_id %d, simd_id %d, wave_id %d, vm_id %d "
6784 					"trap %s, sq_ed_info.source %s.\n",
6785 					type, se_id, sh_id, cu_id,
6786 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SIMD_ID),
6787 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, WAVE_ID),
6788 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, VM_ID),
6789 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, PRIV) ? "true" : "false",
6790 					(sq_edc_source != -1) ? sq_edc_source_names[sq_edc_source] : "unavailable"
6791 				);
6792 			break;
6793 		default:
6794 			DRM_ERROR("SQ invalid encoding type\n.");
6795 	}
6796 }
6797 
6798 static void gfx_v8_0_sq_irq_work_func(struct work_struct *work)
6799 {
6800 
6801 	struct amdgpu_device *adev = container_of(work, struct amdgpu_device, gfx.sq_work.work);
6802 	struct sq_work *sq_work = container_of(work, struct sq_work, work);
6803 
6804 	gfx_v8_0_parse_sq_irq(adev, sq_work->ih_data);
6805 }
6806 
6807 static int gfx_v8_0_sq_irq(struct amdgpu_device *adev,
6808 			   struct amdgpu_irq_src *source,
6809 			   struct amdgpu_iv_entry *entry)
6810 {
6811 	unsigned ih_data = entry->src_data[0];
6812 
6813 	/*
6814 	 * Try to submit work so SQ_EDC_INFO can be accessed from
6815 	 * BH. If previous work submission hasn't finished yet
6816 	 * just print whatever info is possible directly from the ISR.
6817 	 */
6818 	if (work_pending(&adev->gfx.sq_work.work)) {
6819 		gfx_v8_0_parse_sq_irq(adev, ih_data);
6820 	} else {
6821 		adev->gfx.sq_work.ih_data = ih_data;
6822 		schedule_work(&adev->gfx.sq_work.work);
6823 	}
6824 
6825 	return 0;
6826 }
6827 
6828 static const struct amd_ip_funcs gfx_v8_0_ip_funcs = {
6829 	.name = "gfx_v8_0",
6830 	.early_init = gfx_v8_0_early_init,
6831 	.late_init = gfx_v8_0_late_init,
6832 	.sw_init = gfx_v8_0_sw_init,
6833 	.sw_fini = gfx_v8_0_sw_fini,
6834 	.hw_init = gfx_v8_0_hw_init,
6835 	.hw_fini = gfx_v8_0_hw_fini,
6836 	.suspend = gfx_v8_0_suspend,
6837 	.resume = gfx_v8_0_resume,
6838 	.is_idle = gfx_v8_0_is_idle,
6839 	.wait_for_idle = gfx_v8_0_wait_for_idle,
6840 	.check_soft_reset = gfx_v8_0_check_soft_reset,
6841 	.pre_soft_reset = gfx_v8_0_pre_soft_reset,
6842 	.soft_reset = gfx_v8_0_soft_reset,
6843 	.post_soft_reset = gfx_v8_0_post_soft_reset,
6844 	.set_clockgating_state = gfx_v8_0_set_clockgating_state,
6845 	.set_powergating_state = gfx_v8_0_set_powergating_state,
6846 	.get_clockgating_state = gfx_v8_0_get_clockgating_state,
6847 };
6848 
6849 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = {
6850 	.type = AMDGPU_RING_TYPE_GFX,
6851 	.align_mask = 0xff,
6852 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
6853 	.support_64bit_ptrs = false,
6854 	.get_rptr = gfx_v8_0_ring_get_rptr,
6855 	.get_wptr = gfx_v8_0_ring_get_wptr_gfx,
6856 	.set_wptr = gfx_v8_0_ring_set_wptr_gfx,
6857 	.emit_frame_size = /* maximum 215dw if count 16 IBs in */
6858 		5 +  /* COND_EXEC */
6859 		7 +  /* PIPELINE_SYNC */
6860 		VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 9 + /* VM_FLUSH */
6861 		8 +  /* FENCE for VM_FLUSH */
6862 		20 + /* GDS switch */
6863 		4 + /* double SWITCH_BUFFER,
6864 		       the first COND_EXEC jump to the place just
6865 			   prior to this double SWITCH_BUFFER  */
6866 		5 + /* COND_EXEC */
6867 		7 +	 /*	HDP_flush */
6868 		4 +	 /*	VGT_flush */
6869 		14 + /*	CE_META */
6870 		31 + /*	DE_META */
6871 		3 + /* CNTX_CTRL */
6872 		5 + /* HDP_INVL */
6873 		8 + 8 + /* FENCE x2 */
6874 		2, /* SWITCH_BUFFER */
6875 	.emit_ib_size =	4, /* gfx_v8_0_ring_emit_ib_gfx */
6876 	.emit_ib = gfx_v8_0_ring_emit_ib_gfx,
6877 	.emit_fence = gfx_v8_0_ring_emit_fence_gfx,
6878 	.emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync,
6879 	.emit_vm_flush = gfx_v8_0_ring_emit_vm_flush,
6880 	.emit_gds_switch = gfx_v8_0_ring_emit_gds_switch,
6881 	.emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,
6882 	.test_ring = gfx_v8_0_ring_test_ring,
6883 	.test_ib = gfx_v8_0_ring_test_ib,
6884 	.insert_nop = amdgpu_ring_insert_nop,
6885 	.pad_ib = amdgpu_ring_generic_pad_ib,
6886 	.emit_switch_buffer = gfx_v8_ring_emit_sb,
6887 	.emit_cntxcntl = gfx_v8_ring_emit_cntxcntl,
6888 	.init_cond_exec = gfx_v8_0_ring_emit_init_cond_exec,
6889 	.patch_cond_exec = gfx_v8_0_ring_emit_patch_cond_exec,
6890 	.emit_wreg = gfx_v8_0_ring_emit_wreg,
6891 	.soft_recovery = gfx_v8_0_ring_soft_recovery,
6892 };
6893 
6894 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = {
6895 	.type = AMDGPU_RING_TYPE_COMPUTE,
6896 	.align_mask = 0xff,
6897 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
6898 	.support_64bit_ptrs = false,
6899 	.get_rptr = gfx_v8_0_ring_get_rptr,
6900 	.get_wptr = gfx_v8_0_ring_get_wptr_compute,
6901 	.set_wptr = gfx_v8_0_ring_set_wptr_compute,
6902 	.emit_frame_size =
6903 		20 + /* gfx_v8_0_ring_emit_gds_switch */
6904 		7 + /* gfx_v8_0_ring_emit_hdp_flush */
6905 		5 + /* hdp_invalidate */
6906 		7 + /* gfx_v8_0_ring_emit_pipeline_sync */
6907 		VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v8_0_ring_emit_vm_flush */
6908 		7 + 7 + 7, /* gfx_v8_0_ring_emit_fence_compute x3 for user fence, vm fence */
6909 	.emit_ib_size =	7, /* gfx_v8_0_ring_emit_ib_compute */
6910 	.emit_ib = gfx_v8_0_ring_emit_ib_compute,
6911 	.emit_fence = gfx_v8_0_ring_emit_fence_compute,
6912 	.emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync,
6913 	.emit_vm_flush = gfx_v8_0_ring_emit_vm_flush,
6914 	.emit_gds_switch = gfx_v8_0_ring_emit_gds_switch,
6915 	.emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,
6916 	.test_ring = gfx_v8_0_ring_test_ring,
6917 	.test_ib = gfx_v8_0_ring_test_ib,
6918 	.insert_nop = amdgpu_ring_insert_nop,
6919 	.pad_ib = amdgpu_ring_generic_pad_ib,
6920 	.set_priority = gfx_v8_0_ring_set_priority_compute,
6921 	.emit_wreg = gfx_v8_0_ring_emit_wreg,
6922 };
6923 
6924 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = {
6925 	.type = AMDGPU_RING_TYPE_KIQ,
6926 	.align_mask = 0xff,
6927 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
6928 	.support_64bit_ptrs = false,
6929 	.get_rptr = gfx_v8_0_ring_get_rptr,
6930 	.get_wptr = gfx_v8_0_ring_get_wptr_compute,
6931 	.set_wptr = gfx_v8_0_ring_set_wptr_compute,
6932 	.emit_frame_size =
6933 		20 + /* gfx_v8_0_ring_emit_gds_switch */
6934 		7 + /* gfx_v8_0_ring_emit_hdp_flush */
6935 		5 + /* hdp_invalidate */
6936 		7 + /* gfx_v8_0_ring_emit_pipeline_sync */
6937 		17 + /* gfx_v8_0_ring_emit_vm_flush */
6938 		7 + 7 + 7, /* gfx_v8_0_ring_emit_fence_kiq x3 for user fence, vm fence */
6939 	.emit_ib_size =	7, /* gfx_v8_0_ring_emit_ib_compute */
6940 	.emit_fence = gfx_v8_0_ring_emit_fence_kiq,
6941 	.test_ring = gfx_v8_0_ring_test_ring,
6942 	.insert_nop = amdgpu_ring_insert_nop,
6943 	.pad_ib = amdgpu_ring_generic_pad_ib,
6944 	.emit_rreg = gfx_v8_0_ring_emit_rreg,
6945 	.emit_wreg = gfx_v8_0_ring_emit_wreg,
6946 };
6947 
6948 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev)
6949 {
6950 	int i;
6951 
6952 	adev->gfx.kiq.ring.funcs = &gfx_v8_0_ring_funcs_kiq;
6953 
6954 	for (i = 0; i < adev->gfx.num_gfx_rings; i++)
6955 		adev->gfx.gfx_ring[i].funcs = &gfx_v8_0_ring_funcs_gfx;
6956 
6957 	for (i = 0; i < adev->gfx.num_compute_rings; i++)
6958 		adev->gfx.compute_ring[i].funcs = &gfx_v8_0_ring_funcs_compute;
6959 }
6960 
6961 static const struct amdgpu_irq_src_funcs gfx_v8_0_eop_irq_funcs = {
6962 	.set = gfx_v8_0_set_eop_interrupt_state,
6963 	.process = gfx_v8_0_eop_irq,
6964 };
6965 
6966 static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_reg_irq_funcs = {
6967 	.set = gfx_v8_0_set_priv_reg_fault_state,
6968 	.process = gfx_v8_0_priv_reg_irq,
6969 };
6970 
6971 static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_inst_irq_funcs = {
6972 	.set = gfx_v8_0_set_priv_inst_fault_state,
6973 	.process = gfx_v8_0_priv_inst_irq,
6974 };
6975 
6976 static const struct amdgpu_irq_src_funcs gfx_v8_0_cp_ecc_error_irq_funcs = {
6977 	.set = gfx_v8_0_set_cp_ecc_int_state,
6978 	.process = gfx_v8_0_cp_ecc_error_irq,
6979 };
6980 
6981 static const struct amdgpu_irq_src_funcs gfx_v8_0_sq_irq_funcs = {
6982 	.set = gfx_v8_0_set_sq_int_state,
6983 	.process = gfx_v8_0_sq_irq,
6984 };
6985 
6986 static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev)
6987 {
6988 	adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
6989 	adev->gfx.eop_irq.funcs = &gfx_v8_0_eop_irq_funcs;
6990 
6991 	adev->gfx.priv_reg_irq.num_types = 1;
6992 	adev->gfx.priv_reg_irq.funcs = &gfx_v8_0_priv_reg_irq_funcs;
6993 
6994 	adev->gfx.priv_inst_irq.num_types = 1;
6995 	adev->gfx.priv_inst_irq.funcs = &gfx_v8_0_priv_inst_irq_funcs;
6996 
6997 	adev->gfx.cp_ecc_error_irq.num_types = 1;
6998 	adev->gfx.cp_ecc_error_irq.funcs = &gfx_v8_0_cp_ecc_error_irq_funcs;
6999 
7000 	adev->gfx.sq_irq.num_types = 1;
7001 	adev->gfx.sq_irq.funcs = &gfx_v8_0_sq_irq_funcs;
7002 }
7003 
7004 static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev)
7005 {
7006 	adev->gfx.rlc.funcs = &iceland_rlc_funcs;
7007 }
7008 
7009 static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev)
7010 {
7011 	/* init asci gds info */
7012 	adev->gds.mem.total_size = RREG32(mmGDS_VMID0_SIZE);
7013 	adev->gds.gws.total_size = 64;
7014 	adev->gds.oa.total_size = 16;
7015 	adev->gds.gds_compute_max_wave_id = RREG32(mmGDS_COMPUTE_MAX_WAVE_ID);
7016 
7017 	if (adev->gds.mem.total_size == 64 * 1024) {
7018 		adev->gds.mem.gfx_partition_size = 4096;
7019 		adev->gds.mem.cs_partition_size = 4096;
7020 
7021 		adev->gds.gws.gfx_partition_size = 4;
7022 		adev->gds.gws.cs_partition_size = 4;
7023 
7024 		adev->gds.oa.gfx_partition_size = 4;
7025 		adev->gds.oa.cs_partition_size = 1;
7026 	} else {
7027 		adev->gds.mem.gfx_partition_size = 1024;
7028 		adev->gds.mem.cs_partition_size = 1024;
7029 
7030 		adev->gds.gws.gfx_partition_size = 16;
7031 		adev->gds.gws.cs_partition_size = 16;
7032 
7033 		adev->gds.oa.gfx_partition_size = 4;
7034 		adev->gds.oa.cs_partition_size = 4;
7035 	}
7036 }
7037 
7038 static void gfx_v8_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
7039 						 u32 bitmap)
7040 {
7041 	u32 data;
7042 
7043 	if (!bitmap)
7044 		return;
7045 
7046 	data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7047 	data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7048 
7049 	WREG32(mmGC_USER_SHADER_ARRAY_CONFIG, data);
7050 }
7051 
7052 static u32 gfx_v8_0_get_cu_active_bitmap(struct amdgpu_device *adev)
7053 {
7054 	u32 data, mask;
7055 
7056 	data =  RREG32(mmCC_GC_SHADER_ARRAY_CONFIG) |
7057 		RREG32(mmGC_USER_SHADER_ARRAY_CONFIG);
7058 
7059 	mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
7060 
7061 	return ~REG_GET_FIELD(data, CC_GC_SHADER_ARRAY_CONFIG, INACTIVE_CUS) & mask;
7062 }
7063 
7064 static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev)
7065 {
7066 	int i, j, k, counter, active_cu_number = 0;
7067 	u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7068 	struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
7069 	unsigned disable_masks[4 * 2];
7070 	u32 ao_cu_num;
7071 
7072 	memset(cu_info, 0, sizeof(*cu_info));
7073 
7074 	if (adev->flags & AMD_IS_APU)
7075 		ao_cu_num = 2;
7076 	else
7077 		ao_cu_num = adev->gfx.config.max_cu_per_sh;
7078 
7079 	amdgpu_gfx_parse_disable_cu(disable_masks, 4, 2);
7080 
7081 	mutex_lock(&adev->grbm_idx_mutex);
7082 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7083 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7084 			mask = 1;
7085 			ao_bitmap = 0;
7086 			counter = 0;
7087 			gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
7088 			if (i < 4 && j < 2)
7089 				gfx_v8_0_set_user_cu_inactive_bitmap(
7090 					adev, disable_masks[i * 2 + j]);
7091 			bitmap = gfx_v8_0_get_cu_active_bitmap(adev);
7092 			cu_info->bitmap[i][j] = bitmap;
7093 
7094 			for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
7095 				if (bitmap & mask) {
7096 					if (counter < ao_cu_num)
7097 						ao_bitmap |= mask;
7098 					counter ++;
7099 				}
7100 				mask <<= 1;
7101 			}
7102 			active_cu_number += counter;
7103 			if (i < 2 && j < 2)
7104 				ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7105 			cu_info->ao_cu_bitmap[i][j] = ao_bitmap;
7106 		}
7107 	}
7108 	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
7109 	mutex_unlock(&adev->grbm_idx_mutex);
7110 
7111 	cu_info->number = active_cu_number;
7112 	cu_info->ao_cu_mask = ao_cu_mask;
7113 	cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7114 	cu_info->max_waves_per_simd = 10;
7115 	cu_info->max_scratch_slots_per_cu = 32;
7116 	cu_info->wave_front_size = 64;
7117 	cu_info->lds_size = 64;
7118 }
7119 
7120 const struct amdgpu_ip_block_version gfx_v8_0_ip_block =
7121 {
7122 	.type = AMD_IP_BLOCK_TYPE_GFX,
7123 	.major = 8,
7124 	.minor = 0,
7125 	.rev = 0,
7126 	.funcs = &gfx_v8_0_ip_funcs,
7127 };
7128 
7129 const struct amdgpu_ip_block_version gfx_v8_1_ip_block =
7130 {
7131 	.type = AMD_IP_BLOCK_TYPE_GFX,
7132 	.major = 8,
7133 	.minor = 1,
7134 	.rev = 0,
7135 	.funcs = &gfx_v8_0_ip_funcs,
7136 };
7137 
7138 static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
7139 {
7140 	uint64_t ce_payload_addr;
7141 	int cnt_ce;
7142 	union {
7143 		struct vi_ce_ib_state regular;
7144 		struct vi_ce_ib_state_chained_ib chained;
7145 	} ce_payload = {};
7146 
7147 	if (ring->adev->virt.chained_ib_support) {
7148 		ce_payload_addr = amdgpu_csa_vaddr(ring->adev) +
7149 			offsetof(struct vi_gfx_meta_data_chained_ib, ce_payload);
7150 		cnt_ce = (sizeof(ce_payload.chained) >> 2) + 4 - 2;
7151 	} else {
7152 		ce_payload_addr = amdgpu_csa_vaddr(ring->adev) +
7153 			offsetof(struct vi_gfx_meta_data, ce_payload);
7154 		cnt_ce = (sizeof(ce_payload.regular) >> 2) + 4 - 2;
7155 	}
7156 
7157 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_ce));
7158 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
7159 				WRITE_DATA_DST_SEL(8) |
7160 				WR_CONFIRM) |
7161 				WRITE_DATA_CACHE_POLICY(0));
7162 	amdgpu_ring_write(ring, lower_32_bits(ce_payload_addr));
7163 	amdgpu_ring_write(ring, upper_32_bits(ce_payload_addr));
7164 	amdgpu_ring_write_multiple(ring, (void *)&ce_payload, cnt_ce - 2);
7165 }
7166 
7167 static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring)
7168 {
7169 	uint64_t de_payload_addr, gds_addr, csa_addr;
7170 	int cnt_de;
7171 	union {
7172 		struct vi_de_ib_state regular;
7173 		struct vi_de_ib_state_chained_ib chained;
7174 	} de_payload = {};
7175 
7176 	csa_addr = amdgpu_csa_vaddr(ring->adev);
7177 	gds_addr = csa_addr + 4096;
7178 	if (ring->adev->virt.chained_ib_support) {
7179 		de_payload.chained.gds_backup_addrlo = lower_32_bits(gds_addr);
7180 		de_payload.chained.gds_backup_addrhi = upper_32_bits(gds_addr);
7181 		de_payload_addr = csa_addr + offsetof(struct vi_gfx_meta_data_chained_ib, de_payload);
7182 		cnt_de = (sizeof(de_payload.chained) >> 2) + 4 - 2;
7183 	} else {
7184 		de_payload.regular.gds_backup_addrlo = lower_32_bits(gds_addr);
7185 		de_payload.regular.gds_backup_addrhi = upper_32_bits(gds_addr);
7186 		de_payload_addr = csa_addr + offsetof(struct vi_gfx_meta_data, de_payload);
7187 		cnt_de = (sizeof(de_payload.regular) >> 2) + 4 - 2;
7188 	}
7189 
7190 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_de));
7191 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
7192 				WRITE_DATA_DST_SEL(8) |
7193 				WR_CONFIRM) |
7194 				WRITE_DATA_CACHE_POLICY(0));
7195 	amdgpu_ring_write(ring, lower_32_bits(de_payload_addr));
7196 	amdgpu_ring_write(ring, upper_32_bits(de_payload_addr));
7197 	amdgpu_ring_write_multiple(ring, (void *)&de_payload, cnt_de - 2);
7198 }
7199