1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/delay.h> 25 #include <linux/kernel.h> 26 #include <linux/firmware.h> 27 #include <linux/module.h> 28 #include <linux/pci.h> 29 30 #include "amdgpu.h" 31 #include "amdgpu_gfx.h" 32 #include "soc15.h" 33 #include "soc15d.h" 34 #include "amdgpu_atomfirmware.h" 35 #include "amdgpu_pm.h" 36 37 #include "gc/gc_9_0_offset.h" 38 #include "gc/gc_9_0_sh_mask.h" 39 40 #include "vega10_enum.h" 41 42 #include "soc15_common.h" 43 #include "clearstate_gfx9.h" 44 #include "v9_structs.h" 45 46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h" 47 48 #include "amdgpu_ras.h" 49 50 #include "gfx_v9_4.h" 51 #include "gfx_v9_0.h" 52 #include "gfx_v9_4_2.h" 53 54 #include "asic_reg/pwr/pwr_10_0_offset.h" 55 #include "asic_reg/pwr/pwr_10_0_sh_mask.h" 56 #include "asic_reg/gc/gc_9_0_default.h" 57 58 #define GFX9_NUM_GFX_RINGS 1 59 #define GFX9_MEC_HPD_SIZE 4096 60 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L 61 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L 62 63 #define mmGCEA_PROBE_MAP 0x070c 64 #define mmGCEA_PROBE_MAP_BASE_IDX 0 65 66 MODULE_FIRMWARE("amdgpu/vega10_ce.bin"); 67 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin"); 68 MODULE_FIRMWARE("amdgpu/vega10_me.bin"); 69 MODULE_FIRMWARE("amdgpu/vega10_mec.bin"); 70 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin"); 71 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin"); 72 73 MODULE_FIRMWARE("amdgpu/vega12_ce.bin"); 74 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin"); 75 MODULE_FIRMWARE("amdgpu/vega12_me.bin"); 76 MODULE_FIRMWARE("amdgpu/vega12_mec.bin"); 77 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin"); 78 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin"); 79 80 MODULE_FIRMWARE("amdgpu/vega20_ce.bin"); 81 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin"); 82 MODULE_FIRMWARE("amdgpu/vega20_me.bin"); 83 MODULE_FIRMWARE("amdgpu/vega20_mec.bin"); 84 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin"); 85 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin"); 86 87 MODULE_FIRMWARE("amdgpu/raven_ce.bin"); 88 MODULE_FIRMWARE("amdgpu/raven_pfp.bin"); 89 MODULE_FIRMWARE("amdgpu/raven_me.bin"); 90 MODULE_FIRMWARE("amdgpu/raven_mec.bin"); 91 MODULE_FIRMWARE("amdgpu/raven_mec2.bin"); 92 MODULE_FIRMWARE("amdgpu/raven_rlc.bin"); 93 94 MODULE_FIRMWARE("amdgpu/picasso_ce.bin"); 95 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin"); 96 MODULE_FIRMWARE("amdgpu/picasso_me.bin"); 97 MODULE_FIRMWARE("amdgpu/picasso_mec.bin"); 98 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin"); 99 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin"); 100 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin"); 101 102 MODULE_FIRMWARE("amdgpu/raven2_ce.bin"); 103 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin"); 104 MODULE_FIRMWARE("amdgpu/raven2_me.bin"); 105 MODULE_FIRMWARE("amdgpu/raven2_mec.bin"); 106 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin"); 107 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin"); 108 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin"); 109 110 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin"); 111 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin"); 112 113 MODULE_FIRMWARE("amdgpu/renoir_ce.bin"); 114 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin"); 115 MODULE_FIRMWARE("amdgpu/renoir_me.bin"); 116 MODULE_FIRMWARE("amdgpu/renoir_mec.bin"); 117 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin"); 118 119 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin"); 120 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin"); 121 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin"); 122 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin"); 123 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin"); 124 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin"); 125 126 MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin"); 127 MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin"); 128 MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin"); 129 130 #define mmTCP_CHAN_STEER_0_ARCT 0x0b03 131 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0 132 #define mmTCP_CHAN_STEER_1_ARCT 0x0b04 133 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0 134 #define mmTCP_CHAN_STEER_2_ARCT 0x0b09 135 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0 136 #define mmTCP_CHAN_STEER_3_ARCT 0x0b0a 137 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0 138 #define mmTCP_CHAN_STEER_4_ARCT 0x0b0b 139 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0 140 #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c 141 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0 142 143 enum ta_ras_gfx_subblock { 144 /*CPC*/ 145 TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0, 146 TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START, 147 TA_RAS_BLOCK__GFX_CPC_UCODE, 148 TA_RAS_BLOCK__GFX_DC_STATE_ME1, 149 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1, 150 TA_RAS_BLOCK__GFX_DC_RESTORE_ME1, 151 TA_RAS_BLOCK__GFX_DC_STATE_ME2, 152 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2, 153 TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 154 TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 155 /* CPF*/ 156 TA_RAS_BLOCK__GFX_CPF_INDEX_START, 157 TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START, 158 TA_RAS_BLOCK__GFX_CPF_ROQ_ME1, 159 TA_RAS_BLOCK__GFX_CPF_TAG, 160 TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG, 161 /* CPG*/ 162 TA_RAS_BLOCK__GFX_CPG_INDEX_START, 163 TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START, 164 TA_RAS_BLOCK__GFX_CPG_DMA_TAG, 165 TA_RAS_BLOCK__GFX_CPG_TAG, 166 TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG, 167 /* GDS*/ 168 TA_RAS_BLOCK__GFX_GDS_INDEX_START, 169 TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START, 170 TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE, 171 TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM, 172 TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM, 173 TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 174 TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 175 /* SPI*/ 176 TA_RAS_BLOCK__GFX_SPI_SR_MEM, 177 /* SQ*/ 178 TA_RAS_BLOCK__GFX_SQ_INDEX_START, 179 TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START, 180 TA_RAS_BLOCK__GFX_SQ_LDS_D, 181 TA_RAS_BLOCK__GFX_SQ_LDS_I, 182 TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/ 183 TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR, 184 /* SQC (3 ranges)*/ 185 TA_RAS_BLOCK__GFX_SQC_INDEX_START, 186 /* SQC range 0*/ 187 TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START, 188 TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO = 189 TA_RAS_BLOCK__GFX_SQC_INDEX0_START, 190 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 191 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO, 192 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 193 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO, 194 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 195 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 196 TA_RAS_BLOCK__GFX_SQC_INDEX0_END = 197 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 198 /* SQC range 1*/ 199 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 200 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM = 201 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 202 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 203 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO, 204 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM, 205 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM, 206 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO, 207 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO, 208 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 209 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 210 TA_RAS_BLOCK__GFX_SQC_INDEX1_END = 211 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 212 /* SQC range 2*/ 213 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 214 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM = 215 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 216 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 217 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO, 218 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM, 219 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM, 220 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO, 221 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO, 222 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 223 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 224 TA_RAS_BLOCK__GFX_SQC_INDEX2_END = 225 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 226 TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END, 227 /* TA*/ 228 TA_RAS_BLOCK__GFX_TA_INDEX_START, 229 TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START, 230 TA_RAS_BLOCK__GFX_TA_FS_AFIFO, 231 TA_RAS_BLOCK__GFX_TA_FL_LFIFO, 232 TA_RAS_BLOCK__GFX_TA_FX_LFIFO, 233 TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 234 TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 235 /* TCA*/ 236 TA_RAS_BLOCK__GFX_TCA_INDEX_START, 237 TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START, 238 TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 239 TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 240 /* TCC (5 sub-ranges)*/ 241 TA_RAS_BLOCK__GFX_TCC_INDEX_START, 242 /* TCC range 0*/ 243 TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START, 244 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START, 245 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1, 246 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0, 247 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1, 248 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0, 249 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1, 250 TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG, 251 TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 252 TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 253 /* TCC range 1*/ 254 TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 255 TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 256 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 257 TA_RAS_BLOCK__GFX_TCC_INDEX1_END = 258 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 259 /* TCC range 2*/ 260 TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 261 TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 262 TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL, 263 TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO, 264 TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN, 265 TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ, 266 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO, 267 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM, 268 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 269 TA_RAS_BLOCK__GFX_TCC_INDEX2_END = 270 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 271 /* TCC range 3*/ 272 TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 273 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 274 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 275 TA_RAS_BLOCK__GFX_TCC_INDEX3_END = 276 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 277 /* TCC range 4*/ 278 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 279 TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN = 280 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 281 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 282 TA_RAS_BLOCK__GFX_TCC_INDEX4_END = 283 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 284 TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END, 285 /* TCI*/ 286 TA_RAS_BLOCK__GFX_TCI_WRITE_RAM, 287 /* TCP*/ 288 TA_RAS_BLOCK__GFX_TCP_INDEX_START, 289 TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START, 290 TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM, 291 TA_RAS_BLOCK__GFX_TCP_CMD_FIFO, 292 TA_RAS_BLOCK__GFX_TCP_VM_FIFO, 293 TA_RAS_BLOCK__GFX_TCP_DB_RAM, 294 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0, 295 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 296 TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 297 /* TD*/ 298 TA_RAS_BLOCK__GFX_TD_INDEX_START, 299 TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START, 300 TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI, 301 TA_RAS_BLOCK__GFX_TD_CS_FIFO, 302 TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO, 303 /* EA (3 sub-ranges)*/ 304 TA_RAS_BLOCK__GFX_EA_INDEX_START, 305 /* EA range 0*/ 306 TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START, 307 TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START, 308 TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM, 309 TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM, 310 TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM, 311 TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM, 312 TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM, 313 TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM, 314 TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 315 TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 316 /* EA range 1*/ 317 TA_RAS_BLOCK__GFX_EA_INDEX1_START, 318 TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START, 319 TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM, 320 TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM, 321 TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM, 322 TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM, 323 TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM, 324 TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 325 TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 326 /* EA range 2*/ 327 TA_RAS_BLOCK__GFX_EA_INDEX2_START, 328 TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START, 329 TA_RAS_BLOCK__GFX_EA_MAM_D1MEM, 330 TA_RAS_BLOCK__GFX_EA_MAM_D2MEM, 331 TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 332 TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 333 TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END, 334 /* UTC VM L2 bank*/ 335 TA_RAS_BLOCK__UTC_VML2_BANK_CACHE, 336 /* UTC VM walker*/ 337 TA_RAS_BLOCK__UTC_VML2_WALKER, 338 /* UTC ATC L2 2MB cache*/ 339 TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK, 340 /* UTC ATC L2 4KB cache*/ 341 TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK, 342 TA_RAS_BLOCK__GFX_MAX 343 }; 344 345 struct ras_gfx_subblock { 346 unsigned char *name; 347 int ta_subblock; 348 int hw_supported_error_type; 349 int sw_supported_error_type; 350 }; 351 352 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \ 353 [AMDGPU_RAS_BLOCK__##subblock] = { \ 354 #subblock, \ 355 TA_RAS_BLOCK__##subblock, \ 356 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \ 357 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \ 358 } 359 360 static const struct ras_gfx_subblock ras_gfx_subblocks[] = { 361 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1), 362 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1), 363 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 364 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 365 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 366 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 367 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 368 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 369 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 370 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 371 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1), 372 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0), 373 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1), 374 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1), 375 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 376 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0), 377 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0, 378 0), 379 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0, 380 0), 381 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 382 AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0), 383 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0), 384 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1), 385 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0), 386 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0), 387 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1), 388 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 389 0, 0), 390 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 391 0), 392 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 393 0, 0), 394 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0, 395 0), 396 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 397 0, 0), 398 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 399 0), 400 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 401 1), 402 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 403 0, 0, 0), 404 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 405 0), 406 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 407 0), 408 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 409 0), 410 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 411 0), 412 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 413 0), 414 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 415 0, 0), 416 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 417 0), 418 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 419 0), 420 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 421 0, 0, 0), 422 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 423 0), 424 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 425 0), 426 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 427 0), 428 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 429 0), 430 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 431 0), 432 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 433 0, 0), 434 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 435 0), 436 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1), 437 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 438 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 439 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 440 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 441 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0), 442 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 443 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1), 444 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0, 445 1), 446 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0, 447 1), 448 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0, 449 1), 450 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0, 451 0), 452 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0, 453 0), 454 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 455 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 456 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0), 457 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0), 458 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0), 459 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0), 460 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 461 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0), 462 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0), 463 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 464 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0), 465 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0, 466 0), 467 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 468 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0, 469 0), 470 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0, 471 0, 0), 472 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0, 473 0), 474 AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 475 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1), 476 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0), 477 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 478 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 479 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 480 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0), 481 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0), 482 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1), 483 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0), 484 AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 485 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1), 486 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 487 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 488 AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 489 AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 490 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 491 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 492 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 493 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 494 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 495 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 496 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 497 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0), 498 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 499 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 500 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0), 501 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0), 502 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0), 503 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0), 504 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0), 505 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0), 506 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0), 507 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0), 508 }; 509 510 static const struct soc15_reg_golden golden_settings_gc_9_0[] = 511 { 512 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 513 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000), 514 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 515 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 516 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 517 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 518 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 519 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 520 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 521 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87), 522 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f), 523 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 524 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 525 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 526 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 527 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 528 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff), 529 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 530 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 531 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 532 }; 533 534 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] = 535 { 536 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107), 537 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 538 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 539 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 540 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 541 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 542 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042), 543 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 544 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000), 545 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 546 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 547 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 548 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 549 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 550 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 551 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107), 552 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800), 553 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080) 554 }; 555 556 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] = 557 { 558 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080), 559 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 560 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 561 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042), 562 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042), 563 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400), 564 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000), 565 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000), 566 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107), 567 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000), 568 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000) 569 }; 570 571 static const struct soc15_reg_golden golden_settings_gc_9_1[] = 572 { 573 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 574 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 575 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 576 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 577 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 578 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 579 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 580 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 581 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 582 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 583 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 584 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 585 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 586 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 587 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 588 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 589 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 590 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 591 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 592 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff), 593 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080), 594 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 595 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 596 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 597 }; 598 599 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] = 600 { 601 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 602 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042), 603 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042), 604 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000), 605 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000), 606 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 607 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800) 608 }; 609 610 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] = 611 { 612 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000), 613 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 614 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 615 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080), 616 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080), 617 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080), 618 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041), 619 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041), 620 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 621 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 622 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080), 623 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080), 624 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080), 625 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080), 626 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080), 627 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 628 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010), 629 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 630 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 631 }; 632 633 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] = 634 { 635 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 636 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 637 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 638 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042), 639 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042), 640 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 641 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 642 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 643 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 644 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 645 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 646 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc), 647 }; 648 649 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] = 650 { 651 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff), 652 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000), 653 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382) 654 }; 655 656 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] = 657 { 658 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 659 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 660 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 661 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 662 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 663 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 664 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 665 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 666 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87), 667 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f), 668 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 669 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 670 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 671 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 672 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 673 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff) 674 }; 675 676 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] = 677 { 678 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080), 679 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 680 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 681 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041), 682 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041), 683 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 684 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107), 685 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 686 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410), 687 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 688 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 689 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 690 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 691 }; 692 693 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] = 694 { 695 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 696 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000), 697 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e), 698 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca), 699 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098), 700 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3), 701 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1), 702 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135), 703 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000), 704 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00), 705 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000) 706 }; 707 708 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = { 709 {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)}, 710 {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)}, 711 }; 712 713 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] = 714 { 715 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 716 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 717 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 718 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 719 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 720 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 721 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 722 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 723 }; 724 725 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] = 726 { 727 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0, 728 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0, 729 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0, 730 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0, 731 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0, 732 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0, 733 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0, 734 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0, 735 }; 736 737 static void gfx_v9_0_rlcg_w(struct amdgpu_device *adev, u32 offset, u32 v, u32 flag) 738 { 739 static void *scratch_reg0; 740 static void *scratch_reg1; 741 static void *scratch_reg2; 742 static void *scratch_reg3; 743 static void *spare_int; 744 static uint32_t grbm_cntl; 745 static uint32_t grbm_idx; 746 747 scratch_reg0 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0)*4; 748 scratch_reg1 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1)*4; 749 scratch_reg2 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG2)*4; 750 scratch_reg3 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3)*4; 751 spare_int = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT)*4; 752 753 grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL; 754 grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX; 755 756 if (amdgpu_sriov_runtime(adev)) { 757 pr_err("shouldn't call rlcg write register during runtime\n"); 758 return; 759 } 760 761 if (offset == grbm_cntl || offset == grbm_idx) { 762 if (offset == grbm_cntl) 763 writel(v, scratch_reg2); 764 else if (offset == grbm_idx) 765 writel(v, scratch_reg3); 766 767 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4)); 768 } else { 769 uint32_t i = 0; 770 uint32_t retries = 50000; 771 772 writel(v, scratch_reg0); 773 writel(offset | 0x80000000, scratch_reg1); 774 writel(1, spare_int); 775 for (i = 0; i < retries; i++) { 776 u32 tmp; 777 778 tmp = readl(scratch_reg1); 779 if (!(tmp & 0x80000000)) 780 break; 781 782 udelay(10); 783 } 784 if (i >= retries) 785 pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset); 786 } 787 788 } 789 790 static void gfx_v9_0_sriov_wreg(struct amdgpu_device *adev, u32 offset, 791 u32 v, u32 acc_flags, u32 hwip) 792 { 793 if ((acc_flags & AMDGPU_REGS_RLC) && 794 amdgpu_sriov_fullaccess(adev)) { 795 gfx_v9_0_rlcg_w(adev, offset, v, acc_flags); 796 797 return; 798 } 799 800 if (acc_flags & AMDGPU_REGS_NO_KIQ) 801 WREG32_NO_KIQ(offset, v); 802 else 803 WREG32(offset, v); 804 } 805 806 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042 807 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041 808 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042 809 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041 810 811 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev); 812 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev); 813 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev); 814 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev); 815 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 816 struct amdgpu_cu_info *cu_info); 817 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev); 818 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring); 819 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring); 820 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 821 void *ras_error_status); 822 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 823 void *inject_if); 824 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev); 825 826 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring, 827 uint64_t queue_mask) 828 { 829 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 830 amdgpu_ring_write(kiq_ring, 831 PACKET3_SET_RESOURCES_VMID_MASK(0) | 832 /* vmid_mask:0* queue_type:0 (KIQ) */ 833 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); 834 amdgpu_ring_write(kiq_ring, 835 lower_32_bits(queue_mask)); /* queue mask lo */ 836 amdgpu_ring_write(kiq_ring, 837 upper_32_bits(queue_mask)); /* queue mask hi */ 838 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */ 839 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */ 840 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 841 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 842 } 843 844 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring, 845 struct amdgpu_ring *ring) 846 { 847 struct amdgpu_device *adev = kiq_ring->adev; 848 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 849 uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 850 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 851 852 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 853 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 854 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 855 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 856 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ 857 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 858 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 859 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) | 860 /*queue_type: normal compute queue */ 861 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | 862 /* alloc format: all_on_one_pipe */ 863 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | 864 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) | 865 /* num_queues: must be 1 */ 866 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); 867 amdgpu_ring_write(kiq_ring, 868 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); 869 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 870 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 871 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 872 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 873 } 874 875 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, 876 struct amdgpu_ring *ring, 877 enum amdgpu_unmap_queues_action action, 878 u64 gpu_addr, u64 seq) 879 { 880 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 881 882 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 883 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 884 PACKET3_UNMAP_QUEUES_ACTION(action) | 885 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 886 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) | 887 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 888 amdgpu_ring_write(kiq_ring, 889 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 890 891 if (action == PREEMPT_QUEUES_NO_UNMAP) { 892 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr)); 893 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr)); 894 amdgpu_ring_write(kiq_ring, seq); 895 } else { 896 amdgpu_ring_write(kiq_ring, 0); 897 amdgpu_ring_write(kiq_ring, 0); 898 amdgpu_ring_write(kiq_ring, 0); 899 } 900 } 901 902 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring, 903 struct amdgpu_ring *ring, 904 u64 addr, 905 u64 seq) 906 { 907 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 908 909 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5)); 910 amdgpu_ring_write(kiq_ring, 911 PACKET3_QUERY_STATUS_CONTEXT_ID(0) | 912 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) | 913 PACKET3_QUERY_STATUS_COMMAND(2)); 914 /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 915 amdgpu_ring_write(kiq_ring, 916 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) | 917 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel)); 918 amdgpu_ring_write(kiq_ring, lower_32_bits(addr)); 919 amdgpu_ring_write(kiq_ring, upper_32_bits(addr)); 920 amdgpu_ring_write(kiq_ring, lower_32_bits(seq)); 921 amdgpu_ring_write(kiq_ring, upper_32_bits(seq)); 922 } 923 924 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring, 925 uint16_t pasid, uint32_t flush_type, 926 bool all_hub) 927 { 928 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); 929 amdgpu_ring_write(kiq_ring, 930 PACKET3_INVALIDATE_TLBS_DST_SEL(1) | 931 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) | 932 PACKET3_INVALIDATE_TLBS_PASID(pasid) | 933 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type)); 934 } 935 936 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = { 937 .kiq_set_resources = gfx_v9_0_kiq_set_resources, 938 .kiq_map_queues = gfx_v9_0_kiq_map_queues, 939 .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues, 940 .kiq_query_status = gfx_v9_0_kiq_query_status, 941 .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs, 942 .set_resources_size = 8, 943 .map_queues_size = 7, 944 .unmap_queues_size = 6, 945 .query_status_size = 7, 946 .invalidate_tlbs_size = 2, 947 }; 948 949 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) 950 { 951 adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs; 952 } 953 954 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev) 955 { 956 switch (adev->ip_versions[GC_HWIP][0]) { 957 case IP_VERSION(9, 0, 1): 958 soc15_program_register_sequence(adev, 959 golden_settings_gc_9_0, 960 ARRAY_SIZE(golden_settings_gc_9_0)); 961 soc15_program_register_sequence(adev, 962 golden_settings_gc_9_0_vg10, 963 ARRAY_SIZE(golden_settings_gc_9_0_vg10)); 964 break; 965 case IP_VERSION(9, 2, 1): 966 soc15_program_register_sequence(adev, 967 golden_settings_gc_9_2_1, 968 ARRAY_SIZE(golden_settings_gc_9_2_1)); 969 soc15_program_register_sequence(adev, 970 golden_settings_gc_9_2_1_vg12, 971 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12)); 972 break; 973 case IP_VERSION(9, 4, 0): 974 soc15_program_register_sequence(adev, 975 golden_settings_gc_9_0, 976 ARRAY_SIZE(golden_settings_gc_9_0)); 977 soc15_program_register_sequence(adev, 978 golden_settings_gc_9_0_vg20, 979 ARRAY_SIZE(golden_settings_gc_9_0_vg20)); 980 break; 981 case IP_VERSION(9, 4, 1): 982 soc15_program_register_sequence(adev, 983 golden_settings_gc_9_4_1_arct, 984 ARRAY_SIZE(golden_settings_gc_9_4_1_arct)); 985 break; 986 case IP_VERSION(9, 2, 2): 987 case IP_VERSION(9, 1, 0): 988 soc15_program_register_sequence(adev, golden_settings_gc_9_1, 989 ARRAY_SIZE(golden_settings_gc_9_1)); 990 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 991 soc15_program_register_sequence(adev, 992 golden_settings_gc_9_1_rv2, 993 ARRAY_SIZE(golden_settings_gc_9_1_rv2)); 994 else 995 soc15_program_register_sequence(adev, 996 golden_settings_gc_9_1_rv1, 997 ARRAY_SIZE(golden_settings_gc_9_1_rv1)); 998 break; 999 case IP_VERSION(9, 3, 0): 1000 soc15_program_register_sequence(adev, 1001 golden_settings_gc_9_1_rn, 1002 ARRAY_SIZE(golden_settings_gc_9_1_rn)); 1003 return; /* for renoir, don't need common goldensetting */ 1004 case IP_VERSION(9, 4, 2): 1005 gfx_v9_4_2_init_golden_registers(adev, 1006 adev->smuio.funcs->get_die_id(adev)); 1007 break; 1008 default: 1009 break; 1010 } 1011 1012 if ((adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 1)) && 1013 (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 2))) 1014 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common, 1015 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common)); 1016 } 1017 1018 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev) 1019 { 1020 adev->gfx.scratch.num_reg = 8; 1021 adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0); 1022 adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1; 1023 } 1024 1025 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel, 1026 bool wc, uint32_t reg, uint32_t val) 1027 { 1028 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 1029 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) | 1030 WRITE_DATA_DST_SEL(0) | 1031 (wc ? WR_CONFIRM : 0)); 1032 amdgpu_ring_write(ring, reg); 1033 amdgpu_ring_write(ring, 0); 1034 amdgpu_ring_write(ring, val); 1035 } 1036 1037 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, 1038 int mem_space, int opt, uint32_t addr0, 1039 uint32_t addr1, uint32_t ref, uint32_t mask, 1040 uint32_t inv) 1041 { 1042 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 1043 amdgpu_ring_write(ring, 1044 /* memory (1) or register (0) */ 1045 (WAIT_REG_MEM_MEM_SPACE(mem_space) | 1046 WAIT_REG_MEM_OPERATION(opt) | /* wait */ 1047 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 1048 WAIT_REG_MEM_ENGINE(eng_sel))); 1049 1050 if (mem_space) 1051 BUG_ON(addr0 & 0x3); /* Dword align */ 1052 amdgpu_ring_write(ring, addr0); 1053 amdgpu_ring_write(ring, addr1); 1054 amdgpu_ring_write(ring, ref); 1055 amdgpu_ring_write(ring, mask); 1056 amdgpu_ring_write(ring, inv); /* poll interval */ 1057 } 1058 1059 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring) 1060 { 1061 struct amdgpu_device *adev = ring->adev; 1062 uint32_t scratch; 1063 uint32_t tmp = 0; 1064 unsigned i; 1065 int r; 1066 1067 r = amdgpu_gfx_scratch_get(adev, &scratch); 1068 if (r) 1069 return r; 1070 1071 WREG32(scratch, 0xCAFEDEAD); 1072 r = amdgpu_ring_alloc(ring, 3); 1073 if (r) 1074 goto error_free_scratch; 1075 1076 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 1077 amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START)); 1078 amdgpu_ring_write(ring, 0xDEADBEEF); 1079 amdgpu_ring_commit(ring); 1080 1081 for (i = 0; i < adev->usec_timeout; i++) { 1082 tmp = RREG32(scratch); 1083 if (tmp == 0xDEADBEEF) 1084 break; 1085 udelay(1); 1086 } 1087 1088 if (i >= adev->usec_timeout) 1089 r = -ETIMEDOUT; 1090 1091 error_free_scratch: 1092 amdgpu_gfx_scratch_free(adev, scratch); 1093 return r; 1094 } 1095 1096 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 1097 { 1098 struct amdgpu_device *adev = ring->adev; 1099 struct amdgpu_ib ib; 1100 struct dma_fence *f = NULL; 1101 1102 unsigned index; 1103 uint64_t gpu_addr; 1104 uint32_t tmp; 1105 long r; 1106 1107 r = amdgpu_device_wb_get(adev, &index); 1108 if (r) 1109 return r; 1110 1111 gpu_addr = adev->wb.gpu_addr + (index * 4); 1112 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 1113 memset(&ib, 0, sizeof(ib)); 1114 r = amdgpu_ib_get(adev, NULL, 16, 1115 AMDGPU_IB_POOL_DIRECT, &ib); 1116 if (r) 1117 goto err1; 1118 1119 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 1120 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 1121 ib.ptr[2] = lower_32_bits(gpu_addr); 1122 ib.ptr[3] = upper_32_bits(gpu_addr); 1123 ib.ptr[4] = 0xDEADBEEF; 1124 ib.length_dw = 5; 1125 1126 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 1127 if (r) 1128 goto err2; 1129 1130 r = dma_fence_wait_timeout(f, false, timeout); 1131 if (r == 0) { 1132 r = -ETIMEDOUT; 1133 goto err2; 1134 } else if (r < 0) { 1135 goto err2; 1136 } 1137 1138 tmp = adev->wb.wb[index]; 1139 if (tmp == 0xDEADBEEF) 1140 r = 0; 1141 else 1142 r = -EINVAL; 1143 1144 err2: 1145 amdgpu_ib_free(adev, &ib, NULL); 1146 dma_fence_put(f); 1147 err1: 1148 amdgpu_device_wb_free(adev, index); 1149 return r; 1150 } 1151 1152 1153 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev) 1154 { 1155 release_firmware(adev->gfx.pfp_fw); 1156 adev->gfx.pfp_fw = NULL; 1157 release_firmware(adev->gfx.me_fw); 1158 adev->gfx.me_fw = NULL; 1159 release_firmware(adev->gfx.ce_fw); 1160 adev->gfx.ce_fw = NULL; 1161 release_firmware(adev->gfx.rlc_fw); 1162 adev->gfx.rlc_fw = NULL; 1163 release_firmware(adev->gfx.mec_fw); 1164 adev->gfx.mec_fw = NULL; 1165 release_firmware(adev->gfx.mec2_fw); 1166 adev->gfx.mec2_fw = NULL; 1167 1168 kfree(adev->gfx.rlc.register_list_format); 1169 } 1170 1171 static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev) 1172 { 1173 const struct rlc_firmware_header_v2_1 *rlc_hdr; 1174 1175 rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data; 1176 adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver); 1177 adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver); 1178 adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes); 1179 adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes); 1180 adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver); 1181 adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver); 1182 adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes); 1183 adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes); 1184 adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver); 1185 adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver); 1186 adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes); 1187 adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes); 1188 adev->gfx.rlc.reg_list_format_direct_reg_list_length = 1189 le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length); 1190 } 1191 1192 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev) 1193 { 1194 adev->gfx.me_fw_write_wait = false; 1195 adev->gfx.mec_fw_write_wait = false; 1196 1197 if ((adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 1)) && 1198 ((adev->gfx.mec_fw_version < 0x000001a5) || 1199 (adev->gfx.mec_feature_version < 46) || 1200 (adev->gfx.pfp_fw_version < 0x000000b7) || 1201 (adev->gfx.pfp_feature_version < 46))) 1202 DRM_WARN_ONCE("CP firmware version too old, please update!"); 1203 1204 switch (adev->ip_versions[GC_HWIP][0]) { 1205 case IP_VERSION(9, 0, 1): 1206 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1207 (adev->gfx.me_feature_version >= 42) && 1208 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1209 (adev->gfx.pfp_feature_version >= 42)) 1210 adev->gfx.me_fw_write_wait = true; 1211 1212 if ((adev->gfx.mec_fw_version >= 0x00000193) && 1213 (adev->gfx.mec_feature_version >= 42)) 1214 adev->gfx.mec_fw_write_wait = true; 1215 break; 1216 case IP_VERSION(9, 2, 1): 1217 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1218 (adev->gfx.me_feature_version >= 44) && 1219 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1220 (adev->gfx.pfp_feature_version >= 44)) 1221 adev->gfx.me_fw_write_wait = true; 1222 1223 if ((adev->gfx.mec_fw_version >= 0x00000196) && 1224 (adev->gfx.mec_feature_version >= 44)) 1225 adev->gfx.mec_fw_write_wait = true; 1226 break; 1227 case IP_VERSION(9, 4, 0): 1228 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1229 (adev->gfx.me_feature_version >= 44) && 1230 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1231 (adev->gfx.pfp_feature_version >= 44)) 1232 adev->gfx.me_fw_write_wait = true; 1233 1234 if ((adev->gfx.mec_fw_version >= 0x00000197) && 1235 (adev->gfx.mec_feature_version >= 44)) 1236 adev->gfx.mec_fw_write_wait = true; 1237 break; 1238 case IP_VERSION(9, 1, 0): 1239 case IP_VERSION(9, 2, 2): 1240 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1241 (adev->gfx.me_feature_version >= 42) && 1242 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1243 (adev->gfx.pfp_feature_version >= 42)) 1244 adev->gfx.me_fw_write_wait = true; 1245 1246 if ((adev->gfx.mec_fw_version >= 0x00000192) && 1247 (adev->gfx.mec_feature_version >= 42)) 1248 adev->gfx.mec_fw_write_wait = true; 1249 break; 1250 default: 1251 adev->gfx.me_fw_write_wait = true; 1252 adev->gfx.mec_fw_write_wait = true; 1253 break; 1254 } 1255 } 1256 1257 struct amdgpu_gfxoff_quirk { 1258 u16 chip_vendor; 1259 u16 chip_device; 1260 u16 subsys_vendor; 1261 u16 subsys_device; 1262 u8 revision; 1263 }; 1264 1265 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = { 1266 /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */ 1267 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 }, 1268 /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */ 1269 { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 }, 1270 /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */ 1271 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 }, 1272 { 0, 0, 0, 0, 0 }, 1273 }; 1274 1275 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev) 1276 { 1277 const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list; 1278 1279 while (p && p->chip_device != 0) { 1280 if (pdev->vendor == p->chip_vendor && 1281 pdev->device == p->chip_device && 1282 pdev->subsystem_vendor == p->subsys_vendor && 1283 pdev->subsystem_device == p->subsys_device && 1284 pdev->revision == p->revision) { 1285 return true; 1286 } 1287 ++p; 1288 } 1289 return false; 1290 } 1291 1292 static bool is_raven_kicker(struct amdgpu_device *adev) 1293 { 1294 if (adev->pm.fw_version >= 0x41e2b) 1295 return true; 1296 else 1297 return false; 1298 } 1299 1300 static bool check_if_enlarge_doorbell_range(struct amdgpu_device *adev) 1301 { 1302 if ((adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 3, 0)) && 1303 (adev->gfx.me_fw_version >= 0x000000a5) && 1304 (adev->gfx.me_feature_version >= 52)) 1305 return true; 1306 else 1307 return false; 1308 } 1309 1310 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev) 1311 { 1312 if (gfx_v9_0_should_disable_gfxoff(adev->pdev)) 1313 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1314 1315 switch (adev->ip_versions[GC_HWIP][0]) { 1316 case IP_VERSION(9, 0, 1): 1317 case IP_VERSION(9, 2, 1): 1318 case IP_VERSION(9, 4, 0): 1319 break; 1320 case IP_VERSION(9, 2, 2): 1321 case IP_VERSION(9, 1, 0): 1322 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) || 1323 (adev->apu_flags & AMD_APU_IS_PICASSO)) && 1324 ((!is_raven_kicker(adev) && 1325 adev->gfx.rlc_fw_version < 531) || 1326 (adev->gfx.rlc_feature_version < 1) || 1327 !adev->gfx.rlc.is_rlc_v2_1)) 1328 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1329 1330 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1331 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1332 AMD_PG_SUPPORT_CP | 1333 AMD_PG_SUPPORT_RLC_SMU_HS; 1334 break; 1335 case IP_VERSION(9, 3, 0): 1336 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1337 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1338 AMD_PG_SUPPORT_CP | 1339 AMD_PG_SUPPORT_RLC_SMU_HS; 1340 break; 1341 default: 1342 break; 1343 } 1344 } 1345 1346 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev, 1347 const char *chip_name) 1348 { 1349 char fw_name[30]; 1350 int err; 1351 struct amdgpu_firmware_info *info = NULL; 1352 const struct common_firmware_header *header = NULL; 1353 const struct gfx_firmware_header_v1_0 *cp_hdr; 1354 1355 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name); 1356 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev); 1357 if (err) 1358 goto out; 1359 err = amdgpu_ucode_validate(adev->gfx.pfp_fw); 1360 if (err) 1361 goto out; 1362 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; 1363 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1364 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1365 1366 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name); 1367 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev); 1368 if (err) 1369 goto out; 1370 err = amdgpu_ucode_validate(adev->gfx.me_fw); 1371 if (err) 1372 goto out; 1373 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; 1374 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1375 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1376 1377 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name); 1378 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev); 1379 if (err) 1380 goto out; 1381 err = amdgpu_ucode_validate(adev->gfx.ce_fw); 1382 if (err) 1383 goto out; 1384 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; 1385 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1386 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1387 1388 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1389 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP]; 1390 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP; 1391 info->fw = adev->gfx.pfp_fw; 1392 header = (const struct common_firmware_header *)info->fw->data; 1393 adev->firmware.fw_size += 1394 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1395 1396 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME]; 1397 info->ucode_id = AMDGPU_UCODE_ID_CP_ME; 1398 info->fw = adev->gfx.me_fw; 1399 header = (const struct common_firmware_header *)info->fw->data; 1400 adev->firmware.fw_size += 1401 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1402 1403 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE]; 1404 info->ucode_id = AMDGPU_UCODE_ID_CP_CE; 1405 info->fw = adev->gfx.ce_fw; 1406 header = (const struct common_firmware_header *)info->fw->data; 1407 adev->firmware.fw_size += 1408 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1409 } 1410 1411 out: 1412 if (err) { 1413 dev_err(adev->dev, 1414 "gfx9: Failed to load firmware \"%s\"\n", 1415 fw_name); 1416 release_firmware(adev->gfx.pfp_fw); 1417 adev->gfx.pfp_fw = NULL; 1418 release_firmware(adev->gfx.me_fw); 1419 adev->gfx.me_fw = NULL; 1420 release_firmware(adev->gfx.ce_fw); 1421 adev->gfx.ce_fw = NULL; 1422 } 1423 return err; 1424 } 1425 1426 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev, 1427 const char *chip_name) 1428 { 1429 char fw_name[30]; 1430 int err; 1431 struct amdgpu_firmware_info *info = NULL; 1432 const struct common_firmware_header *header = NULL; 1433 const struct rlc_firmware_header_v2_0 *rlc_hdr; 1434 unsigned int *tmp = NULL; 1435 unsigned int i = 0; 1436 uint16_t version_major; 1437 uint16_t version_minor; 1438 uint32_t smu_version; 1439 1440 /* 1441 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin 1442 * instead of picasso_rlc.bin. 1443 * Judgment method: 1444 * PCO AM4: revision >= 0xC8 && revision <= 0xCF 1445 * or revision >= 0xD8 && revision <= 0xDF 1446 * otherwise is PCO FP5 1447 */ 1448 if (!strcmp(chip_name, "picasso") && 1449 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) || 1450 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF)))) 1451 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name); 1452 else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) && 1453 (smu_version >= 0x41e2b)) 1454 /** 1455 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly. 1456 */ 1457 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name); 1458 else 1459 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name); 1460 err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev); 1461 if (err) 1462 goto out; 1463 err = amdgpu_ucode_validate(adev->gfx.rlc_fw); 1464 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 1465 1466 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 1467 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 1468 if (version_major == 2 && version_minor == 1) 1469 adev->gfx.rlc.is_rlc_v2_1 = true; 1470 1471 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); 1472 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); 1473 adev->gfx.rlc.save_and_restore_offset = 1474 le32_to_cpu(rlc_hdr->save_and_restore_offset); 1475 adev->gfx.rlc.clear_state_descriptor_offset = 1476 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset); 1477 adev->gfx.rlc.avail_scratch_ram_locations = 1478 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations); 1479 adev->gfx.rlc.reg_restore_list_size = 1480 le32_to_cpu(rlc_hdr->reg_restore_list_size); 1481 adev->gfx.rlc.reg_list_format_start = 1482 le32_to_cpu(rlc_hdr->reg_list_format_start); 1483 adev->gfx.rlc.reg_list_format_separate_start = 1484 le32_to_cpu(rlc_hdr->reg_list_format_separate_start); 1485 adev->gfx.rlc.starting_offsets_start = 1486 le32_to_cpu(rlc_hdr->starting_offsets_start); 1487 adev->gfx.rlc.reg_list_format_size_bytes = 1488 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes); 1489 adev->gfx.rlc.reg_list_size_bytes = 1490 le32_to_cpu(rlc_hdr->reg_list_size_bytes); 1491 adev->gfx.rlc.register_list_format = 1492 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes + 1493 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL); 1494 if (!adev->gfx.rlc.register_list_format) { 1495 err = -ENOMEM; 1496 goto out; 1497 } 1498 1499 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1500 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); 1501 for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++) 1502 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]); 1503 1504 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i; 1505 1506 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1507 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); 1508 for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++) 1509 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); 1510 1511 if (adev->gfx.rlc.is_rlc_v2_1) 1512 gfx_v9_0_init_rlc_ext_microcode(adev); 1513 1514 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1515 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G]; 1516 info->ucode_id = AMDGPU_UCODE_ID_RLC_G; 1517 info->fw = adev->gfx.rlc_fw; 1518 header = (const struct common_firmware_header *)info->fw->data; 1519 adev->firmware.fw_size += 1520 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1521 1522 if (adev->gfx.rlc.is_rlc_v2_1 && 1523 adev->gfx.rlc.save_restore_list_cntl_size_bytes && 1524 adev->gfx.rlc.save_restore_list_gpm_size_bytes && 1525 adev->gfx.rlc.save_restore_list_srm_size_bytes) { 1526 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL]; 1527 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL; 1528 info->fw = adev->gfx.rlc_fw; 1529 adev->firmware.fw_size += 1530 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE); 1531 1532 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM]; 1533 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM; 1534 info->fw = adev->gfx.rlc_fw; 1535 adev->firmware.fw_size += 1536 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE); 1537 1538 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM]; 1539 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM; 1540 info->fw = adev->gfx.rlc_fw; 1541 adev->firmware.fw_size += 1542 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE); 1543 } 1544 } 1545 1546 out: 1547 if (err) { 1548 dev_err(adev->dev, 1549 "gfx9: Failed to load firmware \"%s\"\n", 1550 fw_name); 1551 release_firmware(adev->gfx.rlc_fw); 1552 adev->gfx.rlc_fw = NULL; 1553 } 1554 return err; 1555 } 1556 1557 static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev) 1558 { 1559 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) || 1560 adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1) || 1561 adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 3, 0)) 1562 return false; 1563 1564 return true; 1565 } 1566 1567 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev, 1568 const char *chip_name) 1569 { 1570 char fw_name[30]; 1571 int err; 1572 struct amdgpu_firmware_info *info = NULL; 1573 const struct common_firmware_header *header = NULL; 1574 const struct gfx_firmware_header_v1_0 *cp_hdr; 1575 1576 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name); 1577 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev); 1578 if (err) 1579 goto out; 1580 err = amdgpu_ucode_validate(adev->gfx.mec_fw); 1581 if (err) 1582 goto out; 1583 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1584 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1585 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1586 1587 1588 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) { 1589 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name); 1590 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev); 1591 if (!err) { 1592 err = amdgpu_ucode_validate(adev->gfx.mec2_fw); 1593 if (err) 1594 goto out; 1595 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1596 adev->gfx.mec2_fw->data; 1597 adev->gfx.mec2_fw_version = 1598 le32_to_cpu(cp_hdr->header.ucode_version); 1599 adev->gfx.mec2_feature_version = 1600 le32_to_cpu(cp_hdr->ucode_feature_version); 1601 } else { 1602 err = 0; 1603 adev->gfx.mec2_fw = NULL; 1604 } 1605 } else { 1606 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version; 1607 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version; 1608 } 1609 1610 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1611 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1]; 1612 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1; 1613 info->fw = adev->gfx.mec_fw; 1614 header = (const struct common_firmware_header *)info->fw->data; 1615 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1616 adev->firmware.fw_size += 1617 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1618 1619 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT]; 1620 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT; 1621 info->fw = adev->gfx.mec_fw; 1622 adev->firmware.fw_size += 1623 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1624 1625 if (adev->gfx.mec2_fw) { 1626 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2]; 1627 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2; 1628 info->fw = adev->gfx.mec2_fw; 1629 header = (const struct common_firmware_header *)info->fw->data; 1630 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1631 adev->firmware.fw_size += 1632 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1633 1634 /* TODO: Determine if MEC2 JT FW loading can be removed 1635 for all GFX V9 asic and above */ 1636 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) { 1637 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT]; 1638 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT; 1639 info->fw = adev->gfx.mec2_fw; 1640 adev->firmware.fw_size += 1641 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, 1642 PAGE_SIZE); 1643 } 1644 } 1645 } 1646 1647 out: 1648 gfx_v9_0_check_if_need_gfxoff(adev); 1649 gfx_v9_0_check_fw_write_wait(adev); 1650 if (err) { 1651 dev_err(adev->dev, 1652 "gfx9: Failed to load firmware \"%s\"\n", 1653 fw_name); 1654 release_firmware(adev->gfx.mec_fw); 1655 adev->gfx.mec_fw = NULL; 1656 release_firmware(adev->gfx.mec2_fw); 1657 adev->gfx.mec2_fw = NULL; 1658 } 1659 return err; 1660 } 1661 1662 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev) 1663 { 1664 const char *chip_name; 1665 int r; 1666 1667 DRM_DEBUG("\n"); 1668 1669 switch (adev->ip_versions[GC_HWIP][0]) { 1670 case IP_VERSION(9, 0, 1): 1671 chip_name = "vega10"; 1672 break; 1673 case IP_VERSION(9, 2, 1): 1674 chip_name = "vega12"; 1675 break; 1676 case IP_VERSION(9, 4, 0): 1677 chip_name = "vega20"; 1678 break; 1679 case IP_VERSION(9, 2, 2): 1680 case IP_VERSION(9, 1, 0): 1681 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 1682 chip_name = "raven2"; 1683 else if (adev->apu_flags & AMD_APU_IS_PICASSO) 1684 chip_name = "picasso"; 1685 else 1686 chip_name = "raven"; 1687 break; 1688 case IP_VERSION(9, 4, 1): 1689 chip_name = "arcturus"; 1690 break; 1691 case IP_VERSION(9, 3, 0): 1692 if (adev->apu_flags & AMD_APU_IS_RENOIR) 1693 chip_name = "renoir"; 1694 else 1695 chip_name = "green_sardine"; 1696 break; 1697 case IP_VERSION(9, 4, 2): 1698 chip_name = "aldebaran"; 1699 break; 1700 default: 1701 BUG(); 1702 } 1703 1704 /* No CPG in Arcturus */ 1705 if (adev->gfx.num_gfx_rings) { 1706 r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name); 1707 if (r) 1708 return r; 1709 } 1710 1711 r = gfx_v9_0_init_rlc_microcode(adev, chip_name); 1712 if (r) 1713 return r; 1714 1715 r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name); 1716 if (r) 1717 return r; 1718 1719 return r; 1720 } 1721 1722 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev) 1723 { 1724 u32 count = 0; 1725 const struct cs_section_def *sect = NULL; 1726 const struct cs_extent_def *ext = NULL; 1727 1728 /* begin clear state */ 1729 count += 2; 1730 /* context control state */ 1731 count += 3; 1732 1733 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 1734 for (ext = sect->section; ext->extent != NULL; ++ext) { 1735 if (sect->id == SECT_CONTEXT) 1736 count += 2 + ext->reg_count; 1737 else 1738 return 0; 1739 } 1740 } 1741 1742 /* end clear state */ 1743 count += 2; 1744 /* clear state */ 1745 count += 2; 1746 1747 return count; 1748 } 1749 1750 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev, 1751 volatile u32 *buffer) 1752 { 1753 u32 count = 0, i; 1754 const struct cs_section_def *sect = NULL; 1755 const struct cs_extent_def *ext = NULL; 1756 1757 if (adev->gfx.rlc.cs_data == NULL) 1758 return; 1759 if (buffer == NULL) 1760 return; 1761 1762 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1763 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 1764 1765 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 1766 buffer[count++] = cpu_to_le32(0x80000000); 1767 buffer[count++] = cpu_to_le32(0x80000000); 1768 1769 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { 1770 for (ext = sect->section; ext->extent != NULL; ++ext) { 1771 if (sect->id == SECT_CONTEXT) { 1772 buffer[count++] = 1773 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 1774 buffer[count++] = cpu_to_le32(ext->reg_index - 1775 PACKET3_SET_CONTEXT_REG_START); 1776 for (i = 0; i < ext->reg_count; i++) 1777 buffer[count++] = cpu_to_le32(ext->extent[i]); 1778 } else { 1779 return; 1780 } 1781 } 1782 } 1783 1784 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1785 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 1786 1787 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 1788 buffer[count++] = cpu_to_le32(0); 1789 } 1790 1791 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev) 1792 { 1793 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; 1794 uint32_t pg_always_on_cu_num = 2; 1795 uint32_t always_on_cu_num; 1796 uint32_t i, j, k; 1797 uint32_t mask, cu_bitmap, counter; 1798 1799 if (adev->flags & AMD_IS_APU) 1800 always_on_cu_num = 4; 1801 else if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 2, 1)) 1802 always_on_cu_num = 8; 1803 else 1804 always_on_cu_num = 12; 1805 1806 mutex_lock(&adev->grbm_idx_mutex); 1807 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 1808 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 1809 mask = 1; 1810 cu_bitmap = 0; 1811 counter = 0; 1812 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 1813 1814 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 1815 if (cu_info->bitmap[i][j] & mask) { 1816 if (counter == pg_always_on_cu_num) 1817 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap); 1818 if (counter < always_on_cu_num) 1819 cu_bitmap |= mask; 1820 else 1821 break; 1822 counter++; 1823 } 1824 mask <<= 1; 1825 } 1826 1827 WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap); 1828 cu_info->ao_cu_bitmap[i][j] = cu_bitmap; 1829 } 1830 } 1831 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1832 mutex_unlock(&adev->grbm_idx_mutex); 1833 } 1834 1835 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev) 1836 { 1837 uint32_t data; 1838 1839 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1840 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1841 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7); 1842 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1843 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16)); 1844 1845 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1846 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1847 1848 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1849 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500); 1850 1851 mutex_lock(&adev->grbm_idx_mutex); 1852 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1853 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1854 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1855 1856 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1857 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1858 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1859 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1860 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1861 1862 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1863 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1864 data &= 0x0000FFFF; 1865 data |= 0x00C00000; 1866 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1867 1868 /* 1869 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven), 1870 * programmed in gfx_v9_0_init_always_on_cu_mask() 1871 */ 1872 1873 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1874 * but used for RLC_LB_CNTL configuration */ 1875 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1876 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1877 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1878 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1879 mutex_unlock(&adev->grbm_idx_mutex); 1880 1881 gfx_v9_0_init_always_on_cu_mask(adev); 1882 } 1883 1884 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev) 1885 { 1886 uint32_t data; 1887 1888 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1889 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1890 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8); 1891 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1892 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16)); 1893 1894 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1895 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1896 1897 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1898 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800); 1899 1900 mutex_lock(&adev->grbm_idx_mutex); 1901 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1902 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1903 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1904 1905 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1906 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1907 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1908 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1909 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1910 1911 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1912 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1913 data &= 0x0000FFFF; 1914 data |= 0x00C00000; 1915 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1916 1917 /* 1918 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON), 1919 * programmed in gfx_v9_0_init_always_on_cu_mask() 1920 */ 1921 1922 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1923 * but used for RLC_LB_CNTL configuration */ 1924 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1925 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1926 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1927 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1928 mutex_unlock(&adev->grbm_idx_mutex); 1929 1930 gfx_v9_0_init_always_on_cu_mask(adev); 1931 } 1932 1933 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable) 1934 { 1935 WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0); 1936 } 1937 1938 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev) 1939 { 1940 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) 1941 return 5; 1942 else 1943 return 4; 1944 } 1945 1946 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev) 1947 { 1948 const struct cs_section_def *cs_data; 1949 int r; 1950 1951 adev->gfx.rlc.cs_data = gfx9_cs_data; 1952 1953 cs_data = adev->gfx.rlc.cs_data; 1954 1955 if (cs_data) { 1956 /* init clear state block */ 1957 r = amdgpu_gfx_rlc_init_csb(adev); 1958 if (r) 1959 return r; 1960 } 1961 1962 if (adev->flags & AMD_IS_APU) { 1963 /* TODO: double check the cp_table_size for RV */ 1964 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ 1965 r = amdgpu_gfx_rlc_init_cpt(adev); 1966 if (r) 1967 return r; 1968 } 1969 1970 switch (adev->ip_versions[GC_HWIP][0]) { 1971 case IP_VERSION(9, 2, 2): 1972 case IP_VERSION(9, 1, 0): 1973 gfx_v9_0_init_lbpw(adev); 1974 break; 1975 case IP_VERSION(9, 4, 0): 1976 gfx_v9_4_init_lbpw(adev); 1977 break; 1978 default: 1979 break; 1980 } 1981 1982 /* init spm vmid with 0xf */ 1983 if (adev->gfx.rlc.funcs->update_spm_vmid) 1984 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf); 1985 1986 return 0; 1987 } 1988 1989 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev) 1990 { 1991 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 1992 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); 1993 } 1994 1995 static int gfx_v9_0_mec_init(struct amdgpu_device *adev) 1996 { 1997 int r; 1998 u32 *hpd; 1999 const __le32 *fw_data; 2000 unsigned fw_size; 2001 u32 *fw; 2002 size_t mec_hpd_size; 2003 2004 const struct gfx_firmware_header_v1_0 *mec_hdr; 2005 2006 bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 2007 2008 /* take ownership of the relevant compute queues */ 2009 amdgpu_gfx_compute_queue_acquire(adev); 2010 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE; 2011 if (mec_hpd_size) { 2012 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 2013 AMDGPU_GEM_DOMAIN_VRAM, 2014 &adev->gfx.mec.hpd_eop_obj, 2015 &adev->gfx.mec.hpd_eop_gpu_addr, 2016 (void **)&hpd); 2017 if (r) { 2018 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 2019 gfx_v9_0_mec_fini(adev); 2020 return r; 2021 } 2022 2023 memset(hpd, 0, mec_hpd_size); 2024 2025 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 2026 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 2027 } 2028 2029 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 2030 2031 fw_data = (const __le32 *) 2032 (adev->gfx.mec_fw->data + 2033 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 2034 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes); 2035 2036 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes, 2037 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 2038 &adev->gfx.mec.mec_fw_obj, 2039 &adev->gfx.mec.mec_fw_gpu_addr, 2040 (void **)&fw); 2041 if (r) { 2042 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r); 2043 gfx_v9_0_mec_fini(adev); 2044 return r; 2045 } 2046 2047 memcpy(fw, fw_data, fw_size); 2048 2049 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 2050 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 2051 2052 return 0; 2053 } 2054 2055 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address) 2056 { 2057 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 2058 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 2059 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 2060 (address << SQ_IND_INDEX__INDEX__SHIFT) | 2061 (SQ_IND_INDEX__FORCE_READ_MASK)); 2062 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 2063 } 2064 2065 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd, 2066 uint32_t wave, uint32_t thread, 2067 uint32_t regno, uint32_t num, uint32_t *out) 2068 { 2069 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 2070 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 2071 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 2072 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 2073 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) | 2074 (SQ_IND_INDEX__FORCE_READ_MASK) | 2075 (SQ_IND_INDEX__AUTO_INCR_MASK)); 2076 while (num--) 2077 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 2078 } 2079 2080 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 2081 { 2082 /* type 1 wave data */ 2083 dst[(*no_fields)++] = 1; 2084 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS); 2085 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO); 2086 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI); 2087 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO); 2088 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI); 2089 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID); 2090 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0); 2091 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1); 2092 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC); 2093 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC); 2094 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS); 2095 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS); 2096 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); 2097 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); 2098 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE); 2099 } 2100 2101 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd, 2102 uint32_t wave, uint32_t start, 2103 uint32_t size, uint32_t *dst) 2104 { 2105 wave_read_regs( 2106 adev, simd, wave, 0, 2107 start + SQIND_WAVE_SGPRS_OFFSET, size, dst); 2108 } 2109 2110 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd, 2111 uint32_t wave, uint32_t thread, 2112 uint32_t start, uint32_t size, 2113 uint32_t *dst) 2114 { 2115 wave_read_regs( 2116 adev, simd, wave, thread, 2117 start + SQIND_WAVE_VGPRS_OFFSET, size, dst); 2118 } 2119 2120 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev, 2121 u32 me, u32 pipe, u32 q, u32 vm) 2122 { 2123 soc15_grbm_select(adev, me, pipe, q, vm); 2124 } 2125 2126 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = { 2127 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 2128 .select_se_sh = &gfx_v9_0_select_se_sh, 2129 .read_wave_data = &gfx_v9_0_read_wave_data, 2130 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 2131 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 2132 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 2133 }; 2134 2135 static const struct amdgpu_gfx_ras_funcs gfx_v9_0_ras_funcs = { 2136 .ras_late_init = amdgpu_gfx_ras_late_init, 2137 .ras_fini = amdgpu_gfx_ras_fini, 2138 .ras_error_inject = &gfx_v9_0_ras_error_inject, 2139 .query_ras_error_count = &gfx_v9_0_query_ras_error_count, 2140 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count, 2141 }; 2142 2143 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev) 2144 { 2145 u32 gb_addr_config; 2146 int err; 2147 2148 adev->gfx.funcs = &gfx_v9_0_gfx_funcs; 2149 2150 switch (adev->ip_versions[GC_HWIP][0]) { 2151 case IP_VERSION(9, 0, 1): 2152 adev->gfx.config.max_hw_contexts = 8; 2153 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2154 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2155 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2156 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2157 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN; 2158 break; 2159 case IP_VERSION(9, 2, 1): 2160 adev->gfx.config.max_hw_contexts = 8; 2161 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2162 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2163 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2164 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2165 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN; 2166 DRM_INFO("fix gfx.config for vega12\n"); 2167 break; 2168 case IP_VERSION(9, 4, 0): 2169 adev->gfx.ras_funcs = &gfx_v9_0_ras_funcs; 2170 adev->gfx.config.max_hw_contexts = 8; 2171 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2172 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2173 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2174 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2175 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2176 gb_addr_config &= ~0xf3e777ff; 2177 gb_addr_config |= 0x22014042; 2178 /* check vbios table if gpu info is not available */ 2179 err = amdgpu_atomfirmware_get_gfx_info(adev); 2180 if (err) 2181 return err; 2182 break; 2183 case IP_VERSION(9, 2, 2): 2184 case IP_VERSION(9, 1, 0): 2185 adev->gfx.config.max_hw_contexts = 8; 2186 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2187 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2188 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2189 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2190 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 2191 gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN; 2192 else 2193 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN; 2194 break; 2195 case IP_VERSION(9, 4, 1): 2196 adev->gfx.ras_funcs = &gfx_v9_4_ras_funcs; 2197 adev->gfx.config.max_hw_contexts = 8; 2198 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2199 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2200 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2201 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2202 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2203 gb_addr_config &= ~0xf3e777ff; 2204 gb_addr_config |= 0x22014042; 2205 break; 2206 case IP_VERSION(9, 3, 0): 2207 adev->gfx.config.max_hw_contexts = 8; 2208 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2209 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2210 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; 2211 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2212 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2213 gb_addr_config &= ~0xf3e777ff; 2214 gb_addr_config |= 0x22010042; 2215 break; 2216 case IP_VERSION(9, 4, 2): 2217 adev->gfx.ras_funcs = &gfx_v9_4_2_ras_funcs; 2218 adev->gfx.config.max_hw_contexts = 8; 2219 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2220 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2221 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2222 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2223 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2224 gb_addr_config &= ~0xf3e777ff; 2225 gb_addr_config |= 0x22014042; 2226 /* check vbios table if gpu info is not available */ 2227 err = amdgpu_atomfirmware_get_gfx_info(adev); 2228 if (err) 2229 return err; 2230 break; 2231 default: 2232 BUG(); 2233 break; 2234 } 2235 2236 adev->gfx.config.gb_addr_config = gb_addr_config; 2237 2238 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << 2239 REG_GET_FIELD( 2240 adev->gfx.config.gb_addr_config, 2241 GB_ADDR_CONFIG, 2242 NUM_PIPES); 2243 2244 adev->gfx.config.max_tile_pipes = 2245 adev->gfx.config.gb_addr_config_fields.num_pipes; 2246 2247 adev->gfx.config.gb_addr_config_fields.num_banks = 1 << 2248 REG_GET_FIELD( 2249 adev->gfx.config.gb_addr_config, 2250 GB_ADDR_CONFIG, 2251 NUM_BANKS); 2252 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << 2253 REG_GET_FIELD( 2254 adev->gfx.config.gb_addr_config, 2255 GB_ADDR_CONFIG, 2256 MAX_COMPRESSED_FRAGS); 2257 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << 2258 REG_GET_FIELD( 2259 adev->gfx.config.gb_addr_config, 2260 GB_ADDR_CONFIG, 2261 NUM_RB_PER_SE); 2262 adev->gfx.config.gb_addr_config_fields.num_se = 1 << 2263 REG_GET_FIELD( 2264 adev->gfx.config.gb_addr_config, 2265 GB_ADDR_CONFIG, 2266 NUM_SHADER_ENGINES); 2267 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + 2268 REG_GET_FIELD( 2269 adev->gfx.config.gb_addr_config, 2270 GB_ADDR_CONFIG, 2271 PIPE_INTERLEAVE_SIZE)); 2272 2273 return 0; 2274 } 2275 2276 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 2277 int mec, int pipe, int queue) 2278 { 2279 unsigned irq_type; 2280 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; 2281 unsigned int hw_prio; 2282 2283 ring = &adev->gfx.compute_ring[ring_id]; 2284 2285 /* mec0 is me1 */ 2286 ring->me = mec + 1; 2287 ring->pipe = pipe; 2288 ring->queue = queue; 2289 2290 ring->ring_obj = NULL; 2291 ring->use_doorbell = true; 2292 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1; 2293 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 2294 + (ring_id * GFX9_MEC_HPD_SIZE); 2295 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 2296 2297 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 2298 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 2299 + ring->pipe; 2300 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? 2301 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 2302 /* type-2 packets are deprecated on MEC, use type-3 instead */ 2303 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 2304 hw_prio, NULL); 2305 } 2306 2307 static int gfx_v9_0_sw_init(void *handle) 2308 { 2309 int i, j, k, r, ring_id; 2310 struct amdgpu_ring *ring; 2311 struct amdgpu_kiq *kiq; 2312 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2313 2314 switch (adev->ip_versions[GC_HWIP][0]) { 2315 case IP_VERSION(9, 0, 1): 2316 case IP_VERSION(9, 2, 1): 2317 case IP_VERSION(9, 4, 0): 2318 case IP_VERSION(9, 2, 2): 2319 case IP_VERSION(9, 1, 0): 2320 case IP_VERSION(9, 4, 1): 2321 case IP_VERSION(9, 3, 0): 2322 case IP_VERSION(9, 4, 2): 2323 adev->gfx.mec.num_mec = 2; 2324 break; 2325 default: 2326 adev->gfx.mec.num_mec = 1; 2327 break; 2328 } 2329 2330 adev->gfx.mec.num_pipe_per_mec = 4; 2331 adev->gfx.mec.num_queue_per_pipe = 8; 2332 2333 /* EOP Event */ 2334 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq); 2335 if (r) 2336 return r; 2337 2338 /* Privileged reg */ 2339 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT, 2340 &adev->gfx.priv_reg_irq); 2341 if (r) 2342 return r; 2343 2344 /* Privileged inst */ 2345 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT, 2346 &adev->gfx.priv_inst_irq); 2347 if (r) 2348 return r; 2349 2350 /* ECC error */ 2351 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR, 2352 &adev->gfx.cp_ecc_error_irq); 2353 if (r) 2354 return r; 2355 2356 /* FUE error */ 2357 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR, 2358 &adev->gfx.cp_ecc_error_irq); 2359 if (r) 2360 return r; 2361 2362 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 2363 2364 gfx_v9_0_scratch_init(adev); 2365 2366 r = gfx_v9_0_init_microcode(adev); 2367 if (r) { 2368 DRM_ERROR("Failed to load gfx firmware!\n"); 2369 return r; 2370 } 2371 2372 r = adev->gfx.rlc.funcs->init(adev); 2373 if (r) { 2374 DRM_ERROR("Failed to init rlc BOs!\n"); 2375 return r; 2376 } 2377 2378 r = gfx_v9_0_mec_init(adev); 2379 if (r) { 2380 DRM_ERROR("Failed to init MEC BOs!\n"); 2381 return r; 2382 } 2383 2384 /* set up the gfx ring */ 2385 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 2386 ring = &adev->gfx.gfx_ring[i]; 2387 ring->ring_obj = NULL; 2388 if (!i) 2389 sprintf(ring->name, "gfx"); 2390 else 2391 sprintf(ring->name, "gfx_%d", i); 2392 ring->use_doorbell = true; 2393 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 2394 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, 2395 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, 2396 AMDGPU_RING_PRIO_DEFAULT, NULL); 2397 if (r) 2398 return r; 2399 } 2400 2401 /* set up the compute queues - allocate horizontally across pipes */ 2402 ring_id = 0; 2403 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 2404 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 2405 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 2406 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j)) 2407 continue; 2408 2409 r = gfx_v9_0_compute_ring_init(adev, 2410 ring_id, 2411 i, k, j); 2412 if (r) 2413 return r; 2414 2415 ring_id++; 2416 } 2417 } 2418 } 2419 2420 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE); 2421 if (r) { 2422 DRM_ERROR("Failed to init KIQ BOs!\n"); 2423 return r; 2424 } 2425 2426 kiq = &adev->gfx.kiq; 2427 r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq); 2428 if (r) 2429 return r; 2430 2431 /* create MQD for all compute queues as wel as KIQ for SRIOV case */ 2432 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation)); 2433 if (r) 2434 return r; 2435 2436 adev->gfx.ce_ram_size = 0x8000; 2437 2438 r = gfx_v9_0_gpu_early_init(adev); 2439 if (r) 2440 return r; 2441 2442 return 0; 2443 } 2444 2445 2446 static int gfx_v9_0_sw_fini(void *handle) 2447 { 2448 int i; 2449 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2450 2451 if (adev->gfx.ras_funcs && 2452 adev->gfx.ras_funcs->ras_fini) 2453 adev->gfx.ras_funcs->ras_fini(adev); 2454 2455 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 2456 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 2457 for (i = 0; i < adev->gfx.num_compute_rings; i++) 2458 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 2459 2460 amdgpu_gfx_mqd_sw_fini(adev); 2461 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring); 2462 amdgpu_gfx_kiq_fini(adev); 2463 2464 gfx_v9_0_mec_fini(adev); 2465 amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj); 2466 if (adev->flags & AMD_IS_APU) { 2467 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 2468 &adev->gfx.rlc.cp_table_gpu_addr, 2469 (void **)&adev->gfx.rlc.cp_table_ptr); 2470 } 2471 gfx_v9_0_free_microcode(adev); 2472 2473 return 0; 2474 } 2475 2476 2477 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev) 2478 { 2479 /* TODO */ 2480 } 2481 2482 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, 2483 u32 instance) 2484 { 2485 u32 data; 2486 2487 if (instance == 0xffffffff) 2488 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); 2489 else 2490 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance); 2491 2492 if (se_num == 0xffffffff) 2493 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); 2494 else 2495 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 2496 2497 if (sh_num == 0xffffffff) 2498 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1); 2499 else 2500 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num); 2501 2502 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data); 2503 } 2504 2505 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev) 2506 { 2507 u32 data, mask; 2508 2509 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE); 2510 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE); 2511 2512 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK; 2513 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT; 2514 2515 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / 2516 adev->gfx.config.max_sh_per_se); 2517 2518 return (~data) & mask; 2519 } 2520 2521 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev) 2522 { 2523 int i, j; 2524 u32 data; 2525 u32 active_rbs = 0; 2526 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / 2527 adev->gfx.config.max_sh_per_se; 2528 2529 mutex_lock(&adev->grbm_idx_mutex); 2530 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2531 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2532 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2533 data = gfx_v9_0_get_rb_active_bitmap(adev); 2534 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * 2535 rb_bitmap_width_per_sh); 2536 } 2537 } 2538 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2539 mutex_unlock(&adev->grbm_idx_mutex); 2540 2541 adev->gfx.config.backend_enable_mask = active_rbs; 2542 adev->gfx.config.num_rbs = hweight32(active_rbs); 2543 } 2544 2545 #define DEFAULT_SH_MEM_BASES (0x6000) 2546 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev) 2547 { 2548 int i; 2549 uint32_t sh_mem_config; 2550 uint32_t sh_mem_bases; 2551 2552 /* 2553 * Configure apertures: 2554 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 2555 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 2556 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 2557 */ 2558 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16); 2559 2560 sh_mem_config = SH_MEM_ADDRESS_MODE_64 | 2561 SH_MEM_ALIGNMENT_MODE_UNALIGNED << 2562 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT; 2563 2564 mutex_lock(&adev->srbm_mutex); 2565 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2566 soc15_grbm_select(adev, 0, 0, 0, i); 2567 /* CP and shaders */ 2568 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config); 2569 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases); 2570 } 2571 soc15_grbm_select(adev, 0, 0, 0, 0); 2572 mutex_unlock(&adev->srbm_mutex); 2573 2574 /* Initialize all compute VMIDs to have no GDS, GWS, or OA 2575 acccess. These should be enabled by FW for target VMIDs. */ 2576 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2577 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0); 2578 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0); 2579 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0); 2580 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0); 2581 } 2582 } 2583 2584 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev) 2585 { 2586 int vmid; 2587 2588 /* 2589 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 2590 * access. Compute VMIDs should be enabled by FW for target VMIDs, 2591 * the driver can enable them for graphics. VMID0 should maintain 2592 * access so that HWS firmware can save/restore entries. 2593 */ 2594 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { 2595 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0); 2596 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0); 2597 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0); 2598 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0); 2599 } 2600 } 2601 2602 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev) 2603 { 2604 uint32_t tmp; 2605 2606 switch (adev->ip_versions[GC_HWIP][0]) { 2607 case IP_VERSION(9, 4, 1): 2608 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG); 2609 tmp = REG_SET_FIELD(tmp, SQ_CONFIG, 2610 DISABLE_BARRIER_WAITCNT, 1); 2611 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp); 2612 break; 2613 default: 2614 break; 2615 } 2616 } 2617 2618 static void gfx_v9_0_constants_init(struct amdgpu_device *adev) 2619 { 2620 u32 tmp; 2621 int i; 2622 2623 WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); 2624 2625 gfx_v9_0_tiling_mode_table_init(adev); 2626 2627 gfx_v9_0_setup_rb(adev); 2628 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info); 2629 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2); 2630 2631 /* XXX SH_MEM regs */ 2632 /* where to put LDS, scratch, GPUVM in FSA64 space */ 2633 mutex_lock(&adev->srbm_mutex); 2634 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) { 2635 soc15_grbm_select(adev, 0, 0, 0, i); 2636 /* CP and shaders */ 2637 if (i == 0) { 2638 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2639 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2640 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2641 !!adev->gmc.noretry); 2642 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2643 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0); 2644 } else { 2645 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2646 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2647 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2648 !!adev->gmc.noretry); 2649 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2650 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 2651 (adev->gmc.private_aperture_start >> 48)); 2652 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE, 2653 (adev->gmc.shared_aperture_start >> 48)); 2654 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp); 2655 } 2656 } 2657 soc15_grbm_select(adev, 0, 0, 0, 0); 2658 2659 mutex_unlock(&adev->srbm_mutex); 2660 2661 gfx_v9_0_init_compute_vmid(adev); 2662 gfx_v9_0_init_gds_vmid(adev); 2663 gfx_v9_0_init_sq_config(adev); 2664 } 2665 2666 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev) 2667 { 2668 u32 i, j, k; 2669 u32 mask; 2670 2671 mutex_lock(&adev->grbm_idx_mutex); 2672 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2673 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2674 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2675 for (k = 0; k < adev->usec_timeout; k++) { 2676 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0) 2677 break; 2678 udelay(1); 2679 } 2680 if (k == adev->usec_timeout) { 2681 gfx_v9_0_select_se_sh(adev, 0xffffffff, 2682 0xffffffff, 0xffffffff); 2683 mutex_unlock(&adev->grbm_idx_mutex); 2684 DRM_INFO("Timeout wait for RLC serdes %u,%u\n", 2685 i, j); 2686 return; 2687 } 2688 } 2689 } 2690 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2691 mutex_unlock(&adev->grbm_idx_mutex); 2692 2693 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | 2694 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK | 2695 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK | 2696 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK; 2697 for (k = 0; k < adev->usec_timeout; k++) { 2698 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 2699 break; 2700 udelay(1); 2701 } 2702 } 2703 2704 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 2705 bool enable) 2706 { 2707 u32 tmp; 2708 2709 /* These interrupts should be enabled to drive DS clock */ 2710 2711 tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0); 2712 2713 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0); 2714 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0); 2715 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0); 2716 if(adev->gfx.num_gfx_rings) 2717 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0); 2718 2719 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp); 2720 } 2721 2722 static void gfx_v9_0_init_csb(struct amdgpu_device *adev) 2723 { 2724 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 2725 /* csib */ 2726 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI), 2727 adev->gfx.rlc.clear_state_gpu_addr >> 32); 2728 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO), 2729 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 2730 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH), 2731 adev->gfx.rlc.clear_state_size); 2732 } 2733 2734 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format, 2735 int indirect_offset, 2736 int list_size, 2737 int *unique_indirect_regs, 2738 int unique_indirect_reg_count, 2739 int *indirect_start_offsets, 2740 int *indirect_start_offsets_count, 2741 int max_start_offsets_count) 2742 { 2743 int idx; 2744 2745 for (; indirect_offset < list_size; indirect_offset++) { 2746 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count); 2747 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset; 2748 *indirect_start_offsets_count = *indirect_start_offsets_count + 1; 2749 2750 while (register_list_format[indirect_offset] != 0xFFFFFFFF) { 2751 indirect_offset += 2; 2752 2753 /* look for the matching indice */ 2754 for (idx = 0; idx < unique_indirect_reg_count; idx++) { 2755 if (unique_indirect_regs[idx] == 2756 register_list_format[indirect_offset] || 2757 !unique_indirect_regs[idx]) 2758 break; 2759 } 2760 2761 BUG_ON(idx >= unique_indirect_reg_count); 2762 2763 if (!unique_indirect_regs[idx]) 2764 unique_indirect_regs[idx] = register_list_format[indirect_offset]; 2765 2766 indirect_offset++; 2767 } 2768 } 2769 } 2770 2771 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev) 2772 { 2773 int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2774 int unique_indirect_reg_count = 0; 2775 2776 int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2777 int indirect_start_offsets_count = 0; 2778 2779 int list_size = 0; 2780 int i = 0, j = 0; 2781 u32 tmp = 0; 2782 2783 u32 *register_list_format = 2784 kmemdup(adev->gfx.rlc.register_list_format, 2785 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); 2786 if (!register_list_format) 2787 return -ENOMEM; 2788 2789 /* setup unique_indirect_regs array and indirect_start_offsets array */ 2790 unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs); 2791 gfx_v9_1_parse_ind_reg_list(register_list_format, 2792 adev->gfx.rlc.reg_list_format_direct_reg_list_length, 2793 adev->gfx.rlc.reg_list_format_size_bytes >> 2, 2794 unique_indirect_regs, 2795 unique_indirect_reg_count, 2796 indirect_start_offsets, 2797 &indirect_start_offsets_count, 2798 ARRAY_SIZE(indirect_start_offsets)); 2799 2800 /* enable auto inc in case it is disabled */ 2801 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL)); 2802 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK; 2803 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp); 2804 2805 /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */ 2806 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR), 2807 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET); 2808 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) 2809 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA), 2810 adev->gfx.rlc.register_restore[i]); 2811 2812 /* load indirect register */ 2813 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2814 adev->gfx.rlc.reg_list_format_start); 2815 2816 /* direct register portion */ 2817 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++) 2818 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2819 register_list_format[i]); 2820 2821 /* indirect register portion */ 2822 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) { 2823 if (register_list_format[i] == 0xFFFFFFFF) { 2824 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2825 continue; 2826 } 2827 2828 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2829 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2830 2831 for (j = 0; j < unique_indirect_reg_count; j++) { 2832 if (register_list_format[i] == unique_indirect_regs[j]) { 2833 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j); 2834 break; 2835 } 2836 } 2837 2838 BUG_ON(j >= unique_indirect_reg_count); 2839 2840 i++; 2841 } 2842 2843 /* set save/restore list size */ 2844 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; 2845 list_size = list_size >> 1; 2846 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2847 adev->gfx.rlc.reg_restore_list_size); 2848 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size); 2849 2850 /* write the starting offsets to RLC scratch ram */ 2851 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2852 adev->gfx.rlc.starting_offsets_start); 2853 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++) 2854 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2855 indirect_start_offsets[i]); 2856 2857 /* load unique indirect regs*/ 2858 for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) { 2859 if (unique_indirect_regs[i] != 0) { 2860 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0) 2861 + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i], 2862 unique_indirect_regs[i] & 0x3FFFF); 2863 2864 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0) 2865 + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i], 2866 unique_indirect_regs[i] >> 20); 2867 } 2868 } 2869 2870 kfree(register_list_format); 2871 return 0; 2872 } 2873 2874 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev) 2875 { 2876 WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1); 2877 } 2878 2879 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev, 2880 bool enable) 2881 { 2882 uint32_t data = 0; 2883 uint32_t default_data = 0; 2884 2885 default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS)); 2886 if (enable) { 2887 /* enable GFXIP control over CGPG */ 2888 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2889 if(default_data != data) 2890 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2891 2892 /* update status */ 2893 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK; 2894 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT); 2895 if(default_data != data) 2896 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2897 } else { 2898 /* restore GFXIP control over GCPG */ 2899 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2900 if(default_data != data) 2901 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2902 } 2903 } 2904 2905 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev) 2906 { 2907 uint32_t data = 0; 2908 2909 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2910 AMD_PG_SUPPORT_GFX_SMG | 2911 AMD_PG_SUPPORT_GFX_DMG)) { 2912 /* init IDLE_POLL_COUNT = 60 */ 2913 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL)); 2914 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK; 2915 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 2916 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data); 2917 2918 /* init RLC PG Delay */ 2919 data = 0; 2920 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT); 2921 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT); 2922 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT); 2923 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT); 2924 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data); 2925 2926 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2)); 2927 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK; 2928 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT); 2929 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data); 2930 2931 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3)); 2932 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK; 2933 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT); 2934 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data); 2935 2936 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL)); 2937 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK; 2938 2939 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */ 2940 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT); 2941 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data); 2942 if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 3, 0)) 2943 pwr_10_0_gfxip_control_over_cgpg(adev, true); 2944 } 2945 } 2946 2947 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev, 2948 bool enable) 2949 { 2950 uint32_t data = 0; 2951 uint32_t default_data = 0; 2952 2953 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2954 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2955 SMU_CLK_SLOWDOWN_ON_PU_ENABLE, 2956 enable ? 1 : 0); 2957 if (default_data != data) 2958 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2959 } 2960 2961 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev, 2962 bool enable) 2963 { 2964 uint32_t data = 0; 2965 uint32_t default_data = 0; 2966 2967 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2968 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2969 SMU_CLK_SLOWDOWN_ON_PD_ENABLE, 2970 enable ? 1 : 0); 2971 if(default_data != data) 2972 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2973 } 2974 2975 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev, 2976 bool enable) 2977 { 2978 uint32_t data = 0; 2979 uint32_t default_data = 0; 2980 2981 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2982 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2983 CP_PG_DISABLE, 2984 enable ? 0 : 1); 2985 if(default_data != data) 2986 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2987 } 2988 2989 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev, 2990 bool enable) 2991 { 2992 uint32_t data, default_data; 2993 2994 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2995 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2996 GFX_POWER_GATING_ENABLE, 2997 enable ? 1 : 0); 2998 if(default_data != data) 2999 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3000 } 3001 3002 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev, 3003 bool enable) 3004 { 3005 uint32_t data, default_data; 3006 3007 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 3008 data = REG_SET_FIELD(data, RLC_PG_CNTL, 3009 GFX_PIPELINE_PG_ENABLE, 3010 enable ? 1 : 0); 3011 if(default_data != data) 3012 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3013 3014 if (!enable) 3015 /* read any GFX register to wake up GFX */ 3016 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL)); 3017 } 3018 3019 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, 3020 bool enable) 3021 { 3022 uint32_t data, default_data; 3023 3024 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 3025 data = REG_SET_FIELD(data, RLC_PG_CNTL, 3026 STATIC_PER_CU_PG_ENABLE, 3027 enable ? 1 : 0); 3028 if(default_data != data) 3029 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3030 } 3031 3032 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev, 3033 bool enable) 3034 { 3035 uint32_t data, default_data; 3036 3037 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 3038 data = REG_SET_FIELD(data, RLC_PG_CNTL, 3039 DYN_PER_CU_PG_ENABLE, 3040 enable ? 1 : 0); 3041 if(default_data != data) 3042 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3043 } 3044 3045 static void gfx_v9_0_init_pg(struct amdgpu_device *adev) 3046 { 3047 gfx_v9_0_init_csb(adev); 3048 3049 /* 3050 * Rlc save restore list is workable since v2_1. 3051 * And it's needed by gfxoff feature. 3052 */ 3053 if (adev->gfx.rlc.is_rlc_v2_1) { 3054 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 2, 1) || 3055 (adev->apu_flags & AMD_APU_IS_RAVEN2)) 3056 gfx_v9_1_init_rlc_save_restore_list(adev); 3057 gfx_v9_0_enable_save_restore_machine(adev); 3058 } 3059 3060 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 3061 AMD_PG_SUPPORT_GFX_SMG | 3062 AMD_PG_SUPPORT_GFX_DMG | 3063 AMD_PG_SUPPORT_CP | 3064 AMD_PG_SUPPORT_GDS | 3065 AMD_PG_SUPPORT_RLC_SMU_HS)) { 3066 WREG32(mmRLC_JUMP_TABLE_RESTORE, 3067 adev->gfx.rlc.cp_table_gpu_addr >> 8); 3068 gfx_v9_0_init_gfx_power_gating(adev); 3069 } 3070 } 3071 3072 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev) 3073 { 3074 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0); 3075 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3076 gfx_v9_0_wait_for_rlc_serdes(adev); 3077 } 3078 3079 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev) 3080 { 3081 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 3082 udelay(50); 3083 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 3084 udelay(50); 3085 } 3086 3087 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev) 3088 { 3089 #ifdef AMDGPU_RLC_DEBUG_RETRY 3090 u32 rlc_ucode_ver; 3091 #endif 3092 3093 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1); 3094 udelay(50); 3095 3096 /* carrizo do enable cp interrupt after cp inited */ 3097 if (!(adev->flags & AMD_IS_APU)) { 3098 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3099 udelay(50); 3100 } 3101 3102 #ifdef AMDGPU_RLC_DEBUG_RETRY 3103 /* RLC_GPM_GENERAL_6 : RLC Ucode version */ 3104 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6); 3105 if(rlc_ucode_ver == 0x108) { 3106 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n", 3107 rlc_ucode_ver, adev->gfx.rlc_fw_version); 3108 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles, 3109 * default is 0x9C4 to create a 100us interval */ 3110 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4); 3111 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr 3112 * to disable the page fault retry interrupts, default is 3113 * 0x100 (256) */ 3114 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100); 3115 } 3116 #endif 3117 } 3118 3119 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev) 3120 { 3121 const struct rlc_firmware_header_v2_0 *hdr; 3122 const __le32 *fw_data; 3123 unsigned i, fw_size; 3124 3125 if (!adev->gfx.rlc_fw) 3126 return -EINVAL; 3127 3128 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 3129 amdgpu_ucode_print_rlc_hdr(&hdr->header); 3130 3131 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 3132 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 3133 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 3134 3135 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, 3136 RLCG_UCODE_LOADING_START_ADDRESS); 3137 for (i = 0; i < fw_size; i++) 3138 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); 3139 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); 3140 3141 return 0; 3142 } 3143 3144 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev) 3145 { 3146 int r; 3147 3148 if (amdgpu_sriov_vf(adev)) { 3149 gfx_v9_0_init_csb(adev); 3150 return 0; 3151 } 3152 3153 adev->gfx.rlc.funcs->stop(adev); 3154 3155 /* disable CG */ 3156 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0); 3157 3158 gfx_v9_0_init_pg(adev); 3159 3160 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3161 /* legacy rlc firmware loading */ 3162 r = gfx_v9_0_rlc_load_microcode(adev); 3163 if (r) 3164 return r; 3165 } 3166 3167 switch (adev->ip_versions[GC_HWIP][0]) { 3168 case IP_VERSION(9, 2, 2): 3169 case IP_VERSION(9, 1, 0): 3170 if (amdgpu_lbpw == 0) 3171 gfx_v9_0_enable_lbpw(adev, false); 3172 else 3173 gfx_v9_0_enable_lbpw(adev, true); 3174 break; 3175 case IP_VERSION(9, 4, 0): 3176 if (amdgpu_lbpw > 0) 3177 gfx_v9_0_enable_lbpw(adev, true); 3178 else 3179 gfx_v9_0_enable_lbpw(adev, false); 3180 break; 3181 default: 3182 break; 3183 } 3184 3185 adev->gfx.rlc.funcs->start(adev); 3186 3187 return 0; 3188 } 3189 3190 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 3191 { 3192 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL); 3193 3194 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); 3195 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); 3196 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1); 3197 WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp); 3198 udelay(50); 3199 } 3200 3201 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev) 3202 { 3203 const struct gfx_firmware_header_v1_0 *pfp_hdr; 3204 const struct gfx_firmware_header_v1_0 *ce_hdr; 3205 const struct gfx_firmware_header_v1_0 *me_hdr; 3206 const __le32 *fw_data; 3207 unsigned i, fw_size; 3208 3209 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw) 3210 return -EINVAL; 3211 3212 pfp_hdr = (const struct gfx_firmware_header_v1_0 *) 3213 adev->gfx.pfp_fw->data; 3214 ce_hdr = (const struct gfx_firmware_header_v1_0 *) 3215 adev->gfx.ce_fw->data; 3216 me_hdr = (const struct gfx_firmware_header_v1_0 *) 3217 adev->gfx.me_fw->data; 3218 3219 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3220 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header); 3221 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3222 3223 gfx_v9_0_cp_gfx_enable(adev, false); 3224 3225 /* PFP */ 3226 fw_data = (const __le32 *) 3227 (adev->gfx.pfp_fw->data + 3228 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 3229 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 3230 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0); 3231 for (i = 0; i < fw_size; i++) 3232 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 3233 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); 3234 3235 /* CE */ 3236 fw_data = (const __le32 *) 3237 (adev->gfx.ce_fw->data + 3238 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 3239 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 3240 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0); 3241 for (i = 0; i < fw_size; i++) 3242 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 3243 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version); 3244 3245 /* ME */ 3246 fw_data = (const __le32 *) 3247 (adev->gfx.me_fw->data + 3248 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 3249 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 3250 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0); 3251 for (i = 0; i < fw_size; i++) 3252 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 3253 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version); 3254 3255 return 0; 3256 } 3257 3258 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev) 3259 { 3260 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; 3261 const struct cs_section_def *sect = NULL; 3262 const struct cs_extent_def *ext = NULL; 3263 int r, i, tmp; 3264 3265 /* init the CP */ 3266 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); 3267 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1); 3268 3269 gfx_v9_0_cp_gfx_enable(adev, true); 3270 3271 r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3); 3272 if (r) { 3273 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r); 3274 return r; 3275 } 3276 3277 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3278 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3279 3280 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 3281 amdgpu_ring_write(ring, 0x80000000); 3282 amdgpu_ring_write(ring, 0x80000000); 3283 3284 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 3285 for (ext = sect->section; ext->extent != NULL; ++ext) { 3286 if (sect->id == SECT_CONTEXT) { 3287 amdgpu_ring_write(ring, 3288 PACKET3(PACKET3_SET_CONTEXT_REG, 3289 ext->reg_count)); 3290 amdgpu_ring_write(ring, 3291 ext->reg_index - PACKET3_SET_CONTEXT_REG_START); 3292 for (i = 0; i < ext->reg_count; i++) 3293 amdgpu_ring_write(ring, ext->extent[i]); 3294 } 3295 } 3296 } 3297 3298 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3299 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3300 3301 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3302 amdgpu_ring_write(ring, 0); 3303 3304 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 3305 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 3306 amdgpu_ring_write(ring, 0x8000); 3307 amdgpu_ring_write(ring, 0x8000); 3308 3309 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1)); 3310 tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE | 3311 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START)); 3312 amdgpu_ring_write(ring, tmp); 3313 amdgpu_ring_write(ring, 0); 3314 3315 amdgpu_ring_commit(ring); 3316 3317 return 0; 3318 } 3319 3320 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev) 3321 { 3322 struct amdgpu_ring *ring; 3323 u32 tmp; 3324 u32 rb_bufsz; 3325 u64 rb_addr, rptr_addr, wptr_gpu_addr; 3326 3327 /* Set the write pointer delay */ 3328 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0); 3329 3330 /* set the RB to use vmid 0 */ 3331 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0); 3332 3333 /* Set ring buffer size */ 3334 ring = &adev->gfx.gfx_ring[0]; 3335 rb_bufsz = order_base_2(ring->ring_size / 8); 3336 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 3337 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 3338 #ifdef __BIG_ENDIAN 3339 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1); 3340 #endif 3341 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3342 3343 /* Initialize the ring buffer's write pointers */ 3344 ring->wptr = 0; 3345 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 3346 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 3347 3348 /* set the wb address wether it's enabled or not */ 3349 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3350 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 3351 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3352 3353 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3354 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr)); 3355 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr)); 3356 3357 mdelay(1); 3358 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3359 3360 rb_addr = ring->gpu_addr >> 8; 3361 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr); 3362 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 3363 3364 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL); 3365 if (ring->use_doorbell) { 3366 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3367 DOORBELL_OFFSET, ring->doorbell_index); 3368 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3369 DOORBELL_EN, 1); 3370 } else { 3371 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0); 3372 } 3373 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp); 3374 3375 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 3376 DOORBELL_RANGE_LOWER, ring->doorbell_index); 3377 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp); 3378 3379 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER, 3380 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 3381 3382 3383 /* start the ring */ 3384 gfx_v9_0_cp_gfx_start(adev); 3385 ring->sched.ready = true; 3386 3387 return 0; 3388 } 3389 3390 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 3391 { 3392 if (enable) { 3393 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0); 3394 } else { 3395 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 3396 (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK)); 3397 adev->gfx.kiq.ring.sched.ready = false; 3398 } 3399 udelay(50); 3400 } 3401 3402 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev) 3403 { 3404 const struct gfx_firmware_header_v1_0 *mec_hdr; 3405 const __le32 *fw_data; 3406 unsigned i; 3407 u32 tmp; 3408 3409 if (!adev->gfx.mec_fw) 3410 return -EINVAL; 3411 3412 gfx_v9_0_cp_compute_enable(adev, false); 3413 3414 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 3415 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3416 3417 fw_data = (const __le32 *) 3418 (adev->gfx.mec_fw->data + 3419 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 3420 tmp = 0; 3421 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 3422 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 3423 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp); 3424 3425 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO, 3426 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000); 3427 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI, 3428 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 3429 3430 /* MEC1 */ 3431 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3432 mec_hdr->jt_offset); 3433 for (i = 0; i < mec_hdr->jt_size; i++) 3434 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA, 3435 le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); 3436 3437 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3438 adev->gfx.mec_fw_version); 3439 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */ 3440 3441 return 0; 3442 } 3443 3444 /* KIQ functions */ 3445 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring) 3446 { 3447 uint32_t tmp; 3448 struct amdgpu_device *adev = ring->adev; 3449 3450 /* tell RLC which is KIQ queue */ 3451 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS); 3452 tmp &= 0xffffff00; 3453 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 3454 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3455 tmp |= 0x80; 3456 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3457 } 3458 3459 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd) 3460 { 3461 struct amdgpu_device *adev = ring->adev; 3462 3463 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 3464 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) { 3465 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; 3466 mqd->cp_hqd_queue_priority = 3467 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; 3468 } 3469 } 3470 } 3471 3472 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring) 3473 { 3474 struct amdgpu_device *adev = ring->adev; 3475 struct v9_mqd *mqd = ring->mqd_ptr; 3476 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 3477 uint32_t tmp; 3478 3479 mqd->header = 0xC0310800; 3480 mqd->compute_pipelinestat_enable = 0x00000001; 3481 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 3482 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 3483 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 3484 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 3485 mqd->compute_static_thread_mgmt_se4 = 0xffffffff; 3486 mqd->compute_static_thread_mgmt_se5 = 0xffffffff; 3487 mqd->compute_static_thread_mgmt_se6 = 0xffffffff; 3488 mqd->compute_static_thread_mgmt_se7 = 0xffffffff; 3489 mqd->compute_misc_reserved = 0x00000003; 3490 3491 mqd->dynamic_cu_mask_addr_lo = 3492 lower_32_bits(ring->mqd_gpu_addr 3493 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3494 mqd->dynamic_cu_mask_addr_hi = 3495 upper_32_bits(ring->mqd_gpu_addr 3496 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3497 3498 eop_base_addr = ring->eop_gpu_addr >> 8; 3499 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 3500 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 3501 3502 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3503 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL); 3504 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 3505 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1)); 3506 3507 mqd->cp_hqd_eop_control = tmp; 3508 3509 /* enable doorbell? */ 3510 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3511 3512 if (ring->use_doorbell) { 3513 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3514 DOORBELL_OFFSET, ring->doorbell_index); 3515 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3516 DOORBELL_EN, 1); 3517 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3518 DOORBELL_SOURCE, 0); 3519 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3520 DOORBELL_HIT, 0); 3521 } else { 3522 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3523 DOORBELL_EN, 0); 3524 } 3525 3526 mqd->cp_hqd_pq_doorbell_control = tmp; 3527 3528 /* disable the queue if it's active */ 3529 ring->wptr = 0; 3530 mqd->cp_hqd_dequeue_request = 0; 3531 mqd->cp_hqd_pq_rptr = 0; 3532 mqd->cp_hqd_pq_wptr_lo = 0; 3533 mqd->cp_hqd_pq_wptr_hi = 0; 3534 3535 /* set the pointer to the MQD */ 3536 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; 3537 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); 3538 3539 /* set MQD vmid to 0 */ 3540 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL); 3541 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 3542 mqd->cp_mqd_control = tmp; 3543 3544 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3545 hqd_gpu_addr = ring->gpu_addr >> 8; 3546 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 3547 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 3548 3549 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3550 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL); 3551 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 3552 (order_base_2(ring->ring_size / 4) - 1)); 3553 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 3554 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8)); 3555 #ifdef __BIG_ENDIAN 3556 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1); 3557 #endif 3558 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0); 3559 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0); 3560 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 3561 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 3562 mqd->cp_hqd_pq_control = tmp; 3563 3564 /* set the wb address whether it's enabled or not */ 3565 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3566 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 3567 mqd->cp_hqd_pq_rptr_report_addr_hi = 3568 upper_32_bits(wb_gpu_addr) & 0xffff; 3569 3570 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3571 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3572 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 3573 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 3574 3575 tmp = 0; 3576 /* enable the doorbell if requested */ 3577 if (ring->use_doorbell) { 3578 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3579 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3580 DOORBELL_OFFSET, ring->doorbell_index); 3581 3582 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3583 DOORBELL_EN, 1); 3584 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3585 DOORBELL_SOURCE, 0); 3586 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3587 DOORBELL_HIT, 0); 3588 } 3589 3590 mqd->cp_hqd_pq_doorbell_control = tmp; 3591 3592 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3593 ring->wptr = 0; 3594 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR); 3595 3596 /* set the vmid for the queue */ 3597 mqd->cp_hqd_vmid = 0; 3598 3599 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE); 3600 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53); 3601 mqd->cp_hqd_persistent_state = tmp; 3602 3603 /* set MIN_IB_AVAIL_SIZE */ 3604 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL); 3605 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 3606 mqd->cp_hqd_ib_control = tmp; 3607 3608 /* set static priority for a queue/ring */ 3609 gfx_v9_0_mqd_set_priority(ring, mqd); 3610 mqd->cp_hqd_quantum = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM); 3611 3612 /* map_queues packet doesn't need activate the queue, 3613 * so only kiq need set this field. 3614 */ 3615 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 3616 mqd->cp_hqd_active = 1; 3617 3618 return 0; 3619 } 3620 3621 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring) 3622 { 3623 struct amdgpu_device *adev = ring->adev; 3624 struct v9_mqd *mqd = ring->mqd_ptr; 3625 int j; 3626 3627 /* disable wptr polling */ 3628 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3629 3630 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR, 3631 mqd->cp_hqd_eop_base_addr_lo); 3632 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI, 3633 mqd->cp_hqd_eop_base_addr_hi); 3634 3635 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3636 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL, 3637 mqd->cp_hqd_eop_control); 3638 3639 /* enable doorbell? */ 3640 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3641 mqd->cp_hqd_pq_doorbell_control); 3642 3643 /* disable the queue if it's active */ 3644 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3645 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3646 for (j = 0; j < adev->usec_timeout; j++) { 3647 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3648 break; 3649 udelay(1); 3650 } 3651 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3652 mqd->cp_hqd_dequeue_request); 3653 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 3654 mqd->cp_hqd_pq_rptr); 3655 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3656 mqd->cp_hqd_pq_wptr_lo); 3657 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3658 mqd->cp_hqd_pq_wptr_hi); 3659 } 3660 3661 /* set the pointer to the MQD */ 3662 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR, 3663 mqd->cp_mqd_base_addr_lo); 3664 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI, 3665 mqd->cp_mqd_base_addr_hi); 3666 3667 /* set MQD vmid to 0 */ 3668 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL, 3669 mqd->cp_mqd_control); 3670 3671 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3672 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE, 3673 mqd->cp_hqd_pq_base_lo); 3674 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI, 3675 mqd->cp_hqd_pq_base_hi); 3676 3677 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3678 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL, 3679 mqd->cp_hqd_pq_control); 3680 3681 /* set the wb address whether it's enabled or not */ 3682 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR, 3683 mqd->cp_hqd_pq_rptr_report_addr_lo); 3684 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 3685 mqd->cp_hqd_pq_rptr_report_addr_hi); 3686 3687 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3688 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR, 3689 mqd->cp_hqd_pq_wptr_poll_addr_lo); 3690 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI, 3691 mqd->cp_hqd_pq_wptr_poll_addr_hi); 3692 3693 /* enable the doorbell if requested */ 3694 if (ring->use_doorbell) { 3695 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER, 3696 (adev->doorbell_index.kiq * 2) << 2); 3697 /* If GC has entered CGPG, ringing doorbell > first page 3698 * doesn't wakeup GC. Enlarge CP_MEC_DOORBELL_RANGE_UPPER to 3699 * workaround this issue. And this change has to align with firmware 3700 * update. 3701 */ 3702 if (check_if_enlarge_doorbell_range(adev)) 3703 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3704 (adev->doorbell.size - 4)); 3705 else 3706 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3707 (adev->doorbell_index.userqueue_end * 2) << 2); 3708 } 3709 3710 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3711 mqd->cp_hqd_pq_doorbell_control); 3712 3713 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3714 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3715 mqd->cp_hqd_pq_wptr_lo); 3716 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3717 mqd->cp_hqd_pq_wptr_hi); 3718 3719 /* set the vmid for the queue */ 3720 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid); 3721 3722 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 3723 mqd->cp_hqd_persistent_state); 3724 3725 /* activate the queue */ 3726 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 3727 mqd->cp_hqd_active); 3728 3729 if (ring->use_doorbell) 3730 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1); 3731 3732 return 0; 3733 } 3734 3735 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring) 3736 { 3737 struct amdgpu_device *adev = ring->adev; 3738 int j; 3739 3740 /* disable the queue if it's active */ 3741 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3742 3743 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3744 3745 for (j = 0; j < adev->usec_timeout; j++) { 3746 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3747 break; 3748 udelay(1); 3749 } 3750 3751 if (j == AMDGPU_MAX_USEC_TIMEOUT) { 3752 DRM_DEBUG("KIQ dequeue request failed.\n"); 3753 3754 /* Manual disable if dequeue request times out */ 3755 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0); 3756 } 3757 3758 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3759 0); 3760 } 3761 3762 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0); 3763 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0); 3764 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0); 3765 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000); 3766 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0); 3767 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0); 3768 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0); 3769 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0); 3770 3771 return 0; 3772 } 3773 3774 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring) 3775 { 3776 struct amdgpu_device *adev = ring->adev; 3777 struct v9_mqd *mqd = ring->mqd_ptr; 3778 int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS; 3779 struct v9_mqd *tmp_mqd; 3780 3781 gfx_v9_0_kiq_setting(ring); 3782 3783 /* GPU could be in bad state during probe, driver trigger the reset 3784 * after load the SMU, in this case , the mqd is not be initialized. 3785 * driver need to re-init the mqd. 3786 * check mqd->cp_hqd_pq_control since this value should not be 0 3787 */ 3788 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx]; 3789 if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){ 3790 /* for GPU_RESET case , reset MQD to a clean status */ 3791 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3792 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3793 3794 /* reset ring buffer */ 3795 ring->wptr = 0; 3796 amdgpu_ring_clear_ring(ring); 3797 3798 mutex_lock(&adev->srbm_mutex); 3799 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3800 gfx_v9_0_kiq_init_register(ring); 3801 soc15_grbm_select(adev, 0, 0, 0, 0); 3802 mutex_unlock(&adev->srbm_mutex); 3803 } else { 3804 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3805 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3806 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3807 mutex_lock(&adev->srbm_mutex); 3808 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3809 gfx_v9_0_mqd_init(ring); 3810 gfx_v9_0_kiq_init_register(ring); 3811 soc15_grbm_select(adev, 0, 0, 0, 0); 3812 mutex_unlock(&adev->srbm_mutex); 3813 3814 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3815 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3816 } 3817 3818 return 0; 3819 } 3820 3821 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring) 3822 { 3823 struct amdgpu_device *adev = ring->adev; 3824 struct v9_mqd *mqd = ring->mqd_ptr; 3825 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 3826 struct v9_mqd *tmp_mqd; 3827 3828 /* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control 3829 * is not be initialized before 3830 */ 3831 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx]; 3832 3833 if (!tmp_mqd->cp_hqd_pq_control || 3834 (!amdgpu_in_reset(adev) && !adev->in_suspend)) { 3835 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3836 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3837 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3838 mutex_lock(&adev->srbm_mutex); 3839 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3840 gfx_v9_0_mqd_init(ring); 3841 soc15_grbm_select(adev, 0, 0, 0, 0); 3842 mutex_unlock(&adev->srbm_mutex); 3843 3844 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3845 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3846 } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 3847 /* reset MQD to a clean status */ 3848 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3849 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3850 3851 /* reset ring buffer */ 3852 ring->wptr = 0; 3853 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0); 3854 amdgpu_ring_clear_ring(ring); 3855 } else { 3856 amdgpu_ring_clear_ring(ring); 3857 } 3858 3859 return 0; 3860 } 3861 3862 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev) 3863 { 3864 struct amdgpu_ring *ring; 3865 int r; 3866 3867 ring = &adev->gfx.kiq.ring; 3868 3869 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3870 if (unlikely(r != 0)) 3871 return r; 3872 3873 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3874 if (unlikely(r != 0)) 3875 return r; 3876 3877 gfx_v9_0_kiq_init_queue(ring); 3878 amdgpu_bo_kunmap(ring->mqd_obj); 3879 ring->mqd_ptr = NULL; 3880 amdgpu_bo_unreserve(ring->mqd_obj); 3881 ring->sched.ready = true; 3882 return 0; 3883 } 3884 3885 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev) 3886 { 3887 struct amdgpu_ring *ring = NULL; 3888 int r = 0, i; 3889 3890 gfx_v9_0_cp_compute_enable(adev, true); 3891 3892 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3893 ring = &adev->gfx.compute_ring[i]; 3894 3895 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3896 if (unlikely(r != 0)) 3897 goto done; 3898 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3899 if (!r) { 3900 r = gfx_v9_0_kcq_init_queue(ring); 3901 amdgpu_bo_kunmap(ring->mqd_obj); 3902 ring->mqd_ptr = NULL; 3903 } 3904 amdgpu_bo_unreserve(ring->mqd_obj); 3905 if (r) 3906 goto done; 3907 } 3908 3909 r = amdgpu_gfx_enable_kcq(adev); 3910 done: 3911 return r; 3912 } 3913 3914 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev) 3915 { 3916 int r, i; 3917 struct amdgpu_ring *ring; 3918 3919 if (!(adev->flags & AMD_IS_APU)) 3920 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3921 3922 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3923 if (adev->gfx.num_gfx_rings) { 3924 /* legacy firmware loading */ 3925 r = gfx_v9_0_cp_gfx_load_microcode(adev); 3926 if (r) 3927 return r; 3928 } 3929 3930 r = gfx_v9_0_cp_compute_load_microcode(adev); 3931 if (r) 3932 return r; 3933 } 3934 3935 r = gfx_v9_0_kiq_resume(adev); 3936 if (r) 3937 return r; 3938 3939 if (adev->gfx.num_gfx_rings) { 3940 r = gfx_v9_0_cp_gfx_resume(adev); 3941 if (r) 3942 return r; 3943 } 3944 3945 r = gfx_v9_0_kcq_resume(adev); 3946 if (r) 3947 return r; 3948 3949 if (adev->gfx.num_gfx_rings) { 3950 ring = &adev->gfx.gfx_ring[0]; 3951 r = amdgpu_ring_test_helper(ring); 3952 if (r) 3953 return r; 3954 } 3955 3956 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3957 ring = &adev->gfx.compute_ring[i]; 3958 amdgpu_ring_test_helper(ring); 3959 } 3960 3961 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3962 3963 return 0; 3964 } 3965 3966 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev) 3967 { 3968 u32 tmp; 3969 3970 if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 1) && 3971 adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 2)) 3972 return; 3973 3974 tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG); 3975 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH, 3976 adev->df.hash_status.hash_64k); 3977 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH, 3978 adev->df.hash_status.hash_2m); 3979 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH, 3980 adev->df.hash_status.hash_1g); 3981 WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp); 3982 } 3983 3984 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable) 3985 { 3986 if (adev->gfx.num_gfx_rings) 3987 gfx_v9_0_cp_gfx_enable(adev, enable); 3988 gfx_v9_0_cp_compute_enable(adev, enable); 3989 } 3990 3991 static int gfx_v9_0_hw_init(void *handle) 3992 { 3993 int r; 3994 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3995 3996 if (!amdgpu_sriov_vf(adev)) 3997 gfx_v9_0_init_golden_registers(adev); 3998 3999 gfx_v9_0_constants_init(adev); 4000 4001 gfx_v9_0_init_tcp_config(adev); 4002 4003 r = adev->gfx.rlc.funcs->resume(adev); 4004 if (r) 4005 return r; 4006 4007 r = gfx_v9_0_cp_resume(adev); 4008 if (r) 4009 return r; 4010 4011 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2)) 4012 gfx_v9_4_2_set_power_brake_sequence(adev); 4013 4014 return r; 4015 } 4016 4017 static int gfx_v9_0_hw_fini(void *handle) 4018 { 4019 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4020 4021 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); 4022 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 4023 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 4024 4025 /* DF freeze and kcq disable will fail */ 4026 if (!amdgpu_ras_intr_triggered()) 4027 /* disable KCQ to avoid CPC touch memory not valid anymore */ 4028 amdgpu_gfx_disable_kcq(adev); 4029 4030 if (amdgpu_sriov_vf(adev)) { 4031 gfx_v9_0_cp_gfx_enable(adev, false); 4032 /* must disable polling for SRIOV when hw finished, otherwise 4033 * CPC engine may still keep fetching WB address which is already 4034 * invalid after sw finished and trigger DMAR reading error in 4035 * hypervisor side. 4036 */ 4037 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 4038 return 0; 4039 } 4040 4041 /* Use deinitialize sequence from CAIL when unbinding device from driver, 4042 * otherwise KIQ is hanging when binding back 4043 */ 4044 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 4045 mutex_lock(&adev->srbm_mutex); 4046 soc15_grbm_select(adev, adev->gfx.kiq.ring.me, 4047 adev->gfx.kiq.ring.pipe, 4048 adev->gfx.kiq.ring.queue, 0); 4049 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring); 4050 soc15_grbm_select(adev, 0, 0, 0, 0); 4051 mutex_unlock(&adev->srbm_mutex); 4052 } 4053 4054 gfx_v9_0_cp_enable(adev, false); 4055 4056 /* Skip suspend with A+A reset */ 4057 if (adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) { 4058 dev_dbg(adev->dev, "Device in reset. Skipping RLC halt\n"); 4059 return 0; 4060 } 4061 4062 adev->gfx.rlc.funcs->stop(adev); 4063 return 0; 4064 } 4065 4066 static int gfx_v9_0_suspend(void *handle) 4067 { 4068 return gfx_v9_0_hw_fini(handle); 4069 } 4070 4071 static int gfx_v9_0_resume(void *handle) 4072 { 4073 return gfx_v9_0_hw_init(handle); 4074 } 4075 4076 static bool gfx_v9_0_is_idle(void *handle) 4077 { 4078 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4079 4080 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS), 4081 GRBM_STATUS, GUI_ACTIVE)) 4082 return false; 4083 else 4084 return true; 4085 } 4086 4087 static int gfx_v9_0_wait_for_idle(void *handle) 4088 { 4089 unsigned i; 4090 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4091 4092 for (i = 0; i < adev->usec_timeout; i++) { 4093 if (gfx_v9_0_is_idle(handle)) 4094 return 0; 4095 udelay(1); 4096 } 4097 return -ETIMEDOUT; 4098 } 4099 4100 static int gfx_v9_0_soft_reset(void *handle) 4101 { 4102 u32 grbm_soft_reset = 0; 4103 u32 tmp; 4104 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4105 4106 /* GRBM_STATUS */ 4107 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS); 4108 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK | 4109 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK | 4110 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK | 4111 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK | 4112 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK | 4113 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) { 4114 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4115 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4116 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4117 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1); 4118 } 4119 4120 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) { 4121 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4122 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4123 } 4124 4125 /* GRBM_STATUS2 */ 4126 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2); 4127 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY)) 4128 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4129 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 4130 4131 4132 if (grbm_soft_reset) { 4133 /* stop the rlc */ 4134 adev->gfx.rlc.funcs->stop(adev); 4135 4136 if (adev->gfx.num_gfx_rings) 4137 /* Disable GFX parsing/prefetching */ 4138 gfx_v9_0_cp_gfx_enable(adev, false); 4139 4140 /* Disable MEC parsing/prefetching */ 4141 gfx_v9_0_cp_compute_enable(adev, false); 4142 4143 if (grbm_soft_reset) { 4144 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4145 tmp |= grbm_soft_reset; 4146 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 4147 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4148 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4149 4150 udelay(50); 4151 4152 tmp &= ~grbm_soft_reset; 4153 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4154 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4155 } 4156 4157 /* Wait a little for things to settle down */ 4158 udelay(50); 4159 } 4160 return 0; 4161 } 4162 4163 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev) 4164 { 4165 signed long r, cnt = 0; 4166 unsigned long flags; 4167 uint32_t seq, reg_val_offs = 0; 4168 uint64_t value = 0; 4169 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 4170 struct amdgpu_ring *ring = &kiq->ring; 4171 4172 BUG_ON(!ring->funcs->emit_rreg); 4173 4174 spin_lock_irqsave(&kiq->ring_lock, flags); 4175 if (amdgpu_device_wb_get(adev, ®_val_offs)) { 4176 pr_err("critical bug! too many kiq readers\n"); 4177 goto failed_unlock; 4178 } 4179 amdgpu_ring_alloc(ring, 32); 4180 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 4181 amdgpu_ring_write(ring, 9 | /* src: register*/ 4182 (5 << 8) | /* dst: memory */ 4183 (1 << 16) | /* count sel */ 4184 (1 << 20)); /* write confirm */ 4185 amdgpu_ring_write(ring, 0); 4186 amdgpu_ring_write(ring, 0); 4187 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 4188 reg_val_offs * 4)); 4189 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 4190 reg_val_offs * 4)); 4191 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 4192 if (r) 4193 goto failed_undo; 4194 4195 amdgpu_ring_commit(ring); 4196 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4197 4198 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4199 4200 /* don't wait anymore for gpu reset case because this way may 4201 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 4202 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 4203 * never return if we keep waiting in virt_kiq_rreg, which cause 4204 * gpu_recover() hang there. 4205 * 4206 * also don't wait anymore for IRQ context 4207 * */ 4208 if (r < 1 && (amdgpu_in_reset(adev))) 4209 goto failed_kiq_read; 4210 4211 might_sleep(); 4212 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 4213 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 4214 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4215 } 4216 4217 if (cnt > MAX_KIQ_REG_TRY) 4218 goto failed_kiq_read; 4219 4220 mb(); 4221 value = (uint64_t)adev->wb.wb[reg_val_offs] | 4222 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL; 4223 amdgpu_device_wb_free(adev, reg_val_offs); 4224 return value; 4225 4226 failed_undo: 4227 amdgpu_ring_undo(ring); 4228 failed_unlock: 4229 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4230 failed_kiq_read: 4231 if (reg_val_offs) 4232 amdgpu_device_wb_free(adev, reg_val_offs); 4233 pr_err("failed to read gpu clock\n"); 4234 return ~0; 4235 } 4236 4237 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev) 4238 { 4239 uint64_t clock; 4240 4241 amdgpu_gfx_off_ctrl(adev, false); 4242 mutex_lock(&adev->gfx.gpu_clock_mutex); 4243 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 0, 1) && amdgpu_sriov_runtime(adev)) { 4244 clock = gfx_v9_0_kiq_read_clock(adev); 4245 } else { 4246 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1); 4247 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) | 4248 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 4249 } 4250 mutex_unlock(&adev->gfx.gpu_clock_mutex); 4251 amdgpu_gfx_off_ctrl(adev, true); 4252 return clock; 4253 } 4254 4255 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 4256 uint32_t vmid, 4257 uint32_t gds_base, uint32_t gds_size, 4258 uint32_t gws_base, uint32_t gws_size, 4259 uint32_t oa_base, uint32_t oa_size) 4260 { 4261 struct amdgpu_device *adev = ring->adev; 4262 4263 /* GDS Base */ 4264 gfx_v9_0_write_data_to_reg(ring, 0, false, 4265 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid, 4266 gds_base); 4267 4268 /* GDS Size */ 4269 gfx_v9_0_write_data_to_reg(ring, 0, false, 4270 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid, 4271 gds_size); 4272 4273 /* GWS */ 4274 gfx_v9_0_write_data_to_reg(ring, 0, false, 4275 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid, 4276 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 4277 4278 /* OA */ 4279 gfx_v9_0_write_data_to_reg(ring, 0, false, 4280 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid, 4281 (1 << (oa_size + oa_base)) - (1 << oa_base)); 4282 } 4283 4284 static const u32 vgpr_init_compute_shader[] = 4285 { 4286 0xb07c0000, 0xbe8000ff, 4287 0x000000f8, 0xbf110800, 4288 0x7e000280, 0x7e020280, 4289 0x7e040280, 0x7e060280, 4290 0x7e080280, 0x7e0a0280, 4291 0x7e0c0280, 0x7e0e0280, 4292 0x80808800, 0xbe803200, 4293 0xbf84fff5, 0xbf9c0000, 4294 0xd28c0001, 0x0001007f, 4295 0xd28d0001, 0x0002027e, 4296 0x10020288, 0xb8810904, 4297 0xb7814000, 0xd1196a01, 4298 0x00000301, 0xbe800087, 4299 0xbefc00c1, 0xd89c4000, 4300 0x00020201, 0xd89cc080, 4301 0x00040401, 0x320202ff, 4302 0x00000800, 0x80808100, 4303 0xbf84fff8, 0x7e020280, 4304 0xbf810000, 0x00000000, 4305 }; 4306 4307 static const u32 sgpr_init_compute_shader[] = 4308 { 4309 0xb07c0000, 0xbe8000ff, 4310 0x0000005f, 0xbee50080, 4311 0xbe812c65, 0xbe822c65, 4312 0xbe832c65, 0xbe842c65, 4313 0xbe852c65, 0xb77c0005, 4314 0x80808500, 0xbf84fff8, 4315 0xbe800080, 0xbf810000, 4316 }; 4317 4318 static const u32 vgpr_init_compute_shader_arcturus[] = { 4319 0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080, 4320 0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080, 4321 0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080, 4322 0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080, 4323 0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080, 4324 0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080, 4325 0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080, 4326 0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080, 4327 0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080, 4328 0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080, 4329 0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080, 4330 0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080, 4331 0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080, 4332 0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080, 4333 0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080, 4334 0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080, 4335 0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080, 4336 0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080, 4337 0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080, 4338 0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080, 4339 0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080, 4340 0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080, 4341 0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080, 4342 0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080, 4343 0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080, 4344 0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080, 4345 0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080, 4346 0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080, 4347 0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080, 4348 0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080, 4349 0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080, 4350 0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080, 4351 0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080, 4352 0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080, 4353 0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080, 4354 0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080, 4355 0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080, 4356 0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080, 4357 0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080, 4358 0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080, 4359 0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080, 4360 0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080, 4361 0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080, 4362 0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080, 4363 0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080, 4364 0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080, 4365 0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080, 4366 0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080, 4367 0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080, 4368 0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080, 4369 0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080, 4370 0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080, 4371 0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080, 4372 0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080, 4373 0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080, 4374 0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080, 4375 0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080, 4376 0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080, 4377 0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080, 4378 0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080, 4379 0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080, 4380 0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080, 4381 0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080, 4382 0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080, 4383 0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080, 4384 0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080, 4385 0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080, 4386 0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080, 4387 0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080, 4388 0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080, 4389 0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080, 4390 0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080, 4391 0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080, 4392 0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080, 4393 0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080, 4394 0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080, 4395 0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080, 4396 0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080, 4397 0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080, 4398 0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080, 4399 0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080, 4400 0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080, 4401 0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080, 4402 0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080, 4403 0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080, 4404 0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a, 4405 0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280, 4406 0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000, 4407 0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904, 4408 0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000, 4409 0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a, 4410 0xbf84fff8, 0xbf810000, 4411 }; 4412 4413 /* When below register arrays changed, please update gpr_reg_size, 4414 and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds, 4415 to cover all gfx9 ASICs */ 4416 static const struct soc15_reg_entry vgpr_init_regs[] = { 4417 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4418 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4419 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4420 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4421 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f }, 4422 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4423 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4424 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4425 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4426 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4427 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4428 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4429 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4430 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4431 }; 4432 4433 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = { 4434 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4435 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4436 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4437 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4438 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf }, 4439 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4440 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4441 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4442 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4443 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4444 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4445 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4446 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4447 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4448 }; 4449 4450 static const struct soc15_reg_entry sgpr1_init_regs[] = { 4451 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4452 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4453 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4454 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4455 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4456 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4457 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff }, 4458 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff }, 4459 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff }, 4460 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff }, 4461 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff }, 4462 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff }, 4463 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff }, 4464 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff }, 4465 }; 4466 4467 static const struct soc15_reg_entry sgpr2_init_regs[] = { 4468 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4469 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4470 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4471 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4472 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4473 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4474 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 }, 4475 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 }, 4476 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 }, 4477 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 }, 4478 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 }, 4479 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 }, 4480 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 }, 4481 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 }, 4482 }; 4483 4484 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = { 4485 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1}, 4486 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1}, 4487 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1}, 4488 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1}, 4489 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1}, 4490 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1}, 4491 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1}, 4492 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1}, 4493 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1}, 4494 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1}, 4495 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1}, 4496 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1}, 4497 { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1}, 4498 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6}, 4499 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16}, 4500 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16}, 4501 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16}, 4502 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16}, 4503 { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16}, 4504 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16}, 4505 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16}, 4506 { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16}, 4507 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6}, 4508 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16}, 4509 { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16}, 4510 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1}, 4511 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1}, 4512 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32}, 4513 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32}, 4514 { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72}, 4515 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16}, 4516 { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2}, 4517 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6}, 4518 }; 4519 4520 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev) 4521 { 4522 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4523 int i, r; 4524 4525 /* only support when RAS is enabled */ 4526 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4527 return 0; 4528 4529 r = amdgpu_ring_alloc(ring, 7); 4530 if (r) { 4531 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n", 4532 ring->name, r); 4533 return r; 4534 } 4535 4536 WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000); 4537 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size); 4538 4539 amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 4540 amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC | 4541 PACKET3_DMA_DATA_DST_SEL(1) | 4542 PACKET3_DMA_DATA_SRC_SEL(2) | 4543 PACKET3_DMA_DATA_ENGINE(0))); 4544 amdgpu_ring_write(ring, 0); 4545 amdgpu_ring_write(ring, 0); 4546 amdgpu_ring_write(ring, 0); 4547 amdgpu_ring_write(ring, 0); 4548 amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT | 4549 adev->gds.gds_size); 4550 4551 amdgpu_ring_commit(ring); 4552 4553 for (i = 0; i < adev->usec_timeout; i++) { 4554 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring)) 4555 break; 4556 udelay(1); 4557 } 4558 4559 if (i >= adev->usec_timeout) 4560 r = -ETIMEDOUT; 4561 4562 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000); 4563 4564 return r; 4565 } 4566 4567 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) 4568 { 4569 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4570 struct amdgpu_ib ib; 4571 struct dma_fence *f = NULL; 4572 int r, i; 4573 unsigned total_size, vgpr_offset, sgpr_offset; 4574 u64 gpu_addr; 4575 4576 int compute_dim_x = adev->gfx.config.max_shader_engines * 4577 adev->gfx.config.max_cu_per_sh * 4578 adev->gfx.config.max_sh_per_se; 4579 int sgpr_work_group_size = 5; 4580 int gpr_reg_size = adev->gfx.config.max_shader_engines + 6; 4581 int vgpr_init_shader_size; 4582 const u32 *vgpr_init_shader_ptr; 4583 const struct soc15_reg_entry *vgpr_init_regs_ptr; 4584 4585 /* only support when RAS is enabled */ 4586 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4587 return 0; 4588 4589 /* bail if the compute ring is not ready */ 4590 if (!ring->sched.ready) 4591 return 0; 4592 4593 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1)) { 4594 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus; 4595 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus); 4596 vgpr_init_regs_ptr = vgpr_init_regs_arcturus; 4597 } else { 4598 vgpr_init_shader_ptr = vgpr_init_compute_shader; 4599 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader); 4600 vgpr_init_regs_ptr = vgpr_init_regs; 4601 } 4602 4603 total_size = 4604 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */ 4605 total_size += 4606 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */ 4607 total_size += 4608 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */ 4609 total_size = ALIGN(total_size, 256); 4610 vgpr_offset = total_size; 4611 total_size += ALIGN(vgpr_init_shader_size, 256); 4612 sgpr_offset = total_size; 4613 total_size += sizeof(sgpr_init_compute_shader); 4614 4615 /* allocate an indirect buffer to put the commands in */ 4616 memset(&ib, 0, sizeof(ib)); 4617 r = amdgpu_ib_get(adev, NULL, total_size, 4618 AMDGPU_IB_POOL_DIRECT, &ib); 4619 if (r) { 4620 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); 4621 return r; 4622 } 4623 4624 /* load the compute shaders */ 4625 for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++) 4626 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i]; 4627 4628 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++) 4629 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i]; 4630 4631 /* init the ib length to 0 */ 4632 ib.length_dw = 0; 4633 4634 /* VGPR */ 4635 /* write the register state for the compute dispatch */ 4636 for (i = 0; i < gpr_reg_size; i++) { 4637 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4638 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i]) 4639 - PACKET3_SET_SH_REG_START; 4640 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value; 4641 } 4642 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4643 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8; 4644 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4645 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4646 - PACKET3_SET_SH_REG_START; 4647 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4648 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4649 4650 /* write dispatch packet */ 4651 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4652 ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */ 4653 ib.ptr[ib.length_dw++] = 1; /* y */ 4654 ib.ptr[ib.length_dw++] = 1; /* z */ 4655 ib.ptr[ib.length_dw++] = 4656 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4657 4658 /* write CS partial flush packet */ 4659 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4660 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4661 4662 /* SGPR1 */ 4663 /* write the register state for the compute dispatch */ 4664 for (i = 0; i < gpr_reg_size; i++) { 4665 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4666 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i]) 4667 - PACKET3_SET_SH_REG_START; 4668 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value; 4669 } 4670 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4671 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4672 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4673 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4674 - PACKET3_SET_SH_REG_START; 4675 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4676 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4677 4678 /* write dispatch packet */ 4679 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4680 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4681 ib.ptr[ib.length_dw++] = 1; /* y */ 4682 ib.ptr[ib.length_dw++] = 1; /* z */ 4683 ib.ptr[ib.length_dw++] = 4684 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4685 4686 /* write CS partial flush packet */ 4687 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4688 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4689 4690 /* SGPR2 */ 4691 /* write the register state for the compute dispatch */ 4692 for (i = 0; i < gpr_reg_size; i++) { 4693 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4694 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i]) 4695 - PACKET3_SET_SH_REG_START; 4696 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value; 4697 } 4698 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4699 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4700 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4701 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4702 - PACKET3_SET_SH_REG_START; 4703 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4704 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4705 4706 /* write dispatch packet */ 4707 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4708 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4709 ib.ptr[ib.length_dw++] = 1; /* y */ 4710 ib.ptr[ib.length_dw++] = 1; /* z */ 4711 ib.ptr[ib.length_dw++] = 4712 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4713 4714 /* write CS partial flush packet */ 4715 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4716 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4717 4718 /* shedule the ib on the ring */ 4719 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 4720 if (r) { 4721 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r); 4722 goto fail; 4723 } 4724 4725 /* wait for the GPU to finish processing the IB */ 4726 r = dma_fence_wait(f, false); 4727 if (r) { 4728 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); 4729 goto fail; 4730 } 4731 4732 fail: 4733 amdgpu_ib_free(adev, &ib, NULL); 4734 dma_fence_put(f); 4735 4736 return r; 4737 } 4738 4739 static int gfx_v9_0_early_init(void *handle) 4740 { 4741 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4742 4743 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1) || 4744 adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2)) 4745 adev->gfx.num_gfx_rings = 0; 4746 else 4747 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS; 4748 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), 4749 AMDGPU_MAX_COMPUTE_RINGS); 4750 gfx_v9_0_set_kiq_pm4_funcs(adev); 4751 gfx_v9_0_set_ring_funcs(adev); 4752 gfx_v9_0_set_irq_funcs(adev); 4753 gfx_v9_0_set_gds_init(adev); 4754 gfx_v9_0_set_rlc_funcs(adev); 4755 4756 return 0; 4757 } 4758 4759 static int gfx_v9_0_ecc_late_init(void *handle) 4760 { 4761 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4762 int r; 4763 4764 /* 4765 * Temp workaround to fix the issue that CP firmware fails to 4766 * update read pointer when CPDMA is writing clearing operation 4767 * to GDS in suspend/resume sequence on several cards. So just 4768 * limit this operation in cold boot sequence. 4769 */ 4770 if ((!adev->in_suspend) && 4771 (adev->gds.gds_size)) { 4772 r = gfx_v9_0_do_edc_gds_workarounds(adev); 4773 if (r) 4774 return r; 4775 } 4776 4777 /* requires IBs so do in late init after IB pool is initialized */ 4778 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2)) 4779 r = gfx_v9_4_2_do_edc_gpr_workarounds(adev); 4780 else 4781 r = gfx_v9_0_do_edc_gpr_workarounds(adev); 4782 4783 if (r) 4784 return r; 4785 4786 if (adev->gfx.ras_funcs && 4787 adev->gfx.ras_funcs->ras_late_init) { 4788 r = adev->gfx.ras_funcs->ras_late_init(adev); 4789 if (r) 4790 return r; 4791 } 4792 4793 if (adev->gfx.ras_funcs && 4794 adev->gfx.ras_funcs->enable_watchdog_timer) 4795 adev->gfx.ras_funcs->enable_watchdog_timer(adev); 4796 4797 return 0; 4798 } 4799 4800 static int gfx_v9_0_late_init(void *handle) 4801 { 4802 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4803 int r; 4804 4805 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 4806 if (r) 4807 return r; 4808 4809 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 4810 if (r) 4811 return r; 4812 4813 r = gfx_v9_0_ecc_late_init(handle); 4814 if (r) 4815 return r; 4816 4817 return 0; 4818 } 4819 4820 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev) 4821 { 4822 uint32_t rlc_setting; 4823 4824 /* if RLC is not enabled, do nothing */ 4825 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL); 4826 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK)) 4827 return false; 4828 4829 return true; 4830 } 4831 4832 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev) 4833 { 4834 uint32_t data; 4835 unsigned i; 4836 4837 data = RLC_SAFE_MODE__CMD_MASK; 4838 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 4839 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4840 4841 /* wait for RLC_SAFE_MODE */ 4842 for (i = 0; i < adev->usec_timeout; i++) { 4843 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 4844 break; 4845 udelay(1); 4846 } 4847 } 4848 4849 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev) 4850 { 4851 uint32_t data; 4852 4853 data = RLC_SAFE_MODE__CMD_MASK; 4854 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4855 } 4856 4857 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev, 4858 bool enable) 4859 { 4860 amdgpu_gfx_rlc_enter_safe_mode(adev); 4861 4862 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { 4863 gfx_v9_0_enable_gfx_cg_power_gating(adev, true); 4864 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4865 gfx_v9_0_enable_gfx_pipeline_powergating(adev, true); 4866 } else { 4867 gfx_v9_0_enable_gfx_cg_power_gating(adev, false); 4868 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4869 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false); 4870 } 4871 4872 amdgpu_gfx_rlc_exit_safe_mode(adev); 4873 } 4874 4875 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev, 4876 bool enable) 4877 { 4878 /* TODO: double check if we need to perform under safe mode */ 4879 /* gfx_v9_0_enter_rlc_safe_mode(adev); */ 4880 4881 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 4882 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true); 4883 else 4884 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false); 4885 4886 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 4887 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true); 4888 else 4889 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false); 4890 4891 /* gfx_v9_0_exit_rlc_safe_mode(adev); */ 4892 } 4893 4894 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 4895 bool enable) 4896 { 4897 uint32_t data, def; 4898 4899 amdgpu_gfx_rlc_enter_safe_mode(adev); 4900 4901 /* It is disabled by HW by default */ 4902 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { 4903 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 4904 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4905 4906 if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 2, 1)) 4907 data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4908 4909 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4910 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4911 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4912 4913 /* only for Vega10 & Raven1 */ 4914 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK; 4915 4916 if (def != data) 4917 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4918 4919 /* MGLS is a global flag to control all MGLS in GFX */ 4920 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 4921 /* 2 - RLC memory Light sleep */ 4922 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { 4923 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4924 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4925 if (def != data) 4926 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4927 } 4928 /* 3 - CP memory Light sleep */ 4929 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { 4930 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4931 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4932 if (def != data) 4933 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4934 } 4935 } 4936 } else { 4937 /* 1 - MGCG_OVERRIDE */ 4938 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4939 4940 if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 2, 1)) 4941 data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4942 4943 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 4944 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4945 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4946 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4947 4948 if (def != data) 4949 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4950 4951 /* 2 - disable MGLS in RLC */ 4952 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4953 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) { 4954 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4955 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4956 } 4957 4958 /* 3 - disable MGLS in CP */ 4959 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4960 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) { 4961 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4962 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4963 } 4964 } 4965 4966 amdgpu_gfx_rlc_exit_safe_mode(adev); 4967 } 4968 4969 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev, 4970 bool enable) 4971 { 4972 uint32_t data, def; 4973 4974 if (!adev->gfx.num_gfx_rings) 4975 return; 4976 4977 amdgpu_gfx_rlc_enter_safe_mode(adev); 4978 4979 /* Enable 3D CGCG/CGLS */ 4980 if (enable) { 4981 /* write cmd to clear cgcg/cgls ov */ 4982 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4983 /* unset CGCG override */ 4984 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 4985 /* update CGCG and CGLS override bits */ 4986 if (def != data) 4987 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4988 4989 /* enable 3Dcgcg FSM(0x0000363f) */ 4990 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4991 4992 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) 4993 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4994 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 4995 else 4996 data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT; 4997 4998 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 4999 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 5000 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 5001 if (def != data) 5002 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 5003 5004 /* set IDLE_POLL_COUNT(0x00900100) */ 5005 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 5006 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 5007 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 5008 if (def != data) 5009 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 5010 } else { 5011 /* Disable CGCG/CGLS */ 5012 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 5013 /* disable cgcg, cgls should be disabled */ 5014 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK | 5015 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK); 5016 /* disable cgcg and cgls in FSM */ 5017 if (def != data) 5018 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 5019 } 5020 5021 amdgpu_gfx_rlc_exit_safe_mode(adev); 5022 } 5023 5024 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 5025 bool enable) 5026 { 5027 uint32_t def, data; 5028 5029 amdgpu_gfx_rlc_enter_safe_mode(adev); 5030 5031 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { 5032 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 5033 /* unset CGCG override */ 5034 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 5035 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5036 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 5037 else 5038 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 5039 /* update CGCG and CGLS override bits */ 5040 if (def != data) 5041 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 5042 5043 /* enable cgcg FSM(0x0000363F) */ 5044 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 5045 5046 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1)) 5047 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5048 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5049 else 5050 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5051 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5052 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5053 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 5054 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5055 if (def != data) 5056 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 5057 5058 /* set IDLE_POLL_COUNT(0x00900100) */ 5059 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 5060 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 5061 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 5062 if (def != data) 5063 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 5064 } else { 5065 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 5066 /* reset CGCG/CGLS bits */ 5067 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK); 5068 /* disable cgcg and cgls in FSM */ 5069 if (def != data) 5070 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 5071 } 5072 5073 amdgpu_gfx_rlc_exit_safe_mode(adev); 5074 } 5075 5076 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev, 5077 bool enable) 5078 { 5079 if (enable) { 5080 /* CGCG/CGLS should be enabled after MGCG/MGLS 5081 * === MGCG + MGLS === 5082 */ 5083 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 5084 /* === CGCG /CGLS for GFX 3D Only === */ 5085 gfx_v9_0_update_3d_clock_gating(adev, enable); 5086 /* === CGCG + CGLS === */ 5087 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 5088 } else { 5089 /* CGCG/CGLS should be disabled before MGCG/MGLS 5090 * === CGCG + CGLS === 5091 */ 5092 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 5093 /* === CGCG /CGLS for GFX 3D Only === */ 5094 gfx_v9_0_update_3d_clock_gating(adev, enable); 5095 /* === MGCG + MGLS === */ 5096 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 5097 } 5098 return 0; 5099 } 5100 5101 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid) 5102 { 5103 u32 reg, data; 5104 5105 reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL); 5106 if (amdgpu_sriov_is_pp_one_vf(adev)) 5107 data = RREG32_NO_KIQ(reg); 5108 else 5109 data = RREG32(reg); 5110 5111 data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK; 5112 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; 5113 5114 if (amdgpu_sriov_is_pp_one_vf(adev)) 5115 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data); 5116 else 5117 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data); 5118 } 5119 5120 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev, 5121 uint32_t offset, 5122 struct soc15_reg_rlcg *entries, int arr_size) 5123 { 5124 int i; 5125 uint32_t reg; 5126 5127 if (!entries) 5128 return false; 5129 5130 for (i = 0; i < arr_size; i++) { 5131 const struct soc15_reg_rlcg *entry; 5132 5133 entry = &entries[i]; 5134 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg; 5135 if (offset == reg) 5136 return true; 5137 } 5138 5139 return false; 5140 } 5141 5142 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset) 5143 { 5144 return gfx_v9_0_check_rlcg_range(adev, offset, 5145 (void *)rlcg_access_gc_9_0, 5146 ARRAY_SIZE(rlcg_access_gc_9_0)); 5147 } 5148 5149 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = { 5150 .is_rlc_enabled = gfx_v9_0_is_rlc_enabled, 5151 .set_safe_mode = gfx_v9_0_set_safe_mode, 5152 .unset_safe_mode = gfx_v9_0_unset_safe_mode, 5153 .init = gfx_v9_0_rlc_init, 5154 .get_csb_size = gfx_v9_0_get_csb_size, 5155 .get_csb_buffer = gfx_v9_0_get_csb_buffer, 5156 .get_cp_table_num = gfx_v9_0_cp_jump_table_num, 5157 .resume = gfx_v9_0_rlc_resume, 5158 .stop = gfx_v9_0_rlc_stop, 5159 .reset = gfx_v9_0_rlc_reset, 5160 .start = gfx_v9_0_rlc_start, 5161 .update_spm_vmid = gfx_v9_0_update_spm_vmid, 5162 .sriov_wreg = gfx_v9_0_sriov_wreg, 5163 .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range, 5164 }; 5165 5166 static int gfx_v9_0_set_powergating_state(void *handle, 5167 enum amd_powergating_state state) 5168 { 5169 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5170 bool enable = (state == AMD_PG_STATE_GATE); 5171 5172 switch (adev->ip_versions[GC_HWIP][0]) { 5173 case IP_VERSION(9, 2, 2): 5174 case IP_VERSION(9, 1, 0): 5175 case IP_VERSION(9, 3, 0): 5176 if (!enable) 5177 amdgpu_gfx_off_ctrl(adev, false); 5178 5179 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { 5180 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true); 5181 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true); 5182 } else { 5183 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false); 5184 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false); 5185 } 5186 5187 if (adev->pg_flags & AMD_PG_SUPPORT_CP) 5188 gfx_v9_0_enable_cp_power_gating(adev, true); 5189 else 5190 gfx_v9_0_enable_cp_power_gating(adev, false); 5191 5192 /* update gfx cgpg state */ 5193 gfx_v9_0_update_gfx_cg_power_gating(adev, enable); 5194 5195 /* update mgcg state */ 5196 gfx_v9_0_update_gfx_mg_power_gating(adev, enable); 5197 5198 if (enable) 5199 amdgpu_gfx_off_ctrl(adev, true); 5200 break; 5201 case IP_VERSION(9, 2, 1): 5202 amdgpu_gfx_off_ctrl(adev, enable); 5203 break; 5204 default: 5205 break; 5206 } 5207 5208 return 0; 5209 } 5210 5211 static int gfx_v9_0_set_clockgating_state(void *handle, 5212 enum amd_clockgating_state state) 5213 { 5214 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5215 5216 if (amdgpu_sriov_vf(adev)) 5217 return 0; 5218 5219 switch (adev->ip_versions[GC_HWIP][0]) { 5220 case IP_VERSION(9, 0, 1): 5221 case IP_VERSION(9, 2, 1): 5222 case IP_VERSION(9, 4, 0): 5223 case IP_VERSION(9, 2, 2): 5224 case IP_VERSION(9, 1, 0): 5225 case IP_VERSION(9, 4, 1): 5226 case IP_VERSION(9, 3, 0): 5227 case IP_VERSION(9, 4, 2): 5228 gfx_v9_0_update_gfx_clock_gating(adev, 5229 state == AMD_CG_STATE_GATE); 5230 break; 5231 default: 5232 break; 5233 } 5234 return 0; 5235 } 5236 5237 static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags) 5238 { 5239 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5240 int data; 5241 5242 if (amdgpu_sriov_vf(adev)) 5243 *flags = 0; 5244 5245 /* AMD_CG_SUPPORT_GFX_MGCG */ 5246 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE)); 5247 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 5248 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5249 5250 /* AMD_CG_SUPPORT_GFX_CGCG */ 5251 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL)); 5252 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5253 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5254 5255 /* AMD_CG_SUPPORT_GFX_CGLS */ 5256 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5257 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5258 5259 /* AMD_CG_SUPPORT_GFX_RLC_LS */ 5260 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL)); 5261 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) 5262 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS; 5263 5264 /* AMD_CG_SUPPORT_GFX_CP_LS */ 5265 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL)); 5266 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) 5267 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS; 5268 5269 if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 1)) { 5270 /* AMD_CG_SUPPORT_GFX_3D_CGCG */ 5271 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D)); 5272 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) 5273 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; 5274 5275 /* AMD_CG_SUPPORT_GFX_3D_CGLS */ 5276 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK) 5277 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS; 5278 } 5279 } 5280 5281 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) 5282 { 5283 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/ 5284 } 5285 5286 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5287 { 5288 struct amdgpu_device *adev = ring->adev; 5289 u64 wptr; 5290 5291 /* XXX check if swapping is necessary on BE */ 5292 if (ring->use_doorbell) { 5293 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]); 5294 } else { 5295 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR); 5296 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32; 5297 } 5298 5299 return wptr; 5300 } 5301 5302 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5303 { 5304 struct amdgpu_device *adev = ring->adev; 5305 5306 if (ring->use_doorbell) { 5307 /* XXX check if swapping is necessary on BE */ 5308 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5309 WDOORBELL64(ring->doorbell_index, ring->wptr); 5310 } else { 5311 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 5312 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 5313 } 5314 } 5315 5316 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5317 { 5318 struct amdgpu_device *adev = ring->adev; 5319 u32 ref_and_mask, reg_mem_engine; 5320 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg; 5321 5322 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 5323 switch (ring->me) { 5324 case 1: 5325 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe; 5326 break; 5327 case 2: 5328 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe; 5329 break; 5330 default: 5331 return; 5332 } 5333 reg_mem_engine = 0; 5334 } else { 5335 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0; 5336 reg_mem_engine = 1; /* pfp */ 5337 } 5338 5339 gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1, 5340 adev->nbio.funcs->get_hdp_flush_req_offset(adev), 5341 adev->nbio.funcs->get_hdp_flush_done_offset(adev), 5342 ref_and_mask, ref_and_mask, 0x20); 5343 } 5344 5345 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 5346 struct amdgpu_job *job, 5347 struct amdgpu_ib *ib, 5348 uint32_t flags) 5349 { 5350 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5351 u32 header, control = 0; 5352 5353 if (ib->flags & AMDGPU_IB_FLAG_CE) 5354 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 5355 else 5356 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 5357 5358 control |= ib->length_dw | (vmid << 24); 5359 5360 if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { 5361 control |= INDIRECT_BUFFER_PRE_ENB(1); 5362 5363 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid) 5364 gfx_v9_0_ring_emit_de_meta(ring); 5365 } 5366 5367 amdgpu_ring_write(ring, header); 5368 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5369 amdgpu_ring_write(ring, 5370 #ifdef __BIG_ENDIAN 5371 (2 << 0) | 5372 #endif 5373 lower_32_bits(ib->gpu_addr)); 5374 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5375 amdgpu_ring_write(ring, control); 5376 } 5377 5378 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 5379 struct amdgpu_job *job, 5380 struct amdgpu_ib *ib, 5381 uint32_t flags) 5382 { 5383 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5384 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 5385 5386 /* Currently, there is a high possibility to get wave ID mismatch 5387 * between ME and GDS, leading to a hw deadlock, because ME generates 5388 * different wave IDs than the GDS expects. This situation happens 5389 * randomly when at least 5 compute pipes use GDS ordered append. 5390 * The wave IDs generated by ME are also wrong after suspend/resume. 5391 * Those are probably bugs somewhere else in the kernel driver. 5392 * 5393 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 5394 * GDS to 0 for this ring (me/pipe). 5395 */ 5396 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 5397 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 5398 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID); 5399 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 5400 } 5401 5402 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 5403 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5404 amdgpu_ring_write(ring, 5405 #ifdef __BIG_ENDIAN 5406 (2 << 0) | 5407 #endif 5408 lower_32_bits(ib->gpu_addr)); 5409 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5410 amdgpu_ring_write(ring, control); 5411 } 5412 5413 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 5414 u64 seq, unsigned flags) 5415 { 5416 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 5417 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 5418 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY; 5419 5420 /* RELEASE_MEM - flush caches, send int */ 5421 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); 5422 amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN | 5423 EOP_TC_NC_ACTION_EN) : 5424 (EOP_TCL1_ACTION_EN | 5425 EOP_TC_ACTION_EN | 5426 EOP_TC_WB_ACTION_EN | 5427 EOP_TC_MD_ACTION_EN)) | 5428 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 5429 EVENT_INDEX(5))); 5430 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 5431 5432 /* 5433 * the address should be Qword aligned if 64bit write, Dword 5434 * aligned if only send 32bit data low (discard data high) 5435 */ 5436 if (write64bit) 5437 BUG_ON(addr & 0x7); 5438 else 5439 BUG_ON(addr & 0x3); 5440 amdgpu_ring_write(ring, lower_32_bits(addr)); 5441 amdgpu_ring_write(ring, upper_32_bits(addr)); 5442 amdgpu_ring_write(ring, lower_32_bits(seq)); 5443 amdgpu_ring_write(ring, upper_32_bits(seq)); 5444 amdgpu_ring_write(ring, 0); 5445 } 5446 5447 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 5448 { 5449 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5450 uint32_t seq = ring->fence_drv.sync_seq; 5451 uint64_t addr = ring->fence_drv.gpu_addr; 5452 5453 gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0, 5454 lower_32_bits(addr), upper_32_bits(addr), 5455 seq, 0xffffffff, 4); 5456 } 5457 5458 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 5459 unsigned vmid, uint64_t pd_addr) 5460 { 5461 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 5462 5463 /* compute doesn't have PFP */ 5464 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 5465 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 5466 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 5467 amdgpu_ring_write(ring, 0x0); 5468 } 5469 } 5470 5471 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring) 5472 { 5473 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */ 5474 } 5475 5476 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 5477 { 5478 u64 wptr; 5479 5480 /* XXX check if swapping is necessary on BE */ 5481 if (ring->use_doorbell) 5482 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]); 5483 else 5484 BUG(); 5485 return wptr; 5486 } 5487 5488 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 5489 { 5490 struct amdgpu_device *adev = ring->adev; 5491 5492 /* XXX check if swapping is necessary on BE */ 5493 if (ring->use_doorbell) { 5494 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5495 WDOORBELL64(ring->doorbell_index, ring->wptr); 5496 } else{ 5497 BUG(); /* only DOORBELL method supported on gfx9 now */ 5498 } 5499 } 5500 5501 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 5502 u64 seq, unsigned int flags) 5503 { 5504 struct amdgpu_device *adev = ring->adev; 5505 5506 /* we only allocate 32bit for each seq wb address */ 5507 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); 5508 5509 /* write fence seq to the "addr" */ 5510 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5511 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5512 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 5513 amdgpu_ring_write(ring, lower_32_bits(addr)); 5514 amdgpu_ring_write(ring, upper_32_bits(addr)); 5515 amdgpu_ring_write(ring, lower_32_bits(seq)); 5516 5517 if (flags & AMDGPU_FENCE_FLAG_INT) { 5518 /* set register to trigger INT */ 5519 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5520 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5521 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 5522 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS)); 5523 amdgpu_ring_write(ring, 0); 5524 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 5525 } 5526 } 5527 5528 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring) 5529 { 5530 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 5531 amdgpu_ring_write(ring, 0); 5532 } 5533 5534 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring) 5535 { 5536 struct v9_ce_ib_state ce_payload = {0}; 5537 uint64_t csa_addr; 5538 int cnt; 5539 5540 cnt = (sizeof(ce_payload) >> 2) + 4 - 2; 5541 csa_addr = amdgpu_csa_vaddr(ring->adev); 5542 5543 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5544 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) | 5545 WRITE_DATA_DST_SEL(8) | 5546 WR_CONFIRM) | 5547 WRITE_DATA_CACHE_POLICY(0)); 5548 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5549 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5550 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2); 5551 } 5552 5553 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring) 5554 { 5555 struct v9_de_ib_state de_payload = {0}; 5556 uint64_t csa_addr, gds_addr; 5557 int cnt; 5558 5559 csa_addr = amdgpu_csa_vaddr(ring->adev); 5560 gds_addr = csa_addr + 4096; 5561 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr); 5562 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr); 5563 5564 cnt = (sizeof(de_payload) >> 2) + 4 - 2; 5565 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5566 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 5567 WRITE_DATA_DST_SEL(8) | 5568 WR_CONFIRM) | 5569 WRITE_DATA_CACHE_POLICY(0)); 5570 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5571 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5572 amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2); 5573 } 5574 5575 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, 5576 bool secure) 5577 { 5578 uint32_t v = secure ? FRAME_TMZ : 0; 5579 5580 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); 5581 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1)); 5582 } 5583 5584 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) 5585 { 5586 uint32_t dw2 = 0; 5587 5588 if (amdgpu_sriov_vf(ring->adev)) 5589 gfx_v9_0_ring_emit_ce_meta(ring); 5590 5591 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 5592 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 5593 /* set load_global_config & load_global_uconfig */ 5594 dw2 |= 0x8001; 5595 /* set load_cs_sh_regs */ 5596 dw2 |= 0x01000000; 5597 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 5598 dw2 |= 0x10002; 5599 5600 /* set load_ce_ram if preamble presented */ 5601 if (AMDGPU_PREAMBLE_IB_PRESENT & flags) 5602 dw2 |= 0x10000000; 5603 } else { 5604 /* still load_ce_ram if this is the first time preamble presented 5605 * although there is no context switch happens. 5606 */ 5607 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags) 5608 dw2 |= 0x10000000; 5609 } 5610 5611 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 5612 amdgpu_ring_write(ring, dw2); 5613 amdgpu_ring_write(ring, 0); 5614 } 5615 5616 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring) 5617 { 5618 unsigned ret; 5619 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 5620 amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr)); 5621 amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr)); 5622 amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */ 5623 ret = ring->wptr & ring->buf_mask; 5624 amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */ 5625 return ret; 5626 } 5627 5628 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset) 5629 { 5630 unsigned cur; 5631 BUG_ON(offset > ring->buf_mask); 5632 BUG_ON(ring->ring[offset] != 0x55aa55aa); 5633 5634 cur = (ring->wptr & ring->buf_mask) - 1; 5635 if (likely(cur > offset)) 5636 ring->ring[offset] = cur - offset; 5637 else 5638 ring->ring[offset] = (ring->ring_size>>2) - offset + cur; 5639 } 5640 5641 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 5642 uint32_t reg_val_offs) 5643 { 5644 struct amdgpu_device *adev = ring->adev; 5645 5646 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 5647 amdgpu_ring_write(ring, 0 | /* src: register*/ 5648 (5 << 8) | /* dst: memory */ 5649 (1 << 20)); /* write confirm */ 5650 amdgpu_ring_write(ring, reg); 5651 amdgpu_ring_write(ring, 0); 5652 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 5653 reg_val_offs * 4)); 5654 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 5655 reg_val_offs * 4)); 5656 } 5657 5658 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 5659 uint32_t val) 5660 { 5661 uint32_t cmd = 0; 5662 5663 switch (ring->funcs->type) { 5664 case AMDGPU_RING_TYPE_GFX: 5665 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 5666 break; 5667 case AMDGPU_RING_TYPE_KIQ: 5668 cmd = (1 << 16); /* no inc addr */ 5669 break; 5670 default: 5671 cmd = WR_CONFIRM; 5672 break; 5673 } 5674 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5675 amdgpu_ring_write(ring, cmd); 5676 amdgpu_ring_write(ring, reg); 5677 amdgpu_ring_write(ring, 0); 5678 amdgpu_ring_write(ring, val); 5679 } 5680 5681 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 5682 uint32_t val, uint32_t mask) 5683 { 5684 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); 5685 } 5686 5687 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, 5688 uint32_t reg0, uint32_t reg1, 5689 uint32_t ref, uint32_t mask) 5690 { 5691 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5692 struct amdgpu_device *adev = ring->adev; 5693 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ? 5694 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait; 5695 5696 if (fw_version_ok) 5697 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, 5698 ref, mask, 0x20); 5699 else 5700 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1, 5701 ref, mask); 5702 } 5703 5704 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid) 5705 { 5706 struct amdgpu_device *adev = ring->adev; 5707 uint32_t value = 0; 5708 5709 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03); 5710 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01); 5711 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1); 5712 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid); 5713 WREG32_SOC15(GC, 0, mmSQ_CMD, value); 5714 } 5715 5716 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 5717 enum amdgpu_interrupt_state state) 5718 { 5719 switch (state) { 5720 case AMDGPU_IRQ_STATE_DISABLE: 5721 case AMDGPU_IRQ_STATE_ENABLE: 5722 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5723 TIME_STAMP_INT_ENABLE, 5724 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5725 break; 5726 default: 5727 break; 5728 } 5729 } 5730 5731 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 5732 int me, int pipe, 5733 enum amdgpu_interrupt_state state) 5734 { 5735 u32 mec_int_cntl, mec_int_cntl_reg; 5736 5737 /* 5738 * amdgpu controls only the first MEC. That's why this function only 5739 * handles the setting of interrupts for this specific MEC. All other 5740 * pipes' interrupts are set by amdkfd. 5741 */ 5742 5743 if (me == 1) { 5744 switch (pipe) { 5745 case 0: 5746 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL); 5747 break; 5748 case 1: 5749 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL); 5750 break; 5751 case 2: 5752 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL); 5753 break; 5754 case 3: 5755 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL); 5756 break; 5757 default: 5758 DRM_DEBUG("invalid pipe %d\n", pipe); 5759 return; 5760 } 5761 } else { 5762 DRM_DEBUG("invalid me %d\n", me); 5763 return; 5764 } 5765 5766 switch (state) { 5767 case AMDGPU_IRQ_STATE_DISABLE: 5768 mec_int_cntl = RREG32(mec_int_cntl_reg); 5769 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5770 TIME_STAMP_INT_ENABLE, 0); 5771 WREG32(mec_int_cntl_reg, mec_int_cntl); 5772 break; 5773 case AMDGPU_IRQ_STATE_ENABLE: 5774 mec_int_cntl = RREG32(mec_int_cntl_reg); 5775 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5776 TIME_STAMP_INT_ENABLE, 1); 5777 WREG32(mec_int_cntl_reg, mec_int_cntl); 5778 break; 5779 default: 5780 break; 5781 } 5782 } 5783 5784 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 5785 struct amdgpu_irq_src *source, 5786 unsigned type, 5787 enum amdgpu_interrupt_state state) 5788 { 5789 switch (state) { 5790 case AMDGPU_IRQ_STATE_DISABLE: 5791 case AMDGPU_IRQ_STATE_ENABLE: 5792 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5793 PRIV_REG_INT_ENABLE, 5794 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5795 break; 5796 default: 5797 break; 5798 } 5799 5800 return 0; 5801 } 5802 5803 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 5804 struct amdgpu_irq_src *source, 5805 unsigned type, 5806 enum amdgpu_interrupt_state state) 5807 { 5808 switch (state) { 5809 case AMDGPU_IRQ_STATE_DISABLE: 5810 case AMDGPU_IRQ_STATE_ENABLE: 5811 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5812 PRIV_INSTR_INT_ENABLE, 5813 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5814 break; 5815 default: 5816 break; 5817 } 5818 5819 return 0; 5820 } 5821 5822 #define ENABLE_ECC_ON_ME_PIPE(me, pipe) \ 5823 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5824 CP_ECC_ERROR_INT_ENABLE, 1) 5825 5826 #define DISABLE_ECC_ON_ME_PIPE(me, pipe) \ 5827 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5828 CP_ECC_ERROR_INT_ENABLE, 0) 5829 5830 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev, 5831 struct amdgpu_irq_src *source, 5832 unsigned type, 5833 enum amdgpu_interrupt_state state) 5834 { 5835 switch (state) { 5836 case AMDGPU_IRQ_STATE_DISABLE: 5837 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5838 CP_ECC_ERROR_INT_ENABLE, 0); 5839 DISABLE_ECC_ON_ME_PIPE(1, 0); 5840 DISABLE_ECC_ON_ME_PIPE(1, 1); 5841 DISABLE_ECC_ON_ME_PIPE(1, 2); 5842 DISABLE_ECC_ON_ME_PIPE(1, 3); 5843 break; 5844 5845 case AMDGPU_IRQ_STATE_ENABLE: 5846 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5847 CP_ECC_ERROR_INT_ENABLE, 1); 5848 ENABLE_ECC_ON_ME_PIPE(1, 0); 5849 ENABLE_ECC_ON_ME_PIPE(1, 1); 5850 ENABLE_ECC_ON_ME_PIPE(1, 2); 5851 ENABLE_ECC_ON_ME_PIPE(1, 3); 5852 break; 5853 default: 5854 break; 5855 } 5856 5857 return 0; 5858 } 5859 5860 5861 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev, 5862 struct amdgpu_irq_src *src, 5863 unsigned type, 5864 enum amdgpu_interrupt_state state) 5865 { 5866 switch (type) { 5867 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 5868 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state); 5869 break; 5870 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 5871 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 5872 break; 5873 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 5874 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 5875 break; 5876 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 5877 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 5878 break; 5879 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 5880 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 5881 break; 5882 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP: 5883 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state); 5884 break; 5885 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP: 5886 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state); 5887 break; 5888 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP: 5889 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state); 5890 break; 5891 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP: 5892 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state); 5893 break; 5894 default: 5895 break; 5896 } 5897 return 0; 5898 } 5899 5900 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev, 5901 struct amdgpu_irq_src *source, 5902 struct amdgpu_iv_entry *entry) 5903 { 5904 int i; 5905 u8 me_id, pipe_id, queue_id; 5906 struct amdgpu_ring *ring; 5907 5908 DRM_DEBUG("IH: CP EOP\n"); 5909 me_id = (entry->ring_id & 0x0c) >> 2; 5910 pipe_id = (entry->ring_id & 0x03) >> 0; 5911 queue_id = (entry->ring_id & 0x70) >> 4; 5912 5913 switch (me_id) { 5914 case 0: 5915 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); 5916 break; 5917 case 1: 5918 case 2: 5919 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5920 ring = &adev->gfx.compute_ring[i]; 5921 /* Per-queue interrupt is supported for MEC starting from VI. 5922 * The interrupt can only be enabled/disabled per pipe instead of per queue. 5923 */ 5924 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) 5925 amdgpu_fence_process(ring); 5926 } 5927 break; 5928 } 5929 return 0; 5930 } 5931 5932 static void gfx_v9_0_fault(struct amdgpu_device *adev, 5933 struct amdgpu_iv_entry *entry) 5934 { 5935 u8 me_id, pipe_id, queue_id; 5936 struct amdgpu_ring *ring; 5937 int i; 5938 5939 me_id = (entry->ring_id & 0x0c) >> 2; 5940 pipe_id = (entry->ring_id & 0x03) >> 0; 5941 queue_id = (entry->ring_id & 0x70) >> 4; 5942 5943 switch (me_id) { 5944 case 0: 5945 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); 5946 break; 5947 case 1: 5948 case 2: 5949 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5950 ring = &adev->gfx.compute_ring[i]; 5951 if (ring->me == me_id && ring->pipe == pipe_id && 5952 ring->queue == queue_id) 5953 drm_sched_fault(&ring->sched); 5954 } 5955 break; 5956 } 5957 } 5958 5959 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev, 5960 struct amdgpu_irq_src *source, 5961 struct amdgpu_iv_entry *entry) 5962 { 5963 DRM_ERROR("Illegal register access in command stream\n"); 5964 gfx_v9_0_fault(adev, entry); 5965 return 0; 5966 } 5967 5968 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev, 5969 struct amdgpu_irq_src *source, 5970 struct amdgpu_iv_entry *entry) 5971 { 5972 DRM_ERROR("Illegal instruction in command stream\n"); 5973 gfx_v9_0_fault(adev, entry); 5974 return 0; 5975 } 5976 5977 5978 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = { 5979 { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 5980 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT), 5981 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT) 5982 }, 5983 { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 5984 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT), 5985 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT) 5986 }, 5987 { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5988 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1), 5989 0, 0 5990 }, 5991 { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5992 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2), 5993 0, 0 5994 }, 5995 { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 5996 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT), 5997 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT) 5998 }, 5999 { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 6000 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT), 6001 0, 0 6002 }, 6003 { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 6004 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT), 6005 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT) 6006 }, 6007 { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 6008 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT), 6009 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT) 6010 }, 6011 { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 6012 SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1), 6013 0, 0 6014 }, 6015 { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 6016 SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1), 6017 0, 0 6018 }, 6019 { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 6020 SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1), 6021 0, 0 6022 }, 6023 { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 6024 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC), 6025 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED) 6026 }, 6027 { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 6028 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED), 6029 0, 0 6030 }, 6031 { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 6032 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC), 6033 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED) 6034 }, 6035 { "GDS_OA_PHY_PHY_CMD_RAM_MEM", 6036 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 6037 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC), 6038 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED) 6039 }, 6040 { "GDS_OA_PHY_PHY_DATA_RAM_MEM", 6041 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 6042 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED), 6043 0, 0 6044 }, 6045 { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM", 6046 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6047 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC), 6048 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED) 6049 }, 6050 { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM", 6051 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6052 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC), 6053 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED) 6054 }, 6055 { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM", 6056 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6057 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC), 6058 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED) 6059 }, 6060 { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM", 6061 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6062 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC), 6063 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED) 6064 }, 6065 { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 6066 SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT), 6067 0, 0 6068 }, 6069 { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6070 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT), 6071 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT) 6072 }, 6073 { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6074 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT), 6075 0, 0 6076 }, 6077 { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6078 SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT), 6079 0, 0 6080 }, 6081 { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6082 SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT), 6083 0, 0 6084 }, 6085 { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6086 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT), 6087 0, 0 6088 }, 6089 { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 6090 SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT), 6091 0, 0 6092 }, 6093 { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 6094 SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT), 6095 0, 0 6096 }, 6097 { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6098 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT), 6099 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT) 6100 }, 6101 { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6102 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT), 6103 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT) 6104 }, 6105 { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6106 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT), 6107 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT) 6108 }, 6109 { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6110 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT), 6111 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT) 6112 }, 6113 { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6114 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT), 6115 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT) 6116 }, 6117 { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6118 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT), 6119 0, 0 6120 }, 6121 { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6122 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT), 6123 0, 0 6124 }, 6125 { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6126 SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT), 6127 0, 0 6128 }, 6129 { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6130 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT), 6131 0, 0 6132 }, 6133 { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6134 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT), 6135 0, 0 6136 }, 6137 { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6138 SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT), 6139 0, 0 6140 }, 6141 { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6142 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT), 6143 0, 0 6144 }, 6145 { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6146 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT), 6147 0, 0 6148 }, 6149 { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6150 SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT), 6151 0, 0 6152 }, 6153 { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6154 SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT), 6155 0, 0 6156 }, 6157 { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6158 SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT), 6159 0, 0 6160 }, 6161 { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6162 SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT), 6163 0, 0 6164 }, 6165 { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6166 SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT), 6167 0, 0 6168 }, 6169 { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 6170 SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT), 6171 0, 0 6172 }, 6173 { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6174 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT), 6175 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT) 6176 }, 6177 { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6178 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT), 6179 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT) 6180 }, 6181 { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6182 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT), 6183 0, 0 6184 }, 6185 { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6186 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT), 6187 0, 0 6188 }, 6189 { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6190 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT), 6191 0, 0 6192 }, 6193 { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6194 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT), 6195 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT) 6196 }, 6197 { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6198 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT), 6199 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT) 6200 }, 6201 { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6202 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT), 6203 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT) 6204 }, 6205 { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6206 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT), 6207 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT) 6208 }, 6209 { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6210 SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT), 6211 0, 0 6212 }, 6213 { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6214 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT), 6215 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT) 6216 }, 6217 { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6218 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT), 6219 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT) 6220 }, 6221 { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6222 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT), 6223 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT) 6224 }, 6225 { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6226 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT), 6227 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT) 6228 }, 6229 { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6230 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT), 6231 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT) 6232 }, 6233 { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6234 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT), 6235 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT) 6236 }, 6237 { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6238 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT), 6239 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT) 6240 }, 6241 { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6242 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT), 6243 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT) 6244 }, 6245 { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6246 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT), 6247 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT) 6248 }, 6249 { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6250 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT), 6251 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT) 6252 }, 6253 { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6254 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT), 6255 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT) 6256 }, 6257 { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6258 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT), 6259 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT) 6260 }, 6261 { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6262 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT), 6263 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT) 6264 }, 6265 { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6266 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT), 6267 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT) 6268 }, 6269 { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6270 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT), 6271 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT) 6272 }, 6273 { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6274 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT), 6275 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT) 6276 }, 6277 { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6278 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT), 6279 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT) 6280 }, 6281 { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6282 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT), 6283 0, 0 6284 }, 6285 { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6286 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT), 6287 0, 0 6288 }, 6289 { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6290 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT), 6291 0, 0 6292 }, 6293 { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6294 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT), 6295 0, 0 6296 }, 6297 { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6298 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT), 6299 0, 0 6300 }, 6301 { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6302 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT), 6303 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT) 6304 }, 6305 { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6306 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT), 6307 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT) 6308 }, 6309 { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6310 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT), 6311 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT) 6312 }, 6313 { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6314 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT), 6315 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT) 6316 }, 6317 { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6318 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT), 6319 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT) 6320 }, 6321 { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6322 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT), 6323 0, 0 6324 }, 6325 { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6326 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT), 6327 0, 0 6328 }, 6329 { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6330 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT), 6331 0, 0 6332 }, 6333 { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6334 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT), 6335 0, 0 6336 }, 6337 { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6338 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT), 6339 0, 0 6340 }, 6341 { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6342 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), 6343 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT) 6344 }, 6345 { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6346 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), 6347 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT) 6348 }, 6349 { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6350 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), 6351 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT) 6352 }, 6353 { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6354 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT), 6355 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT) 6356 }, 6357 { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6358 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT), 6359 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT) 6360 }, 6361 { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6362 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), 6363 0, 0 6364 }, 6365 { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6366 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), 6367 0, 0 6368 }, 6369 { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6370 SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT), 6371 0, 0 6372 }, 6373 { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6374 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT), 6375 0, 0 6376 }, 6377 { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6378 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT), 6379 0, 0 6380 }, 6381 { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6382 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), 6383 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT) 6384 }, 6385 { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6386 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), 6387 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT) 6388 }, 6389 { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6390 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), 6391 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT) 6392 }, 6393 { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6394 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), 6395 0, 0 6396 }, 6397 { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6398 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), 6399 0, 0 6400 }, 6401 { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6402 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT), 6403 0, 0 6404 }, 6405 { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6406 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT), 6407 0, 0 6408 }, 6409 { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6410 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT), 6411 0, 0 6412 }, 6413 { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6414 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT), 6415 0, 0 6416 } 6417 }; 6418 6419 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 6420 void *inject_if) 6421 { 6422 struct ras_inject_if *info = (struct ras_inject_if *)inject_if; 6423 int ret; 6424 struct ta_ras_trigger_error_input block_info = { 0 }; 6425 6426 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6427 return -EINVAL; 6428 6429 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks)) 6430 return -EINVAL; 6431 6432 if (!ras_gfx_subblocks[info->head.sub_block_index].name) 6433 return -EPERM; 6434 6435 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type & 6436 info->head.type)) { 6437 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n", 6438 ras_gfx_subblocks[info->head.sub_block_index].name, 6439 info->head.type); 6440 return -EPERM; 6441 } 6442 6443 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type & 6444 info->head.type)) { 6445 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n", 6446 ras_gfx_subblocks[info->head.sub_block_index].name, 6447 info->head.type); 6448 return -EPERM; 6449 } 6450 6451 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block); 6452 block_info.sub_block_index = 6453 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock; 6454 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type); 6455 block_info.address = info->address; 6456 block_info.value = info->value; 6457 6458 mutex_lock(&adev->grbm_idx_mutex); 6459 ret = psp_ras_trigger_error(&adev->psp, &block_info); 6460 mutex_unlock(&adev->grbm_idx_mutex); 6461 6462 return ret; 6463 } 6464 6465 static const char *vml2_mems[] = { 6466 "UTC_VML2_BANK_CACHE_0_BIGK_MEM0", 6467 "UTC_VML2_BANK_CACHE_0_BIGK_MEM1", 6468 "UTC_VML2_BANK_CACHE_0_4K_MEM0", 6469 "UTC_VML2_BANK_CACHE_0_4K_MEM1", 6470 "UTC_VML2_BANK_CACHE_1_BIGK_MEM0", 6471 "UTC_VML2_BANK_CACHE_1_BIGK_MEM1", 6472 "UTC_VML2_BANK_CACHE_1_4K_MEM0", 6473 "UTC_VML2_BANK_CACHE_1_4K_MEM1", 6474 "UTC_VML2_BANK_CACHE_2_BIGK_MEM0", 6475 "UTC_VML2_BANK_CACHE_2_BIGK_MEM1", 6476 "UTC_VML2_BANK_CACHE_2_4K_MEM0", 6477 "UTC_VML2_BANK_CACHE_2_4K_MEM1", 6478 "UTC_VML2_BANK_CACHE_3_BIGK_MEM0", 6479 "UTC_VML2_BANK_CACHE_3_BIGK_MEM1", 6480 "UTC_VML2_BANK_CACHE_3_4K_MEM0", 6481 "UTC_VML2_BANK_CACHE_3_4K_MEM1", 6482 }; 6483 6484 static const char *vml2_walker_mems[] = { 6485 "UTC_VML2_CACHE_PDE0_MEM0", 6486 "UTC_VML2_CACHE_PDE0_MEM1", 6487 "UTC_VML2_CACHE_PDE1_MEM0", 6488 "UTC_VML2_CACHE_PDE1_MEM1", 6489 "UTC_VML2_CACHE_PDE2_MEM0", 6490 "UTC_VML2_CACHE_PDE2_MEM1", 6491 "UTC_VML2_RDIF_LOG_FIFO", 6492 }; 6493 6494 static const char *atc_l2_cache_2m_mems[] = { 6495 "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM", 6496 "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM", 6497 "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM", 6498 "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM", 6499 }; 6500 6501 static const char *atc_l2_cache_4k_mems[] = { 6502 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0", 6503 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1", 6504 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2", 6505 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3", 6506 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4", 6507 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5", 6508 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6", 6509 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7", 6510 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0", 6511 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1", 6512 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2", 6513 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3", 6514 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4", 6515 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5", 6516 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6", 6517 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7", 6518 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0", 6519 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1", 6520 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2", 6521 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3", 6522 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4", 6523 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5", 6524 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6", 6525 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7", 6526 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0", 6527 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1", 6528 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2", 6529 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3", 6530 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4", 6531 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5", 6532 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6", 6533 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7", 6534 }; 6535 6536 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev, 6537 struct ras_err_data *err_data) 6538 { 6539 uint32_t i, data; 6540 uint32_t sec_count, ded_count; 6541 6542 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6543 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6544 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6545 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6546 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6547 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6548 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6549 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6550 6551 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6552 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6553 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6554 6555 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT); 6556 if (sec_count) { 6557 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6558 "SEC %d\n", i, vml2_mems[i], sec_count); 6559 err_data->ce_count += sec_count; 6560 } 6561 6562 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT); 6563 if (ded_count) { 6564 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6565 "DED %d\n", i, vml2_mems[i], ded_count); 6566 err_data->ue_count += ded_count; 6567 } 6568 } 6569 6570 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6571 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6572 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6573 6574 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6575 SEC_COUNT); 6576 if (sec_count) { 6577 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6578 "SEC %d\n", i, vml2_walker_mems[i], sec_count); 6579 err_data->ce_count += sec_count; 6580 } 6581 6582 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6583 DED_COUNT); 6584 if (ded_count) { 6585 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6586 "DED %d\n", i, vml2_walker_mems[i], ded_count); 6587 err_data->ue_count += ded_count; 6588 } 6589 } 6590 6591 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6592 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6593 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6594 6595 sec_count = (data & 0x00006000L) >> 0xd; 6596 if (sec_count) { 6597 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6598 "SEC %d\n", i, atc_l2_cache_2m_mems[i], 6599 sec_count); 6600 err_data->ce_count += sec_count; 6601 } 6602 } 6603 6604 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6605 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6606 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6607 6608 sec_count = (data & 0x00006000L) >> 0xd; 6609 if (sec_count) { 6610 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6611 "SEC %d\n", i, atc_l2_cache_4k_mems[i], 6612 sec_count); 6613 err_data->ce_count += sec_count; 6614 } 6615 6616 ded_count = (data & 0x00018000L) >> 0xf; 6617 if (ded_count) { 6618 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6619 "DED %d\n", i, atc_l2_cache_4k_mems[i], 6620 ded_count); 6621 err_data->ue_count += ded_count; 6622 } 6623 } 6624 6625 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6626 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6627 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6628 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6629 6630 return 0; 6631 } 6632 6633 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev, 6634 const struct soc15_reg_entry *reg, 6635 uint32_t se_id, uint32_t inst_id, uint32_t value, 6636 uint32_t *sec_count, uint32_t *ded_count) 6637 { 6638 uint32_t i; 6639 uint32_t sec_cnt, ded_cnt; 6640 6641 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) { 6642 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset || 6643 gfx_v9_0_ras_fields[i].seg != reg->seg || 6644 gfx_v9_0_ras_fields[i].inst != reg->inst) 6645 continue; 6646 6647 sec_cnt = (value & 6648 gfx_v9_0_ras_fields[i].sec_count_mask) >> 6649 gfx_v9_0_ras_fields[i].sec_count_shift; 6650 if (sec_cnt) { 6651 dev_info(adev->dev, "GFX SubBlock %s, " 6652 "Instance[%d][%d], SEC %d\n", 6653 gfx_v9_0_ras_fields[i].name, 6654 se_id, inst_id, 6655 sec_cnt); 6656 *sec_count += sec_cnt; 6657 } 6658 6659 ded_cnt = (value & 6660 gfx_v9_0_ras_fields[i].ded_count_mask) >> 6661 gfx_v9_0_ras_fields[i].ded_count_shift; 6662 if (ded_cnt) { 6663 dev_info(adev->dev, "GFX SubBlock %s, " 6664 "Instance[%d][%d], DED %d\n", 6665 gfx_v9_0_ras_fields[i].name, 6666 se_id, inst_id, 6667 ded_cnt); 6668 *ded_count += ded_cnt; 6669 } 6670 } 6671 6672 return 0; 6673 } 6674 6675 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev) 6676 { 6677 int i, j, k; 6678 6679 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6680 return; 6681 6682 /* read back registers to clear the counters */ 6683 mutex_lock(&adev->grbm_idx_mutex); 6684 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6685 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6686 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6687 gfx_v9_0_select_se_sh(adev, j, 0x0, k); 6688 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6689 } 6690 } 6691 } 6692 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000); 6693 mutex_unlock(&adev->grbm_idx_mutex); 6694 6695 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6696 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6697 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6698 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6699 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6700 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6701 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6702 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6703 6704 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6705 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6706 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6707 } 6708 6709 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6710 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6711 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6712 } 6713 6714 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6715 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6716 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6717 } 6718 6719 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6720 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6721 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6722 } 6723 6724 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6725 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6726 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6727 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6728 } 6729 6730 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 6731 void *ras_error_status) 6732 { 6733 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; 6734 uint32_t sec_count = 0, ded_count = 0; 6735 uint32_t i, j, k; 6736 uint32_t reg_value; 6737 6738 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6739 return -EINVAL; 6740 6741 err_data->ue_count = 0; 6742 err_data->ce_count = 0; 6743 6744 mutex_lock(&adev->grbm_idx_mutex); 6745 6746 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6747 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6748 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6749 gfx_v9_0_select_se_sh(adev, j, 0, k); 6750 reg_value = 6751 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6752 if (reg_value) 6753 gfx_v9_0_ras_error_count(adev, 6754 &gfx_v9_0_edc_counter_regs[i], 6755 j, k, reg_value, 6756 &sec_count, &ded_count); 6757 } 6758 } 6759 } 6760 6761 err_data->ce_count += sec_count; 6762 err_data->ue_count += ded_count; 6763 6764 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 6765 mutex_unlock(&adev->grbm_idx_mutex); 6766 6767 gfx_v9_0_query_utc_edc_status(adev, err_data); 6768 6769 return 0; 6770 } 6771 6772 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring) 6773 { 6774 const unsigned int cp_coher_cntl = 6775 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) | 6776 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) | 6777 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) | 6778 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) | 6779 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1); 6780 6781 /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */ 6782 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5)); 6783 amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */ 6784 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6785 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */ 6786 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6787 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 6788 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ 6789 } 6790 6791 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring, 6792 uint32_t pipe, bool enable) 6793 { 6794 struct amdgpu_device *adev = ring->adev; 6795 uint32_t val; 6796 uint32_t wcl_cs_reg; 6797 6798 /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */ 6799 val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT; 6800 6801 switch (pipe) { 6802 case 0: 6803 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0); 6804 break; 6805 case 1: 6806 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1); 6807 break; 6808 case 2: 6809 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2); 6810 break; 6811 case 3: 6812 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3); 6813 break; 6814 default: 6815 DRM_DEBUG("invalid pipe %d\n", pipe); 6816 return; 6817 } 6818 6819 amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val); 6820 6821 } 6822 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable) 6823 { 6824 struct amdgpu_device *adev = ring->adev; 6825 uint32_t val; 6826 int i; 6827 6828 6829 /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit 6830 * number of gfx waves. Setting 5 bit will make sure gfx only gets 6831 * around 25% of gpu resources. 6832 */ 6833 val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT; 6834 amdgpu_ring_emit_wreg(ring, 6835 SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX), 6836 val); 6837 6838 /* Restrict waves for normal/low priority compute queues as well 6839 * to get best QoS for high priority compute jobs. 6840 * 6841 * amdgpu controls only 1st ME(0-3 CS pipes). 6842 */ 6843 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 6844 if (i != ring->pipe) 6845 gfx_v9_0_emit_wave_limit_cs(ring, i, enable); 6846 6847 } 6848 } 6849 6850 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = { 6851 .name = "gfx_v9_0", 6852 .early_init = gfx_v9_0_early_init, 6853 .late_init = gfx_v9_0_late_init, 6854 .sw_init = gfx_v9_0_sw_init, 6855 .sw_fini = gfx_v9_0_sw_fini, 6856 .hw_init = gfx_v9_0_hw_init, 6857 .hw_fini = gfx_v9_0_hw_fini, 6858 .suspend = gfx_v9_0_suspend, 6859 .resume = gfx_v9_0_resume, 6860 .is_idle = gfx_v9_0_is_idle, 6861 .wait_for_idle = gfx_v9_0_wait_for_idle, 6862 .soft_reset = gfx_v9_0_soft_reset, 6863 .set_clockgating_state = gfx_v9_0_set_clockgating_state, 6864 .set_powergating_state = gfx_v9_0_set_powergating_state, 6865 .get_clockgating_state = gfx_v9_0_get_clockgating_state, 6866 }; 6867 6868 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = { 6869 .type = AMDGPU_RING_TYPE_GFX, 6870 .align_mask = 0xff, 6871 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6872 .support_64bit_ptrs = true, 6873 .vmhub = AMDGPU_GFXHUB_0, 6874 .get_rptr = gfx_v9_0_ring_get_rptr_gfx, 6875 .get_wptr = gfx_v9_0_ring_get_wptr_gfx, 6876 .set_wptr = gfx_v9_0_ring_set_wptr_gfx, 6877 .emit_frame_size = /* totally 242 maximum if 16 IBs */ 6878 5 + /* COND_EXEC */ 6879 7 + /* PIPELINE_SYNC */ 6880 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6881 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6882 2 + /* VM_FLUSH */ 6883 8 + /* FENCE for VM_FLUSH */ 6884 20 + /* GDS switch */ 6885 4 + /* double SWITCH_BUFFER, 6886 the first COND_EXEC jump to the place just 6887 prior to this double SWITCH_BUFFER */ 6888 5 + /* COND_EXEC */ 6889 7 + /* HDP_flush */ 6890 4 + /* VGT_flush */ 6891 14 + /* CE_META */ 6892 31 + /* DE_META */ 6893 3 + /* CNTX_CTRL */ 6894 5 + /* HDP_INVL */ 6895 8 + 8 + /* FENCE x2 */ 6896 2 + /* SWITCH_BUFFER */ 6897 7, /* gfx_v9_0_emit_mem_sync */ 6898 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */ 6899 .emit_ib = gfx_v9_0_ring_emit_ib_gfx, 6900 .emit_fence = gfx_v9_0_ring_emit_fence, 6901 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6902 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6903 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6904 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6905 .test_ring = gfx_v9_0_ring_test_ring, 6906 .test_ib = gfx_v9_0_ring_test_ib, 6907 .insert_nop = amdgpu_ring_insert_nop, 6908 .pad_ib = amdgpu_ring_generic_pad_ib, 6909 .emit_switch_buffer = gfx_v9_ring_emit_sb, 6910 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl, 6911 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec, 6912 .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec, 6913 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl, 6914 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6915 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6916 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6917 .soft_recovery = gfx_v9_0_ring_soft_recovery, 6918 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6919 }; 6920 6921 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { 6922 .type = AMDGPU_RING_TYPE_COMPUTE, 6923 .align_mask = 0xff, 6924 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6925 .support_64bit_ptrs = true, 6926 .vmhub = AMDGPU_GFXHUB_0, 6927 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6928 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6929 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6930 .emit_frame_size = 6931 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6932 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6933 5 + /* hdp invalidate */ 6934 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6935 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6936 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6937 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6938 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */ 6939 7 + /* gfx_v9_0_emit_mem_sync */ 6940 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */ 6941 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */ 6942 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6943 .emit_ib = gfx_v9_0_ring_emit_ib_compute, 6944 .emit_fence = gfx_v9_0_ring_emit_fence, 6945 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6946 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6947 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6948 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6949 .test_ring = gfx_v9_0_ring_test_ring, 6950 .test_ib = gfx_v9_0_ring_test_ib, 6951 .insert_nop = amdgpu_ring_insert_nop, 6952 .pad_ib = amdgpu_ring_generic_pad_ib, 6953 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6954 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6955 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6956 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6957 .emit_wave_limit = gfx_v9_0_emit_wave_limit, 6958 }; 6959 6960 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = { 6961 .type = AMDGPU_RING_TYPE_KIQ, 6962 .align_mask = 0xff, 6963 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6964 .support_64bit_ptrs = true, 6965 .vmhub = AMDGPU_GFXHUB_0, 6966 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6967 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6968 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6969 .emit_frame_size = 6970 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6971 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6972 5 + /* hdp invalidate */ 6973 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6974 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6975 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6976 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6977 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 6978 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6979 .emit_fence = gfx_v9_0_ring_emit_fence_kiq, 6980 .test_ring = gfx_v9_0_ring_test_ring, 6981 .insert_nop = amdgpu_ring_insert_nop, 6982 .pad_ib = amdgpu_ring_generic_pad_ib, 6983 .emit_rreg = gfx_v9_0_ring_emit_rreg, 6984 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6985 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6986 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6987 }; 6988 6989 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev) 6990 { 6991 int i; 6992 6993 adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq; 6994 6995 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 6996 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx; 6997 6998 for (i = 0; i < adev->gfx.num_compute_rings; i++) 6999 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute; 7000 } 7001 7002 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = { 7003 .set = gfx_v9_0_set_eop_interrupt_state, 7004 .process = gfx_v9_0_eop_irq, 7005 }; 7006 7007 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = { 7008 .set = gfx_v9_0_set_priv_reg_fault_state, 7009 .process = gfx_v9_0_priv_reg_irq, 7010 }; 7011 7012 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = { 7013 .set = gfx_v9_0_set_priv_inst_fault_state, 7014 .process = gfx_v9_0_priv_inst_irq, 7015 }; 7016 7017 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = { 7018 .set = gfx_v9_0_set_cp_ecc_error_state, 7019 .process = amdgpu_gfx_cp_ecc_error_irq, 7020 }; 7021 7022 7023 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev) 7024 { 7025 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 7026 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs; 7027 7028 adev->gfx.priv_reg_irq.num_types = 1; 7029 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs; 7030 7031 adev->gfx.priv_inst_irq.num_types = 1; 7032 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs; 7033 7034 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/ 7035 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs; 7036 } 7037 7038 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev) 7039 { 7040 switch (adev->ip_versions[GC_HWIP][0]) { 7041 case IP_VERSION(9, 0, 1): 7042 case IP_VERSION(9, 2, 1): 7043 case IP_VERSION(9, 4, 0): 7044 case IP_VERSION(9, 2, 2): 7045 case IP_VERSION(9, 1, 0): 7046 case IP_VERSION(9, 4, 1): 7047 case IP_VERSION(9, 3, 0): 7048 case IP_VERSION(9, 4, 2): 7049 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs; 7050 break; 7051 default: 7052 break; 7053 } 7054 } 7055 7056 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev) 7057 { 7058 /* init asci gds info */ 7059 switch (adev->ip_versions[GC_HWIP][0]) { 7060 case IP_VERSION(9, 0, 1): 7061 case IP_VERSION(9, 2, 1): 7062 case IP_VERSION(9, 4, 0): 7063 adev->gds.gds_size = 0x10000; 7064 break; 7065 case IP_VERSION(9, 2, 2): 7066 case IP_VERSION(9, 1, 0): 7067 case IP_VERSION(9, 4, 1): 7068 adev->gds.gds_size = 0x1000; 7069 break; 7070 case IP_VERSION(9, 4, 2): 7071 /* aldebaran removed all the GDS internal memory, 7072 * only support GWS opcode in kernel, like barrier 7073 * semaphore.etc */ 7074 adev->gds.gds_size = 0; 7075 break; 7076 default: 7077 adev->gds.gds_size = 0x10000; 7078 break; 7079 } 7080 7081 switch (adev->ip_versions[GC_HWIP][0]) { 7082 case IP_VERSION(9, 0, 1): 7083 case IP_VERSION(9, 4, 0): 7084 adev->gds.gds_compute_max_wave_id = 0x7ff; 7085 break; 7086 case IP_VERSION(9, 2, 1): 7087 adev->gds.gds_compute_max_wave_id = 0x27f; 7088 break; 7089 case IP_VERSION(9, 2, 2): 7090 case IP_VERSION(9, 1, 0): 7091 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 7092 adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */ 7093 else 7094 adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */ 7095 break; 7096 case IP_VERSION(9, 4, 1): 7097 adev->gds.gds_compute_max_wave_id = 0xfff; 7098 break; 7099 case IP_VERSION(9, 4, 2): 7100 /* deprecated for Aldebaran, no usage at all */ 7101 adev->gds.gds_compute_max_wave_id = 0; 7102 break; 7103 default: 7104 /* this really depends on the chip */ 7105 adev->gds.gds_compute_max_wave_id = 0x7ff; 7106 break; 7107 } 7108 7109 adev->gds.gws_size = 64; 7110 adev->gds.oa_size = 16; 7111 } 7112 7113 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev, 7114 u32 bitmap) 7115 { 7116 u32 data; 7117 7118 if (!bitmap) 7119 return; 7120 7121 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 7122 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 7123 7124 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data); 7125 } 7126 7127 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev) 7128 { 7129 u32 data, mask; 7130 7131 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG); 7132 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG); 7133 7134 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 7135 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 7136 7137 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); 7138 7139 return (~data) & mask; 7140 } 7141 7142 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 7143 struct amdgpu_cu_info *cu_info) 7144 { 7145 int i, j, k, counter, active_cu_number = 0; 7146 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0; 7147 unsigned disable_masks[4 * 4]; 7148 7149 if (!adev || !cu_info) 7150 return -EINVAL; 7151 7152 /* 7153 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs 7154 */ 7155 if (adev->gfx.config.max_shader_engines * 7156 adev->gfx.config.max_sh_per_se > 16) 7157 return -EINVAL; 7158 7159 amdgpu_gfx_parse_disable_cu(disable_masks, 7160 adev->gfx.config.max_shader_engines, 7161 adev->gfx.config.max_sh_per_se); 7162 7163 mutex_lock(&adev->grbm_idx_mutex); 7164 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 7165 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 7166 mask = 1; 7167 ao_bitmap = 0; 7168 counter = 0; 7169 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 7170 gfx_v9_0_set_user_cu_inactive_bitmap( 7171 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]); 7172 bitmap = gfx_v9_0_get_cu_active_bitmap(adev); 7173 7174 /* 7175 * The bitmap(and ao_cu_bitmap) in cu_info structure is 7176 * 4x4 size array, and it's usually suitable for Vega 7177 * ASICs which has 4*2 SE/SH layout. 7178 * But for Arcturus, SE/SH layout is changed to 8*1. 7179 * To mostly reduce the impact, we make it compatible 7180 * with current bitmap array as below: 7181 * SE4,SH0 --> bitmap[0][1] 7182 * SE5,SH0 --> bitmap[1][1] 7183 * SE6,SH0 --> bitmap[2][1] 7184 * SE7,SH0 --> bitmap[3][1] 7185 */ 7186 cu_info->bitmap[i % 4][j + i / 4] = bitmap; 7187 7188 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 7189 if (bitmap & mask) { 7190 if (counter < adev->gfx.config.max_cu_per_sh) 7191 ao_bitmap |= mask; 7192 counter ++; 7193 } 7194 mask <<= 1; 7195 } 7196 active_cu_number += counter; 7197 if (i < 2 && j < 2) 7198 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8)); 7199 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap; 7200 } 7201 } 7202 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 7203 mutex_unlock(&adev->grbm_idx_mutex); 7204 7205 cu_info->number = active_cu_number; 7206 cu_info->ao_cu_mask = ao_cu_mask; 7207 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 7208 7209 return 0; 7210 } 7211 7212 const struct amdgpu_ip_block_version gfx_v9_0_ip_block = 7213 { 7214 .type = AMD_IP_BLOCK_TYPE_GFX, 7215 .major = 9, 7216 .minor = 0, 7217 .rev = 0, 7218 .funcs = &gfx_v9_0_ip_funcs, 7219 }; 7220