1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/delay.h> 25 #include <linux/kernel.h> 26 #include <linux/firmware.h> 27 #include <linux/module.h> 28 #include <linux/pci.h> 29 30 #include "amdgpu.h" 31 #include "amdgpu_gfx.h" 32 #include "soc15.h" 33 #include "soc15d.h" 34 #include "amdgpu_atomfirmware.h" 35 #include "amdgpu_pm.h" 36 37 #include "gc/gc_9_0_offset.h" 38 #include "gc/gc_9_0_sh_mask.h" 39 40 #include "vega10_enum.h" 41 #include "hdp/hdp_4_0_offset.h" 42 43 #include "soc15_common.h" 44 #include "clearstate_gfx9.h" 45 #include "v9_structs.h" 46 47 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h" 48 49 #include "amdgpu_ras.h" 50 51 #define GFX9_NUM_GFX_RINGS 1 52 #define GFX9_MEC_HPD_SIZE 4096 53 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L 54 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L 55 56 #define mmPWR_MISC_CNTL_STATUS 0x0183 57 #define mmPWR_MISC_CNTL_STATUS_BASE_IDX 0 58 #define PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN__SHIFT 0x0 59 #define PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT 0x1 60 #define PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK 0x00000001L 61 #define PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK 0x00000006L 62 63 #define mmGCEA_PROBE_MAP 0x070c 64 #define mmGCEA_PROBE_MAP_BASE_IDX 0 65 66 MODULE_FIRMWARE("amdgpu/vega10_ce.bin"); 67 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin"); 68 MODULE_FIRMWARE("amdgpu/vega10_me.bin"); 69 MODULE_FIRMWARE("amdgpu/vega10_mec.bin"); 70 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin"); 71 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin"); 72 73 MODULE_FIRMWARE("amdgpu/vega12_ce.bin"); 74 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin"); 75 MODULE_FIRMWARE("amdgpu/vega12_me.bin"); 76 MODULE_FIRMWARE("amdgpu/vega12_mec.bin"); 77 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin"); 78 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin"); 79 80 MODULE_FIRMWARE("amdgpu/vega20_ce.bin"); 81 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin"); 82 MODULE_FIRMWARE("amdgpu/vega20_me.bin"); 83 MODULE_FIRMWARE("amdgpu/vega20_mec.bin"); 84 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin"); 85 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin"); 86 87 MODULE_FIRMWARE("amdgpu/raven_ce.bin"); 88 MODULE_FIRMWARE("amdgpu/raven_pfp.bin"); 89 MODULE_FIRMWARE("amdgpu/raven_me.bin"); 90 MODULE_FIRMWARE("amdgpu/raven_mec.bin"); 91 MODULE_FIRMWARE("amdgpu/raven_mec2.bin"); 92 MODULE_FIRMWARE("amdgpu/raven_rlc.bin"); 93 94 MODULE_FIRMWARE("amdgpu/picasso_ce.bin"); 95 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin"); 96 MODULE_FIRMWARE("amdgpu/picasso_me.bin"); 97 MODULE_FIRMWARE("amdgpu/picasso_mec.bin"); 98 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin"); 99 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin"); 100 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin"); 101 102 MODULE_FIRMWARE("amdgpu/raven2_ce.bin"); 103 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin"); 104 MODULE_FIRMWARE("amdgpu/raven2_me.bin"); 105 MODULE_FIRMWARE("amdgpu/raven2_mec.bin"); 106 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin"); 107 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin"); 108 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin"); 109 110 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin"); 111 MODULE_FIRMWARE("amdgpu/arcturus_mec2.bin"); 112 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin"); 113 114 MODULE_FIRMWARE("amdgpu/renoir_ce.bin"); 115 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin"); 116 MODULE_FIRMWARE("amdgpu/renoir_me.bin"); 117 MODULE_FIRMWARE("amdgpu/renoir_mec.bin"); 118 MODULE_FIRMWARE("amdgpu/renoir_mec2.bin"); 119 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin"); 120 121 #define mmTCP_CHAN_STEER_0_ARCT 0x0b03 122 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0 123 #define mmTCP_CHAN_STEER_1_ARCT 0x0b04 124 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0 125 #define mmTCP_CHAN_STEER_2_ARCT 0x0b09 126 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0 127 #define mmTCP_CHAN_STEER_3_ARCT 0x0b0a 128 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0 129 #define mmTCP_CHAN_STEER_4_ARCT 0x0b0b 130 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0 131 #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c 132 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0 133 134 struct ras_gfx_subblock_reg { 135 const char *name; 136 uint32_t hwip; 137 uint32_t inst; 138 uint32_t seg; 139 uint32_t reg_offset; 140 uint32_t sec_count_mask; 141 uint32_t sec_count_shift; 142 uint32_t ded_count_mask; 143 uint32_t ded_count_shift; 144 }; 145 146 enum ta_ras_gfx_subblock { 147 /*CPC*/ 148 TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0, 149 TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START, 150 TA_RAS_BLOCK__GFX_CPC_UCODE, 151 TA_RAS_BLOCK__GFX_DC_STATE_ME1, 152 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1, 153 TA_RAS_BLOCK__GFX_DC_RESTORE_ME1, 154 TA_RAS_BLOCK__GFX_DC_STATE_ME2, 155 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2, 156 TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 157 TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 158 /* CPF*/ 159 TA_RAS_BLOCK__GFX_CPF_INDEX_START, 160 TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START, 161 TA_RAS_BLOCK__GFX_CPF_ROQ_ME1, 162 TA_RAS_BLOCK__GFX_CPF_TAG, 163 TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG, 164 /* CPG*/ 165 TA_RAS_BLOCK__GFX_CPG_INDEX_START, 166 TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START, 167 TA_RAS_BLOCK__GFX_CPG_DMA_TAG, 168 TA_RAS_BLOCK__GFX_CPG_TAG, 169 TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG, 170 /* GDS*/ 171 TA_RAS_BLOCK__GFX_GDS_INDEX_START, 172 TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START, 173 TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE, 174 TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM, 175 TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM, 176 TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 177 TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 178 /* SPI*/ 179 TA_RAS_BLOCK__GFX_SPI_SR_MEM, 180 /* SQ*/ 181 TA_RAS_BLOCK__GFX_SQ_INDEX_START, 182 TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START, 183 TA_RAS_BLOCK__GFX_SQ_LDS_D, 184 TA_RAS_BLOCK__GFX_SQ_LDS_I, 185 TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/ 186 TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR, 187 /* SQC (3 ranges)*/ 188 TA_RAS_BLOCK__GFX_SQC_INDEX_START, 189 /* SQC range 0*/ 190 TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START, 191 TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO = 192 TA_RAS_BLOCK__GFX_SQC_INDEX0_START, 193 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 194 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO, 195 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 196 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO, 197 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 198 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 199 TA_RAS_BLOCK__GFX_SQC_INDEX0_END = 200 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 201 /* SQC range 1*/ 202 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 203 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM = 204 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 205 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 206 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO, 207 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM, 208 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM, 209 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO, 210 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO, 211 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 212 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 213 TA_RAS_BLOCK__GFX_SQC_INDEX1_END = 214 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 215 /* SQC range 2*/ 216 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 217 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM = 218 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 219 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 220 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO, 221 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM, 222 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM, 223 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO, 224 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO, 225 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 226 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 227 TA_RAS_BLOCK__GFX_SQC_INDEX2_END = 228 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 229 TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END, 230 /* TA*/ 231 TA_RAS_BLOCK__GFX_TA_INDEX_START, 232 TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START, 233 TA_RAS_BLOCK__GFX_TA_FS_AFIFO, 234 TA_RAS_BLOCK__GFX_TA_FL_LFIFO, 235 TA_RAS_BLOCK__GFX_TA_FX_LFIFO, 236 TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 237 TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 238 /* TCA*/ 239 TA_RAS_BLOCK__GFX_TCA_INDEX_START, 240 TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START, 241 TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 242 TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 243 /* TCC (5 sub-ranges)*/ 244 TA_RAS_BLOCK__GFX_TCC_INDEX_START, 245 /* TCC range 0*/ 246 TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START, 247 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START, 248 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1, 249 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0, 250 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1, 251 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0, 252 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1, 253 TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG, 254 TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 255 TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 256 /* TCC range 1*/ 257 TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 258 TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 259 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 260 TA_RAS_BLOCK__GFX_TCC_INDEX1_END = 261 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 262 /* TCC range 2*/ 263 TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 264 TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 265 TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL, 266 TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO, 267 TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN, 268 TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ, 269 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO, 270 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM, 271 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 272 TA_RAS_BLOCK__GFX_TCC_INDEX2_END = 273 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 274 /* TCC range 3*/ 275 TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 276 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 277 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 278 TA_RAS_BLOCK__GFX_TCC_INDEX3_END = 279 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 280 /* TCC range 4*/ 281 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 282 TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN = 283 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 284 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 285 TA_RAS_BLOCK__GFX_TCC_INDEX4_END = 286 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 287 TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END, 288 /* TCI*/ 289 TA_RAS_BLOCK__GFX_TCI_WRITE_RAM, 290 /* TCP*/ 291 TA_RAS_BLOCK__GFX_TCP_INDEX_START, 292 TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START, 293 TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM, 294 TA_RAS_BLOCK__GFX_TCP_CMD_FIFO, 295 TA_RAS_BLOCK__GFX_TCP_VM_FIFO, 296 TA_RAS_BLOCK__GFX_TCP_DB_RAM, 297 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0, 298 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 299 TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 300 /* TD*/ 301 TA_RAS_BLOCK__GFX_TD_INDEX_START, 302 TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START, 303 TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI, 304 TA_RAS_BLOCK__GFX_TD_CS_FIFO, 305 TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO, 306 /* EA (3 sub-ranges)*/ 307 TA_RAS_BLOCK__GFX_EA_INDEX_START, 308 /* EA range 0*/ 309 TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START, 310 TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START, 311 TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM, 312 TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM, 313 TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM, 314 TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM, 315 TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM, 316 TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM, 317 TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 318 TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 319 /* EA range 1*/ 320 TA_RAS_BLOCK__GFX_EA_INDEX1_START, 321 TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START, 322 TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM, 323 TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM, 324 TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM, 325 TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM, 326 TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM, 327 TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 328 TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 329 /* EA range 2*/ 330 TA_RAS_BLOCK__GFX_EA_INDEX2_START, 331 TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START, 332 TA_RAS_BLOCK__GFX_EA_MAM_D1MEM, 333 TA_RAS_BLOCK__GFX_EA_MAM_D2MEM, 334 TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 335 TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 336 TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END, 337 /* UTC VM L2 bank*/ 338 TA_RAS_BLOCK__UTC_VML2_BANK_CACHE, 339 /* UTC VM walker*/ 340 TA_RAS_BLOCK__UTC_VML2_WALKER, 341 /* UTC ATC L2 2MB cache*/ 342 TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK, 343 /* UTC ATC L2 4KB cache*/ 344 TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK, 345 TA_RAS_BLOCK__GFX_MAX 346 }; 347 348 struct ras_gfx_subblock { 349 unsigned char *name; 350 int ta_subblock; 351 int hw_supported_error_type; 352 int sw_supported_error_type; 353 }; 354 355 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \ 356 [AMDGPU_RAS_BLOCK__##subblock] = { \ 357 #subblock, \ 358 TA_RAS_BLOCK__##subblock, \ 359 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \ 360 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \ 361 } 362 363 static const struct ras_gfx_subblock ras_gfx_subblocks[] = { 364 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1), 365 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1), 366 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 367 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 368 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 369 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 370 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 371 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 372 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 373 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 374 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1), 375 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0), 376 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1), 377 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1), 378 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 379 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0), 380 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0, 381 0), 382 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0, 383 0), 384 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 385 AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0), 386 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0), 387 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1), 388 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0), 389 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0), 390 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1), 391 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 392 0, 0), 393 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 394 0), 395 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 396 0, 0), 397 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0, 398 0), 399 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 400 0, 0), 401 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 402 0), 403 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 404 1), 405 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 406 0, 0, 0), 407 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 408 0), 409 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 410 0), 411 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 412 0), 413 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 414 0), 415 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 416 0), 417 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 418 0, 0), 419 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 420 0), 421 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 422 0), 423 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 424 0, 0, 0), 425 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 426 0), 427 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 428 0), 429 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 430 0), 431 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 432 0), 433 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 434 0), 435 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 436 0, 0), 437 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 438 0), 439 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1), 440 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 441 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 442 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 443 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 444 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0), 445 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 446 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1), 447 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0, 448 1), 449 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0, 450 1), 451 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0, 452 1), 453 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0, 454 0), 455 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0, 456 0), 457 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 458 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 459 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0), 460 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0), 461 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0), 462 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0), 463 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 464 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0), 465 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0), 466 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 467 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0), 468 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0, 469 0), 470 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 471 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0, 472 0), 473 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0, 474 0, 0), 475 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0, 476 0), 477 AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 478 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1), 479 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0), 480 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 481 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 482 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 483 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0), 484 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0), 485 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1), 486 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0), 487 AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 488 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1), 489 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 490 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 491 AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 492 AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 493 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 494 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 495 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 496 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 497 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 498 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 499 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 500 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0), 501 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 502 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 503 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0), 504 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0), 505 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0), 506 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0), 507 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0), 508 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0), 509 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0), 510 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0), 511 }; 512 513 static const struct soc15_reg_golden golden_settings_gc_9_0[] = 514 { 515 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 516 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000), 517 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 518 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 519 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 520 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 521 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 522 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 523 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 524 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87), 525 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f), 526 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 527 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 528 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 529 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 530 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 531 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff), 532 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 533 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 534 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 535 }; 536 537 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] = 538 { 539 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107), 540 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 541 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 542 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 543 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 544 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 545 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042), 546 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 547 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000), 548 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 549 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 550 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 551 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 552 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 553 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 554 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107), 555 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800), 556 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080) 557 }; 558 559 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] = 560 { 561 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080), 562 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 563 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 564 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042), 565 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042), 566 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400), 567 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000), 568 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000), 569 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107), 570 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000), 571 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000) 572 }; 573 574 static const struct soc15_reg_golden golden_settings_gc_9_1[] = 575 { 576 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 577 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 578 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 579 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 580 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 581 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 582 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 583 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 584 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 585 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 586 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 587 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 588 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 589 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 590 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 591 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 592 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 593 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 594 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 595 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff), 596 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080), 597 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 598 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 599 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 600 }; 601 602 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] = 603 { 604 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 605 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042), 606 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042), 607 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000), 608 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000), 609 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 610 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800) 611 }; 612 613 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] = 614 { 615 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000), 616 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 617 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 618 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080), 619 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080), 620 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080), 621 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041), 622 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041), 623 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 624 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 625 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080), 626 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080), 627 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080), 628 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080), 629 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080), 630 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 631 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010), 632 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 633 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 634 }; 635 636 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] = 637 { 638 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 639 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 640 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 641 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042), 642 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042), 643 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 644 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 645 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 646 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 647 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 648 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 649 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc), 650 }; 651 652 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] = 653 { 654 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff), 655 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000), 656 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382) 657 }; 658 659 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] = 660 { 661 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 662 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 663 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 664 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 665 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 666 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 667 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 668 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 669 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87), 670 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f), 671 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 672 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 673 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 674 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 675 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 676 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff) 677 }; 678 679 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] = 680 { 681 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080), 682 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 683 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 684 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041), 685 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041), 686 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 687 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107), 688 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 689 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410), 690 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 691 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 692 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 693 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 694 }; 695 696 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] = 697 { 698 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 699 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000), 700 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e), 701 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca), 702 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098), 703 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3), 704 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1), 705 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135), 706 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000), 707 }; 708 709 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] = 710 { 711 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 712 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 713 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 714 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 715 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 716 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 717 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 718 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 719 }; 720 721 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] = 722 { 723 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0, 724 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0, 725 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0, 726 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0, 727 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0, 728 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0, 729 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0, 730 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0, 731 }; 732 733 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042 734 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041 735 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042 736 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041 737 738 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev); 739 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev); 740 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev); 741 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev); 742 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 743 struct amdgpu_cu_info *cu_info); 744 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev); 745 static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance); 746 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring); 747 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring); 748 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 749 void *ras_error_status); 750 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 751 void *inject_if); 752 753 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev) 754 { 755 switch (adev->asic_type) { 756 case CHIP_VEGA10: 757 soc15_program_register_sequence(adev, 758 golden_settings_gc_9_0, 759 ARRAY_SIZE(golden_settings_gc_9_0)); 760 soc15_program_register_sequence(adev, 761 golden_settings_gc_9_0_vg10, 762 ARRAY_SIZE(golden_settings_gc_9_0_vg10)); 763 break; 764 case CHIP_VEGA12: 765 soc15_program_register_sequence(adev, 766 golden_settings_gc_9_2_1, 767 ARRAY_SIZE(golden_settings_gc_9_2_1)); 768 soc15_program_register_sequence(adev, 769 golden_settings_gc_9_2_1_vg12, 770 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12)); 771 break; 772 case CHIP_VEGA20: 773 soc15_program_register_sequence(adev, 774 golden_settings_gc_9_0, 775 ARRAY_SIZE(golden_settings_gc_9_0)); 776 soc15_program_register_sequence(adev, 777 golden_settings_gc_9_0_vg20, 778 ARRAY_SIZE(golden_settings_gc_9_0_vg20)); 779 break; 780 case CHIP_ARCTURUS: 781 soc15_program_register_sequence(adev, 782 golden_settings_gc_9_4_1_arct, 783 ARRAY_SIZE(golden_settings_gc_9_4_1_arct)); 784 break; 785 case CHIP_RAVEN: 786 soc15_program_register_sequence(adev, golden_settings_gc_9_1, 787 ARRAY_SIZE(golden_settings_gc_9_1)); 788 if (adev->rev_id >= 8) 789 soc15_program_register_sequence(adev, 790 golden_settings_gc_9_1_rv2, 791 ARRAY_SIZE(golden_settings_gc_9_1_rv2)); 792 else 793 soc15_program_register_sequence(adev, 794 golden_settings_gc_9_1_rv1, 795 ARRAY_SIZE(golden_settings_gc_9_1_rv1)); 796 break; 797 case CHIP_RENOIR: 798 soc15_program_register_sequence(adev, 799 golden_settings_gc_9_1_rn, 800 ARRAY_SIZE(golden_settings_gc_9_1_rn)); 801 return; /* for renoir, don't need common goldensetting */ 802 default: 803 break; 804 } 805 806 if (adev->asic_type != CHIP_ARCTURUS) 807 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common, 808 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common)); 809 } 810 811 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev) 812 { 813 adev->gfx.scratch.num_reg = 8; 814 adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0); 815 adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1; 816 } 817 818 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel, 819 bool wc, uint32_t reg, uint32_t val) 820 { 821 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 822 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) | 823 WRITE_DATA_DST_SEL(0) | 824 (wc ? WR_CONFIRM : 0)); 825 amdgpu_ring_write(ring, reg); 826 amdgpu_ring_write(ring, 0); 827 amdgpu_ring_write(ring, val); 828 } 829 830 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, 831 int mem_space, int opt, uint32_t addr0, 832 uint32_t addr1, uint32_t ref, uint32_t mask, 833 uint32_t inv) 834 { 835 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 836 amdgpu_ring_write(ring, 837 /* memory (1) or register (0) */ 838 (WAIT_REG_MEM_MEM_SPACE(mem_space) | 839 WAIT_REG_MEM_OPERATION(opt) | /* wait */ 840 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 841 WAIT_REG_MEM_ENGINE(eng_sel))); 842 843 if (mem_space) 844 BUG_ON(addr0 & 0x3); /* Dword align */ 845 amdgpu_ring_write(ring, addr0); 846 amdgpu_ring_write(ring, addr1); 847 amdgpu_ring_write(ring, ref); 848 amdgpu_ring_write(ring, mask); 849 amdgpu_ring_write(ring, inv); /* poll interval */ 850 } 851 852 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring) 853 { 854 struct amdgpu_device *adev = ring->adev; 855 uint32_t scratch; 856 uint32_t tmp = 0; 857 unsigned i; 858 int r; 859 860 r = amdgpu_gfx_scratch_get(adev, &scratch); 861 if (r) 862 return r; 863 864 WREG32(scratch, 0xCAFEDEAD); 865 r = amdgpu_ring_alloc(ring, 3); 866 if (r) 867 goto error_free_scratch; 868 869 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 870 amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START)); 871 amdgpu_ring_write(ring, 0xDEADBEEF); 872 amdgpu_ring_commit(ring); 873 874 for (i = 0; i < adev->usec_timeout; i++) { 875 tmp = RREG32(scratch); 876 if (tmp == 0xDEADBEEF) 877 break; 878 udelay(1); 879 } 880 881 if (i >= adev->usec_timeout) 882 r = -ETIMEDOUT; 883 884 error_free_scratch: 885 amdgpu_gfx_scratch_free(adev, scratch); 886 return r; 887 } 888 889 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 890 { 891 struct amdgpu_device *adev = ring->adev; 892 struct amdgpu_ib ib; 893 struct dma_fence *f = NULL; 894 895 unsigned index; 896 uint64_t gpu_addr; 897 uint32_t tmp; 898 long r; 899 900 r = amdgpu_device_wb_get(adev, &index); 901 if (r) 902 return r; 903 904 gpu_addr = adev->wb.gpu_addr + (index * 4); 905 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 906 memset(&ib, 0, sizeof(ib)); 907 r = amdgpu_ib_get(adev, NULL, 16, &ib); 908 if (r) 909 goto err1; 910 911 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 912 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 913 ib.ptr[2] = lower_32_bits(gpu_addr); 914 ib.ptr[3] = upper_32_bits(gpu_addr); 915 ib.ptr[4] = 0xDEADBEEF; 916 ib.length_dw = 5; 917 918 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 919 if (r) 920 goto err2; 921 922 r = dma_fence_wait_timeout(f, false, timeout); 923 if (r == 0) { 924 r = -ETIMEDOUT; 925 goto err2; 926 } else if (r < 0) { 927 goto err2; 928 } 929 930 tmp = adev->wb.wb[index]; 931 if (tmp == 0xDEADBEEF) 932 r = 0; 933 else 934 r = -EINVAL; 935 936 err2: 937 amdgpu_ib_free(adev, &ib, NULL); 938 dma_fence_put(f); 939 err1: 940 amdgpu_device_wb_free(adev, index); 941 return r; 942 } 943 944 945 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev) 946 { 947 release_firmware(adev->gfx.pfp_fw); 948 adev->gfx.pfp_fw = NULL; 949 release_firmware(adev->gfx.me_fw); 950 adev->gfx.me_fw = NULL; 951 release_firmware(adev->gfx.ce_fw); 952 adev->gfx.ce_fw = NULL; 953 release_firmware(adev->gfx.rlc_fw); 954 adev->gfx.rlc_fw = NULL; 955 release_firmware(adev->gfx.mec_fw); 956 adev->gfx.mec_fw = NULL; 957 release_firmware(adev->gfx.mec2_fw); 958 adev->gfx.mec2_fw = NULL; 959 960 kfree(adev->gfx.rlc.register_list_format); 961 } 962 963 static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev) 964 { 965 const struct rlc_firmware_header_v2_1 *rlc_hdr; 966 967 rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data; 968 adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver); 969 adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver); 970 adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes); 971 adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes); 972 adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver); 973 adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver); 974 adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes); 975 adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes); 976 adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver); 977 adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver); 978 adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes); 979 adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes); 980 adev->gfx.rlc.reg_list_format_direct_reg_list_length = 981 le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length); 982 } 983 984 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev) 985 { 986 adev->gfx.me_fw_write_wait = false; 987 adev->gfx.mec_fw_write_wait = false; 988 989 if ((adev->gfx.mec_fw_version < 0x000001a5) || 990 (adev->gfx.mec_feature_version < 46) || 991 (adev->gfx.pfp_fw_version < 0x000000b7) || 992 (adev->gfx.pfp_feature_version < 46)) 993 DRM_WARN_ONCE("Warning: check cp_fw_version and update it to realize \ 994 GRBM requires 1-cycle delay in cp firmware\n"); 995 996 switch (adev->asic_type) { 997 case CHIP_VEGA10: 998 if ((adev->gfx.me_fw_version >= 0x0000009c) && 999 (adev->gfx.me_feature_version >= 42) && 1000 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1001 (adev->gfx.pfp_feature_version >= 42)) 1002 adev->gfx.me_fw_write_wait = true; 1003 1004 if ((adev->gfx.mec_fw_version >= 0x00000193) && 1005 (adev->gfx.mec_feature_version >= 42)) 1006 adev->gfx.mec_fw_write_wait = true; 1007 break; 1008 case CHIP_VEGA12: 1009 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1010 (adev->gfx.me_feature_version >= 44) && 1011 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1012 (adev->gfx.pfp_feature_version >= 44)) 1013 adev->gfx.me_fw_write_wait = true; 1014 1015 if ((adev->gfx.mec_fw_version >= 0x00000196) && 1016 (adev->gfx.mec_feature_version >= 44)) 1017 adev->gfx.mec_fw_write_wait = true; 1018 break; 1019 case CHIP_VEGA20: 1020 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1021 (adev->gfx.me_feature_version >= 44) && 1022 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1023 (adev->gfx.pfp_feature_version >= 44)) 1024 adev->gfx.me_fw_write_wait = true; 1025 1026 if ((adev->gfx.mec_fw_version >= 0x00000197) && 1027 (adev->gfx.mec_feature_version >= 44)) 1028 adev->gfx.mec_fw_write_wait = true; 1029 break; 1030 case CHIP_RAVEN: 1031 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1032 (adev->gfx.me_feature_version >= 42) && 1033 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1034 (adev->gfx.pfp_feature_version >= 42)) 1035 adev->gfx.me_fw_write_wait = true; 1036 1037 if ((adev->gfx.mec_fw_version >= 0x00000192) && 1038 (adev->gfx.mec_feature_version >= 42)) 1039 adev->gfx.mec_fw_write_wait = true; 1040 break; 1041 default: 1042 break; 1043 } 1044 } 1045 1046 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev) 1047 { 1048 switch (adev->asic_type) { 1049 case CHIP_VEGA10: 1050 case CHIP_VEGA12: 1051 case CHIP_VEGA20: 1052 break; 1053 case CHIP_RAVEN: 1054 if (!(adev->rev_id >= 0x8 || adev->pdev->device == 0x15d8) 1055 &&((adev->gfx.rlc_fw_version != 106 && 1056 adev->gfx.rlc_fw_version < 531) || 1057 (adev->gfx.rlc_fw_version == 53815) || 1058 (adev->gfx.rlc_feature_version < 1) || 1059 !adev->gfx.rlc.is_rlc_v2_1)) 1060 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1061 1062 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1063 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1064 AMD_PG_SUPPORT_CP | 1065 AMD_PG_SUPPORT_RLC_SMU_HS; 1066 break; 1067 case CHIP_RENOIR: 1068 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1069 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1070 AMD_PG_SUPPORT_CP | 1071 AMD_PG_SUPPORT_RLC_SMU_HS; 1072 break; 1073 default: 1074 break; 1075 } 1076 } 1077 1078 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev, 1079 const char *chip_name) 1080 { 1081 char fw_name[30]; 1082 int err; 1083 struct amdgpu_firmware_info *info = NULL; 1084 const struct common_firmware_header *header = NULL; 1085 const struct gfx_firmware_header_v1_0 *cp_hdr; 1086 1087 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name); 1088 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev); 1089 if (err) 1090 goto out; 1091 err = amdgpu_ucode_validate(adev->gfx.pfp_fw); 1092 if (err) 1093 goto out; 1094 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; 1095 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1096 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1097 1098 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name); 1099 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev); 1100 if (err) 1101 goto out; 1102 err = amdgpu_ucode_validate(adev->gfx.me_fw); 1103 if (err) 1104 goto out; 1105 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; 1106 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1107 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1108 1109 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name); 1110 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev); 1111 if (err) 1112 goto out; 1113 err = amdgpu_ucode_validate(adev->gfx.ce_fw); 1114 if (err) 1115 goto out; 1116 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; 1117 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1118 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1119 1120 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1121 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP]; 1122 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP; 1123 info->fw = adev->gfx.pfp_fw; 1124 header = (const struct common_firmware_header *)info->fw->data; 1125 adev->firmware.fw_size += 1126 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1127 1128 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME]; 1129 info->ucode_id = AMDGPU_UCODE_ID_CP_ME; 1130 info->fw = adev->gfx.me_fw; 1131 header = (const struct common_firmware_header *)info->fw->data; 1132 adev->firmware.fw_size += 1133 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1134 1135 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE]; 1136 info->ucode_id = AMDGPU_UCODE_ID_CP_CE; 1137 info->fw = adev->gfx.ce_fw; 1138 header = (const struct common_firmware_header *)info->fw->data; 1139 adev->firmware.fw_size += 1140 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1141 } 1142 1143 out: 1144 if (err) { 1145 dev_err(adev->dev, 1146 "gfx9: Failed to load firmware \"%s\"\n", 1147 fw_name); 1148 release_firmware(adev->gfx.pfp_fw); 1149 adev->gfx.pfp_fw = NULL; 1150 release_firmware(adev->gfx.me_fw); 1151 adev->gfx.me_fw = NULL; 1152 release_firmware(adev->gfx.ce_fw); 1153 adev->gfx.ce_fw = NULL; 1154 } 1155 return err; 1156 } 1157 1158 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev, 1159 const char *chip_name) 1160 { 1161 char fw_name[30]; 1162 int err; 1163 struct amdgpu_firmware_info *info = NULL; 1164 const struct common_firmware_header *header = NULL; 1165 const struct rlc_firmware_header_v2_0 *rlc_hdr; 1166 unsigned int *tmp = NULL; 1167 unsigned int i = 0; 1168 uint16_t version_major; 1169 uint16_t version_minor; 1170 uint32_t smu_version; 1171 1172 /* 1173 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin 1174 * instead of picasso_rlc.bin. 1175 * Judgment method: 1176 * PCO AM4: revision >= 0xC8 && revision <= 0xCF 1177 * or revision >= 0xD8 && revision <= 0xDF 1178 * otherwise is PCO FP5 1179 */ 1180 if (!strcmp(chip_name, "picasso") && 1181 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) || 1182 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF)))) 1183 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name); 1184 else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) && 1185 (smu_version >= 0x41e2b)) 1186 /** 1187 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly. 1188 */ 1189 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name); 1190 else 1191 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name); 1192 err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev); 1193 if (err) 1194 goto out; 1195 err = amdgpu_ucode_validate(adev->gfx.rlc_fw); 1196 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 1197 1198 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 1199 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 1200 if (version_major == 2 && version_minor == 1) 1201 adev->gfx.rlc.is_rlc_v2_1 = true; 1202 1203 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); 1204 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); 1205 adev->gfx.rlc.save_and_restore_offset = 1206 le32_to_cpu(rlc_hdr->save_and_restore_offset); 1207 adev->gfx.rlc.clear_state_descriptor_offset = 1208 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset); 1209 adev->gfx.rlc.avail_scratch_ram_locations = 1210 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations); 1211 adev->gfx.rlc.reg_restore_list_size = 1212 le32_to_cpu(rlc_hdr->reg_restore_list_size); 1213 adev->gfx.rlc.reg_list_format_start = 1214 le32_to_cpu(rlc_hdr->reg_list_format_start); 1215 adev->gfx.rlc.reg_list_format_separate_start = 1216 le32_to_cpu(rlc_hdr->reg_list_format_separate_start); 1217 adev->gfx.rlc.starting_offsets_start = 1218 le32_to_cpu(rlc_hdr->starting_offsets_start); 1219 adev->gfx.rlc.reg_list_format_size_bytes = 1220 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes); 1221 adev->gfx.rlc.reg_list_size_bytes = 1222 le32_to_cpu(rlc_hdr->reg_list_size_bytes); 1223 adev->gfx.rlc.register_list_format = 1224 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes + 1225 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL); 1226 if (!adev->gfx.rlc.register_list_format) { 1227 err = -ENOMEM; 1228 goto out; 1229 } 1230 1231 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1232 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); 1233 for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++) 1234 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]); 1235 1236 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i; 1237 1238 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1239 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); 1240 for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++) 1241 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); 1242 1243 if (adev->gfx.rlc.is_rlc_v2_1) 1244 gfx_v9_0_init_rlc_ext_microcode(adev); 1245 1246 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1247 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G]; 1248 info->ucode_id = AMDGPU_UCODE_ID_RLC_G; 1249 info->fw = adev->gfx.rlc_fw; 1250 header = (const struct common_firmware_header *)info->fw->data; 1251 adev->firmware.fw_size += 1252 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1253 1254 if (adev->gfx.rlc.is_rlc_v2_1 && 1255 adev->gfx.rlc.save_restore_list_cntl_size_bytes && 1256 adev->gfx.rlc.save_restore_list_gpm_size_bytes && 1257 adev->gfx.rlc.save_restore_list_srm_size_bytes) { 1258 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL]; 1259 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL; 1260 info->fw = adev->gfx.rlc_fw; 1261 adev->firmware.fw_size += 1262 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE); 1263 1264 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM]; 1265 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM; 1266 info->fw = adev->gfx.rlc_fw; 1267 adev->firmware.fw_size += 1268 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE); 1269 1270 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM]; 1271 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM; 1272 info->fw = adev->gfx.rlc_fw; 1273 adev->firmware.fw_size += 1274 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE); 1275 } 1276 } 1277 1278 out: 1279 if (err) { 1280 dev_err(adev->dev, 1281 "gfx9: Failed to load firmware \"%s\"\n", 1282 fw_name); 1283 release_firmware(adev->gfx.rlc_fw); 1284 adev->gfx.rlc_fw = NULL; 1285 } 1286 return err; 1287 } 1288 1289 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev, 1290 const char *chip_name) 1291 { 1292 char fw_name[30]; 1293 int err; 1294 struct amdgpu_firmware_info *info = NULL; 1295 const struct common_firmware_header *header = NULL; 1296 const struct gfx_firmware_header_v1_0 *cp_hdr; 1297 1298 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name); 1299 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev); 1300 if (err) 1301 goto out; 1302 err = amdgpu_ucode_validate(adev->gfx.mec_fw); 1303 if (err) 1304 goto out; 1305 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1306 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1307 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1308 1309 1310 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name); 1311 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev); 1312 if (!err) { 1313 err = amdgpu_ucode_validate(adev->gfx.mec2_fw); 1314 if (err) 1315 goto out; 1316 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1317 adev->gfx.mec2_fw->data; 1318 adev->gfx.mec2_fw_version = 1319 le32_to_cpu(cp_hdr->header.ucode_version); 1320 adev->gfx.mec2_feature_version = 1321 le32_to_cpu(cp_hdr->ucode_feature_version); 1322 } else { 1323 err = 0; 1324 adev->gfx.mec2_fw = NULL; 1325 } 1326 1327 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1328 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1]; 1329 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1; 1330 info->fw = adev->gfx.mec_fw; 1331 header = (const struct common_firmware_header *)info->fw->data; 1332 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1333 adev->firmware.fw_size += 1334 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1335 1336 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT]; 1337 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT; 1338 info->fw = adev->gfx.mec_fw; 1339 adev->firmware.fw_size += 1340 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1341 1342 if (adev->gfx.mec2_fw) { 1343 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2]; 1344 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2; 1345 info->fw = adev->gfx.mec2_fw; 1346 header = (const struct common_firmware_header *)info->fw->data; 1347 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1348 adev->firmware.fw_size += 1349 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1350 1351 /* TODO: Determine if MEC2 JT FW loading can be removed 1352 for all GFX V9 asic and above */ 1353 if (adev->asic_type != CHIP_ARCTURUS && 1354 adev->asic_type != CHIP_RENOIR) { 1355 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT]; 1356 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT; 1357 info->fw = adev->gfx.mec2_fw; 1358 adev->firmware.fw_size += 1359 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, 1360 PAGE_SIZE); 1361 } 1362 } 1363 } 1364 1365 out: 1366 gfx_v9_0_check_if_need_gfxoff(adev); 1367 gfx_v9_0_check_fw_write_wait(adev); 1368 if (err) { 1369 dev_err(adev->dev, 1370 "gfx9: Failed to load firmware \"%s\"\n", 1371 fw_name); 1372 release_firmware(adev->gfx.mec_fw); 1373 adev->gfx.mec_fw = NULL; 1374 release_firmware(adev->gfx.mec2_fw); 1375 adev->gfx.mec2_fw = NULL; 1376 } 1377 return err; 1378 } 1379 1380 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev) 1381 { 1382 const char *chip_name; 1383 int r; 1384 1385 DRM_DEBUG("\n"); 1386 1387 switch (adev->asic_type) { 1388 case CHIP_VEGA10: 1389 chip_name = "vega10"; 1390 break; 1391 case CHIP_VEGA12: 1392 chip_name = "vega12"; 1393 break; 1394 case CHIP_VEGA20: 1395 chip_name = "vega20"; 1396 break; 1397 case CHIP_RAVEN: 1398 if (adev->rev_id >= 8) 1399 chip_name = "raven2"; 1400 else if (adev->pdev->device == 0x15d8) 1401 chip_name = "picasso"; 1402 else 1403 chip_name = "raven"; 1404 break; 1405 case CHIP_ARCTURUS: 1406 chip_name = "arcturus"; 1407 break; 1408 case CHIP_RENOIR: 1409 chip_name = "renoir"; 1410 break; 1411 default: 1412 BUG(); 1413 } 1414 1415 /* No CPG in Arcturus */ 1416 if (adev->asic_type != CHIP_ARCTURUS) { 1417 r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name); 1418 if (r) 1419 return r; 1420 } 1421 1422 r = gfx_v9_0_init_rlc_microcode(adev, chip_name); 1423 if (r) 1424 return r; 1425 1426 r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name); 1427 if (r) 1428 return r; 1429 1430 return r; 1431 } 1432 1433 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev) 1434 { 1435 u32 count = 0; 1436 const struct cs_section_def *sect = NULL; 1437 const struct cs_extent_def *ext = NULL; 1438 1439 /* begin clear state */ 1440 count += 2; 1441 /* context control state */ 1442 count += 3; 1443 1444 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 1445 for (ext = sect->section; ext->extent != NULL; ++ext) { 1446 if (sect->id == SECT_CONTEXT) 1447 count += 2 + ext->reg_count; 1448 else 1449 return 0; 1450 } 1451 } 1452 1453 /* end clear state */ 1454 count += 2; 1455 /* clear state */ 1456 count += 2; 1457 1458 return count; 1459 } 1460 1461 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev, 1462 volatile u32 *buffer) 1463 { 1464 u32 count = 0, i; 1465 const struct cs_section_def *sect = NULL; 1466 const struct cs_extent_def *ext = NULL; 1467 1468 if (adev->gfx.rlc.cs_data == NULL) 1469 return; 1470 if (buffer == NULL) 1471 return; 1472 1473 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1474 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 1475 1476 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 1477 buffer[count++] = cpu_to_le32(0x80000000); 1478 buffer[count++] = cpu_to_le32(0x80000000); 1479 1480 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { 1481 for (ext = sect->section; ext->extent != NULL; ++ext) { 1482 if (sect->id == SECT_CONTEXT) { 1483 buffer[count++] = 1484 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 1485 buffer[count++] = cpu_to_le32(ext->reg_index - 1486 PACKET3_SET_CONTEXT_REG_START); 1487 for (i = 0; i < ext->reg_count; i++) 1488 buffer[count++] = cpu_to_le32(ext->extent[i]); 1489 } else { 1490 return; 1491 } 1492 } 1493 } 1494 1495 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1496 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 1497 1498 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 1499 buffer[count++] = cpu_to_le32(0); 1500 } 1501 1502 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev) 1503 { 1504 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; 1505 uint32_t pg_always_on_cu_num = 2; 1506 uint32_t always_on_cu_num; 1507 uint32_t i, j, k; 1508 uint32_t mask, cu_bitmap, counter; 1509 1510 if (adev->flags & AMD_IS_APU) 1511 always_on_cu_num = 4; 1512 else if (adev->asic_type == CHIP_VEGA12) 1513 always_on_cu_num = 8; 1514 else 1515 always_on_cu_num = 12; 1516 1517 mutex_lock(&adev->grbm_idx_mutex); 1518 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 1519 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 1520 mask = 1; 1521 cu_bitmap = 0; 1522 counter = 0; 1523 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 1524 1525 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 1526 if (cu_info->bitmap[i][j] & mask) { 1527 if (counter == pg_always_on_cu_num) 1528 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap); 1529 if (counter < always_on_cu_num) 1530 cu_bitmap |= mask; 1531 else 1532 break; 1533 counter++; 1534 } 1535 mask <<= 1; 1536 } 1537 1538 WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap); 1539 cu_info->ao_cu_bitmap[i][j] = cu_bitmap; 1540 } 1541 } 1542 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1543 mutex_unlock(&adev->grbm_idx_mutex); 1544 } 1545 1546 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev) 1547 { 1548 uint32_t data; 1549 1550 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1551 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1552 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7); 1553 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1554 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16)); 1555 1556 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1557 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1558 1559 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1560 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500); 1561 1562 mutex_lock(&adev->grbm_idx_mutex); 1563 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1564 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1565 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1566 1567 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1568 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1569 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1570 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1571 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1572 1573 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1574 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1575 data &= 0x0000FFFF; 1576 data |= 0x00C00000; 1577 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1578 1579 /* 1580 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven), 1581 * programmed in gfx_v9_0_init_always_on_cu_mask() 1582 */ 1583 1584 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1585 * but used for RLC_LB_CNTL configuration */ 1586 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1587 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1588 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1589 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1590 mutex_unlock(&adev->grbm_idx_mutex); 1591 1592 gfx_v9_0_init_always_on_cu_mask(adev); 1593 } 1594 1595 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev) 1596 { 1597 uint32_t data; 1598 1599 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1600 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1601 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8); 1602 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1603 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16)); 1604 1605 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1606 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1607 1608 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1609 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800); 1610 1611 mutex_lock(&adev->grbm_idx_mutex); 1612 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1613 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1614 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1615 1616 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1617 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1618 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1619 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1620 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1621 1622 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1623 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1624 data &= 0x0000FFFF; 1625 data |= 0x00C00000; 1626 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1627 1628 /* 1629 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON), 1630 * programmed in gfx_v9_0_init_always_on_cu_mask() 1631 */ 1632 1633 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1634 * but used for RLC_LB_CNTL configuration */ 1635 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1636 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1637 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1638 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1639 mutex_unlock(&adev->grbm_idx_mutex); 1640 1641 gfx_v9_0_init_always_on_cu_mask(adev); 1642 } 1643 1644 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable) 1645 { 1646 WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0); 1647 } 1648 1649 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev) 1650 { 1651 return 5; 1652 } 1653 1654 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev) 1655 { 1656 const struct cs_section_def *cs_data; 1657 int r; 1658 1659 adev->gfx.rlc.cs_data = gfx9_cs_data; 1660 1661 cs_data = adev->gfx.rlc.cs_data; 1662 1663 if (cs_data) { 1664 /* init clear state block */ 1665 r = amdgpu_gfx_rlc_init_csb(adev); 1666 if (r) 1667 return r; 1668 } 1669 1670 if (adev->asic_type == CHIP_RAVEN || adev->asic_type == CHIP_RENOIR) { 1671 /* TODO: double check the cp_table_size for RV */ 1672 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ 1673 r = amdgpu_gfx_rlc_init_cpt(adev); 1674 if (r) 1675 return r; 1676 } 1677 1678 switch (adev->asic_type) { 1679 case CHIP_RAVEN: 1680 gfx_v9_0_init_lbpw(adev); 1681 break; 1682 case CHIP_VEGA20: 1683 gfx_v9_4_init_lbpw(adev); 1684 break; 1685 default: 1686 break; 1687 } 1688 1689 return 0; 1690 } 1691 1692 static int gfx_v9_0_csb_vram_pin(struct amdgpu_device *adev) 1693 { 1694 int r; 1695 1696 r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false); 1697 if (unlikely(r != 0)) 1698 return r; 1699 1700 r = amdgpu_bo_pin(adev->gfx.rlc.clear_state_obj, 1701 AMDGPU_GEM_DOMAIN_VRAM); 1702 if (!r) 1703 adev->gfx.rlc.clear_state_gpu_addr = 1704 amdgpu_bo_gpu_offset(adev->gfx.rlc.clear_state_obj); 1705 1706 amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj); 1707 1708 return r; 1709 } 1710 1711 static void gfx_v9_0_csb_vram_unpin(struct amdgpu_device *adev) 1712 { 1713 int r; 1714 1715 if (!adev->gfx.rlc.clear_state_obj) 1716 return; 1717 1718 r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, true); 1719 if (likely(r == 0)) { 1720 amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj); 1721 amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj); 1722 } 1723 } 1724 1725 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev) 1726 { 1727 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 1728 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); 1729 } 1730 1731 static int gfx_v9_0_mec_init(struct amdgpu_device *adev) 1732 { 1733 int r; 1734 u32 *hpd; 1735 const __le32 *fw_data; 1736 unsigned fw_size; 1737 u32 *fw; 1738 size_t mec_hpd_size; 1739 1740 const struct gfx_firmware_header_v1_0 *mec_hdr; 1741 1742 bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 1743 1744 /* take ownership of the relevant compute queues */ 1745 amdgpu_gfx_compute_queue_acquire(adev); 1746 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE; 1747 1748 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 1749 AMDGPU_GEM_DOMAIN_VRAM, 1750 &adev->gfx.mec.hpd_eop_obj, 1751 &adev->gfx.mec.hpd_eop_gpu_addr, 1752 (void **)&hpd); 1753 if (r) { 1754 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 1755 gfx_v9_0_mec_fini(adev); 1756 return r; 1757 } 1758 1759 memset(hpd, 0, adev->gfx.mec.hpd_eop_obj->tbo.mem.size); 1760 1761 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 1762 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 1763 1764 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1765 1766 fw_data = (const __le32 *) 1767 (adev->gfx.mec_fw->data + 1768 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 1769 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4; 1770 1771 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes, 1772 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 1773 &adev->gfx.mec.mec_fw_obj, 1774 &adev->gfx.mec.mec_fw_gpu_addr, 1775 (void **)&fw); 1776 if (r) { 1777 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r); 1778 gfx_v9_0_mec_fini(adev); 1779 return r; 1780 } 1781 1782 memcpy(fw, fw_data, fw_size); 1783 1784 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 1785 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 1786 1787 return 0; 1788 } 1789 1790 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address) 1791 { 1792 WREG32_SOC15(GC, 0, mmSQ_IND_INDEX, 1793 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 1794 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 1795 (address << SQ_IND_INDEX__INDEX__SHIFT) | 1796 (SQ_IND_INDEX__FORCE_READ_MASK)); 1797 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 1798 } 1799 1800 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd, 1801 uint32_t wave, uint32_t thread, 1802 uint32_t regno, uint32_t num, uint32_t *out) 1803 { 1804 WREG32_SOC15(GC, 0, mmSQ_IND_INDEX, 1805 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 1806 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 1807 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 1808 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) | 1809 (SQ_IND_INDEX__FORCE_READ_MASK) | 1810 (SQ_IND_INDEX__AUTO_INCR_MASK)); 1811 while (num--) 1812 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 1813 } 1814 1815 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 1816 { 1817 /* type 1 wave data */ 1818 dst[(*no_fields)++] = 1; 1819 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS); 1820 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO); 1821 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI); 1822 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO); 1823 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI); 1824 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID); 1825 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0); 1826 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1); 1827 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC); 1828 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC); 1829 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS); 1830 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS); 1831 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); 1832 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); 1833 } 1834 1835 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd, 1836 uint32_t wave, uint32_t start, 1837 uint32_t size, uint32_t *dst) 1838 { 1839 wave_read_regs( 1840 adev, simd, wave, 0, 1841 start + SQIND_WAVE_SGPRS_OFFSET, size, dst); 1842 } 1843 1844 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd, 1845 uint32_t wave, uint32_t thread, 1846 uint32_t start, uint32_t size, 1847 uint32_t *dst) 1848 { 1849 wave_read_regs( 1850 adev, simd, wave, thread, 1851 start + SQIND_WAVE_VGPRS_OFFSET, size, dst); 1852 } 1853 1854 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev, 1855 u32 me, u32 pipe, u32 q, u32 vm) 1856 { 1857 soc15_grbm_select(adev, me, pipe, q, vm); 1858 } 1859 1860 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = { 1861 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 1862 .select_se_sh = &gfx_v9_0_select_se_sh, 1863 .read_wave_data = &gfx_v9_0_read_wave_data, 1864 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 1865 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 1866 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 1867 .ras_error_inject = &gfx_v9_0_ras_error_inject, 1868 .query_ras_error_count = &gfx_v9_0_query_ras_error_count 1869 }; 1870 1871 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev) 1872 { 1873 u32 gb_addr_config; 1874 int err; 1875 1876 adev->gfx.funcs = &gfx_v9_0_gfx_funcs; 1877 1878 switch (adev->asic_type) { 1879 case CHIP_VEGA10: 1880 adev->gfx.config.max_hw_contexts = 8; 1881 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1882 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1883 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1884 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1885 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN; 1886 break; 1887 case CHIP_VEGA12: 1888 adev->gfx.config.max_hw_contexts = 8; 1889 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1890 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1891 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1892 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1893 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN; 1894 DRM_INFO("fix gfx.config for vega12\n"); 1895 break; 1896 case CHIP_VEGA20: 1897 adev->gfx.config.max_hw_contexts = 8; 1898 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1899 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1900 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1901 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1902 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 1903 gb_addr_config &= ~0xf3e777ff; 1904 gb_addr_config |= 0x22014042; 1905 /* check vbios table if gpu info is not available */ 1906 err = amdgpu_atomfirmware_get_gfx_info(adev); 1907 if (err) 1908 return err; 1909 break; 1910 case CHIP_RAVEN: 1911 adev->gfx.config.max_hw_contexts = 8; 1912 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1913 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1914 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1915 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1916 if (adev->rev_id >= 8) 1917 gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN; 1918 else 1919 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN; 1920 break; 1921 case CHIP_ARCTURUS: 1922 adev->gfx.config.max_hw_contexts = 8; 1923 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1924 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1925 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1926 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1927 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 1928 gb_addr_config &= ~0xf3e777ff; 1929 gb_addr_config |= 0x22014042; 1930 break; 1931 case CHIP_RENOIR: 1932 adev->gfx.config.max_hw_contexts = 8; 1933 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1934 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1935 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; 1936 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1937 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 1938 gb_addr_config &= ~0xf3e777ff; 1939 gb_addr_config |= 0x22010042; 1940 break; 1941 default: 1942 BUG(); 1943 break; 1944 } 1945 1946 adev->gfx.config.gb_addr_config = gb_addr_config; 1947 1948 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << 1949 REG_GET_FIELD( 1950 adev->gfx.config.gb_addr_config, 1951 GB_ADDR_CONFIG, 1952 NUM_PIPES); 1953 1954 adev->gfx.config.max_tile_pipes = 1955 adev->gfx.config.gb_addr_config_fields.num_pipes; 1956 1957 adev->gfx.config.gb_addr_config_fields.num_banks = 1 << 1958 REG_GET_FIELD( 1959 adev->gfx.config.gb_addr_config, 1960 GB_ADDR_CONFIG, 1961 NUM_BANKS); 1962 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << 1963 REG_GET_FIELD( 1964 adev->gfx.config.gb_addr_config, 1965 GB_ADDR_CONFIG, 1966 MAX_COMPRESSED_FRAGS); 1967 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << 1968 REG_GET_FIELD( 1969 adev->gfx.config.gb_addr_config, 1970 GB_ADDR_CONFIG, 1971 NUM_RB_PER_SE); 1972 adev->gfx.config.gb_addr_config_fields.num_se = 1 << 1973 REG_GET_FIELD( 1974 adev->gfx.config.gb_addr_config, 1975 GB_ADDR_CONFIG, 1976 NUM_SHADER_ENGINES); 1977 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + 1978 REG_GET_FIELD( 1979 adev->gfx.config.gb_addr_config, 1980 GB_ADDR_CONFIG, 1981 PIPE_INTERLEAVE_SIZE)); 1982 1983 return 0; 1984 } 1985 1986 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 1987 int mec, int pipe, int queue) 1988 { 1989 int r; 1990 unsigned irq_type; 1991 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; 1992 1993 ring = &adev->gfx.compute_ring[ring_id]; 1994 1995 /* mec0 is me1 */ 1996 ring->me = mec + 1; 1997 ring->pipe = pipe; 1998 ring->queue = queue; 1999 2000 ring->ring_obj = NULL; 2001 ring->use_doorbell = true; 2002 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1; 2003 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 2004 + (ring_id * GFX9_MEC_HPD_SIZE); 2005 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 2006 2007 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 2008 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 2009 + ring->pipe; 2010 2011 /* type-2 packets are deprecated on MEC, use type-3 instead */ 2012 r = amdgpu_ring_init(adev, ring, 1024, 2013 &adev->gfx.eop_irq, irq_type); 2014 if (r) 2015 return r; 2016 2017 2018 return 0; 2019 } 2020 2021 static int gfx_v9_0_sw_init(void *handle) 2022 { 2023 int i, j, k, r, ring_id; 2024 struct amdgpu_ring *ring; 2025 struct amdgpu_kiq *kiq; 2026 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2027 2028 switch (adev->asic_type) { 2029 case CHIP_VEGA10: 2030 case CHIP_VEGA12: 2031 case CHIP_VEGA20: 2032 case CHIP_RAVEN: 2033 case CHIP_ARCTURUS: 2034 case CHIP_RENOIR: 2035 adev->gfx.mec.num_mec = 2; 2036 break; 2037 default: 2038 adev->gfx.mec.num_mec = 1; 2039 break; 2040 } 2041 2042 adev->gfx.mec.num_pipe_per_mec = 4; 2043 adev->gfx.mec.num_queue_per_pipe = 8; 2044 2045 /* EOP Event */ 2046 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq); 2047 if (r) 2048 return r; 2049 2050 /* Privileged reg */ 2051 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT, 2052 &adev->gfx.priv_reg_irq); 2053 if (r) 2054 return r; 2055 2056 /* Privileged inst */ 2057 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT, 2058 &adev->gfx.priv_inst_irq); 2059 if (r) 2060 return r; 2061 2062 /* ECC error */ 2063 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR, 2064 &adev->gfx.cp_ecc_error_irq); 2065 if (r) 2066 return r; 2067 2068 /* FUE error */ 2069 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR, 2070 &adev->gfx.cp_ecc_error_irq); 2071 if (r) 2072 return r; 2073 2074 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 2075 2076 gfx_v9_0_scratch_init(adev); 2077 2078 r = gfx_v9_0_init_microcode(adev); 2079 if (r) { 2080 DRM_ERROR("Failed to load gfx firmware!\n"); 2081 return r; 2082 } 2083 2084 r = adev->gfx.rlc.funcs->init(adev); 2085 if (r) { 2086 DRM_ERROR("Failed to init rlc BOs!\n"); 2087 return r; 2088 } 2089 2090 r = gfx_v9_0_mec_init(adev); 2091 if (r) { 2092 DRM_ERROR("Failed to init MEC BOs!\n"); 2093 return r; 2094 } 2095 2096 /* set up the gfx ring */ 2097 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 2098 ring = &adev->gfx.gfx_ring[i]; 2099 ring->ring_obj = NULL; 2100 if (!i) 2101 sprintf(ring->name, "gfx"); 2102 else 2103 sprintf(ring->name, "gfx_%d", i); 2104 ring->use_doorbell = true; 2105 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 2106 r = amdgpu_ring_init(adev, ring, 1024, 2107 &adev->gfx.eop_irq, AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP); 2108 if (r) 2109 return r; 2110 } 2111 2112 /* set up the compute queues - allocate horizontally across pipes */ 2113 ring_id = 0; 2114 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 2115 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 2116 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 2117 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j)) 2118 continue; 2119 2120 r = gfx_v9_0_compute_ring_init(adev, 2121 ring_id, 2122 i, k, j); 2123 if (r) 2124 return r; 2125 2126 ring_id++; 2127 } 2128 } 2129 } 2130 2131 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE); 2132 if (r) { 2133 DRM_ERROR("Failed to init KIQ BOs!\n"); 2134 return r; 2135 } 2136 2137 kiq = &adev->gfx.kiq; 2138 r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq); 2139 if (r) 2140 return r; 2141 2142 /* create MQD for all compute queues as wel as KIQ for SRIOV case */ 2143 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation)); 2144 if (r) 2145 return r; 2146 2147 adev->gfx.ce_ram_size = 0x8000; 2148 2149 r = gfx_v9_0_gpu_early_init(adev); 2150 if (r) 2151 return r; 2152 2153 return 0; 2154 } 2155 2156 2157 static int gfx_v9_0_sw_fini(void *handle) 2158 { 2159 int i; 2160 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2161 2162 amdgpu_gfx_ras_fini(adev); 2163 2164 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 2165 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 2166 for (i = 0; i < adev->gfx.num_compute_rings; i++) 2167 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 2168 2169 amdgpu_gfx_mqd_sw_fini(adev); 2170 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring); 2171 amdgpu_gfx_kiq_fini(adev); 2172 2173 gfx_v9_0_mec_fini(adev); 2174 amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj); 2175 if (adev->asic_type == CHIP_RAVEN || adev->asic_type == CHIP_RENOIR) { 2176 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 2177 &adev->gfx.rlc.cp_table_gpu_addr, 2178 (void **)&adev->gfx.rlc.cp_table_ptr); 2179 } 2180 gfx_v9_0_free_microcode(adev); 2181 2182 return 0; 2183 } 2184 2185 2186 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev) 2187 { 2188 /* TODO */ 2189 } 2190 2191 static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance) 2192 { 2193 u32 data; 2194 2195 if (instance == 0xffffffff) 2196 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); 2197 else 2198 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance); 2199 2200 if (se_num == 0xffffffff) 2201 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); 2202 else 2203 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 2204 2205 if (sh_num == 0xffffffff) 2206 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1); 2207 else 2208 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num); 2209 2210 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data); 2211 } 2212 2213 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev) 2214 { 2215 u32 data, mask; 2216 2217 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE); 2218 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE); 2219 2220 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK; 2221 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT; 2222 2223 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / 2224 adev->gfx.config.max_sh_per_se); 2225 2226 return (~data) & mask; 2227 } 2228 2229 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev) 2230 { 2231 int i, j; 2232 u32 data; 2233 u32 active_rbs = 0; 2234 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / 2235 adev->gfx.config.max_sh_per_se; 2236 2237 mutex_lock(&adev->grbm_idx_mutex); 2238 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2239 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2240 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2241 data = gfx_v9_0_get_rb_active_bitmap(adev); 2242 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * 2243 rb_bitmap_width_per_sh); 2244 } 2245 } 2246 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2247 mutex_unlock(&adev->grbm_idx_mutex); 2248 2249 adev->gfx.config.backend_enable_mask = active_rbs; 2250 adev->gfx.config.num_rbs = hweight32(active_rbs); 2251 } 2252 2253 #define DEFAULT_SH_MEM_BASES (0x6000) 2254 #define FIRST_COMPUTE_VMID (8) 2255 #define LAST_COMPUTE_VMID (16) 2256 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev) 2257 { 2258 int i; 2259 uint32_t sh_mem_config; 2260 uint32_t sh_mem_bases; 2261 2262 /* 2263 * Configure apertures: 2264 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 2265 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 2266 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 2267 */ 2268 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16); 2269 2270 sh_mem_config = SH_MEM_ADDRESS_MODE_64 | 2271 SH_MEM_ALIGNMENT_MODE_UNALIGNED << 2272 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT; 2273 2274 mutex_lock(&adev->srbm_mutex); 2275 for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) { 2276 soc15_grbm_select(adev, 0, 0, 0, i); 2277 /* CP and shaders */ 2278 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config); 2279 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases); 2280 } 2281 soc15_grbm_select(adev, 0, 0, 0, 0); 2282 mutex_unlock(&adev->srbm_mutex); 2283 2284 /* Initialize all compute VMIDs to have no GDS, GWS, or OA 2285 acccess. These should be enabled by FW for target VMIDs. */ 2286 for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) { 2287 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0); 2288 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0); 2289 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0); 2290 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0); 2291 } 2292 } 2293 2294 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev) 2295 { 2296 int vmid; 2297 2298 /* 2299 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 2300 * access. Compute VMIDs should be enabled by FW for target VMIDs, 2301 * the driver can enable them for graphics. VMID0 should maintain 2302 * access so that HWS firmware can save/restore entries. 2303 */ 2304 for (vmid = 1; vmid < 16; vmid++) { 2305 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0); 2306 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0); 2307 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0); 2308 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0); 2309 } 2310 } 2311 2312 static void gfx_v9_0_constants_init(struct amdgpu_device *adev) 2313 { 2314 u32 tmp; 2315 int i; 2316 2317 WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); 2318 2319 gfx_v9_0_tiling_mode_table_init(adev); 2320 2321 gfx_v9_0_setup_rb(adev); 2322 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info); 2323 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2); 2324 2325 /* XXX SH_MEM regs */ 2326 /* where to put LDS, scratch, GPUVM in FSA64 space */ 2327 mutex_lock(&adev->srbm_mutex); 2328 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) { 2329 soc15_grbm_select(adev, 0, 0, 0, i); 2330 /* CP and shaders */ 2331 if (i == 0) { 2332 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2333 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2334 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2335 !!amdgpu_noretry); 2336 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2337 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0); 2338 } else { 2339 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2340 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2341 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2342 !!amdgpu_noretry); 2343 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2344 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 2345 (adev->gmc.private_aperture_start >> 48)); 2346 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE, 2347 (adev->gmc.shared_aperture_start >> 48)); 2348 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp); 2349 } 2350 } 2351 soc15_grbm_select(adev, 0, 0, 0, 0); 2352 2353 mutex_unlock(&adev->srbm_mutex); 2354 2355 gfx_v9_0_init_compute_vmid(adev); 2356 gfx_v9_0_init_gds_vmid(adev); 2357 } 2358 2359 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev) 2360 { 2361 u32 i, j, k; 2362 u32 mask; 2363 2364 mutex_lock(&adev->grbm_idx_mutex); 2365 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2366 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2367 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2368 for (k = 0; k < adev->usec_timeout; k++) { 2369 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0) 2370 break; 2371 udelay(1); 2372 } 2373 if (k == adev->usec_timeout) { 2374 gfx_v9_0_select_se_sh(adev, 0xffffffff, 2375 0xffffffff, 0xffffffff); 2376 mutex_unlock(&adev->grbm_idx_mutex); 2377 DRM_INFO("Timeout wait for RLC serdes %u,%u\n", 2378 i, j); 2379 return; 2380 } 2381 } 2382 } 2383 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2384 mutex_unlock(&adev->grbm_idx_mutex); 2385 2386 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | 2387 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK | 2388 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK | 2389 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK; 2390 for (k = 0; k < adev->usec_timeout; k++) { 2391 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 2392 break; 2393 udelay(1); 2394 } 2395 } 2396 2397 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 2398 bool enable) 2399 { 2400 u32 tmp = RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0); 2401 2402 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0); 2403 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0); 2404 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0); 2405 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0); 2406 2407 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp); 2408 } 2409 2410 static void gfx_v9_0_init_csb(struct amdgpu_device *adev) 2411 { 2412 /* csib */ 2413 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI), 2414 adev->gfx.rlc.clear_state_gpu_addr >> 32); 2415 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO), 2416 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 2417 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH), 2418 adev->gfx.rlc.clear_state_size); 2419 } 2420 2421 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format, 2422 int indirect_offset, 2423 int list_size, 2424 int *unique_indirect_regs, 2425 int unique_indirect_reg_count, 2426 int *indirect_start_offsets, 2427 int *indirect_start_offsets_count, 2428 int max_start_offsets_count) 2429 { 2430 int idx; 2431 2432 for (; indirect_offset < list_size; indirect_offset++) { 2433 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count); 2434 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset; 2435 *indirect_start_offsets_count = *indirect_start_offsets_count + 1; 2436 2437 while (register_list_format[indirect_offset] != 0xFFFFFFFF) { 2438 indirect_offset += 2; 2439 2440 /* look for the matching indice */ 2441 for (idx = 0; idx < unique_indirect_reg_count; idx++) { 2442 if (unique_indirect_regs[idx] == 2443 register_list_format[indirect_offset] || 2444 !unique_indirect_regs[idx]) 2445 break; 2446 } 2447 2448 BUG_ON(idx >= unique_indirect_reg_count); 2449 2450 if (!unique_indirect_regs[idx]) 2451 unique_indirect_regs[idx] = register_list_format[indirect_offset]; 2452 2453 indirect_offset++; 2454 } 2455 } 2456 } 2457 2458 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev) 2459 { 2460 int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2461 int unique_indirect_reg_count = 0; 2462 2463 int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2464 int indirect_start_offsets_count = 0; 2465 2466 int list_size = 0; 2467 int i = 0, j = 0; 2468 u32 tmp = 0; 2469 2470 u32 *register_list_format = 2471 kmemdup(adev->gfx.rlc.register_list_format, 2472 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); 2473 if (!register_list_format) 2474 return -ENOMEM; 2475 2476 /* setup unique_indirect_regs array and indirect_start_offsets array */ 2477 unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs); 2478 gfx_v9_1_parse_ind_reg_list(register_list_format, 2479 adev->gfx.rlc.reg_list_format_direct_reg_list_length, 2480 adev->gfx.rlc.reg_list_format_size_bytes >> 2, 2481 unique_indirect_regs, 2482 unique_indirect_reg_count, 2483 indirect_start_offsets, 2484 &indirect_start_offsets_count, 2485 ARRAY_SIZE(indirect_start_offsets)); 2486 2487 /* enable auto inc in case it is disabled */ 2488 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL)); 2489 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK; 2490 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp); 2491 2492 /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */ 2493 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR), 2494 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET); 2495 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) 2496 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA), 2497 adev->gfx.rlc.register_restore[i]); 2498 2499 /* load indirect register */ 2500 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2501 adev->gfx.rlc.reg_list_format_start); 2502 2503 /* direct register portion */ 2504 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++) 2505 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2506 register_list_format[i]); 2507 2508 /* indirect register portion */ 2509 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) { 2510 if (register_list_format[i] == 0xFFFFFFFF) { 2511 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2512 continue; 2513 } 2514 2515 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2516 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2517 2518 for (j = 0; j < unique_indirect_reg_count; j++) { 2519 if (register_list_format[i] == unique_indirect_regs[j]) { 2520 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j); 2521 break; 2522 } 2523 } 2524 2525 BUG_ON(j >= unique_indirect_reg_count); 2526 2527 i++; 2528 } 2529 2530 /* set save/restore list size */ 2531 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; 2532 list_size = list_size >> 1; 2533 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2534 adev->gfx.rlc.reg_restore_list_size); 2535 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size); 2536 2537 /* write the starting offsets to RLC scratch ram */ 2538 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2539 adev->gfx.rlc.starting_offsets_start); 2540 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++) 2541 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2542 indirect_start_offsets[i]); 2543 2544 /* load unique indirect regs*/ 2545 for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) { 2546 if (unique_indirect_regs[i] != 0) { 2547 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0) 2548 + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i], 2549 unique_indirect_regs[i] & 0x3FFFF); 2550 2551 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0) 2552 + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i], 2553 unique_indirect_regs[i] >> 20); 2554 } 2555 } 2556 2557 kfree(register_list_format); 2558 return 0; 2559 } 2560 2561 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev) 2562 { 2563 WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1); 2564 } 2565 2566 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev, 2567 bool enable) 2568 { 2569 uint32_t data = 0; 2570 uint32_t default_data = 0; 2571 2572 default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS)); 2573 if (enable == true) { 2574 /* enable GFXIP control over CGPG */ 2575 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2576 if(default_data != data) 2577 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2578 2579 /* update status */ 2580 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK; 2581 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT); 2582 if(default_data != data) 2583 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2584 } else { 2585 /* restore GFXIP control over GCPG */ 2586 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2587 if(default_data != data) 2588 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2589 } 2590 } 2591 2592 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev) 2593 { 2594 uint32_t data = 0; 2595 2596 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2597 AMD_PG_SUPPORT_GFX_SMG | 2598 AMD_PG_SUPPORT_GFX_DMG)) { 2599 /* init IDLE_POLL_COUNT = 60 */ 2600 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL)); 2601 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK; 2602 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 2603 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data); 2604 2605 /* init RLC PG Delay */ 2606 data = 0; 2607 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT); 2608 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT); 2609 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT); 2610 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT); 2611 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data); 2612 2613 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2)); 2614 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK; 2615 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT); 2616 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data); 2617 2618 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3)); 2619 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK; 2620 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT); 2621 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data); 2622 2623 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL)); 2624 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK; 2625 2626 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */ 2627 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT); 2628 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data); 2629 2630 pwr_10_0_gfxip_control_over_cgpg(adev, true); 2631 } 2632 } 2633 2634 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev, 2635 bool enable) 2636 { 2637 uint32_t data = 0; 2638 uint32_t default_data = 0; 2639 2640 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2641 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2642 SMU_CLK_SLOWDOWN_ON_PU_ENABLE, 2643 enable ? 1 : 0); 2644 if (default_data != data) 2645 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2646 } 2647 2648 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev, 2649 bool enable) 2650 { 2651 uint32_t data = 0; 2652 uint32_t default_data = 0; 2653 2654 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2655 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2656 SMU_CLK_SLOWDOWN_ON_PD_ENABLE, 2657 enable ? 1 : 0); 2658 if(default_data != data) 2659 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2660 } 2661 2662 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev, 2663 bool enable) 2664 { 2665 uint32_t data = 0; 2666 uint32_t default_data = 0; 2667 2668 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2669 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2670 CP_PG_DISABLE, 2671 enable ? 0 : 1); 2672 if(default_data != data) 2673 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2674 } 2675 2676 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev, 2677 bool enable) 2678 { 2679 uint32_t data, default_data; 2680 2681 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2682 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2683 GFX_POWER_GATING_ENABLE, 2684 enable ? 1 : 0); 2685 if(default_data != data) 2686 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2687 } 2688 2689 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev, 2690 bool enable) 2691 { 2692 uint32_t data, default_data; 2693 2694 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2695 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2696 GFX_PIPELINE_PG_ENABLE, 2697 enable ? 1 : 0); 2698 if(default_data != data) 2699 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2700 2701 if (!enable) 2702 /* read any GFX register to wake up GFX */ 2703 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL)); 2704 } 2705 2706 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, 2707 bool enable) 2708 { 2709 uint32_t data, default_data; 2710 2711 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2712 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2713 STATIC_PER_CU_PG_ENABLE, 2714 enable ? 1 : 0); 2715 if(default_data != data) 2716 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2717 } 2718 2719 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev, 2720 bool enable) 2721 { 2722 uint32_t data, default_data; 2723 2724 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2725 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2726 DYN_PER_CU_PG_ENABLE, 2727 enable ? 1 : 0); 2728 if(default_data != data) 2729 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2730 } 2731 2732 static void gfx_v9_0_init_pg(struct amdgpu_device *adev) 2733 { 2734 gfx_v9_0_init_csb(adev); 2735 2736 /* 2737 * Rlc save restore list is workable since v2_1. 2738 * And it's needed by gfxoff feature. 2739 */ 2740 if (adev->gfx.rlc.is_rlc_v2_1) { 2741 if (adev->asic_type == CHIP_VEGA12 || 2742 (adev->asic_type == CHIP_RAVEN && 2743 adev->rev_id >= 8)) 2744 gfx_v9_1_init_rlc_save_restore_list(adev); 2745 gfx_v9_0_enable_save_restore_machine(adev); 2746 } 2747 2748 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2749 AMD_PG_SUPPORT_GFX_SMG | 2750 AMD_PG_SUPPORT_GFX_DMG | 2751 AMD_PG_SUPPORT_CP | 2752 AMD_PG_SUPPORT_GDS | 2753 AMD_PG_SUPPORT_RLC_SMU_HS)) { 2754 WREG32(mmRLC_JUMP_TABLE_RESTORE, 2755 adev->gfx.rlc.cp_table_gpu_addr >> 8); 2756 gfx_v9_0_init_gfx_power_gating(adev); 2757 } 2758 } 2759 2760 void gfx_v9_0_rlc_stop(struct amdgpu_device *adev) 2761 { 2762 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0); 2763 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 2764 gfx_v9_0_wait_for_rlc_serdes(adev); 2765 } 2766 2767 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev) 2768 { 2769 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 2770 udelay(50); 2771 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 2772 udelay(50); 2773 } 2774 2775 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev) 2776 { 2777 #ifdef AMDGPU_RLC_DEBUG_RETRY 2778 u32 rlc_ucode_ver; 2779 #endif 2780 2781 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1); 2782 udelay(50); 2783 2784 /* carrizo do enable cp interrupt after cp inited */ 2785 if (!(adev->flags & AMD_IS_APU)) { 2786 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 2787 udelay(50); 2788 } 2789 2790 #ifdef AMDGPU_RLC_DEBUG_RETRY 2791 /* RLC_GPM_GENERAL_6 : RLC Ucode version */ 2792 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6); 2793 if(rlc_ucode_ver == 0x108) { 2794 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n", 2795 rlc_ucode_ver, adev->gfx.rlc_fw_version); 2796 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles, 2797 * default is 0x9C4 to create a 100us interval */ 2798 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4); 2799 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr 2800 * to disable the page fault retry interrupts, default is 2801 * 0x100 (256) */ 2802 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100); 2803 } 2804 #endif 2805 } 2806 2807 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev) 2808 { 2809 const struct rlc_firmware_header_v2_0 *hdr; 2810 const __le32 *fw_data; 2811 unsigned i, fw_size; 2812 2813 if (!adev->gfx.rlc_fw) 2814 return -EINVAL; 2815 2816 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 2817 amdgpu_ucode_print_rlc_hdr(&hdr->header); 2818 2819 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2820 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2821 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 2822 2823 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, 2824 RLCG_UCODE_LOADING_START_ADDRESS); 2825 for (i = 0; i < fw_size; i++) 2826 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); 2827 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); 2828 2829 return 0; 2830 } 2831 2832 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev) 2833 { 2834 int r; 2835 2836 if (amdgpu_sriov_vf(adev)) { 2837 gfx_v9_0_init_csb(adev); 2838 return 0; 2839 } 2840 2841 adev->gfx.rlc.funcs->stop(adev); 2842 2843 /* disable CG */ 2844 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0); 2845 2846 gfx_v9_0_init_pg(adev); 2847 2848 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 2849 /* legacy rlc firmware loading */ 2850 r = gfx_v9_0_rlc_load_microcode(adev); 2851 if (r) 2852 return r; 2853 } 2854 2855 switch (adev->asic_type) { 2856 case CHIP_RAVEN: 2857 if (amdgpu_lbpw == 0) 2858 gfx_v9_0_enable_lbpw(adev, false); 2859 else 2860 gfx_v9_0_enable_lbpw(adev, true); 2861 break; 2862 case CHIP_VEGA20: 2863 if (amdgpu_lbpw > 0) 2864 gfx_v9_0_enable_lbpw(adev, true); 2865 else 2866 gfx_v9_0_enable_lbpw(adev, false); 2867 break; 2868 default: 2869 break; 2870 } 2871 2872 adev->gfx.rlc.funcs->start(adev); 2873 2874 return 0; 2875 } 2876 2877 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 2878 { 2879 int i; 2880 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL); 2881 2882 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); 2883 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); 2884 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1); 2885 if (!enable) { 2886 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 2887 adev->gfx.gfx_ring[i].sched.ready = false; 2888 } 2889 WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp); 2890 udelay(50); 2891 } 2892 2893 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev) 2894 { 2895 const struct gfx_firmware_header_v1_0 *pfp_hdr; 2896 const struct gfx_firmware_header_v1_0 *ce_hdr; 2897 const struct gfx_firmware_header_v1_0 *me_hdr; 2898 const __le32 *fw_data; 2899 unsigned i, fw_size; 2900 2901 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw) 2902 return -EINVAL; 2903 2904 pfp_hdr = (const struct gfx_firmware_header_v1_0 *) 2905 adev->gfx.pfp_fw->data; 2906 ce_hdr = (const struct gfx_firmware_header_v1_0 *) 2907 adev->gfx.ce_fw->data; 2908 me_hdr = (const struct gfx_firmware_header_v1_0 *) 2909 adev->gfx.me_fw->data; 2910 2911 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 2912 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header); 2913 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 2914 2915 gfx_v9_0_cp_gfx_enable(adev, false); 2916 2917 /* PFP */ 2918 fw_data = (const __le32 *) 2919 (adev->gfx.pfp_fw->data + 2920 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 2921 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 2922 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0); 2923 for (i = 0; i < fw_size; i++) 2924 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 2925 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); 2926 2927 /* CE */ 2928 fw_data = (const __le32 *) 2929 (adev->gfx.ce_fw->data + 2930 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 2931 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 2932 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0); 2933 for (i = 0; i < fw_size; i++) 2934 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 2935 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version); 2936 2937 /* ME */ 2938 fw_data = (const __le32 *) 2939 (adev->gfx.me_fw->data + 2940 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 2941 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 2942 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0); 2943 for (i = 0; i < fw_size; i++) 2944 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 2945 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version); 2946 2947 return 0; 2948 } 2949 2950 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev) 2951 { 2952 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; 2953 const struct cs_section_def *sect = NULL; 2954 const struct cs_extent_def *ext = NULL; 2955 int r, i, tmp; 2956 2957 /* init the CP */ 2958 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); 2959 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1); 2960 2961 gfx_v9_0_cp_gfx_enable(adev, true); 2962 2963 r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3); 2964 if (r) { 2965 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r); 2966 return r; 2967 } 2968 2969 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 2970 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 2971 2972 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 2973 amdgpu_ring_write(ring, 0x80000000); 2974 amdgpu_ring_write(ring, 0x80000000); 2975 2976 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 2977 for (ext = sect->section; ext->extent != NULL; ++ext) { 2978 if (sect->id == SECT_CONTEXT) { 2979 amdgpu_ring_write(ring, 2980 PACKET3(PACKET3_SET_CONTEXT_REG, 2981 ext->reg_count)); 2982 amdgpu_ring_write(ring, 2983 ext->reg_index - PACKET3_SET_CONTEXT_REG_START); 2984 for (i = 0; i < ext->reg_count; i++) 2985 amdgpu_ring_write(ring, ext->extent[i]); 2986 } 2987 } 2988 } 2989 2990 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 2991 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 2992 2993 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 2994 amdgpu_ring_write(ring, 0); 2995 2996 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 2997 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 2998 amdgpu_ring_write(ring, 0x8000); 2999 amdgpu_ring_write(ring, 0x8000); 3000 3001 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1)); 3002 tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE | 3003 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START)); 3004 amdgpu_ring_write(ring, tmp); 3005 amdgpu_ring_write(ring, 0); 3006 3007 amdgpu_ring_commit(ring); 3008 3009 return 0; 3010 } 3011 3012 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev) 3013 { 3014 struct amdgpu_ring *ring; 3015 u32 tmp; 3016 u32 rb_bufsz; 3017 u64 rb_addr, rptr_addr, wptr_gpu_addr; 3018 3019 /* Set the write pointer delay */ 3020 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0); 3021 3022 /* set the RB to use vmid 0 */ 3023 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0); 3024 3025 /* Set ring buffer size */ 3026 ring = &adev->gfx.gfx_ring[0]; 3027 rb_bufsz = order_base_2(ring->ring_size / 8); 3028 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 3029 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 3030 #ifdef __BIG_ENDIAN 3031 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1); 3032 #endif 3033 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3034 3035 /* Initialize the ring buffer's write pointers */ 3036 ring->wptr = 0; 3037 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 3038 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 3039 3040 /* set the wb address wether it's enabled or not */ 3041 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3042 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 3043 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3044 3045 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3046 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr)); 3047 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr)); 3048 3049 mdelay(1); 3050 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3051 3052 rb_addr = ring->gpu_addr >> 8; 3053 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr); 3054 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 3055 3056 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL); 3057 if (ring->use_doorbell) { 3058 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3059 DOORBELL_OFFSET, ring->doorbell_index); 3060 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3061 DOORBELL_EN, 1); 3062 } else { 3063 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0); 3064 } 3065 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp); 3066 3067 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 3068 DOORBELL_RANGE_LOWER, ring->doorbell_index); 3069 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp); 3070 3071 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER, 3072 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 3073 3074 3075 /* start the ring */ 3076 gfx_v9_0_cp_gfx_start(adev); 3077 ring->sched.ready = true; 3078 3079 return 0; 3080 } 3081 3082 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 3083 { 3084 int i; 3085 3086 if (enable) { 3087 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0); 3088 } else { 3089 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 3090 (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK)); 3091 for (i = 0; i < adev->gfx.num_compute_rings; i++) 3092 adev->gfx.compute_ring[i].sched.ready = false; 3093 adev->gfx.kiq.ring.sched.ready = false; 3094 } 3095 udelay(50); 3096 } 3097 3098 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev) 3099 { 3100 const struct gfx_firmware_header_v1_0 *mec_hdr; 3101 const __le32 *fw_data; 3102 unsigned i; 3103 u32 tmp; 3104 3105 if (!adev->gfx.mec_fw) 3106 return -EINVAL; 3107 3108 gfx_v9_0_cp_compute_enable(adev, false); 3109 3110 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 3111 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3112 3113 fw_data = (const __le32 *) 3114 (adev->gfx.mec_fw->data + 3115 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 3116 tmp = 0; 3117 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 3118 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 3119 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp); 3120 3121 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO, 3122 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000); 3123 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI, 3124 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 3125 3126 /* MEC1 */ 3127 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3128 mec_hdr->jt_offset); 3129 for (i = 0; i < mec_hdr->jt_size; i++) 3130 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA, 3131 le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); 3132 3133 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3134 adev->gfx.mec_fw_version); 3135 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */ 3136 3137 return 0; 3138 } 3139 3140 /* KIQ functions */ 3141 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring) 3142 { 3143 uint32_t tmp; 3144 struct amdgpu_device *adev = ring->adev; 3145 3146 /* tell RLC which is KIQ queue */ 3147 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS); 3148 tmp &= 0xffffff00; 3149 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 3150 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3151 tmp |= 0x80; 3152 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3153 } 3154 3155 static int gfx_v9_0_kiq_kcq_enable(struct amdgpu_device *adev) 3156 { 3157 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; 3158 uint64_t queue_mask = 0; 3159 int r, i; 3160 3161 for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) { 3162 if (!test_bit(i, adev->gfx.mec.queue_bitmap)) 3163 continue; 3164 3165 /* This situation may be hit in the future if a new HW 3166 * generation exposes more than 64 queues. If so, the 3167 * definition of queue_mask needs updating */ 3168 if (WARN_ON(i >= (sizeof(queue_mask)*8))) { 3169 DRM_ERROR("Invalid KCQ enabled: %d\n", i); 3170 break; 3171 } 3172 3173 queue_mask |= (1ull << i); 3174 } 3175 3176 r = amdgpu_ring_alloc(kiq_ring, (7 * adev->gfx.num_compute_rings) + 8); 3177 if (r) { 3178 DRM_ERROR("Failed to lock KIQ (%d).\n", r); 3179 return r; 3180 } 3181 3182 /* set resources */ 3183 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 3184 amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) | 3185 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */ 3186 amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */ 3187 amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */ 3188 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */ 3189 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */ 3190 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 3191 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 3192 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3193 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; 3194 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 3195 uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3196 3197 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 3198 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 3199 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 3200 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 3201 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ 3202 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 3203 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 3204 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) | 3205 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */ 3206 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */ 3207 PACKET3_MAP_QUEUES_ENGINE_SEL(0) | /* engine_sel: compute */ 3208 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */ 3209 amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); 3210 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 3211 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 3212 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 3213 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 3214 } 3215 3216 r = amdgpu_ring_test_helper(kiq_ring); 3217 if (r) 3218 DRM_ERROR("KCQ enable failed\n"); 3219 3220 return r; 3221 } 3222 3223 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring) 3224 { 3225 struct amdgpu_device *adev = ring->adev; 3226 struct v9_mqd *mqd = ring->mqd_ptr; 3227 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 3228 uint32_t tmp; 3229 3230 mqd->header = 0xC0310800; 3231 mqd->compute_pipelinestat_enable = 0x00000001; 3232 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 3233 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 3234 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 3235 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 3236 mqd->compute_static_thread_mgmt_se4 = 0xffffffff; 3237 mqd->compute_static_thread_mgmt_se5 = 0xffffffff; 3238 mqd->compute_static_thread_mgmt_se6 = 0xffffffff; 3239 mqd->compute_static_thread_mgmt_se7 = 0xffffffff; 3240 mqd->compute_misc_reserved = 0x00000003; 3241 3242 mqd->dynamic_cu_mask_addr_lo = 3243 lower_32_bits(ring->mqd_gpu_addr 3244 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3245 mqd->dynamic_cu_mask_addr_hi = 3246 upper_32_bits(ring->mqd_gpu_addr 3247 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3248 3249 eop_base_addr = ring->eop_gpu_addr >> 8; 3250 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 3251 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 3252 3253 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3254 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL); 3255 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 3256 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1)); 3257 3258 mqd->cp_hqd_eop_control = tmp; 3259 3260 /* enable doorbell? */ 3261 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3262 3263 if (ring->use_doorbell) { 3264 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3265 DOORBELL_OFFSET, ring->doorbell_index); 3266 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3267 DOORBELL_EN, 1); 3268 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3269 DOORBELL_SOURCE, 0); 3270 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3271 DOORBELL_HIT, 0); 3272 } else { 3273 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3274 DOORBELL_EN, 0); 3275 } 3276 3277 mqd->cp_hqd_pq_doorbell_control = tmp; 3278 3279 /* disable the queue if it's active */ 3280 ring->wptr = 0; 3281 mqd->cp_hqd_dequeue_request = 0; 3282 mqd->cp_hqd_pq_rptr = 0; 3283 mqd->cp_hqd_pq_wptr_lo = 0; 3284 mqd->cp_hqd_pq_wptr_hi = 0; 3285 3286 /* set the pointer to the MQD */ 3287 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; 3288 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); 3289 3290 /* set MQD vmid to 0 */ 3291 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL); 3292 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 3293 mqd->cp_mqd_control = tmp; 3294 3295 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3296 hqd_gpu_addr = ring->gpu_addr >> 8; 3297 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 3298 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 3299 3300 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3301 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL); 3302 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 3303 (order_base_2(ring->ring_size / 4) - 1)); 3304 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 3305 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8)); 3306 #ifdef __BIG_ENDIAN 3307 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1); 3308 #endif 3309 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0); 3310 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0); 3311 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 3312 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 3313 mqd->cp_hqd_pq_control = tmp; 3314 3315 /* set the wb address whether it's enabled or not */ 3316 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3317 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 3318 mqd->cp_hqd_pq_rptr_report_addr_hi = 3319 upper_32_bits(wb_gpu_addr) & 0xffff; 3320 3321 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3322 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3323 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 3324 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 3325 3326 tmp = 0; 3327 /* enable the doorbell if requested */ 3328 if (ring->use_doorbell) { 3329 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3330 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3331 DOORBELL_OFFSET, ring->doorbell_index); 3332 3333 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3334 DOORBELL_EN, 1); 3335 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3336 DOORBELL_SOURCE, 0); 3337 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3338 DOORBELL_HIT, 0); 3339 } 3340 3341 mqd->cp_hqd_pq_doorbell_control = tmp; 3342 3343 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3344 ring->wptr = 0; 3345 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR); 3346 3347 /* set the vmid for the queue */ 3348 mqd->cp_hqd_vmid = 0; 3349 3350 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE); 3351 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53); 3352 mqd->cp_hqd_persistent_state = tmp; 3353 3354 /* set MIN_IB_AVAIL_SIZE */ 3355 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL); 3356 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 3357 mqd->cp_hqd_ib_control = tmp; 3358 3359 /* activate the queue */ 3360 mqd->cp_hqd_active = 1; 3361 3362 return 0; 3363 } 3364 3365 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring) 3366 { 3367 struct amdgpu_device *adev = ring->adev; 3368 struct v9_mqd *mqd = ring->mqd_ptr; 3369 int j; 3370 3371 /* disable wptr polling */ 3372 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3373 3374 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR, 3375 mqd->cp_hqd_eop_base_addr_lo); 3376 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI, 3377 mqd->cp_hqd_eop_base_addr_hi); 3378 3379 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3380 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL, 3381 mqd->cp_hqd_eop_control); 3382 3383 /* enable doorbell? */ 3384 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3385 mqd->cp_hqd_pq_doorbell_control); 3386 3387 /* disable the queue if it's active */ 3388 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3389 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3390 for (j = 0; j < adev->usec_timeout; j++) { 3391 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3392 break; 3393 udelay(1); 3394 } 3395 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3396 mqd->cp_hqd_dequeue_request); 3397 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 3398 mqd->cp_hqd_pq_rptr); 3399 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3400 mqd->cp_hqd_pq_wptr_lo); 3401 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3402 mqd->cp_hqd_pq_wptr_hi); 3403 } 3404 3405 /* set the pointer to the MQD */ 3406 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR, 3407 mqd->cp_mqd_base_addr_lo); 3408 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI, 3409 mqd->cp_mqd_base_addr_hi); 3410 3411 /* set MQD vmid to 0 */ 3412 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL, 3413 mqd->cp_mqd_control); 3414 3415 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3416 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE, 3417 mqd->cp_hqd_pq_base_lo); 3418 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI, 3419 mqd->cp_hqd_pq_base_hi); 3420 3421 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3422 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL, 3423 mqd->cp_hqd_pq_control); 3424 3425 /* set the wb address whether it's enabled or not */ 3426 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR, 3427 mqd->cp_hqd_pq_rptr_report_addr_lo); 3428 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 3429 mqd->cp_hqd_pq_rptr_report_addr_hi); 3430 3431 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3432 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR, 3433 mqd->cp_hqd_pq_wptr_poll_addr_lo); 3434 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI, 3435 mqd->cp_hqd_pq_wptr_poll_addr_hi); 3436 3437 /* enable the doorbell if requested */ 3438 if (ring->use_doorbell) { 3439 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER, 3440 (adev->doorbell_index.kiq * 2) << 2); 3441 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3442 (adev->doorbell_index.userqueue_end * 2) << 2); 3443 } 3444 3445 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3446 mqd->cp_hqd_pq_doorbell_control); 3447 3448 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3449 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3450 mqd->cp_hqd_pq_wptr_lo); 3451 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3452 mqd->cp_hqd_pq_wptr_hi); 3453 3454 /* set the vmid for the queue */ 3455 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid); 3456 3457 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 3458 mqd->cp_hqd_persistent_state); 3459 3460 /* activate the queue */ 3461 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 3462 mqd->cp_hqd_active); 3463 3464 if (ring->use_doorbell) 3465 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1); 3466 3467 return 0; 3468 } 3469 3470 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring) 3471 { 3472 struct amdgpu_device *adev = ring->adev; 3473 int j; 3474 3475 /* disable the queue if it's active */ 3476 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3477 3478 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3479 3480 for (j = 0; j < adev->usec_timeout; j++) { 3481 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3482 break; 3483 udelay(1); 3484 } 3485 3486 if (j == AMDGPU_MAX_USEC_TIMEOUT) { 3487 DRM_DEBUG("KIQ dequeue request failed.\n"); 3488 3489 /* Manual disable if dequeue request times out */ 3490 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0); 3491 } 3492 3493 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3494 0); 3495 } 3496 3497 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0); 3498 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0); 3499 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0); 3500 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000); 3501 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0); 3502 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0); 3503 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0); 3504 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0); 3505 3506 return 0; 3507 } 3508 3509 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring) 3510 { 3511 struct amdgpu_device *adev = ring->adev; 3512 struct v9_mqd *mqd = ring->mqd_ptr; 3513 int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS; 3514 3515 gfx_v9_0_kiq_setting(ring); 3516 3517 if (adev->in_gpu_reset) { /* for GPU_RESET case */ 3518 /* reset MQD to a clean status */ 3519 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3520 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3521 3522 /* reset ring buffer */ 3523 ring->wptr = 0; 3524 amdgpu_ring_clear_ring(ring); 3525 3526 mutex_lock(&adev->srbm_mutex); 3527 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3528 gfx_v9_0_kiq_init_register(ring); 3529 soc15_grbm_select(adev, 0, 0, 0, 0); 3530 mutex_unlock(&adev->srbm_mutex); 3531 } else { 3532 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3533 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3534 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3535 mutex_lock(&adev->srbm_mutex); 3536 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3537 gfx_v9_0_mqd_init(ring); 3538 gfx_v9_0_kiq_init_register(ring); 3539 soc15_grbm_select(adev, 0, 0, 0, 0); 3540 mutex_unlock(&adev->srbm_mutex); 3541 3542 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3543 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3544 } 3545 3546 return 0; 3547 } 3548 3549 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring) 3550 { 3551 struct amdgpu_device *adev = ring->adev; 3552 struct v9_mqd *mqd = ring->mqd_ptr; 3553 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 3554 3555 if (!adev->in_gpu_reset && !adev->in_suspend) { 3556 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3557 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3558 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3559 mutex_lock(&adev->srbm_mutex); 3560 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3561 gfx_v9_0_mqd_init(ring); 3562 soc15_grbm_select(adev, 0, 0, 0, 0); 3563 mutex_unlock(&adev->srbm_mutex); 3564 3565 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3566 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3567 } else if (adev->in_gpu_reset) { /* for GPU_RESET case */ 3568 /* reset MQD to a clean status */ 3569 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3570 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3571 3572 /* reset ring buffer */ 3573 ring->wptr = 0; 3574 amdgpu_ring_clear_ring(ring); 3575 } else { 3576 amdgpu_ring_clear_ring(ring); 3577 } 3578 3579 return 0; 3580 } 3581 3582 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev) 3583 { 3584 struct amdgpu_ring *ring; 3585 int r; 3586 3587 ring = &adev->gfx.kiq.ring; 3588 3589 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3590 if (unlikely(r != 0)) 3591 return r; 3592 3593 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3594 if (unlikely(r != 0)) 3595 return r; 3596 3597 gfx_v9_0_kiq_init_queue(ring); 3598 amdgpu_bo_kunmap(ring->mqd_obj); 3599 ring->mqd_ptr = NULL; 3600 amdgpu_bo_unreserve(ring->mqd_obj); 3601 ring->sched.ready = true; 3602 return 0; 3603 } 3604 3605 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev) 3606 { 3607 struct amdgpu_ring *ring = NULL; 3608 int r = 0, i; 3609 3610 gfx_v9_0_cp_compute_enable(adev, true); 3611 3612 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3613 ring = &adev->gfx.compute_ring[i]; 3614 3615 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3616 if (unlikely(r != 0)) 3617 goto done; 3618 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3619 if (!r) { 3620 r = gfx_v9_0_kcq_init_queue(ring); 3621 amdgpu_bo_kunmap(ring->mqd_obj); 3622 ring->mqd_ptr = NULL; 3623 } 3624 amdgpu_bo_unreserve(ring->mqd_obj); 3625 if (r) 3626 goto done; 3627 } 3628 3629 r = gfx_v9_0_kiq_kcq_enable(adev); 3630 done: 3631 return r; 3632 } 3633 3634 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev) 3635 { 3636 int r, i; 3637 struct amdgpu_ring *ring; 3638 3639 if (!(adev->flags & AMD_IS_APU)) 3640 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3641 3642 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3643 if (adev->asic_type != CHIP_ARCTURUS) { 3644 /* legacy firmware loading */ 3645 r = gfx_v9_0_cp_gfx_load_microcode(adev); 3646 if (r) 3647 return r; 3648 } 3649 3650 r = gfx_v9_0_cp_compute_load_microcode(adev); 3651 if (r) 3652 return r; 3653 } 3654 3655 r = gfx_v9_0_kiq_resume(adev); 3656 if (r) 3657 return r; 3658 3659 if (adev->asic_type != CHIP_ARCTURUS) { 3660 r = gfx_v9_0_cp_gfx_resume(adev); 3661 if (r) 3662 return r; 3663 } 3664 3665 r = gfx_v9_0_kcq_resume(adev); 3666 if (r) 3667 return r; 3668 3669 if (adev->asic_type != CHIP_ARCTURUS) { 3670 ring = &adev->gfx.gfx_ring[0]; 3671 r = amdgpu_ring_test_helper(ring); 3672 if (r) 3673 return r; 3674 } 3675 3676 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3677 ring = &adev->gfx.compute_ring[i]; 3678 amdgpu_ring_test_helper(ring); 3679 } 3680 3681 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3682 3683 return 0; 3684 } 3685 3686 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable) 3687 { 3688 if (adev->asic_type != CHIP_ARCTURUS) 3689 gfx_v9_0_cp_gfx_enable(adev, enable); 3690 gfx_v9_0_cp_compute_enable(adev, enable); 3691 } 3692 3693 static int gfx_v9_0_hw_init(void *handle) 3694 { 3695 int r; 3696 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3697 3698 if (!amdgpu_sriov_vf(adev)) 3699 gfx_v9_0_init_golden_registers(adev); 3700 3701 gfx_v9_0_constants_init(adev); 3702 3703 r = gfx_v9_0_csb_vram_pin(adev); 3704 if (r) 3705 return r; 3706 3707 r = adev->gfx.rlc.funcs->resume(adev); 3708 if (r) 3709 return r; 3710 3711 r = gfx_v9_0_cp_resume(adev); 3712 if (r) 3713 return r; 3714 3715 return r; 3716 } 3717 3718 static int gfx_v9_0_kcq_disable(struct amdgpu_device *adev) 3719 { 3720 int r, i; 3721 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; 3722 3723 r = amdgpu_ring_alloc(kiq_ring, 6 * adev->gfx.num_compute_rings); 3724 if (r) 3725 DRM_ERROR("Failed to lock KIQ (%d).\n", r); 3726 3727 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3728 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; 3729 3730 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 3731 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 3732 PACKET3_UNMAP_QUEUES_ACTION(1) | /* RESET_QUEUES */ 3733 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 3734 PACKET3_UNMAP_QUEUES_ENGINE_SEL(0) | 3735 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 3736 amdgpu_ring_write(kiq_ring, PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 3737 amdgpu_ring_write(kiq_ring, 0); 3738 amdgpu_ring_write(kiq_ring, 0); 3739 amdgpu_ring_write(kiq_ring, 0); 3740 } 3741 r = amdgpu_ring_test_helper(kiq_ring); 3742 if (r) 3743 DRM_ERROR("KCQ disable failed\n"); 3744 3745 return r; 3746 } 3747 3748 static int gfx_v9_0_hw_fini(void *handle) 3749 { 3750 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3751 3752 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); 3753 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 3754 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 3755 3756 /* DF freeze and kcq disable will fail */ 3757 if (!amdgpu_ras_intr_triggered()) 3758 /* disable KCQ to avoid CPC touch memory not valid anymore */ 3759 gfx_v9_0_kcq_disable(adev); 3760 3761 if (amdgpu_sriov_vf(adev)) { 3762 gfx_v9_0_cp_gfx_enable(adev, false); 3763 /* must disable polling for SRIOV when hw finished, otherwise 3764 * CPC engine may still keep fetching WB address which is already 3765 * invalid after sw finished and trigger DMAR reading error in 3766 * hypervisor side. 3767 */ 3768 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3769 return 0; 3770 } 3771 3772 /* Use deinitialize sequence from CAIL when unbinding device from driver, 3773 * otherwise KIQ is hanging when binding back 3774 */ 3775 if (!adev->in_gpu_reset && !adev->in_suspend) { 3776 mutex_lock(&adev->srbm_mutex); 3777 soc15_grbm_select(adev, adev->gfx.kiq.ring.me, 3778 adev->gfx.kiq.ring.pipe, 3779 adev->gfx.kiq.ring.queue, 0); 3780 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring); 3781 soc15_grbm_select(adev, 0, 0, 0, 0); 3782 mutex_unlock(&adev->srbm_mutex); 3783 } 3784 3785 gfx_v9_0_cp_enable(adev, false); 3786 adev->gfx.rlc.funcs->stop(adev); 3787 3788 gfx_v9_0_csb_vram_unpin(adev); 3789 3790 return 0; 3791 } 3792 3793 static int gfx_v9_0_suspend(void *handle) 3794 { 3795 return gfx_v9_0_hw_fini(handle); 3796 } 3797 3798 static int gfx_v9_0_resume(void *handle) 3799 { 3800 return gfx_v9_0_hw_init(handle); 3801 } 3802 3803 static bool gfx_v9_0_is_idle(void *handle) 3804 { 3805 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3806 3807 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS), 3808 GRBM_STATUS, GUI_ACTIVE)) 3809 return false; 3810 else 3811 return true; 3812 } 3813 3814 static int gfx_v9_0_wait_for_idle(void *handle) 3815 { 3816 unsigned i; 3817 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3818 3819 for (i = 0; i < adev->usec_timeout; i++) { 3820 if (gfx_v9_0_is_idle(handle)) 3821 return 0; 3822 udelay(1); 3823 } 3824 return -ETIMEDOUT; 3825 } 3826 3827 static int gfx_v9_0_soft_reset(void *handle) 3828 { 3829 u32 grbm_soft_reset = 0; 3830 u32 tmp; 3831 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3832 3833 /* GRBM_STATUS */ 3834 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS); 3835 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK | 3836 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK | 3837 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK | 3838 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK | 3839 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK | 3840 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) { 3841 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 3842 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 3843 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 3844 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1); 3845 } 3846 3847 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) { 3848 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 3849 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 3850 } 3851 3852 /* GRBM_STATUS2 */ 3853 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2); 3854 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY)) 3855 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 3856 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 3857 3858 3859 if (grbm_soft_reset) { 3860 /* stop the rlc */ 3861 adev->gfx.rlc.funcs->stop(adev); 3862 3863 if (adev->asic_type != CHIP_ARCTURUS) 3864 /* Disable GFX parsing/prefetching */ 3865 gfx_v9_0_cp_gfx_enable(adev, false); 3866 3867 /* Disable MEC parsing/prefetching */ 3868 gfx_v9_0_cp_compute_enable(adev, false); 3869 3870 if (grbm_soft_reset) { 3871 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 3872 tmp |= grbm_soft_reset; 3873 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 3874 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 3875 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 3876 3877 udelay(50); 3878 3879 tmp &= ~grbm_soft_reset; 3880 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 3881 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 3882 } 3883 3884 /* Wait a little for things to settle down */ 3885 udelay(50); 3886 } 3887 return 0; 3888 } 3889 3890 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev) 3891 { 3892 uint64_t clock; 3893 3894 mutex_lock(&adev->gfx.gpu_clock_mutex); 3895 if (adev->asic_type == CHIP_VEGA10 && amdgpu_sriov_runtime(adev)) { 3896 uint32_t tmp, lsb, msb, i = 0; 3897 do { 3898 if (i != 0) 3899 udelay(1); 3900 tmp = RREG32_SOC15(GC, 0, mmRLC_REFCLOCK_TIMESTAMP_MSB); 3901 lsb = RREG32_SOC15(GC, 0, mmRLC_REFCLOCK_TIMESTAMP_LSB); 3902 msb = RREG32_SOC15(GC, 0, mmRLC_REFCLOCK_TIMESTAMP_MSB); 3903 i++; 3904 } while (unlikely(tmp != msb) && (i < adev->usec_timeout)); 3905 clock = (uint64_t)lsb | ((uint64_t)msb << 32ULL); 3906 } else { 3907 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1); 3908 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) | 3909 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 3910 } 3911 mutex_unlock(&adev->gfx.gpu_clock_mutex); 3912 return clock; 3913 } 3914 3915 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 3916 uint32_t vmid, 3917 uint32_t gds_base, uint32_t gds_size, 3918 uint32_t gws_base, uint32_t gws_size, 3919 uint32_t oa_base, uint32_t oa_size) 3920 { 3921 struct amdgpu_device *adev = ring->adev; 3922 3923 /* GDS Base */ 3924 gfx_v9_0_write_data_to_reg(ring, 0, false, 3925 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid, 3926 gds_base); 3927 3928 /* GDS Size */ 3929 gfx_v9_0_write_data_to_reg(ring, 0, false, 3930 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid, 3931 gds_size); 3932 3933 /* GWS */ 3934 gfx_v9_0_write_data_to_reg(ring, 0, false, 3935 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid, 3936 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 3937 3938 /* OA */ 3939 gfx_v9_0_write_data_to_reg(ring, 0, false, 3940 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid, 3941 (1 << (oa_size + oa_base)) - (1 << oa_base)); 3942 } 3943 3944 static const u32 vgpr_init_compute_shader[] = 3945 { 3946 0xb07c0000, 0xbe8000ff, 3947 0x000000f8, 0xbf110800, 3948 0x7e000280, 0x7e020280, 3949 0x7e040280, 0x7e060280, 3950 0x7e080280, 0x7e0a0280, 3951 0x7e0c0280, 0x7e0e0280, 3952 0x80808800, 0xbe803200, 3953 0xbf84fff5, 0xbf9c0000, 3954 0xd28c0001, 0x0001007f, 3955 0xd28d0001, 0x0002027e, 3956 0x10020288, 0xb8810904, 3957 0xb7814000, 0xd1196a01, 3958 0x00000301, 0xbe800087, 3959 0xbefc00c1, 0xd89c4000, 3960 0x00020201, 0xd89cc080, 3961 0x00040401, 0x320202ff, 3962 0x00000800, 0x80808100, 3963 0xbf84fff8, 0x7e020280, 3964 0xbf810000, 0x00000000, 3965 }; 3966 3967 static const u32 sgpr_init_compute_shader[] = 3968 { 3969 0xb07c0000, 0xbe8000ff, 3970 0x0000005f, 0xbee50080, 3971 0xbe812c65, 0xbe822c65, 3972 0xbe832c65, 0xbe842c65, 3973 0xbe852c65, 0xb77c0005, 3974 0x80808500, 0xbf84fff8, 3975 0xbe800080, 0xbf810000, 3976 }; 3977 3978 static const struct soc15_reg_entry vgpr_init_regs[] = { 3979 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 3980 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 3981 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 3982 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 3983 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x1000000 }, /* CU_GROUP_COUNT=1 */ 3984 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 256*2 }, 3985 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 1 }, 3986 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 3987 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x100007f }, /* VGPRS=15 (256 logical VGPRs, SGPRS=1 (16 SGPRs, BULKY=1 */ 3988 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 3989 }; 3990 3991 static const struct soc15_reg_entry sgpr_init_regs[] = { 3992 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 3993 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 3994 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 3995 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 3996 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x1000000 }, /* CU_GROUP_COUNT=1 */ 3997 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 256*2 }, 3998 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 1 }, 3999 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4000 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x340 }, /* SGPRS=13 (112 GPRS) */ 4001 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4002 }; 4003 4004 static const struct soc15_reg_entry sec_ded_counter_registers[] = { 4005 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1}, 4006 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1}, 4007 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1}, 4008 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1}, 4009 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1}, 4010 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1}, 4011 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1}, 4012 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1}, 4013 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1}, 4014 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1}, 4015 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1}, 4016 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1}, 4017 { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1}, 4018 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6}, 4019 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16}, 4020 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16}, 4021 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16}, 4022 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16}, 4023 { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16}, 4024 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16}, 4025 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16}, 4026 { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16}, 4027 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6}, 4028 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16}, 4029 { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16}, 4030 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1}, 4031 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1}, 4032 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32}, 4033 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32}, 4034 { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72}, 4035 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16}, 4036 { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2}, 4037 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6}, 4038 }; 4039 4040 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev) 4041 { 4042 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4043 int i, r; 4044 4045 /* only support when RAS is enabled */ 4046 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4047 return 0; 4048 4049 r = amdgpu_ring_alloc(ring, 7); 4050 if (r) { 4051 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n", 4052 ring->name, r); 4053 return r; 4054 } 4055 4056 WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000); 4057 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size); 4058 4059 amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 4060 amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC | 4061 PACKET3_DMA_DATA_DST_SEL(1) | 4062 PACKET3_DMA_DATA_SRC_SEL(2) | 4063 PACKET3_DMA_DATA_ENGINE(0))); 4064 amdgpu_ring_write(ring, 0); 4065 amdgpu_ring_write(ring, 0); 4066 amdgpu_ring_write(ring, 0); 4067 amdgpu_ring_write(ring, 0); 4068 amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT | 4069 adev->gds.gds_size); 4070 4071 amdgpu_ring_commit(ring); 4072 4073 for (i = 0; i < adev->usec_timeout; i++) { 4074 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring)) 4075 break; 4076 udelay(1); 4077 } 4078 4079 if (i >= adev->usec_timeout) 4080 r = -ETIMEDOUT; 4081 4082 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000); 4083 4084 return r; 4085 } 4086 4087 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) 4088 { 4089 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4090 struct amdgpu_ib ib; 4091 struct dma_fence *f = NULL; 4092 int r, i, j, k; 4093 unsigned total_size, vgpr_offset, sgpr_offset; 4094 u64 gpu_addr; 4095 4096 /* only support when RAS is enabled */ 4097 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4098 return 0; 4099 4100 /* bail if the compute ring is not ready */ 4101 if (!ring->sched.ready) 4102 return 0; 4103 4104 total_size = 4105 ((ARRAY_SIZE(vgpr_init_regs) * 3) + 4 + 5 + 2) * 4; 4106 total_size += 4107 ((ARRAY_SIZE(sgpr_init_regs) * 3) + 4 + 5 + 2) * 4; 4108 total_size = ALIGN(total_size, 256); 4109 vgpr_offset = total_size; 4110 total_size += ALIGN(sizeof(vgpr_init_compute_shader), 256); 4111 sgpr_offset = total_size; 4112 total_size += sizeof(sgpr_init_compute_shader); 4113 4114 /* allocate an indirect buffer to put the commands in */ 4115 memset(&ib, 0, sizeof(ib)); 4116 r = amdgpu_ib_get(adev, NULL, total_size, &ib); 4117 if (r) { 4118 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); 4119 return r; 4120 } 4121 4122 /* load the compute shaders */ 4123 for (i = 0; i < ARRAY_SIZE(vgpr_init_compute_shader); i++) 4124 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_compute_shader[i]; 4125 4126 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++) 4127 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i]; 4128 4129 /* init the ib length to 0 */ 4130 ib.length_dw = 0; 4131 4132 /* VGPR */ 4133 /* write the register state for the compute dispatch */ 4134 for (i = 0; i < ARRAY_SIZE(vgpr_init_regs); i++) { 4135 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4136 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs[i]) 4137 - PACKET3_SET_SH_REG_START; 4138 ib.ptr[ib.length_dw++] = vgpr_init_regs[i].reg_value; 4139 } 4140 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4141 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8; 4142 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4143 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4144 - PACKET3_SET_SH_REG_START; 4145 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4146 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4147 4148 /* write dispatch packet */ 4149 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4150 ib.ptr[ib.length_dw++] = 128; /* x */ 4151 ib.ptr[ib.length_dw++] = 1; /* y */ 4152 ib.ptr[ib.length_dw++] = 1; /* z */ 4153 ib.ptr[ib.length_dw++] = 4154 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4155 4156 /* write CS partial flush packet */ 4157 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4158 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4159 4160 /* SGPR */ 4161 /* write the register state for the compute dispatch */ 4162 for (i = 0; i < ARRAY_SIZE(sgpr_init_regs); i++) { 4163 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4164 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr_init_regs[i]) 4165 - PACKET3_SET_SH_REG_START; 4166 ib.ptr[ib.length_dw++] = sgpr_init_regs[i].reg_value; 4167 } 4168 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4169 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4170 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4171 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4172 - PACKET3_SET_SH_REG_START; 4173 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4174 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4175 4176 /* write dispatch packet */ 4177 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4178 ib.ptr[ib.length_dw++] = 128; /* x */ 4179 ib.ptr[ib.length_dw++] = 1; /* y */ 4180 ib.ptr[ib.length_dw++] = 1; /* z */ 4181 ib.ptr[ib.length_dw++] = 4182 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4183 4184 /* write CS partial flush packet */ 4185 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4186 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4187 4188 /* shedule the ib on the ring */ 4189 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 4190 if (r) { 4191 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r); 4192 goto fail; 4193 } 4194 4195 /* wait for the GPU to finish processing the IB */ 4196 r = dma_fence_wait(f, false); 4197 if (r) { 4198 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); 4199 goto fail; 4200 } 4201 4202 /* read back registers to clear the counters */ 4203 mutex_lock(&adev->grbm_idx_mutex); 4204 for (i = 0; i < ARRAY_SIZE(sec_ded_counter_registers); i++) { 4205 for (j = 0; j < sec_ded_counter_registers[i].se_num; j++) { 4206 for (k = 0; k < sec_ded_counter_registers[i].instance; k++) { 4207 gfx_v9_0_select_se_sh(adev, j, 0x0, k); 4208 RREG32(SOC15_REG_ENTRY_OFFSET(sec_ded_counter_registers[i])); 4209 } 4210 } 4211 } 4212 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000); 4213 mutex_unlock(&adev->grbm_idx_mutex); 4214 4215 fail: 4216 amdgpu_ib_free(adev, &ib, NULL); 4217 dma_fence_put(f); 4218 4219 return r; 4220 } 4221 4222 static int gfx_v9_0_early_init(void *handle) 4223 { 4224 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4225 4226 if (adev->asic_type == CHIP_ARCTURUS) 4227 adev->gfx.num_gfx_rings = 0; 4228 else 4229 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS; 4230 adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS; 4231 gfx_v9_0_set_ring_funcs(adev); 4232 gfx_v9_0_set_irq_funcs(adev); 4233 gfx_v9_0_set_gds_init(adev); 4234 gfx_v9_0_set_rlc_funcs(adev); 4235 4236 return 0; 4237 } 4238 4239 static int gfx_v9_0_ecc_late_init(void *handle) 4240 { 4241 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4242 int r; 4243 4244 r = amdgpu_gfx_ras_late_init(adev); 4245 if (r) 4246 return r; 4247 4248 r = gfx_v9_0_do_edc_gds_workarounds(adev); 4249 if (r) 4250 return r; 4251 4252 /* requires IBs so do in late init after IB pool is initialized */ 4253 r = gfx_v9_0_do_edc_gpr_workarounds(adev); 4254 if (r) 4255 return r; 4256 4257 return 0; 4258 } 4259 4260 static int gfx_v9_0_late_init(void *handle) 4261 { 4262 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4263 int r; 4264 4265 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 4266 if (r) 4267 return r; 4268 4269 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 4270 if (r) 4271 return r; 4272 4273 r = gfx_v9_0_ecc_late_init(handle); 4274 if (r) 4275 return r; 4276 4277 return 0; 4278 } 4279 4280 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev) 4281 { 4282 uint32_t rlc_setting; 4283 4284 /* if RLC is not enabled, do nothing */ 4285 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL); 4286 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK)) 4287 return false; 4288 4289 return true; 4290 } 4291 4292 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev) 4293 { 4294 uint32_t data; 4295 unsigned i; 4296 4297 data = RLC_SAFE_MODE__CMD_MASK; 4298 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 4299 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4300 4301 /* wait for RLC_SAFE_MODE */ 4302 for (i = 0; i < adev->usec_timeout; i++) { 4303 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 4304 break; 4305 udelay(1); 4306 } 4307 } 4308 4309 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev) 4310 { 4311 uint32_t data; 4312 4313 data = RLC_SAFE_MODE__CMD_MASK; 4314 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4315 } 4316 4317 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev, 4318 bool enable) 4319 { 4320 amdgpu_gfx_rlc_enter_safe_mode(adev); 4321 4322 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { 4323 gfx_v9_0_enable_gfx_cg_power_gating(adev, true); 4324 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4325 gfx_v9_0_enable_gfx_pipeline_powergating(adev, true); 4326 } else { 4327 gfx_v9_0_enable_gfx_cg_power_gating(adev, false); 4328 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4329 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false); 4330 } 4331 4332 amdgpu_gfx_rlc_exit_safe_mode(adev); 4333 } 4334 4335 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev, 4336 bool enable) 4337 { 4338 /* TODO: double check if we need to perform under safe mode */ 4339 /* gfx_v9_0_enter_rlc_safe_mode(adev); */ 4340 4341 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 4342 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true); 4343 else 4344 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false); 4345 4346 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 4347 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true); 4348 else 4349 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false); 4350 4351 /* gfx_v9_0_exit_rlc_safe_mode(adev); */ 4352 } 4353 4354 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 4355 bool enable) 4356 { 4357 uint32_t data, def; 4358 4359 amdgpu_gfx_rlc_enter_safe_mode(adev); 4360 4361 /* It is disabled by HW by default */ 4362 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { 4363 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 4364 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4365 4366 if (adev->asic_type != CHIP_VEGA12) 4367 data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4368 4369 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4370 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4371 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4372 4373 /* only for Vega10 & Raven1 */ 4374 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK; 4375 4376 if (def != data) 4377 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4378 4379 /* MGLS is a global flag to control all MGLS in GFX */ 4380 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 4381 /* 2 - RLC memory Light sleep */ 4382 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { 4383 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4384 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4385 if (def != data) 4386 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4387 } 4388 /* 3 - CP memory Light sleep */ 4389 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { 4390 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4391 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4392 if (def != data) 4393 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4394 } 4395 } 4396 } else { 4397 /* 1 - MGCG_OVERRIDE */ 4398 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4399 4400 if (adev->asic_type != CHIP_VEGA12) 4401 data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4402 4403 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 4404 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4405 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4406 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4407 4408 if (def != data) 4409 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4410 4411 /* 2 - disable MGLS in RLC */ 4412 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4413 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) { 4414 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4415 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4416 } 4417 4418 /* 3 - disable MGLS in CP */ 4419 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4420 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) { 4421 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4422 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4423 } 4424 } 4425 4426 amdgpu_gfx_rlc_exit_safe_mode(adev); 4427 } 4428 4429 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev, 4430 bool enable) 4431 { 4432 uint32_t data, def; 4433 4434 if (adev->asic_type == CHIP_ARCTURUS) 4435 return; 4436 4437 amdgpu_gfx_rlc_enter_safe_mode(adev); 4438 4439 /* Enable 3D CGCG/CGLS */ 4440 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)) { 4441 /* write cmd to clear cgcg/cgls ov */ 4442 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4443 /* unset CGCG override */ 4444 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 4445 /* update CGCG and CGLS override bits */ 4446 if (def != data) 4447 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4448 4449 /* enable 3Dcgcg FSM(0x0000363f) */ 4450 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4451 4452 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4453 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 4454 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 4455 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 4456 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 4457 if (def != data) 4458 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4459 4460 /* set IDLE_POLL_COUNT(0x00900100) */ 4461 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 4462 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 4463 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 4464 if (def != data) 4465 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 4466 } else { 4467 /* Disable CGCG/CGLS */ 4468 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4469 /* disable cgcg, cgls should be disabled */ 4470 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK | 4471 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK); 4472 /* disable cgcg and cgls in FSM */ 4473 if (def != data) 4474 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4475 } 4476 4477 amdgpu_gfx_rlc_exit_safe_mode(adev); 4478 } 4479 4480 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 4481 bool enable) 4482 { 4483 uint32_t def, data; 4484 4485 amdgpu_gfx_rlc_enter_safe_mode(adev); 4486 4487 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { 4488 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4489 /* unset CGCG override */ 4490 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 4491 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 4492 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 4493 else 4494 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 4495 /* update CGCG and CGLS override bits */ 4496 if (def != data) 4497 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4498 4499 /* enable cgcg FSM(0x0000363F) */ 4500 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 4501 4502 if (adev->asic_type == CHIP_ARCTURUS) 4503 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4504 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 4505 else 4506 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4507 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 4508 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 4509 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 4510 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 4511 if (def != data) 4512 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 4513 4514 /* set IDLE_POLL_COUNT(0x00900100) */ 4515 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 4516 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 4517 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 4518 if (def != data) 4519 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 4520 } else { 4521 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 4522 /* reset CGCG/CGLS bits */ 4523 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK); 4524 /* disable cgcg and cgls in FSM */ 4525 if (def != data) 4526 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 4527 } 4528 4529 amdgpu_gfx_rlc_exit_safe_mode(adev); 4530 } 4531 4532 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev, 4533 bool enable) 4534 { 4535 if (enable) { 4536 /* CGCG/CGLS should be enabled after MGCG/MGLS 4537 * === MGCG + MGLS === 4538 */ 4539 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 4540 /* === CGCG /CGLS for GFX 3D Only === */ 4541 gfx_v9_0_update_3d_clock_gating(adev, enable); 4542 /* === CGCG + CGLS === */ 4543 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 4544 } else { 4545 /* CGCG/CGLS should be disabled before MGCG/MGLS 4546 * === CGCG + CGLS === 4547 */ 4548 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 4549 /* === CGCG /CGLS for GFX 3D Only === */ 4550 gfx_v9_0_update_3d_clock_gating(adev, enable); 4551 /* === MGCG + MGLS === */ 4552 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 4553 } 4554 return 0; 4555 } 4556 4557 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = { 4558 .is_rlc_enabled = gfx_v9_0_is_rlc_enabled, 4559 .set_safe_mode = gfx_v9_0_set_safe_mode, 4560 .unset_safe_mode = gfx_v9_0_unset_safe_mode, 4561 .init = gfx_v9_0_rlc_init, 4562 .get_csb_size = gfx_v9_0_get_csb_size, 4563 .get_csb_buffer = gfx_v9_0_get_csb_buffer, 4564 .get_cp_table_num = gfx_v9_0_cp_jump_table_num, 4565 .resume = gfx_v9_0_rlc_resume, 4566 .stop = gfx_v9_0_rlc_stop, 4567 .reset = gfx_v9_0_rlc_reset, 4568 .start = gfx_v9_0_rlc_start 4569 }; 4570 4571 static int gfx_v9_0_set_powergating_state(void *handle, 4572 enum amd_powergating_state state) 4573 { 4574 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4575 bool enable = (state == AMD_PG_STATE_GATE) ? true : false; 4576 4577 switch (adev->asic_type) { 4578 case CHIP_RAVEN: 4579 case CHIP_RENOIR: 4580 if (!enable) { 4581 amdgpu_gfx_off_ctrl(adev, false); 4582 cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work); 4583 } 4584 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { 4585 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true); 4586 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true); 4587 } else { 4588 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false); 4589 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false); 4590 } 4591 4592 if (adev->pg_flags & AMD_PG_SUPPORT_CP) 4593 gfx_v9_0_enable_cp_power_gating(adev, true); 4594 else 4595 gfx_v9_0_enable_cp_power_gating(adev, false); 4596 4597 /* update gfx cgpg state */ 4598 gfx_v9_0_update_gfx_cg_power_gating(adev, enable); 4599 4600 /* update mgcg state */ 4601 gfx_v9_0_update_gfx_mg_power_gating(adev, enable); 4602 4603 if (enable) 4604 amdgpu_gfx_off_ctrl(adev, true); 4605 break; 4606 case CHIP_VEGA12: 4607 if (!enable) { 4608 amdgpu_gfx_off_ctrl(adev, false); 4609 cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work); 4610 } else { 4611 amdgpu_gfx_off_ctrl(adev, true); 4612 } 4613 break; 4614 default: 4615 break; 4616 } 4617 4618 return 0; 4619 } 4620 4621 static int gfx_v9_0_set_clockgating_state(void *handle, 4622 enum amd_clockgating_state state) 4623 { 4624 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4625 4626 if (amdgpu_sriov_vf(adev)) 4627 return 0; 4628 4629 switch (adev->asic_type) { 4630 case CHIP_VEGA10: 4631 case CHIP_VEGA12: 4632 case CHIP_VEGA20: 4633 case CHIP_RAVEN: 4634 case CHIP_ARCTURUS: 4635 case CHIP_RENOIR: 4636 gfx_v9_0_update_gfx_clock_gating(adev, 4637 state == AMD_CG_STATE_GATE ? true : false); 4638 break; 4639 default: 4640 break; 4641 } 4642 return 0; 4643 } 4644 4645 static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags) 4646 { 4647 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4648 int data; 4649 4650 if (amdgpu_sriov_vf(adev)) 4651 *flags = 0; 4652 4653 /* AMD_CG_SUPPORT_GFX_MGCG */ 4654 data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4655 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 4656 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 4657 4658 /* AMD_CG_SUPPORT_GFX_CGCG */ 4659 data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 4660 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 4661 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 4662 4663 /* AMD_CG_SUPPORT_GFX_CGLS */ 4664 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 4665 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 4666 4667 /* AMD_CG_SUPPORT_GFX_RLC_LS */ 4668 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4669 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) 4670 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS; 4671 4672 /* AMD_CG_SUPPORT_GFX_CP_LS */ 4673 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4674 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) 4675 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS; 4676 4677 if (adev->asic_type != CHIP_ARCTURUS) { 4678 /* AMD_CG_SUPPORT_GFX_3D_CGCG */ 4679 data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4680 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) 4681 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; 4682 4683 /* AMD_CG_SUPPORT_GFX_3D_CGLS */ 4684 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK) 4685 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS; 4686 } 4687 } 4688 4689 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) 4690 { 4691 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/ 4692 } 4693 4694 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 4695 { 4696 struct amdgpu_device *adev = ring->adev; 4697 u64 wptr; 4698 4699 /* XXX check if swapping is necessary on BE */ 4700 if (ring->use_doorbell) { 4701 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]); 4702 } else { 4703 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR); 4704 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32; 4705 } 4706 4707 return wptr; 4708 } 4709 4710 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 4711 { 4712 struct amdgpu_device *adev = ring->adev; 4713 4714 if (ring->use_doorbell) { 4715 /* XXX check if swapping is necessary on BE */ 4716 atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr); 4717 WDOORBELL64(ring->doorbell_index, ring->wptr); 4718 } else { 4719 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 4720 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 4721 } 4722 } 4723 4724 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 4725 { 4726 struct amdgpu_device *adev = ring->adev; 4727 u32 ref_and_mask, reg_mem_engine; 4728 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg; 4729 4730 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 4731 switch (ring->me) { 4732 case 1: 4733 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe; 4734 break; 4735 case 2: 4736 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe; 4737 break; 4738 default: 4739 return; 4740 } 4741 reg_mem_engine = 0; 4742 } else { 4743 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0; 4744 reg_mem_engine = 1; /* pfp */ 4745 } 4746 4747 gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1, 4748 adev->nbio.funcs->get_hdp_flush_req_offset(adev), 4749 adev->nbio.funcs->get_hdp_flush_done_offset(adev), 4750 ref_and_mask, ref_and_mask, 0x20); 4751 } 4752 4753 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 4754 struct amdgpu_job *job, 4755 struct amdgpu_ib *ib, 4756 uint32_t flags) 4757 { 4758 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 4759 u32 header, control = 0; 4760 4761 if (ib->flags & AMDGPU_IB_FLAG_CE) 4762 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 4763 else 4764 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 4765 4766 control |= ib->length_dw | (vmid << 24); 4767 4768 if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { 4769 control |= INDIRECT_BUFFER_PRE_ENB(1); 4770 4771 if (!(ib->flags & AMDGPU_IB_FLAG_CE)) 4772 gfx_v9_0_ring_emit_de_meta(ring); 4773 } 4774 4775 amdgpu_ring_write(ring, header); 4776 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 4777 amdgpu_ring_write(ring, 4778 #ifdef __BIG_ENDIAN 4779 (2 << 0) | 4780 #endif 4781 lower_32_bits(ib->gpu_addr)); 4782 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 4783 amdgpu_ring_write(ring, control); 4784 } 4785 4786 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 4787 struct amdgpu_job *job, 4788 struct amdgpu_ib *ib, 4789 uint32_t flags) 4790 { 4791 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 4792 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 4793 4794 /* Currently, there is a high possibility to get wave ID mismatch 4795 * between ME and GDS, leading to a hw deadlock, because ME generates 4796 * different wave IDs than the GDS expects. This situation happens 4797 * randomly when at least 5 compute pipes use GDS ordered append. 4798 * The wave IDs generated by ME are also wrong after suspend/resume. 4799 * Those are probably bugs somewhere else in the kernel driver. 4800 * 4801 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 4802 * GDS to 0 for this ring (me/pipe). 4803 */ 4804 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 4805 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 4806 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID); 4807 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 4808 } 4809 4810 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 4811 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 4812 amdgpu_ring_write(ring, 4813 #ifdef __BIG_ENDIAN 4814 (2 << 0) | 4815 #endif 4816 lower_32_bits(ib->gpu_addr)); 4817 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 4818 amdgpu_ring_write(ring, control); 4819 } 4820 4821 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 4822 u64 seq, unsigned flags) 4823 { 4824 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 4825 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 4826 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY; 4827 4828 /* RELEASE_MEM - flush caches, send int */ 4829 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); 4830 amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN | 4831 EOP_TC_NC_ACTION_EN) : 4832 (EOP_TCL1_ACTION_EN | 4833 EOP_TC_ACTION_EN | 4834 EOP_TC_WB_ACTION_EN | 4835 EOP_TC_MD_ACTION_EN)) | 4836 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 4837 EVENT_INDEX(5))); 4838 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 4839 4840 /* 4841 * the address should be Qword aligned if 64bit write, Dword 4842 * aligned if only send 32bit data low (discard data high) 4843 */ 4844 if (write64bit) 4845 BUG_ON(addr & 0x7); 4846 else 4847 BUG_ON(addr & 0x3); 4848 amdgpu_ring_write(ring, lower_32_bits(addr)); 4849 amdgpu_ring_write(ring, upper_32_bits(addr)); 4850 amdgpu_ring_write(ring, lower_32_bits(seq)); 4851 amdgpu_ring_write(ring, upper_32_bits(seq)); 4852 amdgpu_ring_write(ring, 0); 4853 } 4854 4855 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 4856 { 4857 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 4858 uint32_t seq = ring->fence_drv.sync_seq; 4859 uint64_t addr = ring->fence_drv.gpu_addr; 4860 4861 gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0, 4862 lower_32_bits(addr), upper_32_bits(addr), 4863 seq, 0xffffffff, 4); 4864 } 4865 4866 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 4867 unsigned vmid, uint64_t pd_addr) 4868 { 4869 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 4870 4871 /* compute doesn't have PFP */ 4872 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 4873 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 4874 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 4875 amdgpu_ring_write(ring, 0x0); 4876 } 4877 } 4878 4879 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring) 4880 { 4881 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */ 4882 } 4883 4884 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 4885 { 4886 u64 wptr; 4887 4888 /* XXX check if swapping is necessary on BE */ 4889 if (ring->use_doorbell) 4890 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]); 4891 else 4892 BUG(); 4893 return wptr; 4894 } 4895 4896 static void gfx_v9_0_ring_set_pipe_percent(struct amdgpu_ring *ring, 4897 bool acquire) 4898 { 4899 struct amdgpu_device *adev = ring->adev; 4900 int pipe_num, tmp, reg; 4901 int pipe_percent = acquire ? SPI_WCL_PIPE_PERCENT_GFX__VALUE_MASK : 0x1; 4902 4903 pipe_num = ring->me * adev->gfx.mec.num_pipe_per_mec + ring->pipe; 4904 4905 /* first me only has 2 entries, GFX and HP3D */ 4906 if (ring->me > 0) 4907 pipe_num -= 2; 4908 4909 reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX) + pipe_num; 4910 tmp = RREG32(reg); 4911 tmp = REG_SET_FIELD(tmp, SPI_WCL_PIPE_PERCENT_GFX, VALUE, pipe_percent); 4912 WREG32(reg, tmp); 4913 } 4914 4915 static void gfx_v9_0_pipe_reserve_resources(struct amdgpu_device *adev, 4916 struct amdgpu_ring *ring, 4917 bool acquire) 4918 { 4919 int i, pipe; 4920 bool reserve; 4921 struct amdgpu_ring *iring; 4922 4923 mutex_lock(&adev->gfx.pipe_reserve_mutex); 4924 pipe = amdgpu_gfx_mec_queue_to_bit(adev, ring->me, ring->pipe, 0); 4925 if (acquire) 4926 set_bit(pipe, adev->gfx.pipe_reserve_bitmap); 4927 else 4928 clear_bit(pipe, adev->gfx.pipe_reserve_bitmap); 4929 4930 if (!bitmap_weight(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES)) { 4931 /* Clear all reservations - everyone reacquires all resources */ 4932 for (i = 0; i < adev->gfx.num_gfx_rings; ++i) 4933 gfx_v9_0_ring_set_pipe_percent(&adev->gfx.gfx_ring[i], 4934 true); 4935 4936 for (i = 0; i < adev->gfx.num_compute_rings; ++i) 4937 gfx_v9_0_ring_set_pipe_percent(&adev->gfx.compute_ring[i], 4938 true); 4939 } else { 4940 /* Lower all pipes without a current reservation */ 4941 for (i = 0; i < adev->gfx.num_gfx_rings; ++i) { 4942 iring = &adev->gfx.gfx_ring[i]; 4943 pipe = amdgpu_gfx_mec_queue_to_bit(adev, 4944 iring->me, 4945 iring->pipe, 4946 0); 4947 reserve = test_bit(pipe, adev->gfx.pipe_reserve_bitmap); 4948 gfx_v9_0_ring_set_pipe_percent(iring, reserve); 4949 } 4950 4951 for (i = 0; i < adev->gfx.num_compute_rings; ++i) { 4952 iring = &adev->gfx.compute_ring[i]; 4953 pipe = amdgpu_gfx_mec_queue_to_bit(adev, 4954 iring->me, 4955 iring->pipe, 4956 0); 4957 reserve = test_bit(pipe, adev->gfx.pipe_reserve_bitmap); 4958 gfx_v9_0_ring_set_pipe_percent(iring, reserve); 4959 } 4960 } 4961 4962 mutex_unlock(&adev->gfx.pipe_reserve_mutex); 4963 } 4964 4965 static void gfx_v9_0_hqd_set_priority(struct amdgpu_device *adev, 4966 struct amdgpu_ring *ring, 4967 bool acquire) 4968 { 4969 uint32_t pipe_priority = acquire ? 0x2 : 0x0; 4970 uint32_t queue_priority = acquire ? 0xf : 0x0; 4971 4972 mutex_lock(&adev->srbm_mutex); 4973 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4974 4975 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PIPE_PRIORITY, pipe_priority); 4976 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_QUEUE_PRIORITY, queue_priority); 4977 4978 soc15_grbm_select(adev, 0, 0, 0, 0); 4979 mutex_unlock(&adev->srbm_mutex); 4980 } 4981 4982 static void gfx_v9_0_ring_set_priority_compute(struct amdgpu_ring *ring, 4983 enum drm_sched_priority priority) 4984 { 4985 struct amdgpu_device *adev = ring->adev; 4986 bool acquire = priority == DRM_SCHED_PRIORITY_HIGH_HW; 4987 4988 if (ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE) 4989 return; 4990 4991 gfx_v9_0_hqd_set_priority(adev, ring, acquire); 4992 gfx_v9_0_pipe_reserve_resources(adev, ring, acquire); 4993 } 4994 4995 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 4996 { 4997 struct amdgpu_device *adev = ring->adev; 4998 4999 /* XXX check if swapping is necessary on BE */ 5000 if (ring->use_doorbell) { 5001 atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5002 WDOORBELL64(ring->doorbell_index, ring->wptr); 5003 } else{ 5004 BUG(); /* only DOORBELL method supported on gfx9 now */ 5005 } 5006 } 5007 5008 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 5009 u64 seq, unsigned int flags) 5010 { 5011 struct amdgpu_device *adev = ring->adev; 5012 5013 /* we only allocate 32bit for each seq wb address */ 5014 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); 5015 5016 /* write fence seq to the "addr" */ 5017 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5018 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5019 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 5020 amdgpu_ring_write(ring, lower_32_bits(addr)); 5021 amdgpu_ring_write(ring, upper_32_bits(addr)); 5022 amdgpu_ring_write(ring, lower_32_bits(seq)); 5023 5024 if (flags & AMDGPU_FENCE_FLAG_INT) { 5025 /* set register to trigger INT */ 5026 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5027 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5028 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 5029 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS)); 5030 amdgpu_ring_write(ring, 0); 5031 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 5032 } 5033 } 5034 5035 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring) 5036 { 5037 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 5038 amdgpu_ring_write(ring, 0); 5039 } 5040 5041 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring) 5042 { 5043 struct v9_ce_ib_state ce_payload = {0}; 5044 uint64_t csa_addr; 5045 int cnt; 5046 5047 cnt = (sizeof(ce_payload) >> 2) + 4 - 2; 5048 csa_addr = amdgpu_csa_vaddr(ring->adev); 5049 5050 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5051 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) | 5052 WRITE_DATA_DST_SEL(8) | 5053 WR_CONFIRM) | 5054 WRITE_DATA_CACHE_POLICY(0)); 5055 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5056 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5057 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2); 5058 } 5059 5060 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring) 5061 { 5062 struct v9_de_ib_state de_payload = {0}; 5063 uint64_t csa_addr, gds_addr; 5064 int cnt; 5065 5066 csa_addr = amdgpu_csa_vaddr(ring->adev); 5067 gds_addr = csa_addr + 4096; 5068 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr); 5069 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr); 5070 5071 cnt = (sizeof(de_payload) >> 2) + 4 - 2; 5072 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5073 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 5074 WRITE_DATA_DST_SEL(8) | 5075 WR_CONFIRM) | 5076 WRITE_DATA_CACHE_POLICY(0)); 5077 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5078 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5079 amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2); 5080 } 5081 5082 static void gfx_v9_0_ring_emit_tmz(struct amdgpu_ring *ring, bool start) 5083 { 5084 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); 5085 amdgpu_ring_write(ring, FRAME_CMD(start ? 0 : 1)); /* frame_end */ 5086 } 5087 5088 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) 5089 { 5090 uint32_t dw2 = 0; 5091 5092 if (amdgpu_sriov_vf(ring->adev)) 5093 gfx_v9_0_ring_emit_ce_meta(ring); 5094 5095 gfx_v9_0_ring_emit_tmz(ring, true); 5096 5097 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 5098 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 5099 /* set load_global_config & load_global_uconfig */ 5100 dw2 |= 0x8001; 5101 /* set load_cs_sh_regs */ 5102 dw2 |= 0x01000000; 5103 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 5104 dw2 |= 0x10002; 5105 5106 /* set load_ce_ram if preamble presented */ 5107 if (AMDGPU_PREAMBLE_IB_PRESENT & flags) 5108 dw2 |= 0x10000000; 5109 } else { 5110 /* still load_ce_ram if this is the first time preamble presented 5111 * although there is no context switch happens. 5112 */ 5113 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags) 5114 dw2 |= 0x10000000; 5115 } 5116 5117 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 5118 amdgpu_ring_write(ring, dw2); 5119 amdgpu_ring_write(ring, 0); 5120 } 5121 5122 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring) 5123 { 5124 unsigned ret; 5125 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 5126 amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr)); 5127 amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr)); 5128 amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */ 5129 ret = ring->wptr & ring->buf_mask; 5130 amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */ 5131 return ret; 5132 } 5133 5134 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset) 5135 { 5136 unsigned cur; 5137 BUG_ON(offset > ring->buf_mask); 5138 BUG_ON(ring->ring[offset] != 0x55aa55aa); 5139 5140 cur = (ring->wptr & ring->buf_mask) - 1; 5141 if (likely(cur > offset)) 5142 ring->ring[offset] = cur - offset; 5143 else 5144 ring->ring[offset] = (ring->ring_size>>2) - offset + cur; 5145 } 5146 5147 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg) 5148 { 5149 struct amdgpu_device *adev = ring->adev; 5150 5151 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 5152 amdgpu_ring_write(ring, 0 | /* src: register*/ 5153 (5 << 8) | /* dst: memory */ 5154 (1 << 20)); /* write confirm */ 5155 amdgpu_ring_write(ring, reg); 5156 amdgpu_ring_write(ring, 0); 5157 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 5158 adev->virt.reg_val_offs * 4)); 5159 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 5160 adev->virt.reg_val_offs * 4)); 5161 } 5162 5163 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 5164 uint32_t val) 5165 { 5166 uint32_t cmd = 0; 5167 5168 switch (ring->funcs->type) { 5169 case AMDGPU_RING_TYPE_GFX: 5170 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 5171 break; 5172 case AMDGPU_RING_TYPE_KIQ: 5173 cmd = (1 << 16); /* no inc addr */ 5174 break; 5175 default: 5176 cmd = WR_CONFIRM; 5177 break; 5178 } 5179 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5180 amdgpu_ring_write(ring, cmd); 5181 amdgpu_ring_write(ring, reg); 5182 amdgpu_ring_write(ring, 0); 5183 amdgpu_ring_write(ring, val); 5184 } 5185 5186 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 5187 uint32_t val, uint32_t mask) 5188 { 5189 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); 5190 } 5191 5192 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, 5193 uint32_t reg0, uint32_t reg1, 5194 uint32_t ref, uint32_t mask) 5195 { 5196 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5197 struct amdgpu_device *adev = ring->adev; 5198 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ? 5199 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait; 5200 5201 if (fw_version_ok) 5202 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, 5203 ref, mask, 0x20); 5204 else 5205 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1, 5206 ref, mask); 5207 } 5208 5209 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid) 5210 { 5211 struct amdgpu_device *adev = ring->adev; 5212 uint32_t value = 0; 5213 5214 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03); 5215 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01); 5216 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1); 5217 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid); 5218 WREG32_SOC15(GC, 0, mmSQ_CMD, value); 5219 } 5220 5221 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 5222 enum amdgpu_interrupt_state state) 5223 { 5224 switch (state) { 5225 case AMDGPU_IRQ_STATE_DISABLE: 5226 case AMDGPU_IRQ_STATE_ENABLE: 5227 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5228 TIME_STAMP_INT_ENABLE, 5229 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5230 break; 5231 default: 5232 break; 5233 } 5234 } 5235 5236 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 5237 int me, int pipe, 5238 enum amdgpu_interrupt_state state) 5239 { 5240 u32 mec_int_cntl, mec_int_cntl_reg; 5241 5242 /* 5243 * amdgpu controls only the first MEC. That's why this function only 5244 * handles the setting of interrupts for this specific MEC. All other 5245 * pipes' interrupts are set by amdkfd. 5246 */ 5247 5248 if (me == 1) { 5249 switch (pipe) { 5250 case 0: 5251 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL); 5252 break; 5253 case 1: 5254 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL); 5255 break; 5256 case 2: 5257 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL); 5258 break; 5259 case 3: 5260 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL); 5261 break; 5262 default: 5263 DRM_DEBUG("invalid pipe %d\n", pipe); 5264 return; 5265 } 5266 } else { 5267 DRM_DEBUG("invalid me %d\n", me); 5268 return; 5269 } 5270 5271 switch (state) { 5272 case AMDGPU_IRQ_STATE_DISABLE: 5273 mec_int_cntl = RREG32(mec_int_cntl_reg); 5274 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5275 TIME_STAMP_INT_ENABLE, 0); 5276 WREG32(mec_int_cntl_reg, mec_int_cntl); 5277 break; 5278 case AMDGPU_IRQ_STATE_ENABLE: 5279 mec_int_cntl = RREG32(mec_int_cntl_reg); 5280 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5281 TIME_STAMP_INT_ENABLE, 1); 5282 WREG32(mec_int_cntl_reg, mec_int_cntl); 5283 break; 5284 default: 5285 break; 5286 } 5287 } 5288 5289 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 5290 struct amdgpu_irq_src *source, 5291 unsigned type, 5292 enum amdgpu_interrupt_state state) 5293 { 5294 switch (state) { 5295 case AMDGPU_IRQ_STATE_DISABLE: 5296 case AMDGPU_IRQ_STATE_ENABLE: 5297 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5298 PRIV_REG_INT_ENABLE, 5299 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5300 break; 5301 default: 5302 break; 5303 } 5304 5305 return 0; 5306 } 5307 5308 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 5309 struct amdgpu_irq_src *source, 5310 unsigned type, 5311 enum amdgpu_interrupt_state state) 5312 { 5313 switch (state) { 5314 case AMDGPU_IRQ_STATE_DISABLE: 5315 case AMDGPU_IRQ_STATE_ENABLE: 5316 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5317 PRIV_INSTR_INT_ENABLE, 5318 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5319 default: 5320 break; 5321 } 5322 5323 return 0; 5324 } 5325 5326 #define ENABLE_ECC_ON_ME_PIPE(me, pipe) \ 5327 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5328 CP_ECC_ERROR_INT_ENABLE, 1) 5329 5330 #define DISABLE_ECC_ON_ME_PIPE(me, pipe) \ 5331 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5332 CP_ECC_ERROR_INT_ENABLE, 0) 5333 5334 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev, 5335 struct amdgpu_irq_src *source, 5336 unsigned type, 5337 enum amdgpu_interrupt_state state) 5338 { 5339 switch (state) { 5340 case AMDGPU_IRQ_STATE_DISABLE: 5341 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5342 CP_ECC_ERROR_INT_ENABLE, 0); 5343 DISABLE_ECC_ON_ME_PIPE(1, 0); 5344 DISABLE_ECC_ON_ME_PIPE(1, 1); 5345 DISABLE_ECC_ON_ME_PIPE(1, 2); 5346 DISABLE_ECC_ON_ME_PIPE(1, 3); 5347 break; 5348 5349 case AMDGPU_IRQ_STATE_ENABLE: 5350 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5351 CP_ECC_ERROR_INT_ENABLE, 1); 5352 ENABLE_ECC_ON_ME_PIPE(1, 0); 5353 ENABLE_ECC_ON_ME_PIPE(1, 1); 5354 ENABLE_ECC_ON_ME_PIPE(1, 2); 5355 ENABLE_ECC_ON_ME_PIPE(1, 3); 5356 break; 5357 default: 5358 break; 5359 } 5360 5361 return 0; 5362 } 5363 5364 5365 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev, 5366 struct amdgpu_irq_src *src, 5367 unsigned type, 5368 enum amdgpu_interrupt_state state) 5369 { 5370 switch (type) { 5371 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 5372 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state); 5373 break; 5374 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 5375 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 5376 break; 5377 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 5378 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 5379 break; 5380 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 5381 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 5382 break; 5383 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 5384 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 5385 break; 5386 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP: 5387 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state); 5388 break; 5389 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP: 5390 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state); 5391 break; 5392 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP: 5393 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state); 5394 break; 5395 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP: 5396 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state); 5397 break; 5398 default: 5399 break; 5400 } 5401 return 0; 5402 } 5403 5404 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev, 5405 struct amdgpu_irq_src *source, 5406 struct amdgpu_iv_entry *entry) 5407 { 5408 int i; 5409 u8 me_id, pipe_id, queue_id; 5410 struct amdgpu_ring *ring; 5411 5412 DRM_DEBUG("IH: CP EOP\n"); 5413 me_id = (entry->ring_id & 0x0c) >> 2; 5414 pipe_id = (entry->ring_id & 0x03) >> 0; 5415 queue_id = (entry->ring_id & 0x70) >> 4; 5416 5417 switch (me_id) { 5418 case 0: 5419 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); 5420 break; 5421 case 1: 5422 case 2: 5423 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5424 ring = &adev->gfx.compute_ring[i]; 5425 /* Per-queue interrupt is supported for MEC starting from VI. 5426 * The interrupt can only be enabled/disabled per pipe instead of per queue. 5427 */ 5428 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) 5429 amdgpu_fence_process(ring); 5430 } 5431 break; 5432 } 5433 return 0; 5434 } 5435 5436 static void gfx_v9_0_fault(struct amdgpu_device *adev, 5437 struct amdgpu_iv_entry *entry) 5438 { 5439 u8 me_id, pipe_id, queue_id; 5440 struct amdgpu_ring *ring; 5441 int i; 5442 5443 me_id = (entry->ring_id & 0x0c) >> 2; 5444 pipe_id = (entry->ring_id & 0x03) >> 0; 5445 queue_id = (entry->ring_id & 0x70) >> 4; 5446 5447 switch (me_id) { 5448 case 0: 5449 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); 5450 break; 5451 case 1: 5452 case 2: 5453 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5454 ring = &adev->gfx.compute_ring[i]; 5455 if (ring->me == me_id && ring->pipe == pipe_id && 5456 ring->queue == queue_id) 5457 drm_sched_fault(&ring->sched); 5458 } 5459 break; 5460 } 5461 } 5462 5463 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev, 5464 struct amdgpu_irq_src *source, 5465 struct amdgpu_iv_entry *entry) 5466 { 5467 DRM_ERROR("Illegal register access in command stream\n"); 5468 gfx_v9_0_fault(adev, entry); 5469 return 0; 5470 } 5471 5472 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev, 5473 struct amdgpu_irq_src *source, 5474 struct amdgpu_iv_entry *entry) 5475 { 5476 DRM_ERROR("Illegal instruction in command stream\n"); 5477 gfx_v9_0_fault(adev, entry); 5478 return 0; 5479 } 5480 5481 5482 static const struct ras_gfx_subblock_reg ras_subblock_regs[] = { 5483 { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 5484 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT), 5485 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT) 5486 }, 5487 { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 5488 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT), 5489 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT) 5490 }, 5491 { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5492 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1), 5493 0, 0 5494 }, 5495 { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5496 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2), 5497 0, 0 5498 }, 5499 { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 5500 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT), 5501 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT) 5502 }, 5503 { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5504 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT), 5505 0, 0 5506 }, 5507 { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5508 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT), 5509 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT) 5510 }, 5511 { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 5512 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT), 5513 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT) 5514 }, 5515 { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 5516 SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1), 5517 0, 0 5518 }, 5519 { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 5520 SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1), 5521 0, 0 5522 }, 5523 { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 5524 SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1), 5525 0, 0 5526 }, 5527 { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 5528 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC), 5529 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED) 5530 }, 5531 { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 5532 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED), 5533 0, 0 5534 }, 5535 { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5536 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC), 5537 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED) 5538 }, 5539 { "GDS_OA_PHY_PHY_CMD_RAM_MEM", 5540 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5541 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC), 5542 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED) 5543 }, 5544 { "GDS_OA_PHY_PHY_DATA_RAM_MEM", 5545 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5546 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED), 5547 0, 0 5548 }, 5549 { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM", 5550 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5551 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC), 5552 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED) 5553 }, 5554 { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM", 5555 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5556 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC), 5557 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED) 5558 }, 5559 { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM", 5560 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5561 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC), 5562 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED) 5563 }, 5564 { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM", 5565 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5566 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC), 5567 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED) 5568 }, 5569 { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 5570 SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT), 5571 0, 0 5572 }, 5573 { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5574 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT), 5575 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT) 5576 }, 5577 { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5578 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT), 5579 0, 0 5580 }, 5581 { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5582 SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT), 5583 0, 0 5584 }, 5585 { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5586 SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT), 5587 0, 0 5588 }, 5589 { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5590 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT), 5591 0, 0 5592 }, 5593 { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 5594 SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT), 5595 0, 0 5596 }, 5597 { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 5598 SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT), 5599 0, 0 5600 }, 5601 { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5602 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT), 5603 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT) 5604 }, 5605 { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5606 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT), 5607 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT) 5608 }, 5609 { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5610 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT), 5611 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT) 5612 }, 5613 { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5614 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT), 5615 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT) 5616 }, 5617 { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5618 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT), 5619 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT) 5620 }, 5621 { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5622 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT), 5623 0, 0 5624 }, 5625 { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5626 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT), 5627 0, 0 5628 }, 5629 { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5630 SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT), 5631 0, 0 5632 }, 5633 { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5634 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT), 5635 0, 0 5636 }, 5637 { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5638 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT), 5639 0, 0 5640 }, 5641 { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5642 SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT), 5643 0, 0 5644 }, 5645 { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 5646 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT), 5647 0, 0 5648 }, 5649 { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 5650 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT), 5651 0, 0 5652 }, 5653 { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 5654 SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT), 5655 0, 0 5656 }, 5657 { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 5658 SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT), 5659 0, 0 5660 }, 5661 { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 5662 SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT), 5663 0, 0 5664 }, 5665 { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 5666 SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT), 5667 0, 0 5668 }, 5669 { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 5670 SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT), 5671 0, 0 5672 }, 5673 { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 5674 SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT), 5675 0, 0 5676 }, 5677 { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 5678 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT), 5679 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT) 5680 }, 5681 { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 5682 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT), 5683 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT) 5684 }, 5685 { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 5686 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT), 5687 0, 0 5688 }, 5689 { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 5690 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT), 5691 0, 0 5692 }, 5693 { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 5694 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT), 5695 0, 0 5696 }, 5697 { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 5698 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT), 5699 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT) 5700 }, 5701 { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 5702 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT), 5703 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT) 5704 }, 5705 { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 5706 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT), 5707 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT) 5708 }, 5709 { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 5710 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT), 5711 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT) 5712 }, 5713 { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 5714 SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT), 5715 0, 0 5716 }, 5717 { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 5718 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT), 5719 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT) 5720 }, 5721 { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 5722 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT), 5723 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT) 5724 }, 5725 { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 5726 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT), 5727 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT) 5728 }, 5729 { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 5730 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT), 5731 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT) 5732 }, 5733 { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 5734 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT), 5735 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT) 5736 }, 5737 { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 5738 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT), 5739 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT) 5740 }, 5741 { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 5742 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT), 5743 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT) 5744 }, 5745 { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 5746 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT), 5747 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT) 5748 }, 5749 { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 5750 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT), 5751 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT) 5752 }, 5753 { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 5754 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT), 5755 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT) 5756 }, 5757 { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 5758 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT), 5759 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT) 5760 }, 5761 { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 5762 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT), 5763 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT) 5764 }, 5765 { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 5766 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT), 5767 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT) 5768 }, 5769 { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 5770 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT), 5771 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT) 5772 }, 5773 { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 5774 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT), 5775 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT) 5776 }, 5777 { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 5778 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT), 5779 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT) 5780 }, 5781 { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 5782 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT), 5783 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT) 5784 }, 5785 { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 5786 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT), 5787 0, 0 5788 }, 5789 { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 5790 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT), 5791 0, 0 5792 }, 5793 { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 5794 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT), 5795 0, 0 5796 }, 5797 { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 5798 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT), 5799 0, 0 5800 }, 5801 { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 5802 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT), 5803 0, 0 5804 }, 5805 { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 5806 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT), 5807 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT) 5808 }, 5809 { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 5810 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT), 5811 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT) 5812 }, 5813 { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 5814 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT), 5815 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT) 5816 }, 5817 { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 5818 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT), 5819 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT) 5820 }, 5821 { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 5822 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT), 5823 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT) 5824 }, 5825 { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 5826 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT), 5827 0, 0 5828 }, 5829 { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 5830 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT), 5831 0, 0 5832 }, 5833 { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 5834 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT), 5835 0, 0 5836 }, 5837 { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 5838 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT), 5839 0, 0 5840 }, 5841 { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 5842 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT), 5843 0, 0 5844 }, 5845 { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 5846 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), 5847 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT) 5848 }, 5849 { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 5850 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), 5851 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT) 5852 }, 5853 { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 5854 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), 5855 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT) 5856 }, 5857 { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 5858 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT), 5859 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT) 5860 }, 5861 { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 5862 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT), 5863 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT) 5864 }, 5865 { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 5866 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), 5867 0, 0 5868 }, 5869 { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 5870 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), 5871 0, 0 5872 }, 5873 { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 5874 SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT), 5875 0, 0 5876 }, 5877 { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 5878 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT), 5879 0, 0 5880 }, 5881 { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 5882 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT), 5883 0, 0 5884 }, 5885 { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 5886 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), 5887 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT) 5888 }, 5889 { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 5890 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), 5891 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT) 5892 }, 5893 { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 5894 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), 5895 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT) 5896 }, 5897 { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 5898 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), 5899 0, 0 5900 }, 5901 { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 5902 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), 5903 0, 0 5904 }, 5905 { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 5906 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT), 5907 0, 0 5908 }, 5909 { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 5910 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT), 5911 0, 0 5912 }, 5913 { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 5914 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT), 5915 0, 0 5916 }, 5917 { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 5918 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT), 5919 0, 0 5920 } 5921 }; 5922 5923 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 5924 void *inject_if) 5925 { 5926 struct ras_inject_if *info = (struct ras_inject_if *)inject_if; 5927 int ret; 5928 struct ta_ras_trigger_error_input block_info = { 0 }; 5929 5930 if (adev->asic_type != CHIP_VEGA20) 5931 return -EINVAL; 5932 5933 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks)) 5934 return -EINVAL; 5935 5936 if (!ras_gfx_subblocks[info->head.sub_block_index].name) 5937 return -EPERM; 5938 5939 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type & 5940 info->head.type)) { 5941 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n", 5942 ras_gfx_subblocks[info->head.sub_block_index].name, 5943 info->head.type); 5944 return -EPERM; 5945 } 5946 5947 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type & 5948 info->head.type)) { 5949 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n", 5950 ras_gfx_subblocks[info->head.sub_block_index].name, 5951 info->head.type); 5952 return -EPERM; 5953 } 5954 5955 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block); 5956 block_info.sub_block_index = 5957 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock; 5958 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type); 5959 block_info.address = info->address; 5960 block_info.value = info->value; 5961 5962 mutex_lock(&adev->grbm_idx_mutex); 5963 ret = psp_ras_trigger_error(&adev->psp, &block_info); 5964 mutex_unlock(&adev->grbm_idx_mutex); 5965 5966 return ret; 5967 } 5968 5969 static const char *vml2_mems[] = { 5970 "UTC_VML2_BANK_CACHE_0_BIGK_MEM0", 5971 "UTC_VML2_BANK_CACHE_0_BIGK_MEM1", 5972 "UTC_VML2_BANK_CACHE_0_4K_MEM0", 5973 "UTC_VML2_BANK_CACHE_0_4K_MEM1", 5974 "UTC_VML2_BANK_CACHE_1_BIGK_MEM0", 5975 "UTC_VML2_BANK_CACHE_1_BIGK_MEM1", 5976 "UTC_VML2_BANK_CACHE_1_4K_MEM0", 5977 "UTC_VML2_BANK_CACHE_1_4K_MEM1", 5978 "UTC_VML2_BANK_CACHE_2_BIGK_MEM0", 5979 "UTC_VML2_BANK_CACHE_2_BIGK_MEM1", 5980 "UTC_VML2_BANK_CACHE_2_4K_MEM0", 5981 "UTC_VML2_BANK_CACHE_2_4K_MEM1", 5982 "UTC_VML2_BANK_CACHE_3_BIGK_MEM0", 5983 "UTC_VML2_BANK_CACHE_3_BIGK_MEM1", 5984 "UTC_VML2_BANK_CACHE_3_4K_MEM0", 5985 "UTC_VML2_BANK_CACHE_3_4K_MEM1", 5986 }; 5987 5988 static const char *vml2_walker_mems[] = { 5989 "UTC_VML2_CACHE_PDE0_MEM0", 5990 "UTC_VML2_CACHE_PDE0_MEM1", 5991 "UTC_VML2_CACHE_PDE1_MEM0", 5992 "UTC_VML2_CACHE_PDE1_MEM1", 5993 "UTC_VML2_CACHE_PDE2_MEM0", 5994 "UTC_VML2_CACHE_PDE2_MEM1", 5995 "UTC_VML2_RDIF_LOG_FIFO", 5996 }; 5997 5998 static const char *atc_l2_cache_2m_mems[] = { 5999 "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM", 6000 "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM", 6001 "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM", 6002 "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM", 6003 }; 6004 6005 static const char *atc_l2_cache_4k_mems[] = { 6006 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0", 6007 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1", 6008 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2", 6009 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3", 6010 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4", 6011 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5", 6012 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6", 6013 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7", 6014 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0", 6015 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1", 6016 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2", 6017 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3", 6018 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4", 6019 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5", 6020 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6", 6021 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7", 6022 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0", 6023 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1", 6024 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2", 6025 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3", 6026 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4", 6027 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5", 6028 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6", 6029 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7", 6030 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0", 6031 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1", 6032 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2", 6033 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3", 6034 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4", 6035 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5", 6036 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6", 6037 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7", 6038 }; 6039 6040 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev, 6041 struct ras_err_data *err_data) 6042 { 6043 uint32_t i, data; 6044 uint32_t sec_count, ded_count; 6045 6046 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6047 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6048 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6049 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6050 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6051 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6052 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6053 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6054 6055 for (i = 0; i < 16; i++) { 6056 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6057 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6058 6059 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT); 6060 if (sec_count) { 6061 DRM_INFO("Instance[%d]: SubBlock %s, SEC %d\n", i, 6062 vml2_mems[i], sec_count); 6063 err_data->ce_count += sec_count; 6064 } 6065 6066 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT); 6067 if (ded_count) { 6068 DRM_INFO("Instance[%d]: SubBlock %s, DED %d\n", i, 6069 vml2_mems[i], ded_count); 6070 err_data->ue_count += ded_count; 6071 } 6072 } 6073 6074 for (i = 0; i < 7; i++) { 6075 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6076 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6077 6078 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6079 SEC_COUNT); 6080 if (sec_count) { 6081 DRM_INFO("Instance[%d]: SubBlock %s, SEC %d\n", i, 6082 vml2_walker_mems[i], sec_count); 6083 err_data->ce_count += sec_count; 6084 } 6085 6086 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6087 DED_COUNT); 6088 if (ded_count) { 6089 DRM_INFO("Instance[%d]: SubBlock %s, DED %d\n", i, 6090 vml2_walker_mems[i], ded_count); 6091 err_data->ue_count += ded_count; 6092 } 6093 } 6094 6095 for (i = 0; i < 4; i++) { 6096 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6097 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6098 6099 sec_count = (data & 0x00006000L) >> 0xd; 6100 if (sec_count) { 6101 DRM_INFO("Instance[%d]: SubBlock %s, SEC %d\n", i, 6102 atc_l2_cache_2m_mems[i], sec_count); 6103 err_data->ce_count += sec_count; 6104 } 6105 } 6106 6107 for (i = 0; i < 32; i++) { 6108 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6109 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6110 6111 sec_count = (data & 0x00006000L) >> 0xd; 6112 if (sec_count) { 6113 DRM_INFO("Instance[%d]: SubBlock %s, SEC %d\n", i, 6114 atc_l2_cache_4k_mems[i], sec_count); 6115 err_data->ce_count += sec_count; 6116 } 6117 6118 ded_count = (data & 0x00018000L) >> 0xf; 6119 if (ded_count) { 6120 DRM_INFO("Instance[%d]: SubBlock %s, DED %d\n", i, 6121 atc_l2_cache_4k_mems[i], ded_count); 6122 err_data->ue_count += ded_count; 6123 } 6124 } 6125 6126 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6127 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6128 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6129 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6130 6131 return 0; 6132 } 6133 6134 static int __get_ras_error_count(const struct soc15_reg_entry *reg, 6135 uint32_t se_id, uint32_t inst_id, uint32_t value, 6136 uint32_t *sec_count, uint32_t *ded_count) 6137 { 6138 uint32_t i; 6139 uint32_t sec_cnt, ded_cnt; 6140 6141 for (i = 0; i < ARRAY_SIZE(ras_subblock_regs); i++) { 6142 if(ras_subblock_regs[i].reg_offset != reg->reg_offset || 6143 ras_subblock_regs[i].seg != reg->seg || 6144 ras_subblock_regs[i].inst != reg->inst) 6145 continue; 6146 6147 sec_cnt = (value & 6148 ras_subblock_regs[i].sec_count_mask) >> 6149 ras_subblock_regs[i].sec_count_shift; 6150 if (sec_cnt) { 6151 DRM_INFO("GFX SubBlock %s, Instance[%d][%d], SEC %d\n", 6152 ras_subblock_regs[i].name, 6153 se_id, inst_id, 6154 sec_cnt); 6155 *sec_count += sec_cnt; 6156 } 6157 6158 ded_cnt = (value & 6159 ras_subblock_regs[i].ded_count_mask) >> 6160 ras_subblock_regs[i].ded_count_shift; 6161 if (ded_cnt) { 6162 DRM_INFO("GFX SubBlock %s, Instance[%d][%d], DED %d\n", 6163 ras_subblock_regs[i].name, 6164 se_id, inst_id, 6165 ded_cnt); 6166 *ded_count += ded_cnt; 6167 } 6168 } 6169 6170 return 0; 6171 } 6172 6173 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 6174 void *ras_error_status) 6175 { 6176 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; 6177 uint32_t sec_count = 0, ded_count = 0; 6178 uint32_t i, j, k; 6179 uint32_t reg_value; 6180 6181 if (adev->asic_type != CHIP_VEGA20) 6182 return -EINVAL; 6183 6184 err_data->ue_count = 0; 6185 err_data->ce_count = 0; 6186 6187 mutex_lock(&adev->grbm_idx_mutex); 6188 6189 for (i = 0; i < ARRAY_SIZE(sec_ded_counter_registers); i++) { 6190 for (j = 0; j < sec_ded_counter_registers[i].se_num; j++) { 6191 for (k = 0; k < sec_ded_counter_registers[i].instance; k++) { 6192 gfx_v9_0_select_se_sh(adev, j, 0, k); 6193 reg_value = 6194 RREG32(SOC15_REG_ENTRY_OFFSET(sec_ded_counter_registers[i])); 6195 if (reg_value) 6196 __get_ras_error_count(&sec_ded_counter_registers[i], 6197 j, k, reg_value, 6198 &sec_count, &ded_count); 6199 } 6200 } 6201 } 6202 6203 err_data->ce_count += sec_count; 6204 err_data->ue_count += ded_count; 6205 6206 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 6207 mutex_unlock(&adev->grbm_idx_mutex); 6208 6209 gfx_v9_0_query_utc_edc_status(adev, err_data); 6210 6211 return 0; 6212 } 6213 6214 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = { 6215 .name = "gfx_v9_0", 6216 .early_init = gfx_v9_0_early_init, 6217 .late_init = gfx_v9_0_late_init, 6218 .sw_init = gfx_v9_0_sw_init, 6219 .sw_fini = gfx_v9_0_sw_fini, 6220 .hw_init = gfx_v9_0_hw_init, 6221 .hw_fini = gfx_v9_0_hw_fini, 6222 .suspend = gfx_v9_0_suspend, 6223 .resume = gfx_v9_0_resume, 6224 .is_idle = gfx_v9_0_is_idle, 6225 .wait_for_idle = gfx_v9_0_wait_for_idle, 6226 .soft_reset = gfx_v9_0_soft_reset, 6227 .set_clockgating_state = gfx_v9_0_set_clockgating_state, 6228 .set_powergating_state = gfx_v9_0_set_powergating_state, 6229 .get_clockgating_state = gfx_v9_0_get_clockgating_state, 6230 }; 6231 6232 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = { 6233 .type = AMDGPU_RING_TYPE_GFX, 6234 .align_mask = 0xff, 6235 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6236 .support_64bit_ptrs = true, 6237 .vmhub = AMDGPU_GFXHUB_0, 6238 .get_rptr = gfx_v9_0_ring_get_rptr_gfx, 6239 .get_wptr = gfx_v9_0_ring_get_wptr_gfx, 6240 .set_wptr = gfx_v9_0_ring_set_wptr_gfx, 6241 .emit_frame_size = /* totally 242 maximum if 16 IBs */ 6242 5 + /* COND_EXEC */ 6243 7 + /* PIPELINE_SYNC */ 6244 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6245 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6246 2 + /* VM_FLUSH */ 6247 8 + /* FENCE for VM_FLUSH */ 6248 20 + /* GDS switch */ 6249 4 + /* double SWITCH_BUFFER, 6250 the first COND_EXEC jump to the place just 6251 prior to this double SWITCH_BUFFER */ 6252 5 + /* COND_EXEC */ 6253 7 + /* HDP_flush */ 6254 4 + /* VGT_flush */ 6255 14 + /* CE_META */ 6256 31 + /* DE_META */ 6257 3 + /* CNTX_CTRL */ 6258 5 + /* HDP_INVL */ 6259 8 + 8 + /* FENCE x2 */ 6260 2, /* SWITCH_BUFFER */ 6261 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */ 6262 .emit_ib = gfx_v9_0_ring_emit_ib_gfx, 6263 .emit_fence = gfx_v9_0_ring_emit_fence, 6264 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6265 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6266 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6267 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6268 .test_ring = gfx_v9_0_ring_test_ring, 6269 .test_ib = gfx_v9_0_ring_test_ib, 6270 .insert_nop = amdgpu_ring_insert_nop, 6271 .pad_ib = amdgpu_ring_generic_pad_ib, 6272 .emit_switch_buffer = gfx_v9_ring_emit_sb, 6273 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl, 6274 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec, 6275 .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec, 6276 .emit_tmz = gfx_v9_0_ring_emit_tmz, 6277 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6278 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6279 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6280 .soft_recovery = gfx_v9_0_ring_soft_recovery, 6281 }; 6282 6283 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { 6284 .type = AMDGPU_RING_TYPE_COMPUTE, 6285 .align_mask = 0xff, 6286 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6287 .support_64bit_ptrs = true, 6288 .vmhub = AMDGPU_GFXHUB_0, 6289 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6290 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6291 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6292 .emit_frame_size = 6293 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6294 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6295 5 + /* hdp invalidate */ 6296 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6297 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6298 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6299 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6300 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */ 6301 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6302 .emit_ib = gfx_v9_0_ring_emit_ib_compute, 6303 .emit_fence = gfx_v9_0_ring_emit_fence, 6304 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6305 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6306 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6307 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6308 .test_ring = gfx_v9_0_ring_test_ring, 6309 .test_ib = gfx_v9_0_ring_test_ib, 6310 .insert_nop = amdgpu_ring_insert_nop, 6311 .pad_ib = amdgpu_ring_generic_pad_ib, 6312 .set_priority = gfx_v9_0_ring_set_priority_compute, 6313 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6314 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6315 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6316 }; 6317 6318 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = { 6319 .type = AMDGPU_RING_TYPE_KIQ, 6320 .align_mask = 0xff, 6321 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6322 .support_64bit_ptrs = true, 6323 .vmhub = AMDGPU_GFXHUB_0, 6324 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6325 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6326 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6327 .emit_frame_size = 6328 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6329 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6330 5 + /* hdp invalidate */ 6331 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6332 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6333 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6334 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6335 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 6336 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6337 .emit_fence = gfx_v9_0_ring_emit_fence_kiq, 6338 .test_ring = gfx_v9_0_ring_test_ring, 6339 .insert_nop = amdgpu_ring_insert_nop, 6340 .pad_ib = amdgpu_ring_generic_pad_ib, 6341 .emit_rreg = gfx_v9_0_ring_emit_rreg, 6342 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6343 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6344 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6345 }; 6346 6347 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev) 6348 { 6349 int i; 6350 6351 adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq; 6352 6353 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 6354 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx; 6355 6356 for (i = 0; i < adev->gfx.num_compute_rings; i++) 6357 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute; 6358 } 6359 6360 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = { 6361 .set = gfx_v9_0_set_eop_interrupt_state, 6362 .process = gfx_v9_0_eop_irq, 6363 }; 6364 6365 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = { 6366 .set = gfx_v9_0_set_priv_reg_fault_state, 6367 .process = gfx_v9_0_priv_reg_irq, 6368 }; 6369 6370 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = { 6371 .set = gfx_v9_0_set_priv_inst_fault_state, 6372 .process = gfx_v9_0_priv_inst_irq, 6373 }; 6374 6375 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = { 6376 .set = gfx_v9_0_set_cp_ecc_error_state, 6377 .process = amdgpu_gfx_cp_ecc_error_irq, 6378 }; 6379 6380 6381 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev) 6382 { 6383 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 6384 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs; 6385 6386 adev->gfx.priv_reg_irq.num_types = 1; 6387 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs; 6388 6389 adev->gfx.priv_inst_irq.num_types = 1; 6390 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs; 6391 6392 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/ 6393 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs; 6394 } 6395 6396 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev) 6397 { 6398 switch (adev->asic_type) { 6399 case CHIP_VEGA10: 6400 case CHIP_VEGA12: 6401 case CHIP_VEGA20: 6402 case CHIP_RAVEN: 6403 case CHIP_ARCTURUS: 6404 case CHIP_RENOIR: 6405 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs; 6406 break; 6407 default: 6408 break; 6409 } 6410 } 6411 6412 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev) 6413 { 6414 /* init asci gds info */ 6415 switch (adev->asic_type) { 6416 case CHIP_VEGA10: 6417 case CHIP_VEGA12: 6418 case CHIP_VEGA20: 6419 adev->gds.gds_size = 0x10000; 6420 break; 6421 case CHIP_RAVEN: 6422 case CHIP_ARCTURUS: 6423 adev->gds.gds_size = 0x1000; 6424 break; 6425 default: 6426 adev->gds.gds_size = 0x10000; 6427 break; 6428 } 6429 6430 switch (adev->asic_type) { 6431 case CHIP_VEGA10: 6432 case CHIP_VEGA20: 6433 adev->gds.gds_compute_max_wave_id = 0x7ff; 6434 break; 6435 case CHIP_VEGA12: 6436 adev->gds.gds_compute_max_wave_id = 0x27f; 6437 break; 6438 case CHIP_RAVEN: 6439 if (adev->rev_id >= 0x8) 6440 adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */ 6441 else 6442 adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */ 6443 break; 6444 case CHIP_ARCTURUS: 6445 adev->gds.gds_compute_max_wave_id = 0xfff; 6446 break; 6447 default: 6448 /* this really depends on the chip */ 6449 adev->gds.gds_compute_max_wave_id = 0x7ff; 6450 break; 6451 } 6452 6453 adev->gds.gws_size = 64; 6454 adev->gds.oa_size = 16; 6455 } 6456 6457 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev, 6458 u32 bitmap) 6459 { 6460 u32 data; 6461 6462 if (!bitmap) 6463 return; 6464 6465 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 6466 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 6467 6468 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data); 6469 } 6470 6471 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev) 6472 { 6473 u32 data, mask; 6474 6475 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG); 6476 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG); 6477 6478 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 6479 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 6480 6481 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); 6482 6483 return (~data) & mask; 6484 } 6485 6486 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 6487 struct amdgpu_cu_info *cu_info) 6488 { 6489 int i, j, k, counter, active_cu_number = 0; 6490 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0; 6491 unsigned disable_masks[4 * 4]; 6492 6493 if (!adev || !cu_info) 6494 return -EINVAL; 6495 6496 /* 6497 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs 6498 */ 6499 if (adev->gfx.config.max_shader_engines * 6500 adev->gfx.config.max_sh_per_se > 16) 6501 return -EINVAL; 6502 6503 amdgpu_gfx_parse_disable_cu(disable_masks, 6504 adev->gfx.config.max_shader_engines, 6505 adev->gfx.config.max_sh_per_se); 6506 6507 mutex_lock(&adev->grbm_idx_mutex); 6508 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 6509 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 6510 mask = 1; 6511 ao_bitmap = 0; 6512 counter = 0; 6513 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 6514 gfx_v9_0_set_user_cu_inactive_bitmap( 6515 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]); 6516 bitmap = gfx_v9_0_get_cu_active_bitmap(adev); 6517 6518 /* 6519 * The bitmap(and ao_cu_bitmap) in cu_info structure is 6520 * 4x4 size array, and it's usually suitable for Vega 6521 * ASICs which has 4*2 SE/SH layout. 6522 * But for Arcturus, SE/SH layout is changed to 8*1. 6523 * To mostly reduce the impact, we make it compatible 6524 * with current bitmap array as below: 6525 * SE4,SH0 --> bitmap[0][1] 6526 * SE5,SH0 --> bitmap[1][1] 6527 * SE6,SH0 --> bitmap[2][1] 6528 * SE7,SH0 --> bitmap[3][1] 6529 */ 6530 cu_info->bitmap[i % 4][j + i / 4] = bitmap; 6531 6532 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 6533 if (bitmap & mask) { 6534 if (counter < adev->gfx.config.max_cu_per_sh) 6535 ao_bitmap |= mask; 6536 counter ++; 6537 } 6538 mask <<= 1; 6539 } 6540 active_cu_number += counter; 6541 if (i < 2 && j < 2) 6542 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8)); 6543 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap; 6544 } 6545 } 6546 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 6547 mutex_unlock(&adev->grbm_idx_mutex); 6548 6549 cu_info->number = active_cu_number; 6550 cu_info->ao_cu_mask = ao_cu_mask; 6551 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 6552 6553 return 0; 6554 } 6555 6556 const struct amdgpu_ip_block_version gfx_v9_0_ip_block = 6557 { 6558 .type = AMD_IP_BLOCK_TYPE_GFX, 6559 .major = 9, 6560 .minor = 0, 6561 .rev = 0, 6562 .funcs = &gfx_v9_0_ip_funcs, 6563 }; 6564