1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/delay.h> 25 #include <linux/kernel.h> 26 #include <linux/firmware.h> 27 #include <linux/module.h> 28 #include <linux/pci.h> 29 30 #include "amdgpu.h" 31 #include "amdgpu_gfx.h" 32 #include "soc15.h" 33 #include "soc15d.h" 34 #include "amdgpu_atomfirmware.h" 35 #include "amdgpu_pm.h" 36 37 #include "gc/gc_9_0_offset.h" 38 #include "gc/gc_9_0_sh_mask.h" 39 40 #include "vega10_enum.h" 41 #include "hdp/hdp_4_0_offset.h" 42 43 #include "soc15_common.h" 44 #include "clearstate_gfx9.h" 45 #include "v9_structs.h" 46 47 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h" 48 49 #include "amdgpu_ras.h" 50 51 #include "gfx_v9_4.h" 52 53 #include "asic_reg/pwr/pwr_10_0_offset.h" 54 #include "asic_reg/pwr/pwr_10_0_sh_mask.h" 55 56 #define GFX9_NUM_GFX_RINGS 1 57 #define GFX9_MEC_HPD_SIZE 4096 58 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L 59 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L 60 61 #define mmGCEA_PROBE_MAP 0x070c 62 #define mmGCEA_PROBE_MAP_BASE_IDX 0 63 64 MODULE_FIRMWARE("amdgpu/vega10_ce.bin"); 65 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin"); 66 MODULE_FIRMWARE("amdgpu/vega10_me.bin"); 67 MODULE_FIRMWARE("amdgpu/vega10_mec.bin"); 68 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin"); 69 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin"); 70 71 MODULE_FIRMWARE("amdgpu/vega12_ce.bin"); 72 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin"); 73 MODULE_FIRMWARE("amdgpu/vega12_me.bin"); 74 MODULE_FIRMWARE("amdgpu/vega12_mec.bin"); 75 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin"); 76 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin"); 77 78 MODULE_FIRMWARE("amdgpu/vega20_ce.bin"); 79 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin"); 80 MODULE_FIRMWARE("amdgpu/vega20_me.bin"); 81 MODULE_FIRMWARE("amdgpu/vega20_mec.bin"); 82 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin"); 83 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin"); 84 85 MODULE_FIRMWARE("amdgpu/raven_ce.bin"); 86 MODULE_FIRMWARE("amdgpu/raven_pfp.bin"); 87 MODULE_FIRMWARE("amdgpu/raven_me.bin"); 88 MODULE_FIRMWARE("amdgpu/raven_mec.bin"); 89 MODULE_FIRMWARE("amdgpu/raven_mec2.bin"); 90 MODULE_FIRMWARE("amdgpu/raven_rlc.bin"); 91 92 MODULE_FIRMWARE("amdgpu/picasso_ce.bin"); 93 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin"); 94 MODULE_FIRMWARE("amdgpu/picasso_me.bin"); 95 MODULE_FIRMWARE("amdgpu/picasso_mec.bin"); 96 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin"); 97 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin"); 98 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin"); 99 100 MODULE_FIRMWARE("amdgpu/raven2_ce.bin"); 101 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin"); 102 MODULE_FIRMWARE("amdgpu/raven2_me.bin"); 103 MODULE_FIRMWARE("amdgpu/raven2_mec.bin"); 104 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin"); 105 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin"); 106 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin"); 107 108 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin"); 109 MODULE_FIRMWARE("amdgpu/arcturus_mec2.bin"); 110 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin"); 111 112 MODULE_FIRMWARE("amdgpu/renoir_ce.bin"); 113 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin"); 114 MODULE_FIRMWARE("amdgpu/renoir_me.bin"); 115 MODULE_FIRMWARE("amdgpu/renoir_mec.bin"); 116 MODULE_FIRMWARE("amdgpu/renoir_mec2.bin"); 117 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin"); 118 119 #define mmTCP_CHAN_STEER_0_ARCT 0x0b03 120 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0 121 #define mmTCP_CHAN_STEER_1_ARCT 0x0b04 122 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0 123 #define mmTCP_CHAN_STEER_2_ARCT 0x0b09 124 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0 125 #define mmTCP_CHAN_STEER_3_ARCT 0x0b0a 126 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0 127 #define mmTCP_CHAN_STEER_4_ARCT 0x0b0b 128 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0 129 #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c 130 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0 131 132 enum ta_ras_gfx_subblock { 133 /*CPC*/ 134 TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0, 135 TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START, 136 TA_RAS_BLOCK__GFX_CPC_UCODE, 137 TA_RAS_BLOCK__GFX_DC_STATE_ME1, 138 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1, 139 TA_RAS_BLOCK__GFX_DC_RESTORE_ME1, 140 TA_RAS_BLOCK__GFX_DC_STATE_ME2, 141 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2, 142 TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 143 TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 144 /* CPF*/ 145 TA_RAS_BLOCK__GFX_CPF_INDEX_START, 146 TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START, 147 TA_RAS_BLOCK__GFX_CPF_ROQ_ME1, 148 TA_RAS_BLOCK__GFX_CPF_TAG, 149 TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG, 150 /* CPG*/ 151 TA_RAS_BLOCK__GFX_CPG_INDEX_START, 152 TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START, 153 TA_RAS_BLOCK__GFX_CPG_DMA_TAG, 154 TA_RAS_BLOCK__GFX_CPG_TAG, 155 TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG, 156 /* GDS*/ 157 TA_RAS_BLOCK__GFX_GDS_INDEX_START, 158 TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START, 159 TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE, 160 TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM, 161 TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM, 162 TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 163 TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 164 /* SPI*/ 165 TA_RAS_BLOCK__GFX_SPI_SR_MEM, 166 /* SQ*/ 167 TA_RAS_BLOCK__GFX_SQ_INDEX_START, 168 TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START, 169 TA_RAS_BLOCK__GFX_SQ_LDS_D, 170 TA_RAS_BLOCK__GFX_SQ_LDS_I, 171 TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/ 172 TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR, 173 /* SQC (3 ranges)*/ 174 TA_RAS_BLOCK__GFX_SQC_INDEX_START, 175 /* SQC range 0*/ 176 TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START, 177 TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO = 178 TA_RAS_BLOCK__GFX_SQC_INDEX0_START, 179 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 180 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO, 181 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 182 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO, 183 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 184 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 185 TA_RAS_BLOCK__GFX_SQC_INDEX0_END = 186 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 187 /* SQC range 1*/ 188 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 189 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM = 190 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 191 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 192 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO, 193 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM, 194 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM, 195 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO, 196 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO, 197 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 198 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 199 TA_RAS_BLOCK__GFX_SQC_INDEX1_END = 200 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 201 /* SQC range 2*/ 202 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 203 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM = 204 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 205 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 206 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO, 207 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM, 208 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM, 209 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO, 210 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO, 211 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 212 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 213 TA_RAS_BLOCK__GFX_SQC_INDEX2_END = 214 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 215 TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END, 216 /* TA*/ 217 TA_RAS_BLOCK__GFX_TA_INDEX_START, 218 TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START, 219 TA_RAS_BLOCK__GFX_TA_FS_AFIFO, 220 TA_RAS_BLOCK__GFX_TA_FL_LFIFO, 221 TA_RAS_BLOCK__GFX_TA_FX_LFIFO, 222 TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 223 TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 224 /* TCA*/ 225 TA_RAS_BLOCK__GFX_TCA_INDEX_START, 226 TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START, 227 TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 228 TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 229 /* TCC (5 sub-ranges)*/ 230 TA_RAS_BLOCK__GFX_TCC_INDEX_START, 231 /* TCC range 0*/ 232 TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START, 233 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START, 234 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1, 235 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0, 236 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1, 237 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0, 238 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1, 239 TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG, 240 TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 241 TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 242 /* TCC range 1*/ 243 TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 244 TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 245 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 246 TA_RAS_BLOCK__GFX_TCC_INDEX1_END = 247 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 248 /* TCC range 2*/ 249 TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 250 TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 251 TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL, 252 TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO, 253 TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN, 254 TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ, 255 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO, 256 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM, 257 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 258 TA_RAS_BLOCK__GFX_TCC_INDEX2_END = 259 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 260 /* TCC range 3*/ 261 TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 262 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 263 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 264 TA_RAS_BLOCK__GFX_TCC_INDEX3_END = 265 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 266 /* TCC range 4*/ 267 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 268 TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN = 269 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 270 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 271 TA_RAS_BLOCK__GFX_TCC_INDEX4_END = 272 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 273 TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END, 274 /* TCI*/ 275 TA_RAS_BLOCK__GFX_TCI_WRITE_RAM, 276 /* TCP*/ 277 TA_RAS_BLOCK__GFX_TCP_INDEX_START, 278 TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START, 279 TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM, 280 TA_RAS_BLOCK__GFX_TCP_CMD_FIFO, 281 TA_RAS_BLOCK__GFX_TCP_VM_FIFO, 282 TA_RAS_BLOCK__GFX_TCP_DB_RAM, 283 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0, 284 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 285 TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 286 /* TD*/ 287 TA_RAS_BLOCK__GFX_TD_INDEX_START, 288 TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START, 289 TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI, 290 TA_RAS_BLOCK__GFX_TD_CS_FIFO, 291 TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO, 292 /* EA (3 sub-ranges)*/ 293 TA_RAS_BLOCK__GFX_EA_INDEX_START, 294 /* EA range 0*/ 295 TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START, 296 TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START, 297 TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM, 298 TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM, 299 TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM, 300 TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM, 301 TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM, 302 TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM, 303 TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 304 TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 305 /* EA range 1*/ 306 TA_RAS_BLOCK__GFX_EA_INDEX1_START, 307 TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START, 308 TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM, 309 TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM, 310 TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM, 311 TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM, 312 TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM, 313 TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 314 TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 315 /* EA range 2*/ 316 TA_RAS_BLOCK__GFX_EA_INDEX2_START, 317 TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START, 318 TA_RAS_BLOCK__GFX_EA_MAM_D1MEM, 319 TA_RAS_BLOCK__GFX_EA_MAM_D2MEM, 320 TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 321 TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 322 TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END, 323 /* UTC VM L2 bank*/ 324 TA_RAS_BLOCK__UTC_VML2_BANK_CACHE, 325 /* UTC VM walker*/ 326 TA_RAS_BLOCK__UTC_VML2_WALKER, 327 /* UTC ATC L2 2MB cache*/ 328 TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK, 329 /* UTC ATC L2 4KB cache*/ 330 TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK, 331 TA_RAS_BLOCK__GFX_MAX 332 }; 333 334 struct ras_gfx_subblock { 335 unsigned char *name; 336 int ta_subblock; 337 int hw_supported_error_type; 338 int sw_supported_error_type; 339 }; 340 341 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \ 342 [AMDGPU_RAS_BLOCK__##subblock] = { \ 343 #subblock, \ 344 TA_RAS_BLOCK__##subblock, \ 345 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \ 346 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \ 347 } 348 349 static const struct ras_gfx_subblock ras_gfx_subblocks[] = { 350 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1), 351 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1), 352 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 353 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 354 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 355 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 356 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 357 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 358 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 359 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 360 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1), 361 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0), 362 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1), 363 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1), 364 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 365 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0), 366 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0, 367 0), 368 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0, 369 0), 370 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 371 AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0), 372 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0), 373 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1), 374 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0), 375 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0), 376 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1), 377 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 378 0, 0), 379 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 380 0), 381 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 382 0, 0), 383 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0, 384 0), 385 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 386 0, 0), 387 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 388 0), 389 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 390 1), 391 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 392 0, 0, 0), 393 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 394 0), 395 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 396 0), 397 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 398 0), 399 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 400 0), 401 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 402 0), 403 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 404 0, 0), 405 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 406 0), 407 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 408 0), 409 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 410 0, 0, 0), 411 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 412 0), 413 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 414 0), 415 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 416 0), 417 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 418 0), 419 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 420 0), 421 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 422 0, 0), 423 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 424 0), 425 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1), 426 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 427 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 428 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 429 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 430 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0), 431 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 432 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1), 433 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0, 434 1), 435 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0, 436 1), 437 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0, 438 1), 439 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0, 440 0), 441 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0, 442 0), 443 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 444 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 445 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0), 446 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0), 447 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0), 448 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0), 449 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 450 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0), 451 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0), 452 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 453 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0), 454 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0, 455 0), 456 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 457 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0, 458 0), 459 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0, 460 0, 0), 461 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0, 462 0), 463 AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 464 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1), 465 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0), 466 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 467 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 468 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 469 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0), 470 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0), 471 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1), 472 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0), 473 AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 474 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1), 475 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 476 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 477 AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 478 AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 479 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 480 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 481 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 482 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 483 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 484 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 485 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 486 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0), 487 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 488 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 489 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0), 490 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0), 491 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0), 492 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0), 493 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0), 494 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0), 495 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0), 496 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0), 497 }; 498 499 static const struct soc15_reg_golden golden_settings_gc_9_0[] = 500 { 501 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 502 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000), 503 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 504 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 505 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 506 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 507 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 508 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 509 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 510 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87), 511 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f), 512 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 513 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 514 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 515 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 516 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 517 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff), 518 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 519 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 520 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 521 }; 522 523 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] = 524 { 525 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107), 526 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 527 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 528 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 529 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 530 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 531 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042), 532 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 533 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000), 534 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 535 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 536 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 537 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 538 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 539 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 540 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107), 541 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800), 542 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080) 543 }; 544 545 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] = 546 { 547 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080), 548 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 549 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 550 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042), 551 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042), 552 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400), 553 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000), 554 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000), 555 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107), 556 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000), 557 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000) 558 }; 559 560 static const struct soc15_reg_golden golden_settings_gc_9_1[] = 561 { 562 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 563 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 564 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 565 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 566 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 567 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 568 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 569 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 570 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 571 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 572 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 573 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 574 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 575 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 576 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 577 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 578 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 579 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 580 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 581 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff), 582 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080), 583 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 584 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 585 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 586 }; 587 588 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] = 589 { 590 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 591 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042), 592 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042), 593 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000), 594 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000), 595 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 596 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800) 597 }; 598 599 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] = 600 { 601 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000), 602 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 603 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 604 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080), 605 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080), 606 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080), 607 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041), 608 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041), 609 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 610 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 611 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080), 612 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080), 613 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080), 614 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080), 615 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080), 616 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 617 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010), 618 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 619 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 620 }; 621 622 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] = 623 { 624 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 625 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 626 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 627 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042), 628 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042), 629 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 630 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 631 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 632 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 633 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 634 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 635 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc), 636 }; 637 638 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] = 639 { 640 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff), 641 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000), 642 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382) 643 }; 644 645 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] = 646 { 647 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 648 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 649 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 650 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 651 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 652 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 653 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 654 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 655 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87), 656 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f), 657 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 658 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 659 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 660 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 661 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 662 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff) 663 }; 664 665 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] = 666 { 667 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080), 668 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 669 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 670 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041), 671 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041), 672 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 673 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107), 674 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 675 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410), 676 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 677 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 678 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 679 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 680 }; 681 682 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] = 683 { 684 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 685 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000), 686 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e), 687 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca), 688 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098), 689 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3), 690 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1), 691 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135), 692 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000), 693 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00), 694 }; 695 696 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = { 697 {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)}, 698 {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)}, 699 }; 700 701 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] = 702 { 703 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 704 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 705 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 706 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 707 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 708 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 709 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 710 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 711 }; 712 713 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] = 714 { 715 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0, 716 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0, 717 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0, 718 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0, 719 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0, 720 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0, 721 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0, 722 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0, 723 }; 724 725 void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v) 726 { 727 static void *scratch_reg0; 728 static void *scratch_reg1; 729 static void *scratch_reg2; 730 static void *scratch_reg3; 731 static void *spare_int; 732 static uint32_t grbm_cntl; 733 static uint32_t grbm_idx; 734 735 scratch_reg0 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0)*4; 736 scratch_reg1 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1)*4; 737 scratch_reg2 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG2)*4; 738 scratch_reg3 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3)*4; 739 spare_int = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT)*4; 740 741 grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL; 742 grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX; 743 744 if (amdgpu_sriov_runtime(adev)) { 745 pr_err("shouldn't call rlcg write register during runtime\n"); 746 return; 747 } 748 749 if (offset == grbm_cntl || offset == grbm_idx) { 750 if (offset == grbm_cntl) 751 writel(v, scratch_reg2); 752 else if (offset == grbm_idx) 753 writel(v, scratch_reg3); 754 755 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4)); 756 } else { 757 uint32_t i = 0; 758 uint32_t retries = 50000; 759 760 writel(v, scratch_reg0); 761 writel(offset | 0x80000000, scratch_reg1); 762 writel(1, spare_int); 763 for (i = 0; i < retries; i++) { 764 u32 tmp; 765 766 tmp = readl(scratch_reg1); 767 if (!(tmp & 0x80000000)) 768 break; 769 770 udelay(10); 771 } 772 if (i >= retries) 773 pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset); 774 } 775 776 } 777 778 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042 779 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041 780 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042 781 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041 782 783 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev); 784 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev); 785 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev); 786 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev); 787 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 788 struct amdgpu_cu_info *cu_info); 789 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev); 790 static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance); 791 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring); 792 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring); 793 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 794 void *ras_error_status); 795 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 796 void *inject_if); 797 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev); 798 799 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring, 800 uint64_t queue_mask) 801 { 802 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 803 amdgpu_ring_write(kiq_ring, 804 PACKET3_SET_RESOURCES_VMID_MASK(0) | 805 /* vmid_mask:0* queue_type:0 (KIQ) */ 806 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); 807 amdgpu_ring_write(kiq_ring, 808 lower_32_bits(queue_mask)); /* queue mask lo */ 809 amdgpu_ring_write(kiq_ring, 810 upper_32_bits(queue_mask)); /* queue mask hi */ 811 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */ 812 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */ 813 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 814 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 815 } 816 817 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring, 818 struct amdgpu_ring *ring) 819 { 820 struct amdgpu_device *adev = kiq_ring->adev; 821 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 822 uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 823 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 824 825 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 826 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 827 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 828 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 829 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ 830 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 831 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 832 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) | 833 /*queue_type: normal compute queue */ 834 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | 835 /* alloc format: all_on_one_pipe */ 836 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | 837 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) | 838 /* num_queues: must be 1 */ 839 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); 840 amdgpu_ring_write(kiq_ring, 841 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); 842 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 843 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 844 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 845 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 846 } 847 848 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, 849 struct amdgpu_ring *ring, 850 enum amdgpu_unmap_queues_action action, 851 u64 gpu_addr, u64 seq) 852 { 853 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 854 855 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 856 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 857 PACKET3_UNMAP_QUEUES_ACTION(action) | 858 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 859 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) | 860 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 861 amdgpu_ring_write(kiq_ring, 862 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 863 864 if (action == PREEMPT_QUEUES_NO_UNMAP) { 865 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr)); 866 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr)); 867 amdgpu_ring_write(kiq_ring, seq); 868 } else { 869 amdgpu_ring_write(kiq_ring, 0); 870 amdgpu_ring_write(kiq_ring, 0); 871 amdgpu_ring_write(kiq_ring, 0); 872 } 873 } 874 875 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring, 876 struct amdgpu_ring *ring, 877 u64 addr, 878 u64 seq) 879 { 880 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 881 882 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5)); 883 amdgpu_ring_write(kiq_ring, 884 PACKET3_QUERY_STATUS_CONTEXT_ID(0) | 885 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) | 886 PACKET3_QUERY_STATUS_COMMAND(2)); 887 /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 888 amdgpu_ring_write(kiq_ring, 889 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) | 890 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel)); 891 amdgpu_ring_write(kiq_ring, lower_32_bits(addr)); 892 amdgpu_ring_write(kiq_ring, upper_32_bits(addr)); 893 amdgpu_ring_write(kiq_ring, lower_32_bits(seq)); 894 amdgpu_ring_write(kiq_ring, upper_32_bits(seq)); 895 } 896 897 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring, 898 uint16_t pasid, uint32_t flush_type, 899 bool all_hub) 900 { 901 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); 902 amdgpu_ring_write(kiq_ring, 903 PACKET3_INVALIDATE_TLBS_DST_SEL(1) | 904 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) | 905 PACKET3_INVALIDATE_TLBS_PASID(pasid) | 906 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type)); 907 } 908 909 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = { 910 .kiq_set_resources = gfx_v9_0_kiq_set_resources, 911 .kiq_map_queues = gfx_v9_0_kiq_map_queues, 912 .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues, 913 .kiq_query_status = gfx_v9_0_kiq_query_status, 914 .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs, 915 .set_resources_size = 8, 916 .map_queues_size = 7, 917 .unmap_queues_size = 6, 918 .query_status_size = 7, 919 .invalidate_tlbs_size = 2, 920 }; 921 922 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) 923 { 924 adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs; 925 } 926 927 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev) 928 { 929 switch (adev->asic_type) { 930 case CHIP_VEGA10: 931 soc15_program_register_sequence(adev, 932 golden_settings_gc_9_0, 933 ARRAY_SIZE(golden_settings_gc_9_0)); 934 soc15_program_register_sequence(adev, 935 golden_settings_gc_9_0_vg10, 936 ARRAY_SIZE(golden_settings_gc_9_0_vg10)); 937 break; 938 case CHIP_VEGA12: 939 soc15_program_register_sequence(adev, 940 golden_settings_gc_9_2_1, 941 ARRAY_SIZE(golden_settings_gc_9_2_1)); 942 soc15_program_register_sequence(adev, 943 golden_settings_gc_9_2_1_vg12, 944 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12)); 945 break; 946 case CHIP_VEGA20: 947 soc15_program_register_sequence(adev, 948 golden_settings_gc_9_0, 949 ARRAY_SIZE(golden_settings_gc_9_0)); 950 soc15_program_register_sequence(adev, 951 golden_settings_gc_9_0_vg20, 952 ARRAY_SIZE(golden_settings_gc_9_0_vg20)); 953 break; 954 case CHIP_ARCTURUS: 955 soc15_program_register_sequence(adev, 956 golden_settings_gc_9_4_1_arct, 957 ARRAY_SIZE(golden_settings_gc_9_4_1_arct)); 958 break; 959 case CHIP_RAVEN: 960 soc15_program_register_sequence(adev, golden_settings_gc_9_1, 961 ARRAY_SIZE(golden_settings_gc_9_1)); 962 if (adev->rev_id >= 8) 963 soc15_program_register_sequence(adev, 964 golden_settings_gc_9_1_rv2, 965 ARRAY_SIZE(golden_settings_gc_9_1_rv2)); 966 else 967 soc15_program_register_sequence(adev, 968 golden_settings_gc_9_1_rv1, 969 ARRAY_SIZE(golden_settings_gc_9_1_rv1)); 970 break; 971 case CHIP_RENOIR: 972 soc15_program_register_sequence(adev, 973 golden_settings_gc_9_1_rn, 974 ARRAY_SIZE(golden_settings_gc_9_1_rn)); 975 return; /* for renoir, don't need common goldensetting */ 976 default: 977 break; 978 } 979 980 if (adev->asic_type != CHIP_ARCTURUS) 981 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common, 982 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common)); 983 } 984 985 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev) 986 { 987 adev->gfx.scratch.num_reg = 8; 988 adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0); 989 adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1; 990 } 991 992 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel, 993 bool wc, uint32_t reg, uint32_t val) 994 { 995 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 996 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) | 997 WRITE_DATA_DST_SEL(0) | 998 (wc ? WR_CONFIRM : 0)); 999 amdgpu_ring_write(ring, reg); 1000 amdgpu_ring_write(ring, 0); 1001 amdgpu_ring_write(ring, val); 1002 } 1003 1004 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, 1005 int mem_space, int opt, uint32_t addr0, 1006 uint32_t addr1, uint32_t ref, uint32_t mask, 1007 uint32_t inv) 1008 { 1009 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 1010 amdgpu_ring_write(ring, 1011 /* memory (1) or register (0) */ 1012 (WAIT_REG_MEM_MEM_SPACE(mem_space) | 1013 WAIT_REG_MEM_OPERATION(opt) | /* wait */ 1014 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 1015 WAIT_REG_MEM_ENGINE(eng_sel))); 1016 1017 if (mem_space) 1018 BUG_ON(addr0 & 0x3); /* Dword align */ 1019 amdgpu_ring_write(ring, addr0); 1020 amdgpu_ring_write(ring, addr1); 1021 amdgpu_ring_write(ring, ref); 1022 amdgpu_ring_write(ring, mask); 1023 amdgpu_ring_write(ring, inv); /* poll interval */ 1024 } 1025 1026 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring) 1027 { 1028 struct amdgpu_device *adev = ring->adev; 1029 uint32_t scratch; 1030 uint32_t tmp = 0; 1031 unsigned i; 1032 int r; 1033 1034 r = amdgpu_gfx_scratch_get(adev, &scratch); 1035 if (r) 1036 return r; 1037 1038 WREG32(scratch, 0xCAFEDEAD); 1039 r = amdgpu_ring_alloc(ring, 3); 1040 if (r) 1041 goto error_free_scratch; 1042 1043 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 1044 amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START)); 1045 amdgpu_ring_write(ring, 0xDEADBEEF); 1046 amdgpu_ring_commit(ring); 1047 1048 for (i = 0; i < adev->usec_timeout; i++) { 1049 tmp = RREG32(scratch); 1050 if (tmp == 0xDEADBEEF) 1051 break; 1052 udelay(1); 1053 } 1054 1055 if (i >= adev->usec_timeout) 1056 r = -ETIMEDOUT; 1057 1058 error_free_scratch: 1059 amdgpu_gfx_scratch_free(adev, scratch); 1060 return r; 1061 } 1062 1063 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 1064 { 1065 struct amdgpu_device *adev = ring->adev; 1066 struct amdgpu_ib ib; 1067 struct dma_fence *f = NULL; 1068 1069 unsigned index; 1070 uint64_t gpu_addr; 1071 uint32_t tmp; 1072 long r; 1073 1074 r = amdgpu_device_wb_get(adev, &index); 1075 if (r) 1076 return r; 1077 1078 gpu_addr = adev->wb.gpu_addr + (index * 4); 1079 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 1080 memset(&ib, 0, sizeof(ib)); 1081 r = amdgpu_ib_get(adev, NULL, 16, 1082 AMDGPU_IB_POOL_DIRECT, &ib); 1083 if (r) 1084 goto err1; 1085 1086 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 1087 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 1088 ib.ptr[2] = lower_32_bits(gpu_addr); 1089 ib.ptr[3] = upper_32_bits(gpu_addr); 1090 ib.ptr[4] = 0xDEADBEEF; 1091 ib.length_dw = 5; 1092 1093 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 1094 if (r) 1095 goto err2; 1096 1097 r = dma_fence_wait_timeout(f, false, timeout); 1098 if (r == 0) { 1099 r = -ETIMEDOUT; 1100 goto err2; 1101 } else if (r < 0) { 1102 goto err2; 1103 } 1104 1105 tmp = adev->wb.wb[index]; 1106 if (tmp == 0xDEADBEEF) 1107 r = 0; 1108 else 1109 r = -EINVAL; 1110 1111 err2: 1112 amdgpu_ib_free(adev, &ib, NULL); 1113 dma_fence_put(f); 1114 err1: 1115 amdgpu_device_wb_free(adev, index); 1116 return r; 1117 } 1118 1119 1120 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev) 1121 { 1122 release_firmware(adev->gfx.pfp_fw); 1123 adev->gfx.pfp_fw = NULL; 1124 release_firmware(adev->gfx.me_fw); 1125 adev->gfx.me_fw = NULL; 1126 release_firmware(adev->gfx.ce_fw); 1127 adev->gfx.ce_fw = NULL; 1128 release_firmware(adev->gfx.rlc_fw); 1129 adev->gfx.rlc_fw = NULL; 1130 release_firmware(adev->gfx.mec_fw); 1131 adev->gfx.mec_fw = NULL; 1132 release_firmware(adev->gfx.mec2_fw); 1133 adev->gfx.mec2_fw = NULL; 1134 1135 kfree(adev->gfx.rlc.register_list_format); 1136 } 1137 1138 static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev) 1139 { 1140 const struct rlc_firmware_header_v2_1 *rlc_hdr; 1141 1142 rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data; 1143 adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver); 1144 adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver); 1145 adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes); 1146 adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes); 1147 adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver); 1148 adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver); 1149 adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes); 1150 adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes); 1151 adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver); 1152 adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver); 1153 adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes); 1154 adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes); 1155 adev->gfx.rlc.reg_list_format_direct_reg_list_length = 1156 le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length); 1157 } 1158 1159 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev) 1160 { 1161 adev->gfx.me_fw_write_wait = false; 1162 adev->gfx.mec_fw_write_wait = false; 1163 1164 if ((adev->asic_type != CHIP_ARCTURUS) && 1165 ((adev->gfx.mec_fw_version < 0x000001a5) || 1166 (adev->gfx.mec_feature_version < 46) || 1167 (adev->gfx.pfp_fw_version < 0x000000b7) || 1168 (adev->gfx.pfp_feature_version < 46))) 1169 DRM_WARN_ONCE("CP firmware version too old, please update!"); 1170 1171 switch (adev->asic_type) { 1172 case CHIP_VEGA10: 1173 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1174 (adev->gfx.me_feature_version >= 42) && 1175 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1176 (adev->gfx.pfp_feature_version >= 42)) 1177 adev->gfx.me_fw_write_wait = true; 1178 1179 if ((adev->gfx.mec_fw_version >= 0x00000193) && 1180 (adev->gfx.mec_feature_version >= 42)) 1181 adev->gfx.mec_fw_write_wait = true; 1182 break; 1183 case CHIP_VEGA12: 1184 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1185 (adev->gfx.me_feature_version >= 44) && 1186 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1187 (adev->gfx.pfp_feature_version >= 44)) 1188 adev->gfx.me_fw_write_wait = true; 1189 1190 if ((adev->gfx.mec_fw_version >= 0x00000196) && 1191 (adev->gfx.mec_feature_version >= 44)) 1192 adev->gfx.mec_fw_write_wait = true; 1193 break; 1194 case CHIP_VEGA20: 1195 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1196 (adev->gfx.me_feature_version >= 44) && 1197 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1198 (adev->gfx.pfp_feature_version >= 44)) 1199 adev->gfx.me_fw_write_wait = true; 1200 1201 if ((adev->gfx.mec_fw_version >= 0x00000197) && 1202 (adev->gfx.mec_feature_version >= 44)) 1203 adev->gfx.mec_fw_write_wait = true; 1204 break; 1205 case CHIP_RAVEN: 1206 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1207 (adev->gfx.me_feature_version >= 42) && 1208 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1209 (adev->gfx.pfp_feature_version >= 42)) 1210 adev->gfx.me_fw_write_wait = true; 1211 1212 if ((adev->gfx.mec_fw_version >= 0x00000192) && 1213 (adev->gfx.mec_feature_version >= 42)) 1214 adev->gfx.mec_fw_write_wait = true; 1215 break; 1216 default: 1217 adev->gfx.me_fw_write_wait = true; 1218 adev->gfx.mec_fw_write_wait = true; 1219 break; 1220 } 1221 } 1222 1223 struct amdgpu_gfxoff_quirk { 1224 u16 chip_vendor; 1225 u16 chip_device; 1226 u16 subsys_vendor; 1227 u16 subsys_device; 1228 u8 revision; 1229 }; 1230 1231 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = { 1232 /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */ 1233 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 }, 1234 { 0, 0, 0, 0, 0 }, 1235 }; 1236 1237 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev) 1238 { 1239 const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list; 1240 1241 while (p && p->chip_device != 0) { 1242 if (pdev->vendor == p->chip_vendor && 1243 pdev->device == p->chip_device && 1244 pdev->subsystem_vendor == p->subsys_vendor && 1245 pdev->subsystem_device == p->subsys_device && 1246 pdev->revision == p->revision) { 1247 return true; 1248 } 1249 ++p; 1250 } 1251 return false; 1252 } 1253 1254 static bool is_raven_kicker(struct amdgpu_device *adev) 1255 { 1256 if (adev->pm.fw_version >= 0x41e2b) 1257 return true; 1258 else 1259 return false; 1260 } 1261 1262 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev) 1263 { 1264 if (gfx_v9_0_should_disable_gfxoff(adev->pdev)) 1265 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1266 1267 switch (adev->asic_type) { 1268 case CHIP_VEGA10: 1269 case CHIP_VEGA12: 1270 case CHIP_VEGA20: 1271 break; 1272 case CHIP_RAVEN: 1273 if (!(adev->rev_id >= 0x8 || adev->pdev->device == 0x15d8) && 1274 ((!is_raven_kicker(adev) && 1275 adev->gfx.rlc_fw_version < 531) || 1276 (adev->gfx.rlc_feature_version < 1) || 1277 !adev->gfx.rlc.is_rlc_v2_1)) 1278 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1279 1280 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1281 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1282 AMD_PG_SUPPORT_CP | 1283 AMD_PG_SUPPORT_RLC_SMU_HS; 1284 break; 1285 case CHIP_RENOIR: 1286 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1287 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1288 AMD_PG_SUPPORT_CP | 1289 AMD_PG_SUPPORT_RLC_SMU_HS; 1290 break; 1291 default: 1292 break; 1293 } 1294 } 1295 1296 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev, 1297 const char *chip_name) 1298 { 1299 char fw_name[30]; 1300 int err; 1301 struct amdgpu_firmware_info *info = NULL; 1302 const struct common_firmware_header *header = NULL; 1303 const struct gfx_firmware_header_v1_0 *cp_hdr; 1304 1305 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name); 1306 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev); 1307 if (err) 1308 goto out; 1309 err = amdgpu_ucode_validate(adev->gfx.pfp_fw); 1310 if (err) 1311 goto out; 1312 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; 1313 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1314 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1315 1316 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name); 1317 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev); 1318 if (err) 1319 goto out; 1320 err = amdgpu_ucode_validate(adev->gfx.me_fw); 1321 if (err) 1322 goto out; 1323 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; 1324 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1325 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1326 1327 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name); 1328 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev); 1329 if (err) 1330 goto out; 1331 err = amdgpu_ucode_validate(adev->gfx.ce_fw); 1332 if (err) 1333 goto out; 1334 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; 1335 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1336 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1337 1338 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1339 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP]; 1340 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP; 1341 info->fw = adev->gfx.pfp_fw; 1342 header = (const struct common_firmware_header *)info->fw->data; 1343 adev->firmware.fw_size += 1344 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1345 1346 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME]; 1347 info->ucode_id = AMDGPU_UCODE_ID_CP_ME; 1348 info->fw = adev->gfx.me_fw; 1349 header = (const struct common_firmware_header *)info->fw->data; 1350 adev->firmware.fw_size += 1351 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1352 1353 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE]; 1354 info->ucode_id = AMDGPU_UCODE_ID_CP_CE; 1355 info->fw = adev->gfx.ce_fw; 1356 header = (const struct common_firmware_header *)info->fw->data; 1357 adev->firmware.fw_size += 1358 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1359 } 1360 1361 out: 1362 if (err) { 1363 dev_err(adev->dev, 1364 "gfx9: Failed to load firmware \"%s\"\n", 1365 fw_name); 1366 release_firmware(adev->gfx.pfp_fw); 1367 adev->gfx.pfp_fw = NULL; 1368 release_firmware(adev->gfx.me_fw); 1369 adev->gfx.me_fw = NULL; 1370 release_firmware(adev->gfx.ce_fw); 1371 adev->gfx.ce_fw = NULL; 1372 } 1373 return err; 1374 } 1375 1376 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev, 1377 const char *chip_name) 1378 { 1379 char fw_name[30]; 1380 int err; 1381 struct amdgpu_firmware_info *info = NULL; 1382 const struct common_firmware_header *header = NULL; 1383 const struct rlc_firmware_header_v2_0 *rlc_hdr; 1384 unsigned int *tmp = NULL; 1385 unsigned int i = 0; 1386 uint16_t version_major; 1387 uint16_t version_minor; 1388 uint32_t smu_version; 1389 1390 /* 1391 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin 1392 * instead of picasso_rlc.bin. 1393 * Judgment method: 1394 * PCO AM4: revision >= 0xC8 && revision <= 0xCF 1395 * or revision >= 0xD8 && revision <= 0xDF 1396 * otherwise is PCO FP5 1397 */ 1398 if (!strcmp(chip_name, "picasso") && 1399 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) || 1400 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF)))) 1401 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name); 1402 else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) && 1403 (smu_version >= 0x41e2b)) 1404 /** 1405 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly. 1406 */ 1407 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name); 1408 else 1409 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name); 1410 err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev); 1411 if (err) 1412 goto out; 1413 err = amdgpu_ucode_validate(adev->gfx.rlc_fw); 1414 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 1415 1416 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 1417 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 1418 if (version_major == 2 && version_minor == 1) 1419 adev->gfx.rlc.is_rlc_v2_1 = true; 1420 1421 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); 1422 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); 1423 adev->gfx.rlc.save_and_restore_offset = 1424 le32_to_cpu(rlc_hdr->save_and_restore_offset); 1425 adev->gfx.rlc.clear_state_descriptor_offset = 1426 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset); 1427 adev->gfx.rlc.avail_scratch_ram_locations = 1428 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations); 1429 adev->gfx.rlc.reg_restore_list_size = 1430 le32_to_cpu(rlc_hdr->reg_restore_list_size); 1431 adev->gfx.rlc.reg_list_format_start = 1432 le32_to_cpu(rlc_hdr->reg_list_format_start); 1433 adev->gfx.rlc.reg_list_format_separate_start = 1434 le32_to_cpu(rlc_hdr->reg_list_format_separate_start); 1435 adev->gfx.rlc.starting_offsets_start = 1436 le32_to_cpu(rlc_hdr->starting_offsets_start); 1437 adev->gfx.rlc.reg_list_format_size_bytes = 1438 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes); 1439 adev->gfx.rlc.reg_list_size_bytes = 1440 le32_to_cpu(rlc_hdr->reg_list_size_bytes); 1441 adev->gfx.rlc.register_list_format = 1442 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes + 1443 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL); 1444 if (!adev->gfx.rlc.register_list_format) { 1445 err = -ENOMEM; 1446 goto out; 1447 } 1448 1449 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1450 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); 1451 for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++) 1452 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]); 1453 1454 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i; 1455 1456 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1457 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); 1458 for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++) 1459 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); 1460 1461 if (adev->gfx.rlc.is_rlc_v2_1) 1462 gfx_v9_0_init_rlc_ext_microcode(adev); 1463 1464 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1465 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G]; 1466 info->ucode_id = AMDGPU_UCODE_ID_RLC_G; 1467 info->fw = adev->gfx.rlc_fw; 1468 header = (const struct common_firmware_header *)info->fw->data; 1469 adev->firmware.fw_size += 1470 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1471 1472 if (adev->gfx.rlc.is_rlc_v2_1 && 1473 adev->gfx.rlc.save_restore_list_cntl_size_bytes && 1474 adev->gfx.rlc.save_restore_list_gpm_size_bytes && 1475 adev->gfx.rlc.save_restore_list_srm_size_bytes) { 1476 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL]; 1477 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL; 1478 info->fw = adev->gfx.rlc_fw; 1479 adev->firmware.fw_size += 1480 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE); 1481 1482 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM]; 1483 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM; 1484 info->fw = adev->gfx.rlc_fw; 1485 adev->firmware.fw_size += 1486 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE); 1487 1488 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM]; 1489 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM; 1490 info->fw = adev->gfx.rlc_fw; 1491 adev->firmware.fw_size += 1492 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE); 1493 } 1494 } 1495 1496 out: 1497 if (err) { 1498 dev_err(adev->dev, 1499 "gfx9: Failed to load firmware \"%s\"\n", 1500 fw_name); 1501 release_firmware(adev->gfx.rlc_fw); 1502 adev->gfx.rlc_fw = NULL; 1503 } 1504 return err; 1505 } 1506 1507 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev, 1508 const char *chip_name) 1509 { 1510 char fw_name[30]; 1511 int err; 1512 struct amdgpu_firmware_info *info = NULL; 1513 const struct common_firmware_header *header = NULL; 1514 const struct gfx_firmware_header_v1_0 *cp_hdr; 1515 1516 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name); 1517 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev); 1518 if (err) 1519 goto out; 1520 err = amdgpu_ucode_validate(adev->gfx.mec_fw); 1521 if (err) 1522 goto out; 1523 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1524 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1525 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1526 1527 1528 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name); 1529 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev); 1530 if (!err) { 1531 err = amdgpu_ucode_validate(adev->gfx.mec2_fw); 1532 if (err) 1533 goto out; 1534 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1535 adev->gfx.mec2_fw->data; 1536 adev->gfx.mec2_fw_version = 1537 le32_to_cpu(cp_hdr->header.ucode_version); 1538 adev->gfx.mec2_feature_version = 1539 le32_to_cpu(cp_hdr->ucode_feature_version); 1540 } else { 1541 err = 0; 1542 adev->gfx.mec2_fw = NULL; 1543 } 1544 1545 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1546 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1]; 1547 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1; 1548 info->fw = adev->gfx.mec_fw; 1549 header = (const struct common_firmware_header *)info->fw->data; 1550 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1551 adev->firmware.fw_size += 1552 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1553 1554 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT]; 1555 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT; 1556 info->fw = adev->gfx.mec_fw; 1557 adev->firmware.fw_size += 1558 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1559 1560 if (adev->gfx.mec2_fw) { 1561 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2]; 1562 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2; 1563 info->fw = adev->gfx.mec2_fw; 1564 header = (const struct common_firmware_header *)info->fw->data; 1565 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1566 adev->firmware.fw_size += 1567 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1568 1569 /* TODO: Determine if MEC2 JT FW loading can be removed 1570 for all GFX V9 asic and above */ 1571 if (adev->asic_type != CHIP_ARCTURUS && 1572 adev->asic_type != CHIP_RENOIR) { 1573 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT]; 1574 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT; 1575 info->fw = adev->gfx.mec2_fw; 1576 adev->firmware.fw_size += 1577 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, 1578 PAGE_SIZE); 1579 } 1580 } 1581 } 1582 1583 out: 1584 gfx_v9_0_check_if_need_gfxoff(adev); 1585 gfx_v9_0_check_fw_write_wait(adev); 1586 if (err) { 1587 dev_err(adev->dev, 1588 "gfx9: Failed to load firmware \"%s\"\n", 1589 fw_name); 1590 release_firmware(adev->gfx.mec_fw); 1591 adev->gfx.mec_fw = NULL; 1592 release_firmware(adev->gfx.mec2_fw); 1593 adev->gfx.mec2_fw = NULL; 1594 } 1595 return err; 1596 } 1597 1598 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev) 1599 { 1600 const char *chip_name; 1601 int r; 1602 1603 DRM_DEBUG("\n"); 1604 1605 switch (adev->asic_type) { 1606 case CHIP_VEGA10: 1607 chip_name = "vega10"; 1608 break; 1609 case CHIP_VEGA12: 1610 chip_name = "vega12"; 1611 break; 1612 case CHIP_VEGA20: 1613 chip_name = "vega20"; 1614 break; 1615 case CHIP_RAVEN: 1616 if (adev->rev_id >= 8) 1617 chip_name = "raven2"; 1618 else if (adev->pdev->device == 0x15d8) 1619 chip_name = "picasso"; 1620 else 1621 chip_name = "raven"; 1622 break; 1623 case CHIP_ARCTURUS: 1624 chip_name = "arcturus"; 1625 break; 1626 case CHIP_RENOIR: 1627 chip_name = "renoir"; 1628 break; 1629 default: 1630 BUG(); 1631 } 1632 1633 /* No CPG in Arcturus */ 1634 if (adev->asic_type != CHIP_ARCTURUS) { 1635 r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name); 1636 if (r) 1637 return r; 1638 } 1639 1640 r = gfx_v9_0_init_rlc_microcode(adev, chip_name); 1641 if (r) 1642 return r; 1643 1644 r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name); 1645 if (r) 1646 return r; 1647 1648 return r; 1649 } 1650 1651 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev) 1652 { 1653 u32 count = 0; 1654 const struct cs_section_def *sect = NULL; 1655 const struct cs_extent_def *ext = NULL; 1656 1657 /* begin clear state */ 1658 count += 2; 1659 /* context control state */ 1660 count += 3; 1661 1662 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 1663 for (ext = sect->section; ext->extent != NULL; ++ext) { 1664 if (sect->id == SECT_CONTEXT) 1665 count += 2 + ext->reg_count; 1666 else 1667 return 0; 1668 } 1669 } 1670 1671 /* end clear state */ 1672 count += 2; 1673 /* clear state */ 1674 count += 2; 1675 1676 return count; 1677 } 1678 1679 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev, 1680 volatile u32 *buffer) 1681 { 1682 u32 count = 0, i; 1683 const struct cs_section_def *sect = NULL; 1684 const struct cs_extent_def *ext = NULL; 1685 1686 if (adev->gfx.rlc.cs_data == NULL) 1687 return; 1688 if (buffer == NULL) 1689 return; 1690 1691 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1692 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 1693 1694 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 1695 buffer[count++] = cpu_to_le32(0x80000000); 1696 buffer[count++] = cpu_to_le32(0x80000000); 1697 1698 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { 1699 for (ext = sect->section; ext->extent != NULL; ++ext) { 1700 if (sect->id == SECT_CONTEXT) { 1701 buffer[count++] = 1702 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 1703 buffer[count++] = cpu_to_le32(ext->reg_index - 1704 PACKET3_SET_CONTEXT_REG_START); 1705 for (i = 0; i < ext->reg_count; i++) 1706 buffer[count++] = cpu_to_le32(ext->extent[i]); 1707 } else { 1708 return; 1709 } 1710 } 1711 } 1712 1713 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1714 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 1715 1716 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 1717 buffer[count++] = cpu_to_le32(0); 1718 } 1719 1720 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev) 1721 { 1722 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; 1723 uint32_t pg_always_on_cu_num = 2; 1724 uint32_t always_on_cu_num; 1725 uint32_t i, j, k; 1726 uint32_t mask, cu_bitmap, counter; 1727 1728 if (adev->flags & AMD_IS_APU) 1729 always_on_cu_num = 4; 1730 else if (adev->asic_type == CHIP_VEGA12) 1731 always_on_cu_num = 8; 1732 else 1733 always_on_cu_num = 12; 1734 1735 mutex_lock(&adev->grbm_idx_mutex); 1736 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 1737 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 1738 mask = 1; 1739 cu_bitmap = 0; 1740 counter = 0; 1741 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 1742 1743 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 1744 if (cu_info->bitmap[i][j] & mask) { 1745 if (counter == pg_always_on_cu_num) 1746 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap); 1747 if (counter < always_on_cu_num) 1748 cu_bitmap |= mask; 1749 else 1750 break; 1751 counter++; 1752 } 1753 mask <<= 1; 1754 } 1755 1756 WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap); 1757 cu_info->ao_cu_bitmap[i][j] = cu_bitmap; 1758 } 1759 } 1760 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1761 mutex_unlock(&adev->grbm_idx_mutex); 1762 } 1763 1764 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev) 1765 { 1766 uint32_t data; 1767 1768 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1769 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1770 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7); 1771 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1772 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16)); 1773 1774 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1775 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1776 1777 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1778 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500); 1779 1780 mutex_lock(&adev->grbm_idx_mutex); 1781 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1782 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1783 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1784 1785 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1786 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1787 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1788 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1789 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1790 1791 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1792 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1793 data &= 0x0000FFFF; 1794 data |= 0x00C00000; 1795 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1796 1797 /* 1798 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven), 1799 * programmed in gfx_v9_0_init_always_on_cu_mask() 1800 */ 1801 1802 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1803 * but used for RLC_LB_CNTL configuration */ 1804 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1805 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1806 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1807 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1808 mutex_unlock(&adev->grbm_idx_mutex); 1809 1810 gfx_v9_0_init_always_on_cu_mask(adev); 1811 } 1812 1813 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev) 1814 { 1815 uint32_t data; 1816 1817 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1818 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1819 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8); 1820 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1821 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16)); 1822 1823 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1824 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1825 1826 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1827 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800); 1828 1829 mutex_lock(&adev->grbm_idx_mutex); 1830 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1831 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1832 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1833 1834 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1835 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1836 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1837 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1838 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1839 1840 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1841 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1842 data &= 0x0000FFFF; 1843 data |= 0x00C00000; 1844 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1845 1846 /* 1847 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON), 1848 * programmed in gfx_v9_0_init_always_on_cu_mask() 1849 */ 1850 1851 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1852 * but used for RLC_LB_CNTL configuration */ 1853 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1854 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1855 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1856 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1857 mutex_unlock(&adev->grbm_idx_mutex); 1858 1859 gfx_v9_0_init_always_on_cu_mask(adev); 1860 } 1861 1862 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable) 1863 { 1864 WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0); 1865 } 1866 1867 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev) 1868 { 1869 return 5; 1870 } 1871 1872 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev) 1873 { 1874 const struct cs_section_def *cs_data; 1875 int r; 1876 1877 adev->gfx.rlc.cs_data = gfx9_cs_data; 1878 1879 cs_data = adev->gfx.rlc.cs_data; 1880 1881 if (cs_data) { 1882 /* init clear state block */ 1883 r = amdgpu_gfx_rlc_init_csb(adev); 1884 if (r) 1885 return r; 1886 } 1887 1888 if (adev->asic_type == CHIP_RAVEN || adev->asic_type == CHIP_RENOIR) { 1889 /* TODO: double check the cp_table_size for RV */ 1890 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ 1891 r = amdgpu_gfx_rlc_init_cpt(adev); 1892 if (r) 1893 return r; 1894 } 1895 1896 switch (adev->asic_type) { 1897 case CHIP_RAVEN: 1898 gfx_v9_0_init_lbpw(adev); 1899 break; 1900 case CHIP_VEGA20: 1901 gfx_v9_4_init_lbpw(adev); 1902 break; 1903 default: 1904 break; 1905 } 1906 1907 /* init spm vmid with 0xf */ 1908 if (adev->gfx.rlc.funcs->update_spm_vmid) 1909 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf); 1910 1911 return 0; 1912 } 1913 1914 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev) 1915 { 1916 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 1917 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); 1918 } 1919 1920 static int gfx_v9_0_mec_init(struct amdgpu_device *adev) 1921 { 1922 int r; 1923 u32 *hpd; 1924 const __le32 *fw_data; 1925 unsigned fw_size; 1926 u32 *fw; 1927 size_t mec_hpd_size; 1928 1929 const struct gfx_firmware_header_v1_0 *mec_hdr; 1930 1931 bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 1932 1933 /* take ownership of the relevant compute queues */ 1934 amdgpu_gfx_compute_queue_acquire(adev); 1935 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE; 1936 1937 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 1938 AMDGPU_GEM_DOMAIN_VRAM, 1939 &adev->gfx.mec.hpd_eop_obj, 1940 &adev->gfx.mec.hpd_eop_gpu_addr, 1941 (void **)&hpd); 1942 if (r) { 1943 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 1944 gfx_v9_0_mec_fini(adev); 1945 return r; 1946 } 1947 1948 memset(hpd, 0, mec_hpd_size); 1949 1950 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 1951 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 1952 1953 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1954 1955 fw_data = (const __le32 *) 1956 (adev->gfx.mec_fw->data + 1957 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 1958 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4; 1959 1960 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes, 1961 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 1962 &adev->gfx.mec.mec_fw_obj, 1963 &adev->gfx.mec.mec_fw_gpu_addr, 1964 (void **)&fw); 1965 if (r) { 1966 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r); 1967 gfx_v9_0_mec_fini(adev); 1968 return r; 1969 } 1970 1971 memcpy(fw, fw_data, fw_size); 1972 1973 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 1974 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 1975 1976 return 0; 1977 } 1978 1979 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address) 1980 { 1981 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 1982 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 1983 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 1984 (address << SQ_IND_INDEX__INDEX__SHIFT) | 1985 (SQ_IND_INDEX__FORCE_READ_MASK)); 1986 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 1987 } 1988 1989 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd, 1990 uint32_t wave, uint32_t thread, 1991 uint32_t regno, uint32_t num, uint32_t *out) 1992 { 1993 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 1994 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 1995 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 1996 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 1997 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) | 1998 (SQ_IND_INDEX__FORCE_READ_MASK) | 1999 (SQ_IND_INDEX__AUTO_INCR_MASK)); 2000 while (num--) 2001 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 2002 } 2003 2004 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 2005 { 2006 /* type 1 wave data */ 2007 dst[(*no_fields)++] = 1; 2008 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS); 2009 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO); 2010 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI); 2011 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO); 2012 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI); 2013 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID); 2014 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0); 2015 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1); 2016 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC); 2017 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC); 2018 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS); 2019 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS); 2020 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); 2021 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); 2022 } 2023 2024 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd, 2025 uint32_t wave, uint32_t start, 2026 uint32_t size, uint32_t *dst) 2027 { 2028 wave_read_regs( 2029 adev, simd, wave, 0, 2030 start + SQIND_WAVE_SGPRS_OFFSET, size, dst); 2031 } 2032 2033 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd, 2034 uint32_t wave, uint32_t thread, 2035 uint32_t start, uint32_t size, 2036 uint32_t *dst) 2037 { 2038 wave_read_regs( 2039 adev, simd, wave, thread, 2040 start + SQIND_WAVE_VGPRS_OFFSET, size, dst); 2041 } 2042 2043 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev, 2044 u32 me, u32 pipe, u32 q, u32 vm) 2045 { 2046 soc15_grbm_select(adev, me, pipe, q, vm); 2047 } 2048 2049 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = { 2050 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 2051 .select_se_sh = &gfx_v9_0_select_se_sh, 2052 .read_wave_data = &gfx_v9_0_read_wave_data, 2053 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 2054 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 2055 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 2056 .ras_error_inject = &gfx_v9_0_ras_error_inject, 2057 .query_ras_error_count = &gfx_v9_0_query_ras_error_count, 2058 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count, 2059 }; 2060 2061 static const struct amdgpu_gfx_funcs gfx_v9_4_gfx_funcs = { 2062 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 2063 .select_se_sh = &gfx_v9_0_select_se_sh, 2064 .read_wave_data = &gfx_v9_0_read_wave_data, 2065 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 2066 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 2067 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 2068 .ras_error_inject = &gfx_v9_4_ras_error_inject, 2069 .query_ras_error_count = &gfx_v9_4_query_ras_error_count, 2070 .reset_ras_error_count = &gfx_v9_4_reset_ras_error_count, 2071 }; 2072 2073 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev) 2074 { 2075 u32 gb_addr_config; 2076 int err; 2077 2078 adev->gfx.funcs = &gfx_v9_0_gfx_funcs; 2079 2080 switch (adev->asic_type) { 2081 case CHIP_VEGA10: 2082 adev->gfx.config.max_hw_contexts = 8; 2083 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2084 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2085 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2086 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2087 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN; 2088 break; 2089 case CHIP_VEGA12: 2090 adev->gfx.config.max_hw_contexts = 8; 2091 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2092 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2093 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2094 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2095 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN; 2096 DRM_INFO("fix gfx.config for vega12\n"); 2097 break; 2098 case CHIP_VEGA20: 2099 adev->gfx.config.max_hw_contexts = 8; 2100 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2101 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2102 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2103 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2104 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2105 gb_addr_config &= ~0xf3e777ff; 2106 gb_addr_config |= 0x22014042; 2107 /* check vbios table if gpu info is not available */ 2108 err = amdgpu_atomfirmware_get_gfx_info(adev); 2109 if (err) 2110 return err; 2111 break; 2112 case CHIP_RAVEN: 2113 adev->gfx.config.max_hw_contexts = 8; 2114 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2115 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2116 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2117 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2118 if (adev->rev_id >= 8) 2119 gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN; 2120 else 2121 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN; 2122 break; 2123 case CHIP_ARCTURUS: 2124 adev->gfx.funcs = &gfx_v9_4_gfx_funcs; 2125 adev->gfx.config.max_hw_contexts = 8; 2126 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2127 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2128 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2129 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2130 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2131 gb_addr_config &= ~0xf3e777ff; 2132 gb_addr_config |= 0x22014042; 2133 break; 2134 case CHIP_RENOIR: 2135 adev->gfx.config.max_hw_contexts = 8; 2136 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2137 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2138 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; 2139 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2140 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2141 gb_addr_config &= ~0xf3e777ff; 2142 gb_addr_config |= 0x22010042; 2143 break; 2144 default: 2145 BUG(); 2146 break; 2147 } 2148 2149 adev->gfx.config.gb_addr_config = gb_addr_config; 2150 2151 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << 2152 REG_GET_FIELD( 2153 adev->gfx.config.gb_addr_config, 2154 GB_ADDR_CONFIG, 2155 NUM_PIPES); 2156 2157 adev->gfx.config.max_tile_pipes = 2158 adev->gfx.config.gb_addr_config_fields.num_pipes; 2159 2160 adev->gfx.config.gb_addr_config_fields.num_banks = 1 << 2161 REG_GET_FIELD( 2162 adev->gfx.config.gb_addr_config, 2163 GB_ADDR_CONFIG, 2164 NUM_BANKS); 2165 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << 2166 REG_GET_FIELD( 2167 adev->gfx.config.gb_addr_config, 2168 GB_ADDR_CONFIG, 2169 MAX_COMPRESSED_FRAGS); 2170 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << 2171 REG_GET_FIELD( 2172 adev->gfx.config.gb_addr_config, 2173 GB_ADDR_CONFIG, 2174 NUM_RB_PER_SE); 2175 adev->gfx.config.gb_addr_config_fields.num_se = 1 << 2176 REG_GET_FIELD( 2177 adev->gfx.config.gb_addr_config, 2178 GB_ADDR_CONFIG, 2179 NUM_SHADER_ENGINES); 2180 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + 2181 REG_GET_FIELD( 2182 adev->gfx.config.gb_addr_config, 2183 GB_ADDR_CONFIG, 2184 PIPE_INTERLEAVE_SIZE)); 2185 2186 return 0; 2187 } 2188 2189 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 2190 int mec, int pipe, int queue) 2191 { 2192 int r; 2193 unsigned irq_type; 2194 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; 2195 unsigned int hw_prio; 2196 2197 ring = &adev->gfx.compute_ring[ring_id]; 2198 2199 /* mec0 is me1 */ 2200 ring->me = mec + 1; 2201 ring->pipe = pipe; 2202 ring->queue = queue; 2203 2204 ring->ring_obj = NULL; 2205 ring->use_doorbell = true; 2206 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1; 2207 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 2208 + (ring_id * GFX9_MEC_HPD_SIZE); 2209 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 2210 2211 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 2212 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 2213 + ring->pipe; 2214 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring->queue) ? 2215 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 2216 /* type-2 packets are deprecated on MEC, use type-3 instead */ 2217 r = amdgpu_ring_init(adev, ring, 1024, 2218 &adev->gfx.eop_irq, irq_type, hw_prio); 2219 if (r) 2220 return r; 2221 2222 2223 return 0; 2224 } 2225 2226 static int gfx_v9_0_sw_init(void *handle) 2227 { 2228 int i, j, k, r, ring_id; 2229 struct amdgpu_ring *ring; 2230 struct amdgpu_kiq *kiq; 2231 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2232 2233 switch (adev->asic_type) { 2234 case CHIP_VEGA10: 2235 case CHIP_VEGA12: 2236 case CHIP_VEGA20: 2237 case CHIP_RAVEN: 2238 case CHIP_ARCTURUS: 2239 case CHIP_RENOIR: 2240 adev->gfx.mec.num_mec = 2; 2241 break; 2242 default: 2243 adev->gfx.mec.num_mec = 1; 2244 break; 2245 } 2246 2247 adev->gfx.mec.num_pipe_per_mec = 4; 2248 adev->gfx.mec.num_queue_per_pipe = 8; 2249 2250 /* EOP Event */ 2251 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq); 2252 if (r) 2253 return r; 2254 2255 /* Privileged reg */ 2256 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT, 2257 &adev->gfx.priv_reg_irq); 2258 if (r) 2259 return r; 2260 2261 /* Privileged inst */ 2262 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT, 2263 &adev->gfx.priv_inst_irq); 2264 if (r) 2265 return r; 2266 2267 /* ECC error */ 2268 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR, 2269 &adev->gfx.cp_ecc_error_irq); 2270 if (r) 2271 return r; 2272 2273 /* FUE error */ 2274 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR, 2275 &adev->gfx.cp_ecc_error_irq); 2276 if (r) 2277 return r; 2278 2279 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 2280 2281 gfx_v9_0_scratch_init(adev); 2282 2283 r = gfx_v9_0_init_microcode(adev); 2284 if (r) { 2285 DRM_ERROR("Failed to load gfx firmware!\n"); 2286 return r; 2287 } 2288 2289 r = adev->gfx.rlc.funcs->init(adev); 2290 if (r) { 2291 DRM_ERROR("Failed to init rlc BOs!\n"); 2292 return r; 2293 } 2294 2295 r = gfx_v9_0_mec_init(adev); 2296 if (r) { 2297 DRM_ERROR("Failed to init MEC BOs!\n"); 2298 return r; 2299 } 2300 2301 /* set up the gfx ring */ 2302 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 2303 ring = &adev->gfx.gfx_ring[i]; 2304 ring->ring_obj = NULL; 2305 if (!i) 2306 sprintf(ring->name, "gfx"); 2307 else 2308 sprintf(ring->name, "gfx_%d", i); 2309 ring->use_doorbell = true; 2310 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 2311 r = amdgpu_ring_init(adev, ring, 1024, 2312 &adev->gfx.eop_irq, 2313 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, 2314 AMDGPU_RING_PRIO_DEFAULT); 2315 if (r) 2316 return r; 2317 } 2318 2319 /* set up the compute queues - allocate horizontally across pipes */ 2320 ring_id = 0; 2321 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 2322 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 2323 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 2324 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j)) 2325 continue; 2326 2327 r = gfx_v9_0_compute_ring_init(adev, 2328 ring_id, 2329 i, k, j); 2330 if (r) 2331 return r; 2332 2333 ring_id++; 2334 } 2335 } 2336 } 2337 2338 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE); 2339 if (r) { 2340 DRM_ERROR("Failed to init KIQ BOs!\n"); 2341 return r; 2342 } 2343 2344 kiq = &adev->gfx.kiq; 2345 r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq); 2346 if (r) 2347 return r; 2348 2349 /* create MQD for all compute queues as wel as KIQ for SRIOV case */ 2350 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation)); 2351 if (r) 2352 return r; 2353 2354 adev->gfx.ce_ram_size = 0x8000; 2355 2356 r = gfx_v9_0_gpu_early_init(adev); 2357 if (r) 2358 return r; 2359 2360 return 0; 2361 } 2362 2363 2364 static int gfx_v9_0_sw_fini(void *handle) 2365 { 2366 int i; 2367 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2368 2369 amdgpu_gfx_ras_fini(adev); 2370 2371 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 2372 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 2373 for (i = 0; i < adev->gfx.num_compute_rings; i++) 2374 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 2375 2376 amdgpu_gfx_mqd_sw_fini(adev); 2377 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring); 2378 amdgpu_gfx_kiq_fini(adev); 2379 2380 gfx_v9_0_mec_fini(adev); 2381 amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj); 2382 if (adev->asic_type == CHIP_RAVEN || adev->asic_type == CHIP_RENOIR) { 2383 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 2384 &adev->gfx.rlc.cp_table_gpu_addr, 2385 (void **)&adev->gfx.rlc.cp_table_ptr); 2386 } 2387 gfx_v9_0_free_microcode(adev); 2388 2389 return 0; 2390 } 2391 2392 2393 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev) 2394 { 2395 /* TODO */ 2396 } 2397 2398 static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance) 2399 { 2400 u32 data; 2401 2402 if (instance == 0xffffffff) 2403 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); 2404 else 2405 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance); 2406 2407 if (se_num == 0xffffffff) 2408 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); 2409 else 2410 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 2411 2412 if (sh_num == 0xffffffff) 2413 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1); 2414 else 2415 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num); 2416 2417 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data); 2418 } 2419 2420 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev) 2421 { 2422 u32 data, mask; 2423 2424 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE); 2425 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE); 2426 2427 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK; 2428 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT; 2429 2430 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / 2431 adev->gfx.config.max_sh_per_se); 2432 2433 return (~data) & mask; 2434 } 2435 2436 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev) 2437 { 2438 int i, j; 2439 u32 data; 2440 u32 active_rbs = 0; 2441 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / 2442 adev->gfx.config.max_sh_per_se; 2443 2444 mutex_lock(&adev->grbm_idx_mutex); 2445 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2446 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2447 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2448 data = gfx_v9_0_get_rb_active_bitmap(adev); 2449 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * 2450 rb_bitmap_width_per_sh); 2451 } 2452 } 2453 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2454 mutex_unlock(&adev->grbm_idx_mutex); 2455 2456 adev->gfx.config.backend_enable_mask = active_rbs; 2457 adev->gfx.config.num_rbs = hweight32(active_rbs); 2458 } 2459 2460 #define DEFAULT_SH_MEM_BASES (0x6000) 2461 #define FIRST_COMPUTE_VMID (8) 2462 #define LAST_COMPUTE_VMID (16) 2463 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev) 2464 { 2465 int i; 2466 uint32_t sh_mem_config; 2467 uint32_t sh_mem_bases; 2468 2469 /* 2470 * Configure apertures: 2471 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 2472 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 2473 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 2474 */ 2475 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16); 2476 2477 sh_mem_config = SH_MEM_ADDRESS_MODE_64 | 2478 SH_MEM_ALIGNMENT_MODE_UNALIGNED << 2479 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT; 2480 2481 mutex_lock(&adev->srbm_mutex); 2482 for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) { 2483 soc15_grbm_select(adev, 0, 0, 0, i); 2484 /* CP and shaders */ 2485 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config); 2486 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases); 2487 } 2488 soc15_grbm_select(adev, 0, 0, 0, 0); 2489 mutex_unlock(&adev->srbm_mutex); 2490 2491 /* Initialize all compute VMIDs to have no GDS, GWS, or OA 2492 acccess. These should be enabled by FW for target VMIDs. */ 2493 for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) { 2494 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0); 2495 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0); 2496 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0); 2497 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0); 2498 } 2499 } 2500 2501 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev) 2502 { 2503 int vmid; 2504 2505 /* 2506 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 2507 * access. Compute VMIDs should be enabled by FW for target VMIDs, 2508 * the driver can enable them for graphics. VMID0 should maintain 2509 * access so that HWS firmware can save/restore entries. 2510 */ 2511 for (vmid = 1; vmid < 16; vmid++) { 2512 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0); 2513 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0); 2514 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0); 2515 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0); 2516 } 2517 } 2518 2519 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev) 2520 { 2521 uint32_t tmp; 2522 2523 switch (adev->asic_type) { 2524 case CHIP_ARCTURUS: 2525 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG); 2526 tmp = REG_SET_FIELD(tmp, SQ_CONFIG, 2527 DISABLE_BARRIER_WAITCNT, 1); 2528 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp); 2529 break; 2530 default: 2531 break; 2532 }; 2533 } 2534 2535 static void gfx_v9_0_constants_init(struct amdgpu_device *adev) 2536 { 2537 u32 tmp; 2538 int i; 2539 2540 WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); 2541 2542 gfx_v9_0_tiling_mode_table_init(adev); 2543 2544 gfx_v9_0_setup_rb(adev); 2545 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info); 2546 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2); 2547 2548 /* XXX SH_MEM regs */ 2549 /* where to put LDS, scratch, GPUVM in FSA64 space */ 2550 mutex_lock(&adev->srbm_mutex); 2551 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) { 2552 soc15_grbm_select(adev, 0, 0, 0, i); 2553 /* CP and shaders */ 2554 if (i == 0) { 2555 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2556 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2557 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2558 !!amdgpu_noretry); 2559 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2560 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0); 2561 } else { 2562 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2563 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2564 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2565 !!amdgpu_noretry); 2566 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2567 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 2568 (adev->gmc.private_aperture_start >> 48)); 2569 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE, 2570 (adev->gmc.shared_aperture_start >> 48)); 2571 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp); 2572 } 2573 } 2574 soc15_grbm_select(adev, 0, 0, 0, 0); 2575 2576 mutex_unlock(&adev->srbm_mutex); 2577 2578 gfx_v9_0_init_compute_vmid(adev); 2579 gfx_v9_0_init_gds_vmid(adev); 2580 gfx_v9_0_init_sq_config(adev); 2581 } 2582 2583 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev) 2584 { 2585 u32 i, j, k; 2586 u32 mask; 2587 2588 mutex_lock(&adev->grbm_idx_mutex); 2589 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2590 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2591 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2592 for (k = 0; k < adev->usec_timeout; k++) { 2593 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0) 2594 break; 2595 udelay(1); 2596 } 2597 if (k == adev->usec_timeout) { 2598 gfx_v9_0_select_se_sh(adev, 0xffffffff, 2599 0xffffffff, 0xffffffff); 2600 mutex_unlock(&adev->grbm_idx_mutex); 2601 DRM_INFO("Timeout wait for RLC serdes %u,%u\n", 2602 i, j); 2603 return; 2604 } 2605 } 2606 } 2607 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2608 mutex_unlock(&adev->grbm_idx_mutex); 2609 2610 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | 2611 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK | 2612 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK | 2613 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK; 2614 for (k = 0; k < adev->usec_timeout; k++) { 2615 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 2616 break; 2617 udelay(1); 2618 } 2619 } 2620 2621 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 2622 bool enable) 2623 { 2624 u32 tmp = RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0); 2625 2626 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0); 2627 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0); 2628 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0); 2629 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0); 2630 2631 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp); 2632 } 2633 2634 static void gfx_v9_0_init_csb(struct amdgpu_device *adev) 2635 { 2636 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 2637 /* csib */ 2638 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI), 2639 adev->gfx.rlc.clear_state_gpu_addr >> 32); 2640 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO), 2641 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 2642 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH), 2643 adev->gfx.rlc.clear_state_size); 2644 } 2645 2646 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format, 2647 int indirect_offset, 2648 int list_size, 2649 int *unique_indirect_regs, 2650 int unique_indirect_reg_count, 2651 int *indirect_start_offsets, 2652 int *indirect_start_offsets_count, 2653 int max_start_offsets_count) 2654 { 2655 int idx; 2656 2657 for (; indirect_offset < list_size; indirect_offset++) { 2658 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count); 2659 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset; 2660 *indirect_start_offsets_count = *indirect_start_offsets_count + 1; 2661 2662 while (register_list_format[indirect_offset] != 0xFFFFFFFF) { 2663 indirect_offset += 2; 2664 2665 /* look for the matching indice */ 2666 for (idx = 0; idx < unique_indirect_reg_count; idx++) { 2667 if (unique_indirect_regs[idx] == 2668 register_list_format[indirect_offset] || 2669 !unique_indirect_regs[idx]) 2670 break; 2671 } 2672 2673 BUG_ON(idx >= unique_indirect_reg_count); 2674 2675 if (!unique_indirect_regs[idx]) 2676 unique_indirect_regs[idx] = register_list_format[indirect_offset]; 2677 2678 indirect_offset++; 2679 } 2680 } 2681 } 2682 2683 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev) 2684 { 2685 int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2686 int unique_indirect_reg_count = 0; 2687 2688 int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2689 int indirect_start_offsets_count = 0; 2690 2691 int list_size = 0; 2692 int i = 0, j = 0; 2693 u32 tmp = 0; 2694 2695 u32 *register_list_format = 2696 kmemdup(adev->gfx.rlc.register_list_format, 2697 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); 2698 if (!register_list_format) 2699 return -ENOMEM; 2700 2701 /* setup unique_indirect_regs array and indirect_start_offsets array */ 2702 unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs); 2703 gfx_v9_1_parse_ind_reg_list(register_list_format, 2704 adev->gfx.rlc.reg_list_format_direct_reg_list_length, 2705 adev->gfx.rlc.reg_list_format_size_bytes >> 2, 2706 unique_indirect_regs, 2707 unique_indirect_reg_count, 2708 indirect_start_offsets, 2709 &indirect_start_offsets_count, 2710 ARRAY_SIZE(indirect_start_offsets)); 2711 2712 /* enable auto inc in case it is disabled */ 2713 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL)); 2714 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK; 2715 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp); 2716 2717 /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */ 2718 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR), 2719 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET); 2720 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) 2721 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA), 2722 adev->gfx.rlc.register_restore[i]); 2723 2724 /* load indirect register */ 2725 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2726 adev->gfx.rlc.reg_list_format_start); 2727 2728 /* direct register portion */ 2729 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++) 2730 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2731 register_list_format[i]); 2732 2733 /* indirect register portion */ 2734 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) { 2735 if (register_list_format[i] == 0xFFFFFFFF) { 2736 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2737 continue; 2738 } 2739 2740 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2741 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2742 2743 for (j = 0; j < unique_indirect_reg_count; j++) { 2744 if (register_list_format[i] == unique_indirect_regs[j]) { 2745 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j); 2746 break; 2747 } 2748 } 2749 2750 BUG_ON(j >= unique_indirect_reg_count); 2751 2752 i++; 2753 } 2754 2755 /* set save/restore list size */ 2756 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; 2757 list_size = list_size >> 1; 2758 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2759 adev->gfx.rlc.reg_restore_list_size); 2760 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size); 2761 2762 /* write the starting offsets to RLC scratch ram */ 2763 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2764 adev->gfx.rlc.starting_offsets_start); 2765 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++) 2766 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2767 indirect_start_offsets[i]); 2768 2769 /* load unique indirect regs*/ 2770 for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) { 2771 if (unique_indirect_regs[i] != 0) { 2772 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0) 2773 + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i], 2774 unique_indirect_regs[i] & 0x3FFFF); 2775 2776 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0) 2777 + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i], 2778 unique_indirect_regs[i] >> 20); 2779 } 2780 } 2781 2782 kfree(register_list_format); 2783 return 0; 2784 } 2785 2786 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev) 2787 { 2788 WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1); 2789 } 2790 2791 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev, 2792 bool enable) 2793 { 2794 uint32_t data = 0; 2795 uint32_t default_data = 0; 2796 2797 default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS)); 2798 if (enable == true) { 2799 /* enable GFXIP control over CGPG */ 2800 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2801 if(default_data != data) 2802 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2803 2804 /* update status */ 2805 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK; 2806 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT); 2807 if(default_data != data) 2808 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2809 } else { 2810 /* restore GFXIP control over GCPG */ 2811 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2812 if(default_data != data) 2813 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2814 } 2815 } 2816 2817 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev) 2818 { 2819 uint32_t data = 0; 2820 2821 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2822 AMD_PG_SUPPORT_GFX_SMG | 2823 AMD_PG_SUPPORT_GFX_DMG)) { 2824 /* init IDLE_POLL_COUNT = 60 */ 2825 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL)); 2826 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK; 2827 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 2828 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data); 2829 2830 /* init RLC PG Delay */ 2831 data = 0; 2832 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT); 2833 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT); 2834 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT); 2835 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT); 2836 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data); 2837 2838 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2)); 2839 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK; 2840 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT); 2841 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data); 2842 2843 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3)); 2844 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK; 2845 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT); 2846 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data); 2847 2848 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL)); 2849 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK; 2850 2851 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */ 2852 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT); 2853 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data); 2854 2855 pwr_10_0_gfxip_control_over_cgpg(adev, true); 2856 } 2857 } 2858 2859 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev, 2860 bool enable) 2861 { 2862 uint32_t data = 0; 2863 uint32_t default_data = 0; 2864 2865 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2866 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2867 SMU_CLK_SLOWDOWN_ON_PU_ENABLE, 2868 enable ? 1 : 0); 2869 if (default_data != data) 2870 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2871 } 2872 2873 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev, 2874 bool enable) 2875 { 2876 uint32_t data = 0; 2877 uint32_t default_data = 0; 2878 2879 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2880 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2881 SMU_CLK_SLOWDOWN_ON_PD_ENABLE, 2882 enable ? 1 : 0); 2883 if(default_data != data) 2884 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2885 } 2886 2887 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev, 2888 bool enable) 2889 { 2890 uint32_t data = 0; 2891 uint32_t default_data = 0; 2892 2893 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2894 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2895 CP_PG_DISABLE, 2896 enable ? 0 : 1); 2897 if(default_data != data) 2898 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2899 } 2900 2901 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev, 2902 bool enable) 2903 { 2904 uint32_t data, default_data; 2905 2906 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2907 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2908 GFX_POWER_GATING_ENABLE, 2909 enable ? 1 : 0); 2910 if(default_data != data) 2911 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2912 } 2913 2914 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev, 2915 bool enable) 2916 { 2917 uint32_t data, default_data; 2918 2919 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2920 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2921 GFX_PIPELINE_PG_ENABLE, 2922 enable ? 1 : 0); 2923 if(default_data != data) 2924 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2925 2926 if (!enable) 2927 /* read any GFX register to wake up GFX */ 2928 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL)); 2929 } 2930 2931 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, 2932 bool enable) 2933 { 2934 uint32_t data, default_data; 2935 2936 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2937 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2938 STATIC_PER_CU_PG_ENABLE, 2939 enable ? 1 : 0); 2940 if(default_data != data) 2941 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2942 } 2943 2944 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev, 2945 bool enable) 2946 { 2947 uint32_t data, default_data; 2948 2949 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2950 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2951 DYN_PER_CU_PG_ENABLE, 2952 enable ? 1 : 0); 2953 if(default_data != data) 2954 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2955 } 2956 2957 static void gfx_v9_0_init_pg(struct amdgpu_device *adev) 2958 { 2959 gfx_v9_0_init_csb(adev); 2960 2961 /* 2962 * Rlc save restore list is workable since v2_1. 2963 * And it's needed by gfxoff feature. 2964 */ 2965 if (adev->gfx.rlc.is_rlc_v2_1) { 2966 if (adev->asic_type == CHIP_VEGA12 || 2967 (adev->asic_type == CHIP_RAVEN && 2968 adev->rev_id >= 8)) 2969 gfx_v9_1_init_rlc_save_restore_list(adev); 2970 gfx_v9_0_enable_save_restore_machine(adev); 2971 } 2972 2973 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2974 AMD_PG_SUPPORT_GFX_SMG | 2975 AMD_PG_SUPPORT_GFX_DMG | 2976 AMD_PG_SUPPORT_CP | 2977 AMD_PG_SUPPORT_GDS | 2978 AMD_PG_SUPPORT_RLC_SMU_HS)) { 2979 WREG32(mmRLC_JUMP_TABLE_RESTORE, 2980 adev->gfx.rlc.cp_table_gpu_addr >> 8); 2981 gfx_v9_0_init_gfx_power_gating(adev); 2982 } 2983 } 2984 2985 void gfx_v9_0_rlc_stop(struct amdgpu_device *adev) 2986 { 2987 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0); 2988 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 2989 gfx_v9_0_wait_for_rlc_serdes(adev); 2990 } 2991 2992 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev) 2993 { 2994 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 2995 udelay(50); 2996 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 2997 udelay(50); 2998 } 2999 3000 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev) 3001 { 3002 #ifdef AMDGPU_RLC_DEBUG_RETRY 3003 u32 rlc_ucode_ver; 3004 #endif 3005 3006 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1); 3007 udelay(50); 3008 3009 /* carrizo do enable cp interrupt after cp inited */ 3010 if (!(adev->flags & AMD_IS_APU)) { 3011 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3012 udelay(50); 3013 } 3014 3015 #ifdef AMDGPU_RLC_DEBUG_RETRY 3016 /* RLC_GPM_GENERAL_6 : RLC Ucode version */ 3017 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6); 3018 if(rlc_ucode_ver == 0x108) { 3019 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n", 3020 rlc_ucode_ver, adev->gfx.rlc_fw_version); 3021 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles, 3022 * default is 0x9C4 to create a 100us interval */ 3023 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4); 3024 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr 3025 * to disable the page fault retry interrupts, default is 3026 * 0x100 (256) */ 3027 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100); 3028 } 3029 #endif 3030 } 3031 3032 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev) 3033 { 3034 const struct rlc_firmware_header_v2_0 *hdr; 3035 const __le32 *fw_data; 3036 unsigned i, fw_size; 3037 3038 if (!adev->gfx.rlc_fw) 3039 return -EINVAL; 3040 3041 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 3042 amdgpu_ucode_print_rlc_hdr(&hdr->header); 3043 3044 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 3045 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 3046 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 3047 3048 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, 3049 RLCG_UCODE_LOADING_START_ADDRESS); 3050 for (i = 0; i < fw_size; i++) 3051 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); 3052 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); 3053 3054 return 0; 3055 } 3056 3057 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev) 3058 { 3059 int r; 3060 3061 if (amdgpu_sriov_vf(adev)) { 3062 gfx_v9_0_init_csb(adev); 3063 return 0; 3064 } 3065 3066 adev->gfx.rlc.funcs->stop(adev); 3067 3068 /* disable CG */ 3069 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0); 3070 3071 gfx_v9_0_init_pg(adev); 3072 3073 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3074 /* legacy rlc firmware loading */ 3075 r = gfx_v9_0_rlc_load_microcode(adev); 3076 if (r) 3077 return r; 3078 } 3079 3080 switch (adev->asic_type) { 3081 case CHIP_RAVEN: 3082 if (amdgpu_lbpw == 0) 3083 gfx_v9_0_enable_lbpw(adev, false); 3084 else 3085 gfx_v9_0_enable_lbpw(adev, true); 3086 break; 3087 case CHIP_VEGA20: 3088 if (amdgpu_lbpw > 0) 3089 gfx_v9_0_enable_lbpw(adev, true); 3090 else 3091 gfx_v9_0_enable_lbpw(adev, false); 3092 break; 3093 default: 3094 break; 3095 } 3096 3097 adev->gfx.rlc.funcs->start(adev); 3098 3099 return 0; 3100 } 3101 3102 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 3103 { 3104 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL); 3105 3106 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); 3107 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); 3108 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1); 3109 WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp); 3110 udelay(50); 3111 } 3112 3113 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev) 3114 { 3115 const struct gfx_firmware_header_v1_0 *pfp_hdr; 3116 const struct gfx_firmware_header_v1_0 *ce_hdr; 3117 const struct gfx_firmware_header_v1_0 *me_hdr; 3118 const __le32 *fw_data; 3119 unsigned i, fw_size; 3120 3121 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw) 3122 return -EINVAL; 3123 3124 pfp_hdr = (const struct gfx_firmware_header_v1_0 *) 3125 adev->gfx.pfp_fw->data; 3126 ce_hdr = (const struct gfx_firmware_header_v1_0 *) 3127 adev->gfx.ce_fw->data; 3128 me_hdr = (const struct gfx_firmware_header_v1_0 *) 3129 adev->gfx.me_fw->data; 3130 3131 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3132 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header); 3133 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3134 3135 gfx_v9_0_cp_gfx_enable(adev, false); 3136 3137 /* PFP */ 3138 fw_data = (const __le32 *) 3139 (adev->gfx.pfp_fw->data + 3140 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 3141 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 3142 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0); 3143 for (i = 0; i < fw_size; i++) 3144 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 3145 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); 3146 3147 /* CE */ 3148 fw_data = (const __le32 *) 3149 (adev->gfx.ce_fw->data + 3150 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 3151 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 3152 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0); 3153 for (i = 0; i < fw_size; i++) 3154 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 3155 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version); 3156 3157 /* ME */ 3158 fw_data = (const __le32 *) 3159 (adev->gfx.me_fw->data + 3160 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 3161 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 3162 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0); 3163 for (i = 0; i < fw_size; i++) 3164 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 3165 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version); 3166 3167 return 0; 3168 } 3169 3170 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev) 3171 { 3172 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; 3173 const struct cs_section_def *sect = NULL; 3174 const struct cs_extent_def *ext = NULL; 3175 int r, i, tmp; 3176 3177 /* init the CP */ 3178 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); 3179 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1); 3180 3181 gfx_v9_0_cp_gfx_enable(adev, true); 3182 3183 r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3); 3184 if (r) { 3185 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r); 3186 return r; 3187 } 3188 3189 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3190 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3191 3192 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 3193 amdgpu_ring_write(ring, 0x80000000); 3194 amdgpu_ring_write(ring, 0x80000000); 3195 3196 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 3197 for (ext = sect->section; ext->extent != NULL; ++ext) { 3198 if (sect->id == SECT_CONTEXT) { 3199 amdgpu_ring_write(ring, 3200 PACKET3(PACKET3_SET_CONTEXT_REG, 3201 ext->reg_count)); 3202 amdgpu_ring_write(ring, 3203 ext->reg_index - PACKET3_SET_CONTEXT_REG_START); 3204 for (i = 0; i < ext->reg_count; i++) 3205 amdgpu_ring_write(ring, ext->extent[i]); 3206 } 3207 } 3208 } 3209 3210 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3211 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3212 3213 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3214 amdgpu_ring_write(ring, 0); 3215 3216 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 3217 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 3218 amdgpu_ring_write(ring, 0x8000); 3219 amdgpu_ring_write(ring, 0x8000); 3220 3221 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1)); 3222 tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE | 3223 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START)); 3224 amdgpu_ring_write(ring, tmp); 3225 amdgpu_ring_write(ring, 0); 3226 3227 amdgpu_ring_commit(ring); 3228 3229 return 0; 3230 } 3231 3232 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev) 3233 { 3234 struct amdgpu_ring *ring; 3235 u32 tmp; 3236 u32 rb_bufsz; 3237 u64 rb_addr, rptr_addr, wptr_gpu_addr; 3238 3239 /* Set the write pointer delay */ 3240 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0); 3241 3242 /* set the RB to use vmid 0 */ 3243 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0); 3244 3245 /* Set ring buffer size */ 3246 ring = &adev->gfx.gfx_ring[0]; 3247 rb_bufsz = order_base_2(ring->ring_size / 8); 3248 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 3249 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 3250 #ifdef __BIG_ENDIAN 3251 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1); 3252 #endif 3253 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3254 3255 /* Initialize the ring buffer's write pointers */ 3256 ring->wptr = 0; 3257 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 3258 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 3259 3260 /* set the wb address wether it's enabled or not */ 3261 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3262 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 3263 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3264 3265 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3266 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr)); 3267 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr)); 3268 3269 mdelay(1); 3270 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3271 3272 rb_addr = ring->gpu_addr >> 8; 3273 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr); 3274 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 3275 3276 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL); 3277 if (ring->use_doorbell) { 3278 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3279 DOORBELL_OFFSET, ring->doorbell_index); 3280 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3281 DOORBELL_EN, 1); 3282 } else { 3283 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0); 3284 } 3285 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp); 3286 3287 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 3288 DOORBELL_RANGE_LOWER, ring->doorbell_index); 3289 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp); 3290 3291 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER, 3292 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 3293 3294 3295 /* start the ring */ 3296 gfx_v9_0_cp_gfx_start(adev); 3297 ring->sched.ready = true; 3298 3299 return 0; 3300 } 3301 3302 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 3303 { 3304 if (enable) { 3305 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0); 3306 } else { 3307 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 3308 (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK)); 3309 adev->gfx.kiq.ring.sched.ready = false; 3310 } 3311 udelay(50); 3312 } 3313 3314 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev) 3315 { 3316 const struct gfx_firmware_header_v1_0 *mec_hdr; 3317 const __le32 *fw_data; 3318 unsigned i; 3319 u32 tmp; 3320 3321 if (!adev->gfx.mec_fw) 3322 return -EINVAL; 3323 3324 gfx_v9_0_cp_compute_enable(adev, false); 3325 3326 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 3327 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3328 3329 fw_data = (const __le32 *) 3330 (adev->gfx.mec_fw->data + 3331 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 3332 tmp = 0; 3333 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 3334 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 3335 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp); 3336 3337 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO, 3338 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000); 3339 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI, 3340 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 3341 3342 /* MEC1 */ 3343 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3344 mec_hdr->jt_offset); 3345 for (i = 0; i < mec_hdr->jt_size; i++) 3346 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA, 3347 le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); 3348 3349 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3350 adev->gfx.mec_fw_version); 3351 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */ 3352 3353 return 0; 3354 } 3355 3356 /* KIQ functions */ 3357 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring) 3358 { 3359 uint32_t tmp; 3360 struct amdgpu_device *adev = ring->adev; 3361 3362 /* tell RLC which is KIQ queue */ 3363 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS); 3364 tmp &= 0xffffff00; 3365 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 3366 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3367 tmp |= 0x80; 3368 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3369 } 3370 3371 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd) 3372 { 3373 struct amdgpu_device *adev = ring->adev; 3374 3375 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 3376 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring->queue)) { 3377 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; 3378 mqd->cp_hqd_queue_priority = 3379 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; 3380 } 3381 } 3382 } 3383 3384 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring) 3385 { 3386 struct amdgpu_device *adev = ring->adev; 3387 struct v9_mqd *mqd = ring->mqd_ptr; 3388 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 3389 uint32_t tmp; 3390 3391 mqd->header = 0xC0310800; 3392 mqd->compute_pipelinestat_enable = 0x00000001; 3393 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 3394 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 3395 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 3396 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 3397 mqd->compute_static_thread_mgmt_se4 = 0xffffffff; 3398 mqd->compute_static_thread_mgmt_se5 = 0xffffffff; 3399 mqd->compute_static_thread_mgmt_se6 = 0xffffffff; 3400 mqd->compute_static_thread_mgmt_se7 = 0xffffffff; 3401 mqd->compute_misc_reserved = 0x00000003; 3402 3403 mqd->dynamic_cu_mask_addr_lo = 3404 lower_32_bits(ring->mqd_gpu_addr 3405 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3406 mqd->dynamic_cu_mask_addr_hi = 3407 upper_32_bits(ring->mqd_gpu_addr 3408 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3409 3410 eop_base_addr = ring->eop_gpu_addr >> 8; 3411 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 3412 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 3413 3414 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3415 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL); 3416 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 3417 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1)); 3418 3419 mqd->cp_hqd_eop_control = tmp; 3420 3421 /* enable doorbell? */ 3422 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3423 3424 if (ring->use_doorbell) { 3425 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3426 DOORBELL_OFFSET, ring->doorbell_index); 3427 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3428 DOORBELL_EN, 1); 3429 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3430 DOORBELL_SOURCE, 0); 3431 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3432 DOORBELL_HIT, 0); 3433 } else { 3434 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3435 DOORBELL_EN, 0); 3436 } 3437 3438 mqd->cp_hqd_pq_doorbell_control = tmp; 3439 3440 /* disable the queue if it's active */ 3441 ring->wptr = 0; 3442 mqd->cp_hqd_dequeue_request = 0; 3443 mqd->cp_hqd_pq_rptr = 0; 3444 mqd->cp_hqd_pq_wptr_lo = 0; 3445 mqd->cp_hqd_pq_wptr_hi = 0; 3446 3447 /* set the pointer to the MQD */ 3448 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; 3449 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); 3450 3451 /* set MQD vmid to 0 */ 3452 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL); 3453 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 3454 mqd->cp_mqd_control = tmp; 3455 3456 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3457 hqd_gpu_addr = ring->gpu_addr >> 8; 3458 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 3459 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 3460 3461 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3462 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL); 3463 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 3464 (order_base_2(ring->ring_size / 4) - 1)); 3465 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 3466 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8)); 3467 #ifdef __BIG_ENDIAN 3468 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1); 3469 #endif 3470 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0); 3471 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0); 3472 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 3473 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 3474 mqd->cp_hqd_pq_control = tmp; 3475 3476 /* set the wb address whether it's enabled or not */ 3477 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3478 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 3479 mqd->cp_hqd_pq_rptr_report_addr_hi = 3480 upper_32_bits(wb_gpu_addr) & 0xffff; 3481 3482 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3483 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3484 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 3485 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 3486 3487 tmp = 0; 3488 /* enable the doorbell if requested */ 3489 if (ring->use_doorbell) { 3490 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3491 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3492 DOORBELL_OFFSET, ring->doorbell_index); 3493 3494 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3495 DOORBELL_EN, 1); 3496 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3497 DOORBELL_SOURCE, 0); 3498 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3499 DOORBELL_HIT, 0); 3500 } 3501 3502 mqd->cp_hqd_pq_doorbell_control = tmp; 3503 3504 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3505 ring->wptr = 0; 3506 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR); 3507 3508 /* set the vmid for the queue */ 3509 mqd->cp_hqd_vmid = 0; 3510 3511 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE); 3512 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53); 3513 mqd->cp_hqd_persistent_state = tmp; 3514 3515 /* set MIN_IB_AVAIL_SIZE */ 3516 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL); 3517 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 3518 mqd->cp_hqd_ib_control = tmp; 3519 3520 /* set static priority for a queue/ring */ 3521 gfx_v9_0_mqd_set_priority(ring, mqd); 3522 mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM); 3523 3524 /* map_queues packet doesn't need activate the queue, 3525 * so only kiq need set this field. 3526 */ 3527 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 3528 mqd->cp_hqd_active = 1; 3529 3530 return 0; 3531 } 3532 3533 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring) 3534 { 3535 struct amdgpu_device *adev = ring->adev; 3536 struct v9_mqd *mqd = ring->mqd_ptr; 3537 int j; 3538 3539 /* disable wptr polling */ 3540 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3541 3542 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR, 3543 mqd->cp_hqd_eop_base_addr_lo); 3544 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI, 3545 mqd->cp_hqd_eop_base_addr_hi); 3546 3547 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3548 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL, 3549 mqd->cp_hqd_eop_control); 3550 3551 /* enable doorbell? */ 3552 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3553 mqd->cp_hqd_pq_doorbell_control); 3554 3555 /* disable the queue if it's active */ 3556 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3557 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3558 for (j = 0; j < adev->usec_timeout; j++) { 3559 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3560 break; 3561 udelay(1); 3562 } 3563 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3564 mqd->cp_hqd_dequeue_request); 3565 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 3566 mqd->cp_hqd_pq_rptr); 3567 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3568 mqd->cp_hqd_pq_wptr_lo); 3569 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3570 mqd->cp_hqd_pq_wptr_hi); 3571 } 3572 3573 /* set the pointer to the MQD */ 3574 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR, 3575 mqd->cp_mqd_base_addr_lo); 3576 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI, 3577 mqd->cp_mqd_base_addr_hi); 3578 3579 /* set MQD vmid to 0 */ 3580 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL, 3581 mqd->cp_mqd_control); 3582 3583 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3584 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE, 3585 mqd->cp_hqd_pq_base_lo); 3586 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI, 3587 mqd->cp_hqd_pq_base_hi); 3588 3589 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3590 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL, 3591 mqd->cp_hqd_pq_control); 3592 3593 /* set the wb address whether it's enabled or not */ 3594 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR, 3595 mqd->cp_hqd_pq_rptr_report_addr_lo); 3596 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 3597 mqd->cp_hqd_pq_rptr_report_addr_hi); 3598 3599 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3600 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR, 3601 mqd->cp_hqd_pq_wptr_poll_addr_lo); 3602 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI, 3603 mqd->cp_hqd_pq_wptr_poll_addr_hi); 3604 3605 /* enable the doorbell if requested */ 3606 if (ring->use_doorbell) { 3607 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER, 3608 (adev->doorbell_index.kiq * 2) << 2); 3609 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3610 (adev->doorbell_index.userqueue_end * 2) << 2); 3611 } 3612 3613 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3614 mqd->cp_hqd_pq_doorbell_control); 3615 3616 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3617 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3618 mqd->cp_hqd_pq_wptr_lo); 3619 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3620 mqd->cp_hqd_pq_wptr_hi); 3621 3622 /* set the vmid for the queue */ 3623 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid); 3624 3625 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 3626 mqd->cp_hqd_persistent_state); 3627 3628 /* activate the queue */ 3629 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 3630 mqd->cp_hqd_active); 3631 3632 if (ring->use_doorbell) 3633 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1); 3634 3635 return 0; 3636 } 3637 3638 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring) 3639 { 3640 struct amdgpu_device *adev = ring->adev; 3641 int j; 3642 3643 /* disable the queue if it's active */ 3644 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3645 3646 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3647 3648 for (j = 0; j < adev->usec_timeout; j++) { 3649 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3650 break; 3651 udelay(1); 3652 } 3653 3654 if (j == AMDGPU_MAX_USEC_TIMEOUT) { 3655 DRM_DEBUG("KIQ dequeue request failed.\n"); 3656 3657 /* Manual disable if dequeue request times out */ 3658 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0); 3659 } 3660 3661 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3662 0); 3663 } 3664 3665 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0); 3666 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0); 3667 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0); 3668 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000); 3669 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0); 3670 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0); 3671 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0); 3672 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0); 3673 3674 return 0; 3675 } 3676 3677 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring) 3678 { 3679 struct amdgpu_device *adev = ring->adev; 3680 struct v9_mqd *mqd = ring->mqd_ptr; 3681 int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS; 3682 3683 gfx_v9_0_kiq_setting(ring); 3684 3685 if (adev->in_gpu_reset) { /* for GPU_RESET case */ 3686 /* reset MQD to a clean status */ 3687 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3688 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3689 3690 /* reset ring buffer */ 3691 ring->wptr = 0; 3692 amdgpu_ring_clear_ring(ring); 3693 3694 mutex_lock(&adev->srbm_mutex); 3695 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3696 gfx_v9_0_kiq_init_register(ring); 3697 soc15_grbm_select(adev, 0, 0, 0, 0); 3698 mutex_unlock(&adev->srbm_mutex); 3699 } else { 3700 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3701 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3702 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3703 mutex_lock(&adev->srbm_mutex); 3704 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3705 gfx_v9_0_mqd_init(ring); 3706 gfx_v9_0_kiq_init_register(ring); 3707 soc15_grbm_select(adev, 0, 0, 0, 0); 3708 mutex_unlock(&adev->srbm_mutex); 3709 3710 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3711 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3712 } 3713 3714 return 0; 3715 } 3716 3717 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring) 3718 { 3719 struct amdgpu_device *adev = ring->adev; 3720 struct v9_mqd *mqd = ring->mqd_ptr; 3721 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 3722 3723 if (!adev->in_gpu_reset && !adev->in_suspend) { 3724 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3725 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3726 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3727 mutex_lock(&adev->srbm_mutex); 3728 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3729 gfx_v9_0_mqd_init(ring); 3730 soc15_grbm_select(adev, 0, 0, 0, 0); 3731 mutex_unlock(&adev->srbm_mutex); 3732 3733 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3734 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3735 } else if (adev->in_gpu_reset) { /* for GPU_RESET case */ 3736 /* reset MQD to a clean status */ 3737 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3738 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3739 3740 /* reset ring buffer */ 3741 ring->wptr = 0; 3742 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0); 3743 amdgpu_ring_clear_ring(ring); 3744 } else { 3745 amdgpu_ring_clear_ring(ring); 3746 } 3747 3748 return 0; 3749 } 3750 3751 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev) 3752 { 3753 struct amdgpu_ring *ring; 3754 int r; 3755 3756 ring = &adev->gfx.kiq.ring; 3757 3758 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3759 if (unlikely(r != 0)) 3760 return r; 3761 3762 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3763 if (unlikely(r != 0)) 3764 return r; 3765 3766 gfx_v9_0_kiq_init_queue(ring); 3767 amdgpu_bo_kunmap(ring->mqd_obj); 3768 ring->mqd_ptr = NULL; 3769 amdgpu_bo_unreserve(ring->mqd_obj); 3770 ring->sched.ready = true; 3771 return 0; 3772 } 3773 3774 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev) 3775 { 3776 struct amdgpu_ring *ring = NULL; 3777 int r = 0, i; 3778 3779 gfx_v9_0_cp_compute_enable(adev, true); 3780 3781 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3782 ring = &adev->gfx.compute_ring[i]; 3783 3784 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3785 if (unlikely(r != 0)) 3786 goto done; 3787 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3788 if (!r) { 3789 r = gfx_v9_0_kcq_init_queue(ring); 3790 amdgpu_bo_kunmap(ring->mqd_obj); 3791 ring->mqd_ptr = NULL; 3792 } 3793 amdgpu_bo_unreserve(ring->mqd_obj); 3794 if (r) 3795 goto done; 3796 } 3797 3798 r = amdgpu_gfx_enable_kcq(adev); 3799 done: 3800 return r; 3801 } 3802 3803 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev) 3804 { 3805 int r, i; 3806 struct amdgpu_ring *ring; 3807 3808 if (!(adev->flags & AMD_IS_APU)) 3809 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3810 3811 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3812 if (adev->asic_type != CHIP_ARCTURUS) { 3813 /* legacy firmware loading */ 3814 r = gfx_v9_0_cp_gfx_load_microcode(adev); 3815 if (r) 3816 return r; 3817 } 3818 3819 r = gfx_v9_0_cp_compute_load_microcode(adev); 3820 if (r) 3821 return r; 3822 } 3823 3824 r = gfx_v9_0_kiq_resume(adev); 3825 if (r) 3826 return r; 3827 3828 if (adev->asic_type != CHIP_ARCTURUS) { 3829 r = gfx_v9_0_cp_gfx_resume(adev); 3830 if (r) 3831 return r; 3832 } 3833 3834 r = gfx_v9_0_kcq_resume(adev); 3835 if (r) 3836 return r; 3837 3838 if (adev->asic_type != CHIP_ARCTURUS) { 3839 ring = &adev->gfx.gfx_ring[0]; 3840 r = amdgpu_ring_test_helper(ring); 3841 if (r) 3842 return r; 3843 } 3844 3845 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3846 ring = &adev->gfx.compute_ring[i]; 3847 amdgpu_ring_test_helper(ring); 3848 } 3849 3850 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3851 3852 return 0; 3853 } 3854 3855 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev) 3856 { 3857 u32 tmp; 3858 3859 if (adev->asic_type != CHIP_ARCTURUS) 3860 return; 3861 3862 tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG); 3863 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH, 3864 adev->df.hash_status.hash_64k); 3865 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH, 3866 adev->df.hash_status.hash_2m); 3867 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH, 3868 adev->df.hash_status.hash_1g); 3869 WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp); 3870 } 3871 3872 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable) 3873 { 3874 if (adev->asic_type != CHIP_ARCTURUS) 3875 gfx_v9_0_cp_gfx_enable(adev, enable); 3876 gfx_v9_0_cp_compute_enable(adev, enable); 3877 } 3878 3879 static int gfx_v9_0_hw_init(void *handle) 3880 { 3881 int r; 3882 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3883 3884 if (!amdgpu_sriov_vf(adev)) 3885 gfx_v9_0_init_golden_registers(adev); 3886 3887 gfx_v9_0_constants_init(adev); 3888 3889 gfx_v9_0_init_tcp_config(adev); 3890 3891 r = adev->gfx.rlc.funcs->resume(adev); 3892 if (r) 3893 return r; 3894 3895 r = gfx_v9_0_cp_resume(adev); 3896 if (r) 3897 return r; 3898 3899 return r; 3900 } 3901 3902 static int gfx_v9_0_hw_fini(void *handle) 3903 { 3904 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3905 3906 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); 3907 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 3908 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 3909 3910 /* DF freeze and kcq disable will fail */ 3911 if (!amdgpu_ras_intr_triggered()) 3912 /* disable KCQ to avoid CPC touch memory not valid anymore */ 3913 amdgpu_gfx_disable_kcq(adev); 3914 3915 if (amdgpu_sriov_vf(adev)) { 3916 gfx_v9_0_cp_gfx_enable(adev, false); 3917 /* must disable polling for SRIOV when hw finished, otherwise 3918 * CPC engine may still keep fetching WB address which is already 3919 * invalid after sw finished and trigger DMAR reading error in 3920 * hypervisor side. 3921 */ 3922 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3923 return 0; 3924 } 3925 3926 /* Use deinitialize sequence from CAIL when unbinding device from driver, 3927 * otherwise KIQ is hanging when binding back 3928 */ 3929 if (!adev->in_gpu_reset && !adev->in_suspend) { 3930 mutex_lock(&adev->srbm_mutex); 3931 soc15_grbm_select(adev, adev->gfx.kiq.ring.me, 3932 adev->gfx.kiq.ring.pipe, 3933 adev->gfx.kiq.ring.queue, 0); 3934 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring); 3935 soc15_grbm_select(adev, 0, 0, 0, 0); 3936 mutex_unlock(&adev->srbm_mutex); 3937 } 3938 3939 gfx_v9_0_cp_enable(adev, false); 3940 adev->gfx.rlc.funcs->stop(adev); 3941 3942 return 0; 3943 } 3944 3945 static int gfx_v9_0_suspend(void *handle) 3946 { 3947 return gfx_v9_0_hw_fini(handle); 3948 } 3949 3950 static int gfx_v9_0_resume(void *handle) 3951 { 3952 return gfx_v9_0_hw_init(handle); 3953 } 3954 3955 static bool gfx_v9_0_is_idle(void *handle) 3956 { 3957 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3958 3959 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS), 3960 GRBM_STATUS, GUI_ACTIVE)) 3961 return false; 3962 else 3963 return true; 3964 } 3965 3966 static int gfx_v9_0_wait_for_idle(void *handle) 3967 { 3968 unsigned i; 3969 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3970 3971 for (i = 0; i < adev->usec_timeout; i++) { 3972 if (gfx_v9_0_is_idle(handle)) 3973 return 0; 3974 udelay(1); 3975 } 3976 return -ETIMEDOUT; 3977 } 3978 3979 static int gfx_v9_0_soft_reset(void *handle) 3980 { 3981 u32 grbm_soft_reset = 0; 3982 u32 tmp; 3983 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3984 3985 /* GRBM_STATUS */ 3986 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS); 3987 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK | 3988 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK | 3989 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK | 3990 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK | 3991 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK | 3992 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) { 3993 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 3994 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 3995 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 3996 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1); 3997 } 3998 3999 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) { 4000 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4001 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4002 } 4003 4004 /* GRBM_STATUS2 */ 4005 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2); 4006 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY)) 4007 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4008 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 4009 4010 4011 if (grbm_soft_reset) { 4012 /* stop the rlc */ 4013 adev->gfx.rlc.funcs->stop(adev); 4014 4015 if (adev->asic_type != CHIP_ARCTURUS) 4016 /* Disable GFX parsing/prefetching */ 4017 gfx_v9_0_cp_gfx_enable(adev, false); 4018 4019 /* Disable MEC parsing/prefetching */ 4020 gfx_v9_0_cp_compute_enable(adev, false); 4021 4022 if (grbm_soft_reset) { 4023 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4024 tmp |= grbm_soft_reset; 4025 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 4026 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4027 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4028 4029 udelay(50); 4030 4031 tmp &= ~grbm_soft_reset; 4032 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4033 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4034 } 4035 4036 /* Wait a little for things to settle down */ 4037 udelay(50); 4038 } 4039 return 0; 4040 } 4041 4042 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev) 4043 { 4044 signed long r, cnt = 0; 4045 unsigned long flags; 4046 uint32_t seq; 4047 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 4048 struct amdgpu_ring *ring = &kiq->ring; 4049 4050 BUG_ON(!ring->funcs->emit_rreg); 4051 4052 spin_lock_irqsave(&kiq->ring_lock, flags); 4053 amdgpu_ring_alloc(ring, 32); 4054 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 4055 amdgpu_ring_write(ring, 9 | /* src: register*/ 4056 (5 << 8) | /* dst: memory */ 4057 (1 << 16) | /* count sel */ 4058 (1 << 20)); /* write confirm */ 4059 amdgpu_ring_write(ring, 0); 4060 amdgpu_ring_write(ring, 0); 4061 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 4062 kiq->reg_val_offs * 4)); 4063 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 4064 kiq->reg_val_offs * 4)); 4065 amdgpu_fence_emit_polling(ring, &seq); 4066 amdgpu_ring_commit(ring); 4067 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4068 4069 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4070 4071 /* don't wait anymore for gpu reset case because this way may 4072 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 4073 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 4074 * never return if we keep waiting in virt_kiq_rreg, which cause 4075 * gpu_recover() hang there. 4076 * 4077 * also don't wait anymore for IRQ context 4078 * */ 4079 if (r < 1 && (adev->in_gpu_reset || in_interrupt())) 4080 goto failed_kiq_read; 4081 4082 might_sleep(); 4083 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 4084 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 4085 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4086 } 4087 4088 if (cnt > MAX_KIQ_REG_TRY) 4089 goto failed_kiq_read; 4090 4091 return (uint64_t)adev->wb.wb[kiq->reg_val_offs] | 4092 (uint64_t)adev->wb.wb[kiq->reg_val_offs + 1 ] << 32ULL; 4093 4094 failed_kiq_read: 4095 pr_err("failed to read gpu clock\n"); 4096 return ~0; 4097 } 4098 4099 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev) 4100 { 4101 uint64_t clock; 4102 4103 amdgpu_gfx_off_ctrl(adev, false); 4104 mutex_lock(&adev->gfx.gpu_clock_mutex); 4105 if (adev->asic_type == CHIP_VEGA10 && amdgpu_sriov_runtime(adev)) { 4106 clock = gfx_v9_0_kiq_read_clock(adev); 4107 } else { 4108 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1); 4109 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) | 4110 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 4111 } 4112 mutex_unlock(&adev->gfx.gpu_clock_mutex); 4113 amdgpu_gfx_off_ctrl(adev, true); 4114 return clock; 4115 } 4116 4117 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 4118 uint32_t vmid, 4119 uint32_t gds_base, uint32_t gds_size, 4120 uint32_t gws_base, uint32_t gws_size, 4121 uint32_t oa_base, uint32_t oa_size) 4122 { 4123 struct amdgpu_device *adev = ring->adev; 4124 4125 /* GDS Base */ 4126 gfx_v9_0_write_data_to_reg(ring, 0, false, 4127 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid, 4128 gds_base); 4129 4130 /* GDS Size */ 4131 gfx_v9_0_write_data_to_reg(ring, 0, false, 4132 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid, 4133 gds_size); 4134 4135 /* GWS */ 4136 gfx_v9_0_write_data_to_reg(ring, 0, false, 4137 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid, 4138 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 4139 4140 /* OA */ 4141 gfx_v9_0_write_data_to_reg(ring, 0, false, 4142 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid, 4143 (1 << (oa_size + oa_base)) - (1 << oa_base)); 4144 } 4145 4146 static const u32 vgpr_init_compute_shader[] = 4147 { 4148 0xb07c0000, 0xbe8000ff, 4149 0x000000f8, 0xbf110800, 4150 0x7e000280, 0x7e020280, 4151 0x7e040280, 0x7e060280, 4152 0x7e080280, 0x7e0a0280, 4153 0x7e0c0280, 0x7e0e0280, 4154 0x80808800, 0xbe803200, 4155 0xbf84fff5, 0xbf9c0000, 4156 0xd28c0001, 0x0001007f, 4157 0xd28d0001, 0x0002027e, 4158 0x10020288, 0xb8810904, 4159 0xb7814000, 0xd1196a01, 4160 0x00000301, 0xbe800087, 4161 0xbefc00c1, 0xd89c4000, 4162 0x00020201, 0xd89cc080, 4163 0x00040401, 0x320202ff, 4164 0x00000800, 0x80808100, 4165 0xbf84fff8, 0x7e020280, 4166 0xbf810000, 0x00000000, 4167 }; 4168 4169 static const u32 sgpr_init_compute_shader[] = 4170 { 4171 0xb07c0000, 0xbe8000ff, 4172 0x0000005f, 0xbee50080, 4173 0xbe812c65, 0xbe822c65, 4174 0xbe832c65, 0xbe842c65, 4175 0xbe852c65, 0xb77c0005, 4176 0x80808500, 0xbf84fff8, 4177 0xbe800080, 0xbf810000, 4178 }; 4179 4180 static const u32 vgpr_init_compute_shader_arcturus[] = { 4181 0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080, 4182 0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080, 4183 0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080, 4184 0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080, 4185 0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080, 4186 0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080, 4187 0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080, 4188 0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080, 4189 0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080, 4190 0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080, 4191 0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080, 4192 0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080, 4193 0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080, 4194 0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080, 4195 0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080, 4196 0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080, 4197 0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080, 4198 0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080, 4199 0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080, 4200 0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080, 4201 0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080, 4202 0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080, 4203 0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080, 4204 0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080, 4205 0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080, 4206 0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080, 4207 0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080, 4208 0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080, 4209 0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080, 4210 0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080, 4211 0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080, 4212 0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080, 4213 0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080, 4214 0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080, 4215 0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080, 4216 0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080, 4217 0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080, 4218 0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080, 4219 0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080, 4220 0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080, 4221 0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080, 4222 0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080, 4223 0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080, 4224 0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080, 4225 0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080, 4226 0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080, 4227 0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080, 4228 0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080, 4229 0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080, 4230 0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080, 4231 0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080, 4232 0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080, 4233 0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080, 4234 0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080, 4235 0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080, 4236 0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080, 4237 0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080, 4238 0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080, 4239 0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080, 4240 0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080, 4241 0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080, 4242 0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080, 4243 0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080, 4244 0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080, 4245 0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080, 4246 0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080, 4247 0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080, 4248 0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080, 4249 0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080, 4250 0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080, 4251 0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080, 4252 0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080, 4253 0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080, 4254 0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080, 4255 0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080, 4256 0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080, 4257 0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080, 4258 0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080, 4259 0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080, 4260 0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080, 4261 0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080, 4262 0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080, 4263 0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080, 4264 0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080, 4265 0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080, 4266 0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a, 4267 0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280, 4268 0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000, 4269 0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904, 4270 0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000, 4271 0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a, 4272 0xbf84fff8, 0xbf810000, 4273 }; 4274 4275 /* When below register arrays changed, please update gpr_reg_size, 4276 and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds, 4277 to cover all gfx9 ASICs */ 4278 static const struct soc15_reg_entry vgpr_init_regs[] = { 4279 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4280 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4281 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4282 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4283 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f }, 4284 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4285 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4286 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4287 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4288 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4289 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4290 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4291 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4292 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4293 }; 4294 4295 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = { 4296 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4297 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4298 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4299 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4300 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf }, 4301 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4302 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4303 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4304 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4305 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4306 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4307 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4308 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4309 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4310 }; 4311 4312 static const struct soc15_reg_entry sgpr1_init_regs[] = { 4313 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4314 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4315 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4316 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4317 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4318 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4319 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff }, 4320 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff }, 4321 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff }, 4322 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff }, 4323 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff }, 4324 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff }, 4325 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff }, 4326 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff }, 4327 }; 4328 4329 static const struct soc15_reg_entry sgpr2_init_regs[] = { 4330 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4331 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4332 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4333 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4334 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4335 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4336 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 }, 4337 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 }, 4338 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 }, 4339 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 }, 4340 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 }, 4341 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 }, 4342 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 }, 4343 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 }, 4344 }; 4345 4346 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = { 4347 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1}, 4348 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1}, 4349 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1}, 4350 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1}, 4351 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1}, 4352 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1}, 4353 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1}, 4354 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1}, 4355 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1}, 4356 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1}, 4357 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1}, 4358 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1}, 4359 { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1}, 4360 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6}, 4361 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16}, 4362 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16}, 4363 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16}, 4364 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16}, 4365 { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16}, 4366 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16}, 4367 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16}, 4368 { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16}, 4369 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6}, 4370 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16}, 4371 { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16}, 4372 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1}, 4373 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1}, 4374 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32}, 4375 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32}, 4376 { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72}, 4377 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16}, 4378 { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2}, 4379 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6}, 4380 }; 4381 4382 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev) 4383 { 4384 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4385 int i, r; 4386 4387 /* only support when RAS is enabled */ 4388 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4389 return 0; 4390 4391 r = amdgpu_ring_alloc(ring, 7); 4392 if (r) { 4393 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n", 4394 ring->name, r); 4395 return r; 4396 } 4397 4398 WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000); 4399 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size); 4400 4401 amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 4402 amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC | 4403 PACKET3_DMA_DATA_DST_SEL(1) | 4404 PACKET3_DMA_DATA_SRC_SEL(2) | 4405 PACKET3_DMA_DATA_ENGINE(0))); 4406 amdgpu_ring_write(ring, 0); 4407 amdgpu_ring_write(ring, 0); 4408 amdgpu_ring_write(ring, 0); 4409 amdgpu_ring_write(ring, 0); 4410 amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT | 4411 adev->gds.gds_size); 4412 4413 amdgpu_ring_commit(ring); 4414 4415 for (i = 0; i < adev->usec_timeout; i++) { 4416 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring)) 4417 break; 4418 udelay(1); 4419 } 4420 4421 if (i >= adev->usec_timeout) 4422 r = -ETIMEDOUT; 4423 4424 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000); 4425 4426 return r; 4427 } 4428 4429 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) 4430 { 4431 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4432 struct amdgpu_ib ib; 4433 struct dma_fence *f = NULL; 4434 int r, i; 4435 unsigned total_size, vgpr_offset, sgpr_offset; 4436 u64 gpu_addr; 4437 4438 int compute_dim_x = adev->gfx.config.max_shader_engines * 4439 adev->gfx.config.max_cu_per_sh * 4440 adev->gfx.config.max_sh_per_se; 4441 int sgpr_work_group_size = 5; 4442 int gpr_reg_size = adev->gfx.config.max_shader_engines + 6; 4443 int vgpr_init_shader_size; 4444 const u32 *vgpr_init_shader_ptr; 4445 const struct soc15_reg_entry *vgpr_init_regs_ptr; 4446 4447 /* only support when RAS is enabled */ 4448 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4449 return 0; 4450 4451 /* bail if the compute ring is not ready */ 4452 if (!ring->sched.ready) 4453 return 0; 4454 4455 if (adev->asic_type == CHIP_ARCTURUS) { 4456 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus; 4457 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus); 4458 vgpr_init_regs_ptr = vgpr_init_regs_arcturus; 4459 } else { 4460 vgpr_init_shader_ptr = vgpr_init_compute_shader; 4461 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader); 4462 vgpr_init_regs_ptr = vgpr_init_regs; 4463 } 4464 4465 total_size = 4466 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */ 4467 total_size += 4468 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */ 4469 total_size += 4470 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */ 4471 total_size = ALIGN(total_size, 256); 4472 vgpr_offset = total_size; 4473 total_size += ALIGN(vgpr_init_shader_size, 256); 4474 sgpr_offset = total_size; 4475 total_size += sizeof(sgpr_init_compute_shader); 4476 4477 /* allocate an indirect buffer to put the commands in */ 4478 memset(&ib, 0, sizeof(ib)); 4479 r = amdgpu_ib_get(adev, NULL, total_size, 4480 AMDGPU_IB_POOL_DIRECT, &ib); 4481 if (r) { 4482 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); 4483 return r; 4484 } 4485 4486 /* load the compute shaders */ 4487 for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++) 4488 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i]; 4489 4490 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++) 4491 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i]; 4492 4493 /* init the ib length to 0 */ 4494 ib.length_dw = 0; 4495 4496 /* VGPR */ 4497 /* write the register state for the compute dispatch */ 4498 for (i = 0; i < gpr_reg_size; i++) { 4499 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4500 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i]) 4501 - PACKET3_SET_SH_REG_START; 4502 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value; 4503 } 4504 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4505 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8; 4506 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4507 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4508 - PACKET3_SET_SH_REG_START; 4509 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4510 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4511 4512 /* write dispatch packet */ 4513 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4514 ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */ 4515 ib.ptr[ib.length_dw++] = 1; /* y */ 4516 ib.ptr[ib.length_dw++] = 1; /* z */ 4517 ib.ptr[ib.length_dw++] = 4518 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4519 4520 /* write CS partial flush packet */ 4521 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4522 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4523 4524 /* SGPR1 */ 4525 /* write the register state for the compute dispatch */ 4526 for (i = 0; i < gpr_reg_size; i++) { 4527 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4528 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i]) 4529 - PACKET3_SET_SH_REG_START; 4530 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value; 4531 } 4532 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4533 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4534 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4535 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4536 - PACKET3_SET_SH_REG_START; 4537 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4538 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4539 4540 /* write dispatch packet */ 4541 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4542 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4543 ib.ptr[ib.length_dw++] = 1; /* y */ 4544 ib.ptr[ib.length_dw++] = 1; /* z */ 4545 ib.ptr[ib.length_dw++] = 4546 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4547 4548 /* write CS partial flush packet */ 4549 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4550 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4551 4552 /* SGPR2 */ 4553 /* write the register state for the compute dispatch */ 4554 for (i = 0; i < gpr_reg_size; i++) { 4555 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4556 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i]) 4557 - PACKET3_SET_SH_REG_START; 4558 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value; 4559 } 4560 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4561 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4562 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4563 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4564 - PACKET3_SET_SH_REG_START; 4565 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4566 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4567 4568 /* write dispatch packet */ 4569 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4570 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4571 ib.ptr[ib.length_dw++] = 1; /* y */ 4572 ib.ptr[ib.length_dw++] = 1; /* z */ 4573 ib.ptr[ib.length_dw++] = 4574 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4575 4576 /* write CS partial flush packet */ 4577 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4578 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4579 4580 /* shedule the ib on the ring */ 4581 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 4582 if (r) { 4583 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r); 4584 goto fail; 4585 } 4586 4587 /* wait for the GPU to finish processing the IB */ 4588 r = dma_fence_wait(f, false); 4589 if (r) { 4590 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); 4591 goto fail; 4592 } 4593 4594 fail: 4595 amdgpu_ib_free(adev, &ib, NULL); 4596 dma_fence_put(f); 4597 4598 return r; 4599 } 4600 4601 static int gfx_v9_0_early_init(void *handle) 4602 { 4603 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4604 4605 if (adev->asic_type == CHIP_ARCTURUS) 4606 adev->gfx.num_gfx_rings = 0; 4607 else 4608 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS; 4609 adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS; 4610 gfx_v9_0_set_kiq_pm4_funcs(adev); 4611 gfx_v9_0_set_ring_funcs(adev); 4612 gfx_v9_0_set_irq_funcs(adev); 4613 gfx_v9_0_set_gds_init(adev); 4614 gfx_v9_0_set_rlc_funcs(adev); 4615 4616 return 0; 4617 } 4618 4619 static int gfx_v9_0_ecc_late_init(void *handle) 4620 { 4621 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4622 int r; 4623 4624 /* 4625 * Temp workaround to fix the issue that CP firmware fails to 4626 * update read pointer when CPDMA is writing clearing operation 4627 * to GDS in suspend/resume sequence on several cards. So just 4628 * limit this operation in cold boot sequence. 4629 */ 4630 if (!adev->in_suspend) { 4631 r = gfx_v9_0_do_edc_gds_workarounds(adev); 4632 if (r) 4633 return r; 4634 } 4635 4636 /* requires IBs so do in late init after IB pool is initialized */ 4637 r = gfx_v9_0_do_edc_gpr_workarounds(adev); 4638 if (r) 4639 return r; 4640 4641 if (adev->gfx.funcs && 4642 adev->gfx.funcs->reset_ras_error_count) 4643 adev->gfx.funcs->reset_ras_error_count(adev); 4644 4645 r = amdgpu_gfx_ras_late_init(adev); 4646 if (r) 4647 return r; 4648 4649 return 0; 4650 } 4651 4652 static int gfx_v9_0_late_init(void *handle) 4653 { 4654 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4655 int r; 4656 4657 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 4658 if (r) 4659 return r; 4660 4661 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 4662 if (r) 4663 return r; 4664 4665 r = gfx_v9_0_ecc_late_init(handle); 4666 if (r) 4667 return r; 4668 4669 return 0; 4670 } 4671 4672 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev) 4673 { 4674 uint32_t rlc_setting; 4675 4676 /* if RLC is not enabled, do nothing */ 4677 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL); 4678 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK)) 4679 return false; 4680 4681 return true; 4682 } 4683 4684 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev) 4685 { 4686 uint32_t data; 4687 unsigned i; 4688 4689 data = RLC_SAFE_MODE__CMD_MASK; 4690 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 4691 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4692 4693 /* wait for RLC_SAFE_MODE */ 4694 for (i = 0; i < adev->usec_timeout; i++) { 4695 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 4696 break; 4697 udelay(1); 4698 } 4699 } 4700 4701 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev) 4702 { 4703 uint32_t data; 4704 4705 data = RLC_SAFE_MODE__CMD_MASK; 4706 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4707 } 4708 4709 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev, 4710 bool enable) 4711 { 4712 amdgpu_gfx_rlc_enter_safe_mode(adev); 4713 4714 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { 4715 gfx_v9_0_enable_gfx_cg_power_gating(adev, true); 4716 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4717 gfx_v9_0_enable_gfx_pipeline_powergating(adev, true); 4718 } else { 4719 gfx_v9_0_enable_gfx_cg_power_gating(adev, false); 4720 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4721 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false); 4722 } 4723 4724 amdgpu_gfx_rlc_exit_safe_mode(adev); 4725 } 4726 4727 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev, 4728 bool enable) 4729 { 4730 /* TODO: double check if we need to perform under safe mode */ 4731 /* gfx_v9_0_enter_rlc_safe_mode(adev); */ 4732 4733 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 4734 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true); 4735 else 4736 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false); 4737 4738 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 4739 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true); 4740 else 4741 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false); 4742 4743 /* gfx_v9_0_exit_rlc_safe_mode(adev); */ 4744 } 4745 4746 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 4747 bool enable) 4748 { 4749 uint32_t data, def; 4750 4751 amdgpu_gfx_rlc_enter_safe_mode(adev); 4752 4753 /* It is disabled by HW by default */ 4754 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { 4755 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 4756 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4757 4758 if (adev->asic_type != CHIP_VEGA12) 4759 data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4760 4761 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4762 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4763 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4764 4765 /* only for Vega10 & Raven1 */ 4766 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK; 4767 4768 if (def != data) 4769 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4770 4771 /* MGLS is a global flag to control all MGLS in GFX */ 4772 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 4773 /* 2 - RLC memory Light sleep */ 4774 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { 4775 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4776 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4777 if (def != data) 4778 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4779 } 4780 /* 3 - CP memory Light sleep */ 4781 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { 4782 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4783 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4784 if (def != data) 4785 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4786 } 4787 } 4788 } else { 4789 /* 1 - MGCG_OVERRIDE */ 4790 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4791 4792 if (adev->asic_type != CHIP_VEGA12) 4793 data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4794 4795 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 4796 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4797 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4798 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4799 4800 if (def != data) 4801 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4802 4803 /* 2 - disable MGLS in RLC */ 4804 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4805 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) { 4806 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4807 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4808 } 4809 4810 /* 3 - disable MGLS in CP */ 4811 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4812 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) { 4813 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4814 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4815 } 4816 } 4817 4818 amdgpu_gfx_rlc_exit_safe_mode(adev); 4819 } 4820 4821 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev, 4822 bool enable) 4823 { 4824 uint32_t data, def; 4825 4826 if (adev->asic_type == CHIP_ARCTURUS) 4827 return; 4828 4829 amdgpu_gfx_rlc_enter_safe_mode(adev); 4830 4831 /* Enable 3D CGCG/CGLS */ 4832 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)) { 4833 /* write cmd to clear cgcg/cgls ov */ 4834 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4835 /* unset CGCG override */ 4836 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 4837 /* update CGCG and CGLS override bits */ 4838 if (def != data) 4839 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4840 4841 /* enable 3Dcgcg FSM(0x0000363f) */ 4842 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4843 4844 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4845 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 4846 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 4847 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 4848 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 4849 if (def != data) 4850 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4851 4852 /* set IDLE_POLL_COUNT(0x00900100) */ 4853 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 4854 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 4855 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 4856 if (def != data) 4857 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 4858 } else { 4859 /* Disable CGCG/CGLS */ 4860 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4861 /* disable cgcg, cgls should be disabled */ 4862 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK | 4863 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK); 4864 /* disable cgcg and cgls in FSM */ 4865 if (def != data) 4866 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4867 } 4868 4869 amdgpu_gfx_rlc_exit_safe_mode(adev); 4870 } 4871 4872 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 4873 bool enable) 4874 { 4875 uint32_t def, data; 4876 4877 amdgpu_gfx_rlc_enter_safe_mode(adev); 4878 4879 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { 4880 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4881 /* unset CGCG override */ 4882 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 4883 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 4884 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 4885 else 4886 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 4887 /* update CGCG and CGLS override bits */ 4888 if (def != data) 4889 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4890 4891 /* enable cgcg FSM(0x0000363F) */ 4892 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 4893 4894 if (adev->asic_type == CHIP_ARCTURUS) 4895 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4896 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 4897 else 4898 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4899 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 4900 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 4901 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 4902 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 4903 if (def != data) 4904 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 4905 4906 /* set IDLE_POLL_COUNT(0x00900100) */ 4907 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 4908 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 4909 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 4910 if (def != data) 4911 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 4912 } else { 4913 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 4914 /* reset CGCG/CGLS bits */ 4915 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK); 4916 /* disable cgcg and cgls in FSM */ 4917 if (def != data) 4918 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 4919 } 4920 4921 amdgpu_gfx_rlc_exit_safe_mode(adev); 4922 } 4923 4924 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev, 4925 bool enable) 4926 { 4927 if (enable) { 4928 /* CGCG/CGLS should be enabled after MGCG/MGLS 4929 * === MGCG + MGLS === 4930 */ 4931 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 4932 /* === CGCG /CGLS for GFX 3D Only === */ 4933 gfx_v9_0_update_3d_clock_gating(adev, enable); 4934 /* === CGCG + CGLS === */ 4935 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 4936 } else { 4937 /* CGCG/CGLS should be disabled before MGCG/MGLS 4938 * === CGCG + CGLS === 4939 */ 4940 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 4941 /* === CGCG /CGLS for GFX 3D Only === */ 4942 gfx_v9_0_update_3d_clock_gating(adev, enable); 4943 /* === MGCG + MGLS === */ 4944 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 4945 } 4946 return 0; 4947 } 4948 4949 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid) 4950 { 4951 u32 data; 4952 4953 data = RREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL); 4954 4955 data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK; 4956 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; 4957 4958 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data); 4959 } 4960 4961 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev, 4962 uint32_t offset, 4963 struct soc15_reg_rlcg *entries, int arr_size) 4964 { 4965 int i; 4966 uint32_t reg; 4967 4968 if (!entries) 4969 return false; 4970 4971 for (i = 0; i < arr_size; i++) { 4972 const struct soc15_reg_rlcg *entry; 4973 4974 entry = &entries[i]; 4975 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg; 4976 if (offset == reg) 4977 return true; 4978 } 4979 4980 return false; 4981 } 4982 4983 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset) 4984 { 4985 return gfx_v9_0_check_rlcg_range(adev, offset, 4986 (void *)rlcg_access_gc_9_0, 4987 ARRAY_SIZE(rlcg_access_gc_9_0)); 4988 } 4989 4990 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = { 4991 .is_rlc_enabled = gfx_v9_0_is_rlc_enabled, 4992 .set_safe_mode = gfx_v9_0_set_safe_mode, 4993 .unset_safe_mode = gfx_v9_0_unset_safe_mode, 4994 .init = gfx_v9_0_rlc_init, 4995 .get_csb_size = gfx_v9_0_get_csb_size, 4996 .get_csb_buffer = gfx_v9_0_get_csb_buffer, 4997 .get_cp_table_num = gfx_v9_0_cp_jump_table_num, 4998 .resume = gfx_v9_0_rlc_resume, 4999 .stop = gfx_v9_0_rlc_stop, 5000 .reset = gfx_v9_0_rlc_reset, 5001 .start = gfx_v9_0_rlc_start, 5002 .update_spm_vmid = gfx_v9_0_update_spm_vmid, 5003 .rlcg_wreg = gfx_v9_0_rlcg_wreg, 5004 .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range, 5005 }; 5006 5007 static int gfx_v9_0_set_powergating_state(void *handle, 5008 enum amd_powergating_state state) 5009 { 5010 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5011 bool enable = (state == AMD_PG_STATE_GATE); 5012 5013 switch (adev->asic_type) { 5014 case CHIP_RAVEN: 5015 case CHIP_RENOIR: 5016 if (!enable) { 5017 amdgpu_gfx_off_ctrl(adev, false); 5018 cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work); 5019 } 5020 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { 5021 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true); 5022 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true); 5023 } else { 5024 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false); 5025 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false); 5026 } 5027 5028 if (adev->pg_flags & AMD_PG_SUPPORT_CP) 5029 gfx_v9_0_enable_cp_power_gating(adev, true); 5030 else 5031 gfx_v9_0_enable_cp_power_gating(adev, false); 5032 5033 /* update gfx cgpg state */ 5034 gfx_v9_0_update_gfx_cg_power_gating(adev, enable); 5035 5036 /* update mgcg state */ 5037 gfx_v9_0_update_gfx_mg_power_gating(adev, enable); 5038 5039 if (enable) 5040 amdgpu_gfx_off_ctrl(adev, true); 5041 break; 5042 case CHIP_VEGA12: 5043 if (!enable) { 5044 amdgpu_gfx_off_ctrl(adev, false); 5045 cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work); 5046 } else { 5047 amdgpu_gfx_off_ctrl(adev, true); 5048 } 5049 break; 5050 default: 5051 break; 5052 } 5053 5054 return 0; 5055 } 5056 5057 static int gfx_v9_0_set_clockgating_state(void *handle, 5058 enum amd_clockgating_state state) 5059 { 5060 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5061 5062 if (amdgpu_sriov_vf(adev)) 5063 return 0; 5064 5065 switch (adev->asic_type) { 5066 case CHIP_VEGA10: 5067 case CHIP_VEGA12: 5068 case CHIP_VEGA20: 5069 case CHIP_RAVEN: 5070 case CHIP_ARCTURUS: 5071 case CHIP_RENOIR: 5072 gfx_v9_0_update_gfx_clock_gating(adev, 5073 state == AMD_CG_STATE_GATE); 5074 break; 5075 default: 5076 break; 5077 } 5078 return 0; 5079 } 5080 5081 static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags) 5082 { 5083 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5084 int data; 5085 5086 if (amdgpu_sriov_vf(adev)) 5087 *flags = 0; 5088 5089 /* AMD_CG_SUPPORT_GFX_MGCG */ 5090 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE)); 5091 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 5092 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5093 5094 /* AMD_CG_SUPPORT_GFX_CGCG */ 5095 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL)); 5096 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5097 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5098 5099 /* AMD_CG_SUPPORT_GFX_CGLS */ 5100 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5101 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5102 5103 /* AMD_CG_SUPPORT_GFX_RLC_LS */ 5104 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL)); 5105 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) 5106 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS; 5107 5108 /* AMD_CG_SUPPORT_GFX_CP_LS */ 5109 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL)); 5110 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) 5111 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS; 5112 5113 if (adev->asic_type != CHIP_ARCTURUS) { 5114 /* AMD_CG_SUPPORT_GFX_3D_CGCG */ 5115 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D)); 5116 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) 5117 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; 5118 5119 /* AMD_CG_SUPPORT_GFX_3D_CGLS */ 5120 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK) 5121 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS; 5122 } 5123 } 5124 5125 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) 5126 { 5127 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/ 5128 } 5129 5130 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5131 { 5132 struct amdgpu_device *adev = ring->adev; 5133 u64 wptr; 5134 5135 /* XXX check if swapping is necessary on BE */ 5136 if (ring->use_doorbell) { 5137 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]); 5138 } else { 5139 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR); 5140 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32; 5141 } 5142 5143 return wptr; 5144 } 5145 5146 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5147 { 5148 struct amdgpu_device *adev = ring->adev; 5149 5150 if (ring->use_doorbell) { 5151 /* XXX check if swapping is necessary on BE */ 5152 atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5153 WDOORBELL64(ring->doorbell_index, ring->wptr); 5154 } else { 5155 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 5156 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 5157 } 5158 } 5159 5160 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5161 { 5162 struct amdgpu_device *adev = ring->adev; 5163 u32 ref_and_mask, reg_mem_engine; 5164 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg; 5165 5166 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 5167 switch (ring->me) { 5168 case 1: 5169 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe; 5170 break; 5171 case 2: 5172 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe; 5173 break; 5174 default: 5175 return; 5176 } 5177 reg_mem_engine = 0; 5178 } else { 5179 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0; 5180 reg_mem_engine = 1; /* pfp */ 5181 } 5182 5183 gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1, 5184 adev->nbio.funcs->get_hdp_flush_req_offset(adev), 5185 adev->nbio.funcs->get_hdp_flush_done_offset(adev), 5186 ref_and_mask, ref_and_mask, 0x20); 5187 } 5188 5189 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 5190 struct amdgpu_job *job, 5191 struct amdgpu_ib *ib, 5192 uint32_t flags) 5193 { 5194 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5195 u32 header, control = 0; 5196 5197 if (ib->flags & AMDGPU_IB_FLAG_CE) 5198 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 5199 else 5200 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 5201 5202 control |= ib->length_dw | (vmid << 24); 5203 5204 if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { 5205 control |= INDIRECT_BUFFER_PRE_ENB(1); 5206 5207 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid) 5208 gfx_v9_0_ring_emit_de_meta(ring); 5209 } 5210 5211 amdgpu_ring_write(ring, header); 5212 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5213 amdgpu_ring_write(ring, 5214 #ifdef __BIG_ENDIAN 5215 (2 << 0) | 5216 #endif 5217 lower_32_bits(ib->gpu_addr)); 5218 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5219 amdgpu_ring_write(ring, control); 5220 } 5221 5222 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 5223 struct amdgpu_job *job, 5224 struct amdgpu_ib *ib, 5225 uint32_t flags) 5226 { 5227 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5228 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 5229 5230 /* Currently, there is a high possibility to get wave ID mismatch 5231 * between ME and GDS, leading to a hw deadlock, because ME generates 5232 * different wave IDs than the GDS expects. This situation happens 5233 * randomly when at least 5 compute pipes use GDS ordered append. 5234 * The wave IDs generated by ME are also wrong after suspend/resume. 5235 * Those are probably bugs somewhere else in the kernel driver. 5236 * 5237 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 5238 * GDS to 0 for this ring (me/pipe). 5239 */ 5240 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 5241 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 5242 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID); 5243 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 5244 } 5245 5246 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 5247 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5248 amdgpu_ring_write(ring, 5249 #ifdef __BIG_ENDIAN 5250 (2 << 0) | 5251 #endif 5252 lower_32_bits(ib->gpu_addr)); 5253 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5254 amdgpu_ring_write(ring, control); 5255 } 5256 5257 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 5258 u64 seq, unsigned flags) 5259 { 5260 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 5261 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 5262 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY; 5263 5264 /* RELEASE_MEM - flush caches, send int */ 5265 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); 5266 amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN | 5267 EOP_TC_NC_ACTION_EN) : 5268 (EOP_TCL1_ACTION_EN | 5269 EOP_TC_ACTION_EN | 5270 EOP_TC_WB_ACTION_EN | 5271 EOP_TC_MD_ACTION_EN)) | 5272 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 5273 EVENT_INDEX(5))); 5274 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 5275 5276 /* 5277 * the address should be Qword aligned if 64bit write, Dword 5278 * aligned if only send 32bit data low (discard data high) 5279 */ 5280 if (write64bit) 5281 BUG_ON(addr & 0x7); 5282 else 5283 BUG_ON(addr & 0x3); 5284 amdgpu_ring_write(ring, lower_32_bits(addr)); 5285 amdgpu_ring_write(ring, upper_32_bits(addr)); 5286 amdgpu_ring_write(ring, lower_32_bits(seq)); 5287 amdgpu_ring_write(ring, upper_32_bits(seq)); 5288 amdgpu_ring_write(ring, 0); 5289 } 5290 5291 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 5292 { 5293 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5294 uint32_t seq = ring->fence_drv.sync_seq; 5295 uint64_t addr = ring->fence_drv.gpu_addr; 5296 5297 gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0, 5298 lower_32_bits(addr), upper_32_bits(addr), 5299 seq, 0xffffffff, 4); 5300 } 5301 5302 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 5303 unsigned vmid, uint64_t pd_addr) 5304 { 5305 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 5306 5307 /* compute doesn't have PFP */ 5308 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 5309 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 5310 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 5311 amdgpu_ring_write(ring, 0x0); 5312 } 5313 } 5314 5315 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring) 5316 { 5317 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */ 5318 } 5319 5320 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 5321 { 5322 u64 wptr; 5323 5324 /* XXX check if swapping is necessary on BE */ 5325 if (ring->use_doorbell) 5326 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]); 5327 else 5328 BUG(); 5329 return wptr; 5330 } 5331 5332 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 5333 { 5334 struct amdgpu_device *adev = ring->adev; 5335 5336 /* XXX check if swapping is necessary on BE */ 5337 if (ring->use_doorbell) { 5338 atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5339 WDOORBELL64(ring->doorbell_index, ring->wptr); 5340 } else{ 5341 BUG(); /* only DOORBELL method supported on gfx9 now */ 5342 } 5343 } 5344 5345 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 5346 u64 seq, unsigned int flags) 5347 { 5348 struct amdgpu_device *adev = ring->adev; 5349 5350 /* we only allocate 32bit for each seq wb address */ 5351 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); 5352 5353 /* write fence seq to the "addr" */ 5354 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5355 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5356 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 5357 amdgpu_ring_write(ring, lower_32_bits(addr)); 5358 amdgpu_ring_write(ring, upper_32_bits(addr)); 5359 amdgpu_ring_write(ring, lower_32_bits(seq)); 5360 5361 if (flags & AMDGPU_FENCE_FLAG_INT) { 5362 /* set register to trigger INT */ 5363 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5364 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5365 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 5366 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS)); 5367 amdgpu_ring_write(ring, 0); 5368 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 5369 } 5370 } 5371 5372 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring) 5373 { 5374 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 5375 amdgpu_ring_write(ring, 0); 5376 } 5377 5378 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring) 5379 { 5380 struct v9_ce_ib_state ce_payload = {0}; 5381 uint64_t csa_addr; 5382 int cnt; 5383 5384 cnt = (sizeof(ce_payload) >> 2) + 4 - 2; 5385 csa_addr = amdgpu_csa_vaddr(ring->adev); 5386 5387 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5388 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) | 5389 WRITE_DATA_DST_SEL(8) | 5390 WR_CONFIRM) | 5391 WRITE_DATA_CACHE_POLICY(0)); 5392 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5393 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5394 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2); 5395 } 5396 5397 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring) 5398 { 5399 struct v9_de_ib_state de_payload = {0}; 5400 uint64_t csa_addr, gds_addr; 5401 int cnt; 5402 5403 csa_addr = amdgpu_csa_vaddr(ring->adev); 5404 gds_addr = csa_addr + 4096; 5405 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr); 5406 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr); 5407 5408 cnt = (sizeof(de_payload) >> 2) + 4 - 2; 5409 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5410 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 5411 WRITE_DATA_DST_SEL(8) | 5412 WR_CONFIRM) | 5413 WRITE_DATA_CACHE_POLICY(0)); 5414 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5415 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5416 amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2); 5417 } 5418 5419 static void gfx_v9_0_ring_emit_tmz(struct amdgpu_ring *ring, bool start) 5420 { 5421 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); 5422 amdgpu_ring_write(ring, FRAME_CMD(start ? 0 : 1)); /* frame_end */ 5423 } 5424 5425 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) 5426 { 5427 uint32_t dw2 = 0; 5428 5429 if (amdgpu_sriov_vf(ring->adev)) 5430 gfx_v9_0_ring_emit_ce_meta(ring); 5431 5432 gfx_v9_0_ring_emit_tmz(ring, true); 5433 5434 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 5435 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 5436 /* set load_global_config & load_global_uconfig */ 5437 dw2 |= 0x8001; 5438 /* set load_cs_sh_regs */ 5439 dw2 |= 0x01000000; 5440 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 5441 dw2 |= 0x10002; 5442 5443 /* set load_ce_ram if preamble presented */ 5444 if (AMDGPU_PREAMBLE_IB_PRESENT & flags) 5445 dw2 |= 0x10000000; 5446 } else { 5447 /* still load_ce_ram if this is the first time preamble presented 5448 * although there is no context switch happens. 5449 */ 5450 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags) 5451 dw2 |= 0x10000000; 5452 } 5453 5454 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 5455 amdgpu_ring_write(ring, dw2); 5456 amdgpu_ring_write(ring, 0); 5457 } 5458 5459 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring) 5460 { 5461 unsigned ret; 5462 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 5463 amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr)); 5464 amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr)); 5465 amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */ 5466 ret = ring->wptr & ring->buf_mask; 5467 amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */ 5468 return ret; 5469 } 5470 5471 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset) 5472 { 5473 unsigned cur; 5474 BUG_ON(offset > ring->buf_mask); 5475 BUG_ON(ring->ring[offset] != 0x55aa55aa); 5476 5477 cur = (ring->wptr & ring->buf_mask) - 1; 5478 if (likely(cur > offset)) 5479 ring->ring[offset] = cur - offset; 5480 else 5481 ring->ring[offset] = (ring->ring_size>>2) - offset + cur; 5482 } 5483 5484 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg) 5485 { 5486 struct amdgpu_device *adev = ring->adev; 5487 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 5488 5489 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 5490 amdgpu_ring_write(ring, 0 | /* src: register*/ 5491 (5 << 8) | /* dst: memory */ 5492 (1 << 20)); /* write confirm */ 5493 amdgpu_ring_write(ring, reg); 5494 amdgpu_ring_write(ring, 0); 5495 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 5496 kiq->reg_val_offs * 4)); 5497 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 5498 kiq->reg_val_offs * 4)); 5499 } 5500 5501 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 5502 uint32_t val) 5503 { 5504 uint32_t cmd = 0; 5505 5506 switch (ring->funcs->type) { 5507 case AMDGPU_RING_TYPE_GFX: 5508 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 5509 break; 5510 case AMDGPU_RING_TYPE_KIQ: 5511 cmd = (1 << 16); /* no inc addr */ 5512 break; 5513 default: 5514 cmd = WR_CONFIRM; 5515 break; 5516 } 5517 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5518 amdgpu_ring_write(ring, cmd); 5519 amdgpu_ring_write(ring, reg); 5520 amdgpu_ring_write(ring, 0); 5521 amdgpu_ring_write(ring, val); 5522 } 5523 5524 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 5525 uint32_t val, uint32_t mask) 5526 { 5527 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); 5528 } 5529 5530 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, 5531 uint32_t reg0, uint32_t reg1, 5532 uint32_t ref, uint32_t mask) 5533 { 5534 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5535 struct amdgpu_device *adev = ring->adev; 5536 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ? 5537 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait; 5538 5539 if (fw_version_ok) 5540 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, 5541 ref, mask, 0x20); 5542 else 5543 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1, 5544 ref, mask); 5545 } 5546 5547 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid) 5548 { 5549 struct amdgpu_device *adev = ring->adev; 5550 uint32_t value = 0; 5551 5552 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03); 5553 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01); 5554 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1); 5555 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid); 5556 WREG32_SOC15(GC, 0, mmSQ_CMD, value); 5557 } 5558 5559 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 5560 enum amdgpu_interrupt_state state) 5561 { 5562 switch (state) { 5563 case AMDGPU_IRQ_STATE_DISABLE: 5564 case AMDGPU_IRQ_STATE_ENABLE: 5565 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5566 TIME_STAMP_INT_ENABLE, 5567 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5568 break; 5569 default: 5570 break; 5571 } 5572 } 5573 5574 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 5575 int me, int pipe, 5576 enum amdgpu_interrupt_state state) 5577 { 5578 u32 mec_int_cntl, mec_int_cntl_reg; 5579 5580 /* 5581 * amdgpu controls only the first MEC. That's why this function only 5582 * handles the setting of interrupts for this specific MEC. All other 5583 * pipes' interrupts are set by amdkfd. 5584 */ 5585 5586 if (me == 1) { 5587 switch (pipe) { 5588 case 0: 5589 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL); 5590 break; 5591 case 1: 5592 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL); 5593 break; 5594 case 2: 5595 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL); 5596 break; 5597 case 3: 5598 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL); 5599 break; 5600 default: 5601 DRM_DEBUG("invalid pipe %d\n", pipe); 5602 return; 5603 } 5604 } else { 5605 DRM_DEBUG("invalid me %d\n", me); 5606 return; 5607 } 5608 5609 switch (state) { 5610 case AMDGPU_IRQ_STATE_DISABLE: 5611 mec_int_cntl = RREG32(mec_int_cntl_reg); 5612 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5613 TIME_STAMP_INT_ENABLE, 0); 5614 WREG32(mec_int_cntl_reg, mec_int_cntl); 5615 break; 5616 case AMDGPU_IRQ_STATE_ENABLE: 5617 mec_int_cntl = RREG32(mec_int_cntl_reg); 5618 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5619 TIME_STAMP_INT_ENABLE, 1); 5620 WREG32(mec_int_cntl_reg, mec_int_cntl); 5621 break; 5622 default: 5623 break; 5624 } 5625 } 5626 5627 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 5628 struct amdgpu_irq_src *source, 5629 unsigned type, 5630 enum amdgpu_interrupt_state state) 5631 { 5632 switch (state) { 5633 case AMDGPU_IRQ_STATE_DISABLE: 5634 case AMDGPU_IRQ_STATE_ENABLE: 5635 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5636 PRIV_REG_INT_ENABLE, 5637 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5638 break; 5639 default: 5640 break; 5641 } 5642 5643 return 0; 5644 } 5645 5646 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 5647 struct amdgpu_irq_src *source, 5648 unsigned type, 5649 enum amdgpu_interrupt_state state) 5650 { 5651 switch (state) { 5652 case AMDGPU_IRQ_STATE_DISABLE: 5653 case AMDGPU_IRQ_STATE_ENABLE: 5654 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5655 PRIV_INSTR_INT_ENABLE, 5656 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5657 default: 5658 break; 5659 } 5660 5661 return 0; 5662 } 5663 5664 #define ENABLE_ECC_ON_ME_PIPE(me, pipe) \ 5665 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5666 CP_ECC_ERROR_INT_ENABLE, 1) 5667 5668 #define DISABLE_ECC_ON_ME_PIPE(me, pipe) \ 5669 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5670 CP_ECC_ERROR_INT_ENABLE, 0) 5671 5672 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev, 5673 struct amdgpu_irq_src *source, 5674 unsigned type, 5675 enum amdgpu_interrupt_state state) 5676 { 5677 switch (state) { 5678 case AMDGPU_IRQ_STATE_DISABLE: 5679 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5680 CP_ECC_ERROR_INT_ENABLE, 0); 5681 DISABLE_ECC_ON_ME_PIPE(1, 0); 5682 DISABLE_ECC_ON_ME_PIPE(1, 1); 5683 DISABLE_ECC_ON_ME_PIPE(1, 2); 5684 DISABLE_ECC_ON_ME_PIPE(1, 3); 5685 break; 5686 5687 case AMDGPU_IRQ_STATE_ENABLE: 5688 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5689 CP_ECC_ERROR_INT_ENABLE, 1); 5690 ENABLE_ECC_ON_ME_PIPE(1, 0); 5691 ENABLE_ECC_ON_ME_PIPE(1, 1); 5692 ENABLE_ECC_ON_ME_PIPE(1, 2); 5693 ENABLE_ECC_ON_ME_PIPE(1, 3); 5694 break; 5695 default: 5696 break; 5697 } 5698 5699 return 0; 5700 } 5701 5702 5703 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev, 5704 struct amdgpu_irq_src *src, 5705 unsigned type, 5706 enum amdgpu_interrupt_state state) 5707 { 5708 switch (type) { 5709 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 5710 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state); 5711 break; 5712 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 5713 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 5714 break; 5715 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 5716 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 5717 break; 5718 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 5719 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 5720 break; 5721 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 5722 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 5723 break; 5724 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP: 5725 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state); 5726 break; 5727 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP: 5728 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state); 5729 break; 5730 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP: 5731 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state); 5732 break; 5733 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP: 5734 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state); 5735 break; 5736 default: 5737 break; 5738 } 5739 return 0; 5740 } 5741 5742 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev, 5743 struct amdgpu_irq_src *source, 5744 struct amdgpu_iv_entry *entry) 5745 { 5746 int i; 5747 u8 me_id, pipe_id, queue_id; 5748 struct amdgpu_ring *ring; 5749 5750 DRM_DEBUG("IH: CP EOP\n"); 5751 me_id = (entry->ring_id & 0x0c) >> 2; 5752 pipe_id = (entry->ring_id & 0x03) >> 0; 5753 queue_id = (entry->ring_id & 0x70) >> 4; 5754 5755 switch (me_id) { 5756 case 0: 5757 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); 5758 break; 5759 case 1: 5760 case 2: 5761 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5762 ring = &adev->gfx.compute_ring[i]; 5763 /* Per-queue interrupt is supported for MEC starting from VI. 5764 * The interrupt can only be enabled/disabled per pipe instead of per queue. 5765 */ 5766 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) 5767 amdgpu_fence_process(ring); 5768 } 5769 break; 5770 } 5771 return 0; 5772 } 5773 5774 static void gfx_v9_0_fault(struct amdgpu_device *adev, 5775 struct amdgpu_iv_entry *entry) 5776 { 5777 u8 me_id, pipe_id, queue_id; 5778 struct amdgpu_ring *ring; 5779 int i; 5780 5781 me_id = (entry->ring_id & 0x0c) >> 2; 5782 pipe_id = (entry->ring_id & 0x03) >> 0; 5783 queue_id = (entry->ring_id & 0x70) >> 4; 5784 5785 switch (me_id) { 5786 case 0: 5787 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); 5788 break; 5789 case 1: 5790 case 2: 5791 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5792 ring = &adev->gfx.compute_ring[i]; 5793 if (ring->me == me_id && ring->pipe == pipe_id && 5794 ring->queue == queue_id) 5795 drm_sched_fault(&ring->sched); 5796 } 5797 break; 5798 } 5799 } 5800 5801 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev, 5802 struct amdgpu_irq_src *source, 5803 struct amdgpu_iv_entry *entry) 5804 { 5805 DRM_ERROR("Illegal register access in command stream\n"); 5806 gfx_v9_0_fault(adev, entry); 5807 return 0; 5808 } 5809 5810 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev, 5811 struct amdgpu_irq_src *source, 5812 struct amdgpu_iv_entry *entry) 5813 { 5814 DRM_ERROR("Illegal instruction in command stream\n"); 5815 gfx_v9_0_fault(adev, entry); 5816 return 0; 5817 } 5818 5819 5820 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = { 5821 { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 5822 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT), 5823 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT) 5824 }, 5825 { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 5826 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT), 5827 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT) 5828 }, 5829 { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5830 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1), 5831 0, 0 5832 }, 5833 { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5834 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2), 5835 0, 0 5836 }, 5837 { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 5838 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT), 5839 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT) 5840 }, 5841 { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5842 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT), 5843 0, 0 5844 }, 5845 { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5846 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT), 5847 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT) 5848 }, 5849 { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 5850 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT), 5851 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT) 5852 }, 5853 { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 5854 SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1), 5855 0, 0 5856 }, 5857 { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 5858 SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1), 5859 0, 0 5860 }, 5861 { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 5862 SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1), 5863 0, 0 5864 }, 5865 { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 5866 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC), 5867 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED) 5868 }, 5869 { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 5870 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED), 5871 0, 0 5872 }, 5873 { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5874 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC), 5875 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED) 5876 }, 5877 { "GDS_OA_PHY_PHY_CMD_RAM_MEM", 5878 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5879 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC), 5880 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED) 5881 }, 5882 { "GDS_OA_PHY_PHY_DATA_RAM_MEM", 5883 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5884 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED), 5885 0, 0 5886 }, 5887 { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM", 5888 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5889 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC), 5890 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED) 5891 }, 5892 { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM", 5893 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5894 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC), 5895 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED) 5896 }, 5897 { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM", 5898 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5899 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC), 5900 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED) 5901 }, 5902 { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM", 5903 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5904 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC), 5905 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED) 5906 }, 5907 { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 5908 SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT), 5909 0, 0 5910 }, 5911 { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5912 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT), 5913 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT) 5914 }, 5915 { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5916 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT), 5917 0, 0 5918 }, 5919 { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5920 SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT), 5921 0, 0 5922 }, 5923 { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5924 SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT), 5925 0, 0 5926 }, 5927 { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5928 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT), 5929 0, 0 5930 }, 5931 { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 5932 SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT), 5933 0, 0 5934 }, 5935 { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 5936 SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT), 5937 0, 0 5938 }, 5939 { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5940 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT), 5941 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT) 5942 }, 5943 { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5944 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT), 5945 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT) 5946 }, 5947 { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5948 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT), 5949 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT) 5950 }, 5951 { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5952 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT), 5953 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT) 5954 }, 5955 { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5956 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT), 5957 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT) 5958 }, 5959 { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5960 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT), 5961 0, 0 5962 }, 5963 { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5964 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT), 5965 0, 0 5966 }, 5967 { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5968 SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT), 5969 0, 0 5970 }, 5971 { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5972 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT), 5973 0, 0 5974 }, 5975 { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5976 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT), 5977 0, 0 5978 }, 5979 { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5980 SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT), 5981 0, 0 5982 }, 5983 { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 5984 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT), 5985 0, 0 5986 }, 5987 { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 5988 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT), 5989 0, 0 5990 }, 5991 { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 5992 SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT), 5993 0, 0 5994 }, 5995 { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 5996 SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT), 5997 0, 0 5998 }, 5999 { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6000 SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT), 6001 0, 0 6002 }, 6003 { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6004 SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT), 6005 0, 0 6006 }, 6007 { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6008 SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT), 6009 0, 0 6010 }, 6011 { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 6012 SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT), 6013 0, 0 6014 }, 6015 { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6016 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT), 6017 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT) 6018 }, 6019 { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6020 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT), 6021 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT) 6022 }, 6023 { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6024 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT), 6025 0, 0 6026 }, 6027 { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6028 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT), 6029 0, 0 6030 }, 6031 { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6032 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT), 6033 0, 0 6034 }, 6035 { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6036 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT), 6037 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT) 6038 }, 6039 { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6040 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT), 6041 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT) 6042 }, 6043 { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6044 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT), 6045 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT) 6046 }, 6047 { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6048 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT), 6049 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT) 6050 }, 6051 { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6052 SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT), 6053 0, 0 6054 }, 6055 { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6056 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT), 6057 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT) 6058 }, 6059 { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6060 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT), 6061 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT) 6062 }, 6063 { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6064 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT), 6065 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT) 6066 }, 6067 { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6068 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT), 6069 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT) 6070 }, 6071 { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6072 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT), 6073 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT) 6074 }, 6075 { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6076 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT), 6077 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT) 6078 }, 6079 { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6080 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT), 6081 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT) 6082 }, 6083 { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6084 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT), 6085 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT) 6086 }, 6087 { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6088 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT), 6089 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT) 6090 }, 6091 { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6092 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT), 6093 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT) 6094 }, 6095 { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6096 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT), 6097 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT) 6098 }, 6099 { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6100 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT), 6101 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT) 6102 }, 6103 { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6104 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT), 6105 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT) 6106 }, 6107 { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6108 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT), 6109 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT) 6110 }, 6111 { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6112 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT), 6113 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT) 6114 }, 6115 { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6116 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT), 6117 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT) 6118 }, 6119 { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6120 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT), 6121 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT) 6122 }, 6123 { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6124 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT), 6125 0, 0 6126 }, 6127 { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6128 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT), 6129 0, 0 6130 }, 6131 { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6132 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT), 6133 0, 0 6134 }, 6135 { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6136 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT), 6137 0, 0 6138 }, 6139 { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6140 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT), 6141 0, 0 6142 }, 6143 { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6144 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT), 6145 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT) 6146 }, 6147 { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6148 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT), 6149 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT) 6150 }, 6151 { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6152 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT), 6153 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT) 6154 }, 6155 { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6156 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT), 6157 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT) 6158 }, 6159 { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6160 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT), 6161 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT) 6162 }, 6163 { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6164 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT), 6165 0, 0 6166 }, 6167 { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6168 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT), 6169 0, 0 6170 }, 6171 { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6172 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT), 6173 0, 0 6174 }, 6175 { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6176 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT), 6177 0, 0 6178 }, 6179 { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6180 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT), 6181 0, 0 6182 }, 6183 { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6184 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), 6185 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT) 6186 }, 6187 { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6188 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), 6189 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT) 6190 }, 6191 { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6192 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), 6193 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT) 6194 }, 6195 { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6196 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT), 6197 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT) 6198 }, 6199 { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6200 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT), 6201 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT) 6202 }, 6203 { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6204 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), 6205 0, 0 6206 }, 6207 { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6208 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), 6209 0, 0 6210 }, 6211 { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6212 SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT), 6213 0, 0 6214 }, 6215 { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6216 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT), 6217 0, 0 6218 }, 6219 { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6220 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT), 6221 0, 0 6222 }, 6223 { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6224 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), 6225 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT) 6226 }, 6227 { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6228 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), 6229 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT) 6230 }, 6231 { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6232 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), 6233 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT) 6234 }, 6235 { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6236 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), 6237 0, 0 6238 }, 6239 { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6240 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), 6241 0, 0 6242 }, 6243 { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6244 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT), 6245 0, 0 6246 }, 6247 { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6248 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT), 6249 0, 0 6250 }, 6251 { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6252 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT), 6253 0, 0 6254 }, 6255 { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6256 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT), 6257 0, 0 6258 } 6259 }; 6260 6261 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 6262 void *inject_if) 6263 { 6264 struct ras_inject_if *info = (struct ras_inject_if *)inject_if; 6265 int ret; 6266 struct ta_ras_trigger_error_input block_info = { 0 }; 6267 6268 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6269 return -EINVAL; 6270 6271 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks)) 6272 return -EINVAL; 6273 6274 if (!ras_gfx_subblocks[info->head.sub_block_index].name) 6275 return -EPERM; 6276 6277 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type & 6278 info->head.type)) { 6279 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n", 6280 ras_gfx_subblocks[info->head.sub_block_index].name, 6281 info->head.type); 6282 return -EPERM; 6283 } 6284 6285 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type & 6286 info->head.type)) { 6287 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n", 6288 ras_gfx_subblocks[info->head.sub_block_index].name, 6289 info->head.type); 6290 return -EPERM; 6291 } 6292 6293 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block); 6294 block_info.sub_block_index = 6295 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock; 6296 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type); 6297 block_info.address = info->address; 6298 block_info.value = info->value; 6299 6300 mutex_lock(&adev->grbm_idx_mutex); 6301 ret = psp_ras_trigger_error(&adev->psp, &block_info); 6302 mutex_unlock(&adev->grbm_idx_mutex); 6303 6304 return ret; 6305 } 6306 6307 static const char *vml2_mems[] = { 6308 "UTC_VML2_BANK_CACHE_0_BIGK_MEM0", 6309 "UTC_VML2_BANK_CACHE_0_BIGK_MEM1", 6310 "UTC_VML2_BANK_CACHE_0_4K_MEM0", 6311 "UTC_VML2_BANK_CACHE_0_4K_MEM1", 6312 "UTC_VML2_BANK_CACHE_1_BIGK_MEM0", 6313 "UTC_VML2_BANK_CACHE_1_BIGK_MEM1", 6314 "UTC_VML2_BANK_CACHE_1_4K_MEM0", 6315 "UTC_VML2_BANK_CACHE_1_4K_MEM1", 6316 "UTC_VML2_BANK_CACHE_2_BIGK_MEM0", 6317 "UTC_VML2_BANK_CACHE_2_BIGK_MEM1", 6318 "UTC_VML2_BANK_CACHE_2_4K_MEM0", 6319 "UTC_VML2_BANK_CACHE_2_4K_MEM1", 6320 "UTC_VML2_BANK_CACHE_3_BIGK_MEM0", 6321 "UTC_VML2_BANK_CACHE_3_BIGK_MEM1", 6322 "UTC_VML2_BANK_CACHE_3_4K_MEM0", 6323 "UTC_VML2_BANK_CACHE_3_4K_MEM1", 6324 }; 6325 6326 static const char *vml2_walker_mems[] = { 6327 "UTC_VML2_CACHE_PDE0_MEM0", 6328 "UTC_VML2_CACHE_PDE0_MEM1", 6329 "UTC_VML2_CACHE_PDE1_MEM0", 6330 "UTC_VML2_CACHE_PDE1_MEM1", 6331 "UTC_VML2_CACHE_PDE2_MEM0", 6332 "UTC_VML2_CACHE_PDE2_MEM1", 6333 "UTC_VML2_RDIF_LOG_FIFO", 6334 }; 6335 6336 static const char *atc_l2_cache_2m_mems[] = { 6337 "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM", 6338 "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM", 6339 "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM", 6340 "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM", 6341 }; 6342 6343 static const char *atc_l2_cache_4k_mems[] = { 6344 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0", 6345 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1", 6346 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2", 6347 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3", 6348 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4", 6349 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5", 6350 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6", 6351 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7", 6352 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0", 6353 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1", 6354 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2", 6355 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3", 6356 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4", 6357 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5", 6358 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6", 6359 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7", 6360 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0", 6361 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1", 6362 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2", 6363 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3", 6364 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4", 6365 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5", 6366 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6", 6367 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7", 6368 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0", 6369 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1", 6370 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2", 6371 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3", 6372 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4", 6373 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5", 6374 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6", 6375 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7", 6376 }; 6377 6378 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev, 6379 struct ras_err_data *err_data) 6380 { 6381 uint32_t i, data; 6382 uint32_t sec_count, ded_count; 6383 6384 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6385 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6386 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6387 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6388 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6389 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6390 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6391 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6392 6393 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6394 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6395 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6396 6397 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT); 6398 if (sec_count) { 6399 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6400 "SEC %d\n", i, vml2_mems[i], sec_count); 6401 err_data->ce_count += sec_count; 6402 } 6403 6404 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT); 6405 if (ded_count) { 6406 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6407 "DED %d\n", i, vml2_mems[i], ded_count); 6408 err_data->ue_count += ded_count; 6409 } 6410 } 6411 6412 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6413 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6414 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6415 6416 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6417 SEC_COUNT); 6418 if (sec_count) { 6419 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6420 "SEC %d\n", i, vml2_walker_mems[i], sec_count); 6421 err_data->ce_count += sec_count; 6422 } 6423 6424 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6425 DED_COUNT); 6426 if (ded_count) { 6427 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6428 "DED %d\n", i, vml2_walker_mems[i], ded_count); 6429 err_data->ue_count += ded_count; 6430 } 6431 } 6432 6433 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6434 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6435 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6436 6437 sec_count = (data & 0x00006000L) >> 0xd; 6438 if (sec_count) { 6439 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6440 "SEC %d\n", i, atc_l2_cache_2m_mems[i], 6441 sec_count); 6442 err_data->ce_count += sec_count; 6443 } 6444 } 6445 6446 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6447 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6448 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6449 6450 sec_count = (data & 0x00006000L) >> 0xd; 6451 if (sec_count) { 6452 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6453 "SEC %d\n", i, atc_l2_cache_4k_mems[i], 6454 sec_count); 6455 err_data->ce_count += sec_count; 6456 } 6457 6458 ded_count = (data & 0x00018000L) >> 0xf; 6459 if (ded_count) { 6460 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6461 "DED %d\n", i, atc_l2_cache_4k_mems[i], 6462 ded_count); 6463 err_data->ue_count += ded_count; 6464 } 6465 } 6466 6467 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6468 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6469 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6470 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6471 6472 return 0; 6473 } 6474 6475 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev, 6476 const struct soc15_reg_entry *reg, 6477 uint32_t se_id, uint32_t inst_id, uint32_t value, 6478 uint32_t *sec_count, uint32_t *ded_count) 6479 { 6480 uint32_t i; 6481 uint32_t sec_cnt, ded_cnt; 6482 6483 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) { 6484 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset || 6485 gfx_v9_0_ras_fields[i].seg != reg->seg || 6486 gfx_v9_0_ras_fields[i].inst != reg->inst) 6487 continue; 6488 6489 sec_cnt = (value & 6490 gfx_v9_0_ras_fields[i].sec_count_mask) >> 6491 gfx_v9_0_ras_fields[i].sec_count_shift; 6492 if (sec_cnt) { 6493 dev_info(adev->dev, "GFX SubBlock %s, " 6494 "Instance[%d][%d], SEC %d\n", 6495 gfx_v9_0_ras_fields[i].name, 6496 se_id, inst_id, 6497 sec_cnt); 6498 *sec_count += sec_cnt; 6499 } 6500 6501 ded_cnt = (value & 6502 gfx_v9_0_ras_fields[i].ded_count_mask) >> 6503 gfx_v9_0_ras_fields[i].ded_count_shift; 6504 if (ded_cnt) { 6505 dev_info(adev->dev, "GFX SubBlock %s, " 6506 "Instance[%d][%d], DED %d\n", 6507 gfx_v9_0_ras_fields[i].name, 6508 se_id, inst_id, 6509 ded_cnt); 6510 *ded_count += ded_cnt; 6511 } 6512 } 6513 6514 return 0; 6515 } 6516 6517 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev) 6518 { 6519 int i, j, k; 6520 6521 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6522 return; 6523 6524 /* read back registers to clear the counters */ 6525 mutex_lock(&adev->grbm_idx_mutex); 6526 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6527 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6528 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6529 gfx_v9_0_select_se_sh(adev, j, 0x0, k); 6530 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6531 } 6532 } 6533 } 6534 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000); 6535 mutex_unlock(&adev->grbm_idx_mutex); 6536 6537 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6538 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6539 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6540 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6541 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6542 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6543 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6544 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6545 6546 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6547 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6548 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6549 } 6550 6551 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6552 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6553 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6554 } 6555 6556 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6557 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6558 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6559 } 6560 6561 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6562 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6563 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6564 } 6565 6566 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6567 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6568 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6569 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6570 } 6571 6572 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 6573 void *ras_error_status) 6574 { 6575 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; 6576 uint32_t sec_count = 0, ded_count = 0; 6577 uint32_t i, j, k; 6578 uint32_t reg_value; 6579 6580 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6581 return -EINVAL; 6582 6583 err_data->ue_count = 0; 6584 err_data->ce_count = 0; 6585 6586 mutex_lock(&adev->grbm_idx_mutex); 6587 6588 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6589 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6590 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6591 gfx_v9_0_select_se_sh(adev, j, 0, k); 6592 reg_value = 6593 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6594 if (reg_value) 6595 gfx_v9_0_ras_error_count(adev, 6596 &gfx_v9_0_edc_counter_regs[i], 6597 j, k, reg_value, 6598 &sec_count, &ded_count); 6599 } 6600 } 6601 } 6602 6603 err_data->ce_count += sec_count; 6604 err_data->ue_count += ded_count; 6605 6606 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 6607 mutex_unlock(&adev->grbm_idx_mutex); 6608 6609 gfx_v9_0_query_utc_edc_status(adev, err_data); 6610 6611 return 0; 6612 } 6613 6614 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = { 6615 .name = "gfx_v9_0", 6616 .early_init = gfx_v9_0_early_init, 6617 .late_init = gfx_v9_0_late_init, 6618 .sw_init = gfx_v9_0_sw_init, 6619 .sw_fini = gfx_v9_0_sw_fini, 6620 .hw_init = gfx_v9_0_hw_init, 6621 .hw_fini = gfx_v9_0_hw_fini, 6622 .suspend = gfx_v9_0_suspend, 6623 .resume = gfx_v9_0_resume, 6624 .is_idle = gfx_v9_0_is_idle, 6625 .wait_for_idle = gfx_v9_0_wait_for_idle, 6626 .soft_reset = gfx_v9_0_soft_reset, 6627 .set_clockgating_state = gfx_v9_0_set_clockgating_state, 6628 .set_powergating_state = gfx_v9_0_set_powergating_state, 6629 .get_clockgating_state = gfx_v9_0_get_clockgating_state, 6630 }; 6631 6632 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = { 6633 .type = AMDGPU_RING_TYPE_GFX, 6634 .align_mask = 0xff, 6635 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6636 .support_64bit_ptrs = true, 6637 .vmhub = AMDGPU_GFXHUB_0, 6638 .get_rptr = gfx_v9_0_ring_get_rptr_gfx, 6639 .get_wptr = gfx_v9_0_ring_get_wptr_gfx, 6640 .set_wptr = gfx_v9_0_ring_set_wptr_gfx, 6641 .emit_frame_size = /* totally 242 maximum if 16 IBs */ 6642 5 + /* COND_EXEC */ 6643 7 + /* PIPELINE_SYNC */ 6644 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6645 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6646 2 + /* VM_FLUSH */ 6647 8 + /* FENCE for VM_FLUSH */ 6648 20 + /* GDS switch */ 6649 4 + /* double SWITCH_BUFFER, 6650 the first COND_EXEC jump to the place just 6651 prior to this double SWITCH_BUFFER */ 6652 5 + /* COND_EXEC */ 6653 7 + /* HDP_flush */ 6654 4 + /* VGT_flush */ 6655 14 + /* CE_META */ 6656 31 + /* DE_META */ 6657 3 + /* CNTX_CTRL */ 6658 5 + /* HDP_INVL */ 6659 8 + 8 + /* FENCE x2 */ 6660 2, /* SWITCH_BUFFER */ 6661 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */ 6662 .emit_ib = gfx_v9_0_ring_emit_ib_gfx, 6663 .emit_fence = gfx_v9_0_ring_emit_fence, 6664 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6665 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6666 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6667 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6668 .test_ring = gfx_v9_0_ring_test_ring, 6669 .test_ib = gfx_v9_0_ring_test_ib, 6670 .insert_nop = amdgpu_ring_insert_nop, 6671 .pad_ib = amdgpu_ring_generic_pad_ib, 6672 .emit_switch_buffer = gfx_v9_ring_emit_sb, 6673 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl, 6674 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec, 6675 .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec, 6676 .emit_tmz = gfx_v9_0_ring_emit_tmz, 6677 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6678 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6679 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6680 .soft_recovery = gfx_v9_0_ring_soft_recovery, 6681 }; 6682 6683 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { 6684 .type = AMDGPU_RING_TYPE_COMPUTE, 6685 .align_mask = 0xff, 6686 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6687 .support_64bit_ptrs = true, 6688 .vmhub = AMDGPU_GFXHUB_0, 6689 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6690 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6691 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6692 .emit_frame_size = 6693 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6694 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6695 5 + /* hdp invalidate */ 6696 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6697 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6698 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6699 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6700 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */ 6701 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6702 .emit_ib = gfx_v9_0_ring_emit_ib_compute, 6703 .emit_fence = gfx_v9_0_ring_emit_fence, 6704 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6705 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6706 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6707 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6708 .test_ring = gfx_v9_0_ring_test_ring, 6709 .test_ib = gfx_v9_0_ring_test_ib, 6710 .insert_nop = amdgpu_ring_insert_nop, 6711 .pad_ib = amdgpu_ring_generic_pad_ib, 6712 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6713 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6714 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6715 }; 6716 6717 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = { 6718 .type = AMDGPU_RING_TYPE_KIQ, 6719 .align_mask = 0xff, 6720 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6721 .support_64bit_ptrs = true, 6722 .vmhub = AMDGPU_GFXHUB_0, 6723 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6724 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6725 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6726 .emit_frame_size = 6727 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6728 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6729 5 + /* hdp invalidate */ 6730 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6731 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6732 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6733 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6734 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 6735 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6736 .emit_fence = gfx_v9_0_ring_emit_fence_kiq, 6737 .test_ring = gfx_v9_0_ring_test_ring, 6738 .insert_nop = amdgpu_ring_insert_nop, 6739 .pad_ib = amdgpu_ring_generic_pad_ib, 6740 .emit_rreg = gfx_v9_0_ring_emit_rreg, 6741 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6742 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6743 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6744 }; 6745 6746 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev) 6747 { 6748 int i; 6749 6750 adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq; 6751 6752 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 6753 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx; 6754 6755 for (i = 0; i < adev->gfx.num_compute_rings; i++) 6756 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute; 6757 } 6758 6759 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = { 6760 .set = gfx_v9_0_set_eop_interrupt_state, 6761 .process = gfx_v9_0_eop_irq, 6762 }; 6763 6764 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = { 6765 .set = gfx_v9_0_set_priv_reg_fault_state, 6766 .process = gfx_v9_0_priv_reg_irq, 6767 }; 6768 6769 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = { 6770 .set = gfx_v9_0_set_priv_inst_fault_state, 6771 .process = gfx_v9_0_priv_inst_irq, 6772 }; 6773 6774 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = { 6775 .set = gfx_v9_0_set_cp_ecc_error_state, 6776 .process = amdgpu_gfx_cp_ecc_error_irq, 6777 }; 6778 6779 6780 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev) 6781 { 6782 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 6783 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs; 6784 6785 adev->gfx.priv_reg_irq.num_types = 1; 6786 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs; 6787 6788 adev->gfx.priv_inst_irq.num_types = 1; 6789 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs; 6790 6791 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/ 6792 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs; 6793 } 6794 6795 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev) 6796 { 6797 switch (adev->asic_type) { 6798 case CHIP_VEGA10: 6799 case CHIP_VEGA12: 6800 case CHIP_VEGA20: 6801 case CHIP_RAVEN: 6802 case CHIP_ARCTURUS: 6803 case CHIP_RENOIR: 6804 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs; 6805 break; 6806 default: 6807 break; 6808 } 6809 } 6810 6811 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev) 6812 { 6813 /* init asci gds info */ 6814 switch (adev->asic_type) { 6815 case CHIP_VEGA10: 6816 case CHIP_VEGA12: 6817 case CHIP_VEGA20: 6818 adev->gds.gds_size = 0x10000; 6819 break; 6820 case CHIP_RAVEN: 6821 case CHIP_ARCTURUS: 6822 adev->gds.gds_size = 0x1000; 6823 break; 6824 default: 6825 adev->gds.gds_size = 0x10000; 6826 break; 6827 } 6828 6829 switch (adev->asic_type) { 6830 case CHIP_VEGA10: 6831 case CHIP_VEGA20: 6832 adev->gds.gds_compute_max_wave_id = 0x7ff; 6833 break; 6834 case CHIP_VEGA12: 6835 adev->gds.gds_compute_max_wave_id = 0x27f; 6836 break; 6837 case CHIP_RAVEN: 6838 if (adev->rev_id >= 0x8) 6839 adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */ 6840 else 6841 adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */ 6842 break; 6843 case CHIP_ARCTURUS: 6844 adev->gds.gds_compute_max_wave_id = 0xfff; 6845 break; 6846 default: 6847 /* this really depends on the chip */ 6848 adev->gds.gds_compute_max_wave_id = 0x7ff; 6849 break; 6850 } 6851 6852 adev->gds.gws_size = 64; 6853 adev->gds.oa_size = 16; 6854 } 6855 6856 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev, 6857 u32 bitmap) 6858 { 6859 u32 data; 6860 6861 if (!bitmap) 6862 return; 6863 6864 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 6865 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 6866 6867 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data); 6868 } 6869 6870 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev) 6871 { 6872 u32 data, mask; 6873 6874 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG); 6875 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG); 6876 6877 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 6878 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 6879 6880 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); 6881 6882 return (~data) & mask; 6883 } 6884 6885 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 6886 struct amdgpu_cu_info *cu_info) 6887 { 6888 int i, j, k, counter, active_cu_number = 0; 6889 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0; 6890 unsigned disable_masks[4 * 4]; 6891 6892 if (!adev || !cu_info) 6893 return -EINVAL; 6894 6895 /* 6896 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs 6897 */ 6898 if (adev->gfx.config.max_shader_engines * 6899 adev->gfx.config.max_sh_per_se > 16) 6900 return -EINVAL; 6901 6902 amdgpu_gfx_parse_disable_cu(disable_masks, 6903 adev->gfx.config.max_shader_engines, 6904 adev->gfx.config.max_sh_per_se); 6905 6906 mutex_lock(&adev->grbm_idx_mutex); 6907 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 6908 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 6909 mask = 1; 6910 ao_bitmap = 0; 6911 counter = 0; 6912 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 6913 gfx_v9_0_set_user_cu_inactive_bitmap( 6914 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]); 6915 bitmap = gfx_v9_0_get_cu_active_bitmap(adev); 6916 6917 /* 6918 * The bitmap(and ao_cu_bitmap) in cu_info structure is 6919 * 4x4 size array, and it's usually suitable for Vega 6920 * ASICs which has 4*2 SE/SH layout. 6921 * But for Arcturus, SE/SH layout is changed to 8*1. 6922 * To mostly reduce the impact, we make it compatible 6923 * with current bitmap array as below: 6924 * SE4,SH0 --> bitmap[0][1] 6925 * SE5,SH0 --> bitmap[1][1] 6926 * SE6,SH0 --> bitmap[2][1] 6927 * SE7,SH0 --> bitmap[3][1] 6928 */ 6929 cu_info->bitmap[i % 4][j + i / 4] = bitmap; 6930 6931 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 6932 if (bitmap & mask) { 6933 if (counter < adev->gfx.config.max_cu_per_sh) 6934 ao_bitmap |= mask; 6935 counter ++; 6936 } 6937 mask <<= 1; 6938 } 6939 active_cu_number += counter; 6940 if (i < 2 && j < 2) 6941 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8)); 6942 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap; 6943 } 6944 } 6945 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 6946 mutex_unlock(&adev->grbm_idx_mutex); 6947 6948 cu_info->number = active_cu_number; 6949 cu_info->ao_cu_mask = ao_cu_mask; 6950 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 6951 6952 return 0; 6953 } 6954 6955 const struct amdgpu_ip_block_version gfx_v9_0_ip_block = 6956 { 6957 .type = AMD_IP_BLOCK_TYPE_GFX, 6958 .major = 9, 6959 .minor = 0, 6960 .rev = 0, 6961 .funcs = &gfx_v9_0_ip_funcs, 6962 }; 6963