1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/delay.h> 25 #include <linux/kernel.h> 26 #include <linux/firmware.h> 27 #include <linux/module.h> 28 #include <linux/pci.h> 29 30 #include "amdgpu.h" 31 #include "amdgpu_gfx.h" 32 #include "soc15.h" 33 #include "soc15d.h" 34 #include "amdgpu_atomfirmware.h" 35 #include "amdgpu_pm.h" 36 37 #include "gc/gc_9_0_offset.h" 38 #include "gc/gc_9_0_sh_mask.h" 39 40 #include "vega10_enum.h" 41 #include "hdp/hdp_4_0_offset.h" 42 43 #include "soc15_common.h" 44 #include "clearstate_gfx9.h" 45 #include "v9_structs.h" 46 47 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h" 48 49 #include "amdgpu_ras.h" 50 51 #include "gfx_v9_4.h" 52 53 #include "asic_reg/pwr/pwr_10_0_offset.h" 54 #include "asic_reg/pwr/pwr_10_0_sh_mask.h" 55 56 #define GFX9_NUM_GFX_RINGS 1 57 #define GFX9_MEC_HPD_SIZE 4096 58 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L 59 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L 60 61 #define mmGCEA_PROBE_MAP 0x070c 62 #define mmGCEA_PROBE_MAP_BASE_IDX 0 63 64 MODULE_FIRMWARE("amdgpu/vega10_ce.bin"); 65 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin"); 66 MODULE_FIRMWARE("amdgpu/vega10_me.bin"); 67 MODULE_FIRMWARE("amdgpu/vega10_mec.bin"); 68 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin"); 69 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin"); 70 71 MODULE_FIRMWARE("amdgpu/vega12_ce.bin"); 72 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin"); 73 MODULE_FIRMWARE("amdgpu/vega12_me.bin"); 74 MODULE_FIRMWARE("amdgpu/vega12_mec.bin"); 75 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin"); 76 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin"); 77 78 MODULE_FIRMWARE("amdgpu/vega20_ce.bin"); 79 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin"); 80 MODULE_FIRMWARE("amdgpu/vega20_me.bin"); 81 MODULE_FIRMWARE("amdgpu/vega20_mec.bin"); 82 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin"); 83 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin"); 84 85 MODULE_FIRMWARE("amdgpu/raven_ce.bin"); 86 MODULE_FIRMWARE("amdgpu/raven_pfp.bin"); 87 MODULE_FIRMWARE("amdgpu/raven_me.bin"); 88 MODULE_FIRMWARE("amdgpu/raven_mec.bin"); 89 MODULE_FIRMWARE("amdgpu/raven_mec2.bin"); 90 MODULE_FIRMWARE("amdgpu/raven_rlc.bin"); 91 92 MODULE_FIRMWARE("amdgpu/picasso_ce.bin"); 93 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin"); 94 MODULE_FIRMWARE("amdgpu/picasso_me.bin"); 95 MODULE_FIRMWARE("amdgpu/picasso_mec.bin"); 96 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin"); 97 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin"); 98 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin"); 99 100 MODULE_FIRMWARE("amdgpu/raven2_ce.bin"); 101 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin"); 102 MODULE_FIRMWARE("amdgpu/raven2_me.bin"); 103 MODULE_FIRMWARE("amdgpu/raven2_mec.bin"); 104 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin"); 105 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin"); 106 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin"); 107 108 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin"); 109 MODULE_FIRMWARE("amdgpu/arcturus_mec2.bin"); 110 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin"); 111 112 MODULE_FIRMWARE("amdgpu/renoir_ce.bin"); 113 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin"); 114 MODULE_FIRMWARE("amdgpu/renoir_me.bin"); 115 MODULE_FIRMWARE("amdgpu/renoir_mec.bin"); 116 MODULE_FIRMWARE("amdgpu/renoir_mec2.bin"); 117 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin"); 118 119 #define mmTCP_CHAN_STEER_0_ARCT 0x0b03 120 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0 121 #define mmTCP_CHAN_STEER_1_ARCT 0x0b04 122 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0 123 #define mmTCP_CHAN_STEER_2_ARCT 0x0b09 124 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0 125 #define mmTCP_CHAN_STEER_3_ARCT 0x0b0a 126 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0 127 #define mmTCP_CHAN_STEER_4_ARCT 0x0b0b 128 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0 129 #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c 130 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0 131 132 enum ta_ras_gfx_subblock { 133 /*CPC*/ 134 TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0, 135 TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START, 136 TA_RAS_BLOCK__GFX_CPC_UCODE, 137 TA_RAS_BLOCK__GFX_DC_STATE_ME1, 138 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1, 139 TA_RAS_BLOCK__GFX_DC_RESTORE_ME1, 140 TA_RAS_BLOCK__GFX_DC_STATE_ME2, 141 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2, 142 TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 143 TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 144 /* CPF*/ 145 TA_RAS_BLOCK__GFX_CPF_INDEX_START, 146 TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START, 147 TA_RAS_BLOCK__GFX_CPF_ROQ_ME1, 148 TA_RAS_BLOCK__GFX_CPF_TAG, 149 TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG, 150 /* CPG*/ 151 TA_RAS_BLOCK__GFX_CPG_INDEX_START, 152 TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START, 153 TA_RAS_BLOCK__GFX_CPG_DMA_TAG, 154 TA_RAS_BLOCK__GFX_CPG_TAG, 155 TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG, 156 /* GDS*/ 157 TA_RAS_BLOCK__GFX_GDS_INDEX_START, 158 TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START, 159 TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE, 160 TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM, 161 TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM, 162 TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 163 TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 164 /* SPI*/ 165 TA_RAS_BLOCK__GFX_SPI_SR_MEM, 166 /* SQ*/ 167 TA_RAS_BLOCK__GFX_SQ_INDEX_START, 168 TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START, 169 TA_RAS_BLOCK__GFX_SQ_LDS_D, 170 TA_RAS_BLOCK__GFX_SQ_LDS_I, 171 TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/ 172 TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR, 173 /* SQC (3 ranges)*/ 174 TA_RAS_BLOCK__GFX_SQC_INDEX_START, 175 /* SQC range 0*/ 176 TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START, 177 TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO = 178 TA_RAS_BLOCK__GFX_SQC_INDEX0_START, 179 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 180 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO, 181 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 182 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO, 183 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 184 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 185 TA_RAS_BLOCK__GFX_SQC_INDEX0_END = 186 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 187 /* SQC range 1*/ 188 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 189 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM = 190 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 191 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 192 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO, 193 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM, 194 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM, 195 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO, 196 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO, 197 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 198 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 199 TA_RAS_BLOCK__GFX_SQC_INDEX1_END = 200 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 201 /* SQC range 2*/ 202 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 203 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM = 204 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 205 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 206 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO, 207 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM, 208 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM, 209 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO, 210 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO, 211 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 212 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 213 TA_RAS_BLOCK__GFX_SQC_INDEX2_END = 214 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 215 TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END, 216 /* TA*/ 217 TA_RAS_BLOCK__GFX_TA_INDEX_START, 218 TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START, 219 TA_RAS_BLOCK__GFX_TA_FS_AFIFO, 220 TA_RAS_BLOCK__GFX_TA_FL_LFIFO, 221 TA_RAS_BLOCK__GFX_TA_FX_LFIFO, 222 TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 223 TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 224 /* TCA*/ 225 TA_RAS_BLOCK__GFX_TCA_INDEX_START, 226 TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START, 227 TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 228 TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 229 /* TCC (5 sub-ranges)*/ 230 TA_RAS_BLOCK__GFX_TCC_INDEX_START, 231 /* TCC range 0*/ 232 TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START, 233 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START, 234 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1, 235 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0, 236 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1, 237 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0, 238 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1, 239 TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG, 240 TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 241 TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 242 /* TCC range 1*/ 243 TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 244 TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 245 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 246 TA_RAS_BLOCK__GFX_TCC_INDEX1_END = 247 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 248 /* TCC range 2*/ 249 TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 250 TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 251 TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL, 252 TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO, 253 TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN, 254 TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ, 255 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO, 256 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM, 257 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 258 TA_RAS_BLOCK__GFX_TCC_INDEX2_END = 259 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 260 /* TCC range 3*/ 261 TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 262 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 263 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 264 TA_RAS_BLOCK__GFX_TCC_INDEX3_END = 265 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 266 /* TCC range 4*/ 267 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 268 TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN = 269 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 270 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 271 TA_RAS_BLOCK__GFX_TCC_INDEX4_END = 272 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 273 TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END, 274 /* TCI*/ 275 TA_RAS_BLOCK__GFX_TCI_WRITE_RAM, 276 /* TCP*/ 277 TA_RAS_BLOCK__GFX_TCP_INDEX_START, 278 TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START, 279 TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM, 280 TA_RAS_BLOCK__GFX_TCP_CMD_FIFO, 281 TA_RAS_BLOCK__GFX_TCP_VM_FIFO, 282 TA_RAS_BLOCK__GFX_TCP_DB_RAM, 283 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0, 284 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 285 TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 286 /* TD*/ 287 TA_RAS_BLOCK__GFX_TD_INDEX_START, 288 TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START, 289 TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI, 290 TA_RAS_BLOCK__GFX_TD_CS_FIFO, 291 TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO, 292 /* EA (3 sub-ranges)*/ 293 TA_RAS_BLOCK__GFX_EA_INDEX_START, 294 /* EA range 0*/ 295 TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START, 296 TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START, 297 TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM, 298 TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM, 299 TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM, 300 TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM, 301 TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM, 302 TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM, 303 TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 304 TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 305 /* EA range 1*/ 306 TA_RAS_BLOCK__GFX_EA_INDEX1_START, 307 TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START, 308 TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM, 309 TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM, 310 TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM, 311 TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM, 312 TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM, 313 TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 314 TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 315 /* EA range 2*/ 316 TA_RAS_BLOCK__GFX_EA_INDEX2_START, 317 TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START, 318 TA_RAS_BLOCK__GFX_EA_MAM_D1MEM, 319 TA_RAS_BLOCK__GFX_EA_MAM_D2MEM, 320 TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 321 TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 322 TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END, 323 /* UTC VM L2 bank*/ 324 TA_RAS_BLOCK__UTC_VML2_BANK_CACHE, 325 /* UTC VM walker*/ 326 TA_RAS_BLOCK__UTC_VML2_WALKER, 327 /* UTC ATC L2 2MB cache*/ 328 TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK, 329 /* UTC ATC L2 4KB cache*/ 330 TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK, 331 TA_RAS_BLOCK__GFX_MAX 332 }; 333 334 struct ras_gfx_subblock { 335 unsigned char *name; 336 int ta_subblock; 337 int hw_supported_error_type; 338 int sw_supported_error_type; 339 }; 340 341 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \ 342 [AMDGPU_RAS_BLOCK__##subblock] = { \ 343 #subblock, \ 344 TA_RAS_BLOCK__##subblock, \ 345 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \ 346 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \ 347 } 348 349 static const struct ras_gfx_subblock ras_gfx_subblocks[] = { 350 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1), 351 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1), 352 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 353 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 354 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 355 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 356 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 357 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 358 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 359 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 360 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1), 361 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0), 362 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1), 363 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1), 364 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 365 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0), 366 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0, 367 0), 368 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0, 369 0), 370 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 371 AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0), 372 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0), 373 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1), 374 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0), 375 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0), 376 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1), 377 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 378 0, 0), 379 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 380 0), 381 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 382 0, 0), 383 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0, 384 0), 385 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 386 0, 0), 387 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 388 0), 389 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 390 1), 391 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 392 0, 0, 0), 393 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 394 0), 395 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 396 0), 397 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 398 0), 399 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 400 0), 401 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 402 0), 403 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 404 0, 0), 405 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 406 0), 407 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 408 0), 409 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 410 0, 0, 0), 411 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 412 0), 413 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 414 0), 415 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 416 0), 417 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 418 0), 419 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 420 0), 421 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 422 0, 0), 423 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 424 0), 425 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1), 426 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 427 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 428 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 429 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 430 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0), 431 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 432 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1), 433 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0, 434 1), 435 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0, 436 1), 437 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0, 438 1), 439 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0, 440 0), 441 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0, 442 0), 443 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 444 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 445 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0), 446 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0), 447 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0), 448 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0), 449 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 450 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0), 451 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0), 452 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 453 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0), 454 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0, 455 0), 456 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 457 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0, 458 0), 459 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0, 460 0, 0), 461 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0, 462 0), 463 AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 464 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1), 465 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0), 466 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 467 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 468 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 469 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0), 470 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0), 471 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1), 472 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0), 473 AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 474 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1), 475 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 476 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 477 AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 478 AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 479 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 480 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 481 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 482 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 483 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 484 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 485 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 486 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0), 487 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 488 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 489 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0), 490 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0), 491 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0), 492 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0), 493 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0), 494 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0), 495 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0), 496 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0), 497 }; 498 499 static const struct soc15_reg_golden golden_settings_gc_9_0[] = 500 { 501 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 502 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000), 503 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 504 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 505 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 506 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 507 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 508 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 509 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 510 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87), 511 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f), 512 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 513 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 514 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 515 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 516 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 517 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff), 518 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 519 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 520 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 521 }; 522 523 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] = 524 { 525 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107), 526 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 527 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 528 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 529 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 530 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 531 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042), 532 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 533 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000), 534 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 535 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 536 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 537 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 538 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 539 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 540 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107), 541 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800), 542 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080) 543 }; 544 545 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] = 546 { 547 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080), 548 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 549 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 550 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042), 551 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042), 552 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400), 553 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000), 554 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000), 555 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107), 556 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000), 557 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000) 558 }; 559 560 static const struct soc15_reg_golden golden_settings_gc_9_1[] = 561 { 562 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 563 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 564 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 565 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 566 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 567 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 568 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 569 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 570 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 571 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 572 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 573 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 574 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 575 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 576 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 577 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 578 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 579 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 580 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 581 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff), 582 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080), 583 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 584 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 585 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 586 }; 587 588 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] = 589 { 590 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 591 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042), 592 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042), 593 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000), 594 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000), 595 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 596 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800) 597 }; 598 599 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] = 600 { 601 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000), 602 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 603 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 604 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080), 605 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080), 606 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080), 607 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041), 608 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041), 609 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 610 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 611 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080), 612 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080), 613 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080), 614 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080), 615 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080), 616 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 617 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010), 618 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 619 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 620 }; 621 622 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] = 623 { 624 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 625 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 626 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 627 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042), 628 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042), 629 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 630 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 631 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 632 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 633 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 634 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 635 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc), 636 }; 637 638 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] = 639 { 640 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff), 641 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000), 642 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382) 643 }; 644 645 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] = 646 { 647 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 648 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 649 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 650 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 651 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 652 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 653 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 654 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 655 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87), 656 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f), 657 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 658 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 659 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 660 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 661 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 662 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff) 663 }; 664 665 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] = 666 { 667 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080), 668 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 669 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 670 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041), 671 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041), 672 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 673 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107), 674 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 675 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410), 676 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 677 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 678 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 679 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 680 }; 681 682 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] = 683 { 684 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 685 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000), 686 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e), 687 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca), 688 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098), 689 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3), 690 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1), 691 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135), 692 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000), 693 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00), 694 }; 695 696 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = { 697 {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)}, 698 {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)}, 699 }; 700 701 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] = 702 { 703 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 704 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 705 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 706 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 707 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 708 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 709 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 710 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 711 }; 712 713 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] = 714 { 715 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0, 716 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0, 717 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0, 718 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0, 719 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0, 720 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0, 721 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0, 722 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0, 723 }; 724 725 static void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v) 726 { 727 static void *scratch_reg0; 728 static void *scratch_reg1; 729 static void *scratch_reg2; 730 static void *scratch_reg3; 731 static void *spare_int; 732 static uint32_t grbm_cntl; 733 static uint32_t grbm_idx; 734 735 scratch_reg0 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0)*4; 736 scratch_reg1 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1)*4; 737 scratch_reg2 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG2)*4; 738 scratch_reg3 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3)*4; 739 spare_int = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT)*4; 740 741 grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL; 742 grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX; 743 744 if (amdgpu_sriov_runtime(adev)) { 745 pr_err("shouldn't call rlcg write register during runtime\n"); 746 return; 747 } 748 749 if (offset == grbm_cntl || offset == grbm_idx) { 750 if (offset == grbm_cntl) 751 writel(v, scratch_reg2); 752 else if (offset == grbm_idx) 753 writel(v, scratch_reg3); 754 755 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4)); 756 } else { 757 uint32_t i = 0; 758 uint32_t retries = 50000; 759 760 writel(v, scratch_reg0); 761 writel(offset | 0x80000000, scratch_reg1); 762 writel(1, spare_int); 763 for (i = 0; i < retries; i++) { 764 u32 tmp; 765 766 tmp = readl(scratch_reg1); 767 if (!(tmp & 0x80000000)) 768 break; 769 770 udelay(10); 771 } 772 if (i >= retries) 773 pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset); 774 } 775 776 } 777 778 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042 779 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041 780 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042 781 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041 782 783 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev); 784 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev); 785 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev); 786 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev); 787 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 788 struct amdgpu_cu_info *cu_info); 789 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev); 790 static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance); 791 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring); 792 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring); 793 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 794 void *ras_error_status); 795 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 796 void *inject_if); 797 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev); 798 799 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring, 800 uint64_t queue_mask) 801 { 802 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 803 amdgpu_ring_write(kiq_ring, 804 PACKET3_SET_RESOURCES_VMID_MASK(0) | 805 /* vmid_mask:0* queue_type:0 (KIQ) */ 806 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); 807 amdgpu_ring_write(kiq_ring, 808 lower_32_bits(queue_mask)); /* queue mask lo */ 809 amdgpu_ring_write(kiq_ring, 810 upper_32_bits(queue_mask)); /* queue mask hi */ 811 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */ 812 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */ 813 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 814 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 815 } 816 817 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring, 818 struct amdgpu_ring *ring) 819 { 820 struct amdgpu_device *adev = kiq_ring->adev; 821 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 822 uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 823 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 824 825 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 826 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 827 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 828 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 829 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ 830 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 831 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 832 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) | 833 /*queue_type: normal compute queue */ 834 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | 835 /* alloc format: all_on_one_pipe */ 836 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | 837 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) | 838 /* num_queues: must be 1 */ 839 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); 840 amdgpu_ring_write(kiq_ring, 841 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); 842 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 843 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 844 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 845 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 846 } 847 848 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, 849 struct amdgpu_ring *ring, 850 enum amdgpu_unmap_queues_action action, 851 u64 gpu_addr, u64 seq) 852 { 853 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 854 855 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 856 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 857 PACKET3_UNMAP_QUEUES_ACTION(action) | 858 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 859 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) | 860 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 861 amdgpu_ring_write(kiq_ring, 862 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 863 864 if (action == PREEMPT_QUEUES_NO_UNMAP) { 865 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr)); 866 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr)); 867 amdgpu_ring_write(kiq_ring, seq); 868 } else { 869 amdgpu_ring_write(kiq_ring, 0); 870 amdgpu_ring_write(kiq_ring, 0); 871 amdgpu_ring_write(kiq_ring, 0); 872 } 873 } 874 875 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring, 876 struct amdgpu_ring *ring, 877 u64 addr, 878 u64 seq) 879 { 880 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 881 882 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5)); 883 amdgpu_ring_write(kiq_ring, 884 PACKET3_QUERY_STATUS_CONTEXT_ID(0) | 885 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) | 886 PACKET3_QUERY_STATUS_COMMAND(2)); 887 /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 888 amdgpu_ring_write(kiq_ring, 889 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) | 890 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel)); 891 amdgpu_ring_write(kiq_ring, lower_32_bits(addr)); 892 amdgpu_ring_write(kiq_ring, upper_32_bits(addr)); 893 amdgpu_ring_write(kiq_ring, lower_32_bits(seq)); 894 amdgpu_ring_write(kiq_ring, upper_32_bits(seq)); 895 } 896 897 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring, 898 uint16_t pasid, uint32_t flush_type, 899 bool all_hub) 900 { 901 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); 902 amdgpu_ring_write(kiq_ring, 903 PACKET3_INVALIDATE_TLBS_DST_SEL(1) | 904 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) | 905 PACKET3_INVALIDATE_TLBS_PASID(pasid) | 906 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type)); 907 } 908 909 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = { 910 .kiq_set_resources = gfx_v9_0_kiq_set_resources, 911 .kiq_map_queues = gfx_v9_0_kiq_map_queues, 912 .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues, 913 .kiq_query_status = gfx_v9_0_kiq_query_status, 914 .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs, 915 .set_resources_size = 8, 916 .map_queues_size = 7, 917 .unmap_queues_size = 6, 918 .query_status_size = 7, 919 .invalidate_tlbs_size = 2, 920 }; 921 922 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) 923 { 924 adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs; 925 } 926 927 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev) 928 { 929 switch (adev->asic_type) { 930 case CHIP_VEGA10: 931 soc15_program_register_sequence(adev, 932 golden_settings_gc_9_0, 933 ARRAY_SIZE(golden_settings_gc_9_0)); 934 soc15_program_register_sequence(adev, 935 golden_settings_gc_9_0_vg10, 936 ARRAY_SIZE(golden_settings_gc_9_0_vg10)); 937 break; 938 case CHIP_VEGA12: 939 soc15_program_register_sequence(adev, 940 golden_settings_gc_9_2_1, 941 ARRAY_SIZE(golden_settings_gc_9_2_1)); 942 soc15_program_register_sequence(adev, 943 golden_settings_gc_9_2_1_vg12, 944 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12)); 945 break; 946 case CHIP_VEGA20: 947 soc15_program_register_sequence(adev, 948 golden_settings_gc_9_0, 949 ARRAY_SIZE(golden_settings_gc_9_0)); 950 soc15_program_register_sequence(adev, 951 golden_settings_gc_9_0_vg20, 952 ARRAY_SIZE(golden_settings_gc_9_0_vg20)); 953 break; 954 case CHIP_ARCTURUS: 955 soc15_program_register_sequence(adev, 956 golden_settings_gc_9_4_1_arct, 957 ARRAY_SIZE(golden_settings_gc_9_4_1_arct)); 958 break; 959 case CHIP_RAVEN: 960 soc15_program_register_sequence(adev, golden_settings_gc_9_1, 961 ARRAY_SIZE(golden_settings_gc_9_1)); 962 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 963 soc15_program_register_sequence(adev, 964 golden_settings_gc_9_1_rv2, 965 ARRAY_SIZE(golden_settings_gc_9_1_rv2)); 966 else 967 soc15_program_register_sequence(adev, 968 golden_settings_gc_9_1_rv1, 969 ARRAY_SIZE(golden_settings_gc_9_1_rv1)); 970 break; 971 case CHIP_RENOIR: 972 soc15_program_register_sequence(adev, 973 golden_settings_gc_9_1_rn, 974 ARRAY_SIZE(golden_settings_gc_9_1_rn)); 975 return; /* for renoir, don't need common goldensetting */ 976 default: 977 break; 978 } 979 980 if (adev->asic_type != CHIP_ARCTURUS) 981 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common, 982 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common)); 983 } 984 985 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev) 986 { 987 adev->gfx.scratch.num_reg = 8; 988 adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0); 989 adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1; 990 } 991 992 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel, 993 bool wc, uint32_t reg, uint32_t val) 994 { 995 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 996 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) | 997 WRITE_DATA_DST_SEL(0) | 998 (wc ? WR_CONFIRM : 0)); 999 amdgpu_ring_write(ring, reg); 1000 amdgpu_ring_write(ring, 0); 1001 amdgpu_ring_write(ring, val); 1002 } 1003 1004 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, 1005 int mem_space, int opt, uint32_t addr0, 1006 uint32_t addr1, uint32_t ref, uint32_t mask, 1007 uint32_t inv) 1008 { 1009 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 1010 amdgpu_ring_write(ring, 1011 /* memory (1) or register (0) */ 1012 (WAIT_REG_MEM_MEM_SPACE(mem_space) | 1013 WAIT_REG_MEM_OPERATION(opt) | /* wait */ 1014 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 1015 WAIT_REG_MEM_ENGINE(eng_sel))); 1016 1017 if (mem_space) 1018 BUG_ON(addr0 & 0x3); /* Dword align */ 1019 amdgpu_ring_write(ring, addr0); 1020 amdgpu_ring_write(ring, addr1); 1021 amdgpu_ring_write(ring, ref); 1022 amdgpu_ring_write(ring, mask); 1023 amdgpu_ring_write(ring, inv); /* poll interval */ 1024 } 1025 1026 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring) 1027 { 1028 struct amdgpu_device *adev = ring->adev; 1029 uint32_t scratch; 1030 uint32_t tmp = 0; 1031 unsigned i; 1032 int r; 1033 1034 r = amdgpu_gfx_scratch_get(adev, &scratch); 1035 if (r) 1036 return r; 1037 1038 WREG32(scratch, 0xCAFEDEAD); 1039 r = amdgpu_ring_alloc(ring, 3); 1040 if (r) 1041 goto error_free_scratch; 1042 1043 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 1044 amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START)); 1045 amdgpu_ring_write(ring, 0xDEADBEEF); 1046 amdgpu_ring_commit(ring); 1047 1048 for (i = 0; i < adev->usec_timeout; i++) { 1049 tmp = RREG32(scratch); 1050 if (tmp == 0xDEADBEEF) 1051 break; 1052 udelay(1); 1053 } 1054 1055 if (i >= adev->usec_timeout) 1056 r = -ETIMEDOUT; 1057 1058 error_free_scratch: 1059 amdgpu_gfx_scratch_free(adev, scratch); 1060 return r; 1061 } 1062 1063 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 1064 { 1065 struct amdgpu_device *adev = ring->adev; 1066 struct amdgpu_ib ib; 1067 struct dma_fence *f = NULL; 1068 1069 unsigned index; 1070 uint64_t gpu_addr; 1071 uint32_t tmp; 1072 long r; 1073 1074 r = amdgpu_device_wb_get(adev, &index); 1075 if (r) 1076 return r; 1077 1078 gpu_addr = adev->wb.gpu_addr + (index * 4); 1079 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 1080 memset(&ib, 0, sizeof(ib)); 1081 r = amdgpu_ib_get(adev, NULL, 16, 1082 AMDGPU_IB_POOL_DIRECT, &ib); 1083 if (r) 1084 goto err1; 1085 1086 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 1087 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 1088 ib.ptr[2] = lower_32_bits(gpu_addr); 1089 ib.ptr[3] = upper_32_bits(gpu_addr); 1090 ib.ptr[4] = 0xDEADBEEF; 1091 ib.length_dw = 5; 1092 1093 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 1094 if (r) 1095 goto err2; 1096 1097 r = dma_fence_wait_timeout(f, false, timeout); 1098 if (r == 0) { 1099 r = -ETIMEDOUT; 1100 goto err2; 1101 } else if (r < 0) { 1102 goto err2; 1103 } 1104 1105 tmp = adev->wb.wb[index]; 1106 if (tmp == 0xDEADBEEF) 1107 r = 0; 1108 else 1109 r = -EINVAL; 1110 1111 err2: 1112 amdgpu_ib_free(adev, &ib, NULL); 1113 dma_fence_put(f); 1114 err1: 1115 amdgpu_device_wb_free(adev, index); 1116 return r; 1117 } 1118 1119 1120 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev) 1121 { 1122 release_firmware(adev->gfx.pfp_fw); 1123 adev->gfx.pfp_fw = NULL; 1124 release_firmware(adev->gfx.me_fw); 1125 adev->gfx.me_fw = NULL; 1126 release_firmware(adev->gfx.ce_fw); 1127 adev->gfx.ce_fw = NULL; 1128 release_firmware(adev->gfx.rlc_fw); 1129 adev->gfx.rlc_fw = NULL; 1130 release_firmware(adev->gfx.mec_fw); 1131 adev->gfx.mec_fw = NULL; 1132 release_firmware(adev->gfx.mec2_fw); 1133 adev->gfx.mec2_fw = NULL; 1134 1135 kfree(adev->gfx.rlc.register_list_format); 1136 } 1137 1138 static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev) 1139 { 1140 const struct rlc_firmware_header_v2_1 *rlc_hdr; 1141 1142 rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data; 1143 adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver); 1144 adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver); 1145 adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes); 1146 adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes); 1147 adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver); 1148 adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver); 1149 adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes); 1150 adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes); 1151 adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver); 1152 adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver); 1153 adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes); 1154 adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes); 1155 adev->gfx.rlc.reg_list_format_direct_reg_list_length = 1156 le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length); 1157 } 1158 1159 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev) 1160 { 1161 adev->gfx.me_fw_write_wait = false; 1162 adev->gfx.mec_fw_write_wait = false; 1163 1164 if ((adev->asic_type != CHIP_ARCTURUS) && 1165 ((adev->gfx.mec_fw_version < 0x000001a5) || 1166 (adev->gfx.mec_feature_version < 46) || 1167 (adev->gfx.pfp_fw_version < 0x000000b7) || 1168 (adev->gfx.pfp_feature_version < 46))) 1169 DRM_WARN_ONCE("CP firmware version too old, please update!"); 1170 1171 switch (adev->asic_type) { 1172 case CHIP_VEGA10: 1173 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1174 (adev->gfx.me_feature_version >= 42) && 1175 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1176 (adev->gfx.pfp_feature_version >= 42)) 1177 adev->gfx.me_fw_write_wait = true; 1178 1179 if ((adev->gfx.mec_fw_version >= 0x00000193) && 1180 (adev->gfx.mec_feature_version >= 42)) 1181 adev->gfx.mec_fw_write_wait = true; 1182 break; 1183 case CHIP_VEGA12: 1184 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1185 (adev->gfx.me_feature_version >= 44) && 1186 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1187 (adev->gfx.pfp_feature_version >= 44)) 1188 adev->gfx.me_fw_write_wait = true; 1189 1190 if ((adev->gfx.mec_fw_version >= 0x00000196) && 1191 (adev->gfx.mec_feature_version >= 44)) 1192 adev->gfx.mec_fw_write_wait = true; 1193 break; 1194 case CHIP_VEGA20: 1195 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1196 (adev->gfx.me_feature_version >= 44) && 1197 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1198 (adev->gfx.pfp_feature_version >= 44)) 1199 adev->gfx.me_fw_write_wait = true; 1200 1201 if ((adev->gfx.mec_fw_version >= 0x00000197) && 1202 (adev->gfx.mec_feature_version >= 44)) 1203 adev->gfx.mec_fw_write_wait = true; 1204 break; 1205 case CHIP_RAVEN: 1206 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1207 (adev->gfx.me_feature_version >= 42) && 1208 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1209 (adev->gfx.pfp_feature_version >= 42)) 1210 adev->gfx.me_fw_write_wait = true; 1211 1212 if ((adev->gfx.mec_fw_version >= 0x00000192) && 1213 (adev->gfx.mec_feature_version >= 42)) 1214 adev->gfx.mec_fw_write_wait = true; 1215 break; 1216 default: 1217 adev->gfx.me_fw_write_wait = true; 1218 adev->gfx.mec_fw_write_wait = true; 1219 break; 1220 } 1221 } 1222 1223 struct amdgpu_gfxoff_quirk { 1224 u16 chip_vendor; 1225 u16 chip_device; 1226 u16 subsys_vendor; 1227 u16 subsys_device; 1228 u8 revision; 1229 }; 1230 1231 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = { 1232 /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */ 1233 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 }, 1234 /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */ 1235 { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 }, 1236 /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */ 1237 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 }, 1238 { 0, 0, 0, 0, 0 }, 1239 }; 1240 1241 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev) 1242 { 1243 const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list; 1244 1245 while (p && p->chip_device != 0) { 1246 if (pdev->vendor == p->chip_vendor && 1247 pdev->device == p->chip_device && 1248 pdev->subsystem_vendor == p->subsys_vendor && 1249 pdev->subsystem_device == p->subsys_device && 1250 pdev->revision == p->revision) { 1251 return true; 1252 } 1253 ++p; 1254 } 1255 return false; 1256 } 1257 1258 static bool is_raven_kicker(struct amdgpu_device *adev) 1259 { 1260 if (adev->pm.fw_version >= 0x41e2b) 1261 return true; 1262 else 1263 return false; 1264 } 1265 1266 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev) 1267 { 1268 if (gfx_v9_0_should_disable_gfxoff(adev->pdev)) 1269 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1270 1271 switch (adev->asic_type) { 1272 case CHIP_VEGA10: 1273 case CHIP_VEGA12: 1274 case CHIP_VEGA20: 1275 break; 1276 case CHIP_RAVEN: 1277 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) || 1278 (adev->apu_flags & AMD_APU_IS_PICASSO)) && 1279 ((!is_raven_kicker(adev) && 1280 adev->gfx.rlc_fw_version < 531) || 1281 (adev->gfx.rlc_feature_version < 1) || 1282 !adev->gfx.rlc.is_rlc_v2_1)) 1283 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1284 1285 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1286 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1287 AMD_PG_SUPPORT_CP | 1288 AMD_PG_SUPPORT_RLC_SMU_HS; 1289 break; 1290 case CHIP_RENOIR: 1291 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1292 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1293 AMD_PG_SUPPORT_CP | 1294 AMD_PG_SUPPORT_RLC_SMU_HS; 1295 break; 1296 default: 1297 break; 1298 } 1299 } 1300 1301 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev, 1302 const char *chip_name) 1303 { 1304 char fw_name[30]; 1305 int err; 1306 struct amdgpu_firmware_info *info = NULL; 1307 const struct common_firmware_header *header = NULL; 1308 const struct gfx_firmware_header_v1_0 *cp_hdr; 1309 1310 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name); 1311 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev); 1312 if (err) 1313 goto out; 1314 err = amdgpu_ucode_validate(adev->gfx.pfp_fw); 1315 if (err) 1316 goto out; 1317 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; 1318 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1319 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1320 1321 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name); 1322 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev); 1323 if (err) 1324 goto out; 1325 err = amdgpu_ucode_validate(adev->gfx.me_fw); 1326 if (err) 1327 goto out; 1328 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; 1329 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1330 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1331 1332 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name); 1333 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev); 1334 if (err) 1335 goto out; 1336 err = amdgpu_ucode_validate(adev->gfx.ce_fw); 1337 if (err) 1338 goto out; 1339 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; 1340 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1341 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1342 1343 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1344 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP]; 1345 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP; 1346 info->fw = adev->gfx.pfp_fw; 1347 header = (const struct common_firmware_header *)info->fw->data; 1348 adev->firmware.fw_size += 1349 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1350 1351 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME]; 1352 info->ucode_id = AMDGPU_UCODE_ID_CP_ME; 1353 info->fw = adev->gfx.me_fw; 1354 header = (const struct common_firmware_header *)info->fw->data; 1355 adev->firmware.fw_size += 1356 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1357 1358 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE]; 1359 info->ucode_id = AMDGPU_UCODE_ID_CP_CE; 1360 info->fw = adev->gfx.ce_fw; 1361 header = (const struct common_firmware_header *)info->fw->data; 1362 adev->firmware.fw_size += 1363 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1364 } 1365 1366 out: 1367 if (err) { 1368 dev_err(adev->dev, 1369 "gfx9: Failed to load firmware \"%s\"\n", 1370 fw_name); 1371 release_firmware(adev->gfx.pfp_fw); 1372 adev->gfx.pfp_fw = NULL; 1373 release_firmware(adev->gfx.me_fw); 1374 adev->gfx.me_fw = NULL; 1375 release_firmware(adev->gfx.ce_fw); 1376 adev->gfx.ce_fw = NULL; 1377 } 1378 return err; 1379 } 1380 1381 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev, 1382 const char *chip_name) 1383 { 1384 char fw_name[30]; 1385 int err; 1386 struct amdgpu_firmware_info *info = NULL; 1387 const struct common_firmware_header *header = NULL; 1388 const struct rlc_firmware_header_v2_0 *rlc_hdr; 1389 unsigned int *tmp = NULL; 1390 unsigned int i = 0; 1391 uint16_t version_major; 1392 uint16_t version_minor; 1393 uint32_t smu_version; 1394 1395 /* 1396 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin 1397 * instead of picasso_rlc.bin. 1398 * Judgment method: 1399 * PCO AM4: revision >= 0xC8 && revision <= 0xCF 1400 * or revision >= 0xD8 && revision <= 0xDF 1401 * otherwise is PCO FP5 1402 */ 1403 if (!strcmp(chip_name, "picasso") && 1404 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) || 1405 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF)))) 1406 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name); 1407 else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) && 1408 (smu_version >= 0x41e2b)) 1409 /** 1410 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly. 1411 */ 1412 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name); 1413 else 1414 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name); 1415 err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev); 1416 if (err) 1417 goto out; 1418 err = amdgpu_ucode_validate(adev->gfx.rlc_fw); 1419 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 1420 1421 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 1422 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 1423 if (version_major == 2 && version_minor == 1) 1424 adev->gfx.rlc.is_rlc_v2_1 = true; 1425 1426 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); 1427 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); 1428 adev->gfx.rlc.save_and_restore_offset = 1429 le32_to_cpu(rlc_hdr->save_and_restore_offset); 1430 adev->gfx.rlc.clear_state_descriptor_offset = 1431 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset); 1432 adev->gfx.rlc.avail_scratch_ram_locations = 1433 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations); 1434 adev->gfx.rlc.reg_restore_list_size = 1435 le32_to_cpu(rlc_hdr->reg_restore_list_size); 1436 adev->gfx.rlc.reg_list_format_start = 1437 le32_to_cpu(rlc_hdr->reg_list_format_start); 1438 adev->gfx.rlc.reg_list_format_separate_start = 1439 le32_to_cpu(rlc_hdr->reg_list_format_separate_start); 1440 adev->gfx.rlc.starting_offsets_start = 1441 le32_to_cpu(rlc_hdr->starting_offsets_start); 1442 adev->gfx.rlc.reg_list_format_size_bytes = 1443 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes); 1444 adev->gfx.rlc.reg_list_size_bytes = 1445 le32_to_cpu(rlc_hdr->reg_list_size_bytes); 1446 adev->gfx.rlc.register_list_format = 1447 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes + 1448 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL); 1449 if (!adev->gfx.rlc.register_list_format) { 1450 err = -ENOMEM; 1451 goto out; 1452 } 1453 1454 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1455 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); 1456 for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++) 1457 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]); 1458 1459 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i; 1460 1461 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1462 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); 1463 for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++) 1464 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); 1465 1466 if (adev->gfx.rlc.is_rlc_v2_1) 1467 gfx_v9_0_init_rlc_ext_microcode(adev); 1468 1469 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1470 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G]; 1471 info->ucode_id = AMDGPU_UCODE_ID_RLC_G; 1472 info->fw = adev->gfx.rlc_fw; 1473 header = (const struct common_firmware_header *)info->fw->data; 1474 adev->firmware.fw_size += 1475 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1476 1477 if (adev->gfx.rlc.is_rlc_v2_1 && 1478 adev->gfx.rlc.save_restore_list_cntl_size_bytes && 1479 adev->gfx.rlc.save_restore_list_gpm_size_bytes && 1480 adev->gfx.rlc.save_restore_list_srm_size_bytes) { 1481 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL]; 1482 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL; 1483 info->fw = adev->gfx.rlc_fw; 1484 adev->firmware.fw_size += 1485 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE); 1486 1487 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM]; 1488 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM; 1489 info->fw = adev->gfx.rlc_fw; 1490 adev->firmware.fw_size += 1491 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE); 1492 1493 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM]; 1494 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM; 1495 info->fw = adev->gfx.rlc_fw; 1496 adev->firmware.fw_size += 1497 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE); 1498 } 1499 } 1500 1501 out: 1502 if (err) { 1503 dev_err(adev->dev, 1504 "gfx9: Failed to load firmware \"%s\"\n", 1505 fw_name); 1506 release_firmware(adev->gfx.rlc_fw); 1507 adev->gfx.rlc_fw = NULL; 1508 } 1509 return err; 1510 } 1511 1512 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev, 1513 const char *chip_name) 1514 { 1515 char fw_name[30]; 1516 int err; 1517 struct amdgpu_firmware_info *info = NULL; 1518 const struct common_firmware_header *header = NULL; 1519 const struct gfx_firmware_header_v1_0 *cp_hdr; 1520 1521 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name); 1522 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev); 1523 if (err) 1524 goto out; 1525 err = amdgpu_ucode_validate(adev->gfx.mec_fw); 1526 if (err) 1527 goto out; 1528 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1529 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1530 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1531 1532 1533 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name); 1534 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev); 1535 if (!err) { 1536 err = amdgpu_ucode_validate(adev->gfx.mec2_fw); 1537 if (err) 1538 goto out; 1539 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1540 adev->gfx.mec2_fw->data; 1541 adev->gfx.mec2_fw_version = 1542 le32_to_cpu(cp_hdr->header.ucode_version); 1543 adev->gfx.mec2_feature_version = 1544 le32_to_cpu(cp_hdr->ucode_feature_version); 1545 } else { 1546 err = 0; 1547 adev->gfx.mec2_fw = NULL; 1548 } 1549 1550 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1551 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1]; 1552 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1; 1553 info->fw = adev->gfx.mec_fw; 1554 header = (const struct common_firmware_header *)info->fw->data; 1555 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1556 adev->firmware.fw_size += 1557 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1558 1559 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT]; 1560 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT; 1561 info->fw = adev->gfx.mec_fw; 1562 adev->firmware.fw_size += 1563 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1564 1565 if (adev->gfx.mec2_fw) { 1566 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2]; 1567 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2; 1568 info->fw = adev->gfx.mec2_fw; 1569 header = (const struct common_firmware_header *)info->fw->data; 1570 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1571 adev->firmware.fw_size += 1572 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1573 1574 /* TODO: Determine if MEC2 JT FW loading can be removed 1575 for all GFX V9 asic and above */ 1576 if (adev->asic_type != CHIP_ARCTURUS && 1577 adev->asic_type != CHIP_RENOIR) { 1578 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT]; 1579 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT; 1580 info->fw = adev->gfx.mec2_fw; 1581 adev->firmware.fw_size += 1582 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, 1583 PAGE_SIZE); 1584 } 1585 } 1586 } 1587 1588 out: 1589 gfx_v9_0_check_if_need_gfxoff(adev); 1590 gfx_v9_0_check_fw_write_wait(adev); 1591 if (err) { 1592 dev_err(adev->dev, 1593 "gfx9: Failed to load firmware \"%s\"\n", 1594 fw_name); 1595 release_firmware(adev->gfx.mec_fw); 1596 adev->gfx.mec_fw = NULL; 1597 release_firmware(adev->gfx.mec2_fw); 1598 adev->gfx.mec2_fw = NULL; 1599 } 1600 return err; 1601 } 1602 1603 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev) 1604 { 1605 const char *chip_name; 1606 int r; 1607 1608 DRM_DEBUG("\n"); 1609 1610 switch (adev->asic_type) { 1611 case CHIP_VEGA10: 1612 chip_name = "vega10"; 1613 break; 1614 case CHIP_VEGA12: 1615 chip_name = "vega12"; 1616 break; 1617 case CHIP_VEGA20: 1618 chip_name = "vega20"; 1619 break; 1620 case CHIP_RAVEN: 1621 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 1622 chip_name = "raven2"; 1623 else if (adev->apu_flags & AMD_APU_IS_PICASSO) 1624 chip_name = "picasso"; 1625 else 1626 chip_name = "raven"; 1627 break; 1628 case CHIP_ARCTURUS: 1629 chip_name = "arcturus"; 1630 break; 1631 case CHIP_RENOIR: 1632 chip_name = "renoir"; 1633 break; 1634 default: 1635 BUG(); 1636 } 1637 1638 /* No CPG in Arcturus */ 1639 if (adev->asic_type != CHIP_ARCTURUS) { 1640 r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name); 1641 if (r) 1642 return r; 1643 } 1644 1645 r = gfx_v9_0_init_rlc_microcode(adev, chip_name); 1646 if (r) 1647 return r; 1648 1649 r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name); 1650 if (r) 1651 return r; 1652 1653 return r; 1654 } 1655 1656 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev) 1657 { 1658 u32 count = 0; 1659 const struct cs_section_def *sect = NULL; 1660 const struct cs_extent_def *ext = NULL; 1661 1662 /* begin clear state */ 1663 count += 2; 1664 /* context control state */ 1665 count += 3; 1666 1667 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 1668 for (ext = sect->section; ext->extent != NULL; ++ext) { 1669 if (sect->id == SECT_CONTEXT) 1670 count += 2 + ext->reg_count; 1671 else 1672 return 0; 1673 } 1674 } 1675 1676 /* end clear state */ 1677 count += 2; 1678 /* clear state */ 1679 count += 2; 1680 1681 return count; 1682 } 1683 1684 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev, 1685 volatile u32 *buffer) 1686 { 1687 u32 count = 0, i; 1688 const struct cs_section_def *sect = NULL; 1689 const struct cs_extent_def *ext = NULL; 1690 1691 if (adev->gfx.rlc.cs_data == NULL) 1692 return; 1693 if (buffer == NULL) 1694 return; 1695 1696 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1697 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 1698 1699 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 1700 buffer[count++] = cpu_to_le32(0x80000000); 1701 buffer[count++] = cpu_to_le32(0x80000000); 1702 1703 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { 1704 for (ext = sect->section; ext->extent != NULL; ++ext) { 1705 if (sect->id == SECT_CONTEXT) { 1706 buffer[count++] = 1707 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 1708 buffer[count++] = cpu_to_le32(ext->reg_index - 1709 PACKET3_SET_CONTEXT_REG_START); 1710 for (i = 0; i < ext->reg_count; i++) 1711 buffer[count++] = cpu_to_le32(ext->extent[i]); 1712 } else { 1713 return; 1714 } 1715 } 1716 } 1717 1718 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1719 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 1720 1721 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 1722 buffer[count++] = cpu_to_le32(0); 1723 } 1724 1725 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev) 1726 { 1727 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; 1728 uint32_t pg_always_on_cu_num = 2; 1729 uint32_t always_on_cu_num; 1730 uint32_t i, j, k; 1731 uint32_t mask, cu_bitmap, counter; 1732 1733 if (adev->flags & AMD_IS_APU) 1734 always_on_cu_num = 4; 1735 else if (adev->asic_type == CHIP_VEGA12) 1736 always_on_cu_num = 8; 1737 else 1738 always_on_cu_num = 12; 1739 1740 mutex_lock(&adev->grbm_idx_mutex); 1741 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 1742 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 1743 mask = 1; 1744 cu_bitmap = 0; 1745 counter = 0; 1746 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 1747 1748 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 1749 if (cu_info->bitmap[i][j] & mask) { 1750 if (counter == pg_always_on_cu_num) 1751 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap); 1752 if (counter < always_on_cu_num) 1753 cu_bitmap |= mask; 1754 else 1755 break; 1756 counter++; 1757 } 1758 mask <<= 1; 1759 } 1760 1761 WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap); 1762 cu_info->ao_cu_bitmap[i][j] = cu_bitmap; 1763 } 1764 } 1765 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1766 mutex_unlock(&adev->grbm_idx_mutex); 1767 } 1768 1769 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev) 1770 { 1771 uint32_t data; 1772 1773 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1774 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1775 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7); 1776 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1777 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16)); 1778 1779 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1780 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1781 1782 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1783 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500); 1784 1785 mutex_lock(&adev->grbm_idx_mutex); 1786 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1787 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1788 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1789 1790 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1791 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1792 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1793 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1794 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1795 1796 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1797 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1798 data &= 0x0000FFFF; 1799 data |= 0x00C00000; 1800 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1801 1802 /* 1803 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven), 1804 * programmed in gfx_v9_0_init_always_on_cu_mask() 1805 */ 1806 1807 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1808 * but used for RLC_LB_CNTL configuration */ 1809 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1810 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1811 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1812 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1813 mutex_unlock(&adev->grbm_idx_mutex); 1814 1815 gfx_v9_0_init_always_on_cu_mask(adev); 1816 } 1817 1818 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev) 1819 { 1820 uint32_t data; 1821 1822 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1823 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1824 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8); 1825 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1826 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16)); 1827 1828 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1829 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1830 1831 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1832 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800); 1833 1834 mutex_lock(&adev->grbm_idx_mutex); 1835 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1836 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1837 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1838 1839 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1840 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1841 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1842 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1843 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1844 1845 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1846 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1847 data &= 0x0000FFFF; 1848 data |= 0x00C00000; 1849 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1850 1851 /* 1852 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON), 1853 * programmed in gfx_v9_0_init_always_on_cu_mask() 1854 */ 1855 1856 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1857 * but used for RLC_LB_CNTL configuration */ 1858 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1859 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1860 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1861 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1862 mutex_unlock(&adev->grbm_idx_mutex); 1863 1864 gfx_v9_0_init_always_on_cu_mask(adev); 1865 } 1866 1867 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable) 1868 { 1869 WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0); 1870 } 1871 1872 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev) 1873 { 1874 return 5; 1875 } 1876 1877 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev) 1878 { 1879 const struct cs_section_def *cs_data; 1880 int r; 1881 1882 adev->gfx.rlc.cs_data = gfx9_cs_data; 1883 1884 cs_data = adev->gfx.rlc.cs_data; 1885 1886 if (cs_data) { 1887 /* init clear state block */ 1888 r = amdgpu_gfx_rlc_init_csb(adev); 1889 if (r) 1890 return r; 1891 } 1892 1893 if (adev->flags & AMD_IS_APU) { 1894 /* TODO: double check the cp_table_size for RV */ 1895 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ 1896 r = amdgpu_gfx_rlc_init_cpt(adev); 1897 if (r) 1898 return r; 1899 } 1900 1901 switch (adev->asic_type) { 1902 case CHIP_RAVEN: 1903 gfx_v9_0_init_lbpw(adev); 1904 break; 1905 case CHIP_VEGA20: 1906 gfx_v9_4_init_lbpw(adev); 1907 break; 1908 default: 1909 break; 1910 } 1911 1912 /* init spm vmid with 0xf */ 1913 if (adev->gfx.rlc.funcs->update_spm_vmid) 1914 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf); 1915 1916 return 0; 1917 } 1918 1919 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev) 1920 { 1921 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 1922 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); 1923 } 1924 1925 static int gfx_v9_0_mec_init(struct amdgpu_device *adev) 1926 { 1927 int r; 1928 u32 *hpd; 1929 const __le32 *fw_data; 1930 unsigned fw_size; 1931 u32 *fw; 1932 size_t mec_hpd_size; 1933 1934 const struct gfx_firmware_header_v1_0 *mec_hdr; 1935 1936 bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 1937 1938 /* take ownership of the relevant compute queues */ 1939 amdgpu_gfx_compute_queue_acquire(adev); 1940 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE; 1941 1942 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 1943 AMDGPU_GEM_DOMAIN_VRAM, 1944 &adev->gfx.mec.hpd_eop_obj, 1945 &adev->gfx.mec.hpd_eop_gpu_addr, 1946 (void **)&hpd); 1947 if (r) { 1948 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 1949 gfx_v9_0_mec_fini(adev); 1950 return r; 1951 } 1952 1953 memset(hpd, 0, mec_hpd_size); 1954 1955 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 1956 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 1957 1958 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1959 1960 fw_data = (const __le32 *) 1961 (adev->gfx.mec_fw->data + 1962 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 1963 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes); 1964 1965 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes, 1966 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 1967 &adev->gfx.mec.mec_fw_obj, 1968 &adev->gfx.mec.mec_fw_gpu_addr, 1969 (void **)&fw); 1970 if (r) { 1971 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r); 1972 gfx_v9_0_mec_fini(adev); 1973 return r; 1974 } 1975 1976 memcpy(fw, fw_data, fw_size); 1977 1978 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 1979 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 1980 1981 return 0; 1982 } 1983 1984 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address) 1985 { 1986 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 1987 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 1988 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 1989 (address << SQ_IND_INDEX__INDEX__SHIFT) | 1990 (SQ_IND_INDEX__FORCE_READ_MASK)); 1991 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 1992 } 1993 1994 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd, 1995 uint32_t wave, uint32_t thread, 1996 uint32_t regno, uint32_t num, uint32_t *out) 1997 { 1998 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 1999 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 2000 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 2001 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 2002 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) | 2003 (SQ_IND_INDEX__FORCE_READ_MASK) | 2004 (SQ_IND_INDEX__AUTO_INCR_MASK)); 2005 while (num--) 2006 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 2007 } 2008 2009 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 2010 { 2011 /* type 1 wave data */ 2012 dst[(*no_fields)++] = 1; 2013 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS); 2014 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO); 2015 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI); 2016 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO); 2017 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI); 2018 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID); 2019 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0); 2020 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1); 2021 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC); 2022 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC); 2023 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS); 2024 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS); 2025 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); 2026 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); 2027 } 2028 2029 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd, 2030 uint32_t wave, uint32_t start, 2031 uint32_t size, uint32_t *dst) 2032 { 2033 wave_read_regs( 2034 adev, simd, wave, 0, 2035 start + SQIND_WAVE_SGPRS_OFFSET, size, dst); 2036 } 2037 2038 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd, 2039 uint32_t wave, uint32_t thread, 2040 uint32_t start, uint32_t size, 2041 uint32_t *dst) 2042 { 2043 wave_read_regs( 2044 adev, simd, wave, thread, 2045 start + SQIND_WAVE_VGPRS_OFFSET, size, dst); 2046 } 2047 2048 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev, 2049 u32 me, u32 pipe, u32 q, u32 vm) 2050 { 2051 soc15_grbm_select(adev, me, pipe, q, vm); 2052 } 2053 2054 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = { 2055 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 2056 .select_se_sh = &gfx_v9_0_select_se_sh, 2057 .read_wave_data = &gfx_v9_0_read_wave_data, 2058 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 2059 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 2060 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 2061 .ras_error_inject = &gfx_v9_0_ras_error_inject, 2062 .query_ras_error_count = &gfx_v9_0_query_ras_error_count, 2063 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count, 2064 }; 2065 2066 static const struct amdgpu_gfx_funcs gfx_v9_4_gfx_funcs = { 2067 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 2068 .select_se_sh = &gfx_v9_0_select_se_sh, 2069 .read_wave_data = &gfx_v9_0_read_wave_data, 2070 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 2071 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 2072 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 2073 .ras_error_inject = &gfx_v9_4_ras_error_inject, 2074 .query_ras_error_count = &gfx_v9_4_query_ras_error_count, 2075 .reset_ras_error_count = &gfx_v9_4_reset_ras_error_count, 2076 }; 2077 2078 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev) 2079 { 2080 u32 gb_addr_config; 2081 int err; 2082 2083 adev->gfx.funcs = &gfx_v9_0_gfx_funcs; 2084 2085 switch (adev->asic_type) { 2086 case CHIP_VEGA10: 2087 adev->gfx.config.max_hw_contexts = 8; 2088 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2089 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2090 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2091 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2092 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN; 2093 break; 2094 case CHIP_VEGA12: 2095 adev->gfx.config.max_hw_contexts = 8; 2096 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2097 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2098 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2099 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2100 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN; 2101 DRM_INFO("fix gfx.config for vega12\n"); 2102 break; 2103 case CHIP_VEGA20: 2104 adev->gfx.config.max_hw_contexts = 8; 2105 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2106 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2107 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2108 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2109 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2110 gb_addr_config &= ~0xf3e777ff; 2111 gb_addr_config |= 0x22014042; 2112 /* check vbios table if gpu info is not available */ 2113 err = amdgpu_atomfirmware_get_gfx_info(adev); 2114 if (err) 2115 return err; 2116 break; 2117 case CHIP_RAVEN: 2118 adev->gfx.config.max_hw_contexts = 8; 2119 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2120 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2121 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2122 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2123 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 2124 gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN; 2125 else 2126 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN; 2127 break; 2128 case CHIP_ARCTURUS: 2129 adev->gfx.funcs = &gfx_v9_4_gfx_funcs; 2130 adev->gfx.config.max_hw_contexts = 8; 2131 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2132 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2133 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2134 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2135 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2136 gb_addr_config &= ~0xf3e777ff; 2137 gb_addr_config |= 0x22014042; 2138 break; 2139 case CHIP_RENOIR: 2140 adev->gfx.config.max_hw_contexts = 8; 2141 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2142 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2143 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; 2144 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2145 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2146 gb_addr_config &= ~0xf3e777ff; 2147 gb_addr_config |= 0x22010042; 2148 break; 2149 default: 2150 BUG(); 2151 break; 2152 } 2153 2154 adev->gfx.config.gb_addr_config = gb_addr_config; 2155 2156 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << 2157 REG_GET_FIELD( 2158 adev->gfx.config.gb_addr_config, 2159 GB_ADDR_CONFIG, 2160 NUM_PIPES); 2161 2162 adev->gfx.config.max_tile_pipes = 2163 adev->gfx.config.gb_addr_config_fields.num_pipes; 2164 2165 adev->gfx.config.gb_addr_config_fields.num_banks = 1 << 2166 REG_GET_FIELD( 2167 adev->gfx.config.gb_addr_config, 2168 GB_ADDR_CONFIG, 2169 NUM_BANKS); 2170 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << 2171 REG_GET_FIELD( 2172 adev->gfx.config.gb_addr_config, 2173 GB_ADDR_CONFIG, 2174 MAX_COMPRESSED_FRAGS); 2175 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << 2176 REG_GET_FIELD( 2177 adev->gfx.config.gb_addr_config, 2178 GB_ADDR_CONFIG, 2179 NUM_RB_PER_SE); 2180 adev->gfx.config.gb_addr_config_fields.num_se = 1 << 2181 REG_GET_FIELD( 2182 adev->gfx.config.gb_addr_config, 2183 GB_ADDR_CONFIG, 2184 NUM_SHADER_ENGINES); 2185 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + 2186 REG_GET_FIELD( 2187 adev->gfx.config.gb_addr_config, 2188 GB_ADDR_CONFIG, 2189 PIPE_INTERLEAVE_SIZE)); 2190 2191 return 0; 2192 } 2193 2194 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 2195 int mec, int pipe, int queue) 2196 { 2197 int r; 2198 unsigned irq_type; 2199 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; 2200 unsigned int hw_prio; 2201 2202 ring = &adev->gfx.compute_ring[ring_id]; 2203 2204 /* mec0 is me1 */ 2205 ring->me = mec + 1; 2206 ring->pipe = pipe; 2207 ring->queue = queue; 2208 2209 ring->ring_obj = NULL; 2210 ring->use_doorbell = true; 2211 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1; 2212 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 2213 + (ring_id * GFX9_MEC_HPD_SIZE); 2214 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 2215 2216 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 2217 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 2218 + ring->pipe; 2219 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring->queue) ? 2220 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 2221 /* type-2 packets are deprecated on MEC, use type-3 instead */ 2222 r = amdgpu_ring_init(adev, ring, 1024, 2223 &adev->gfx.eop_irq, irq_type, hw_prio); 2224 if (r) 2225 return r; 2226 2227 2228 return 0; 2229 } 2230 2231 static int gfx_v9_0_sw_init(void *handle) 2232 { 2233 int i, j, k, r, ring_id; 2234 struct amdgpu_ring *ring; 2235 struct amdgpu_kiq *kiq; 2236 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2237 2238 switch (adev->asic_type) { 2239 case CHIP_VEGA10: 2240 case CHIP_VEGA12: 2241 case CHIP_VEGA20: 2242 case CHIP_RAVEN: 2243 case CHIP_ARCTURUS: 2244 case CHIP_RENOIR: 2245 adev->gfx.mec.num_mec = 2; 2246 break; 2247 default: 2248 adev->gfx.mec.num_mec = 1; 2249 break; 2250 } 2251 2252 adev->gfx.mec.num_pipe_per_mec = 4; 2253 adev->gfx.mec.num_queue_per_pipe = 8; 2254 2255 /* EOP Event */ 2256 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq); 2257 if (r) 2258 return r; 2259 2260 /* Privileged reg */ 2261 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT, 2262 &adev->gfx.priv_reg_irq); 2263 if (r) 2264 return r; 2265 2266 /* Privileged inst */ 2267 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT, 2268 &adev->gfx.priv_inst_irq); 2269 if (r) 2270 return r; 2271 2272 /* ECC error */ 2273 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR, 2274 &adev->gfx.cp_ecc_error_irq); 2275 if (r) 2276 return r; 2277 2278 /* FUE error */ 2279 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR, 2280 &adev->gfx.cp_ecc_error_irq); 2281 if (r) 2282 return r; 2283 2284 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 2285 2286 gfx_v9_0_scratch_init(adev); 2287 2288 r = gfx_v9_0_init_microcode(adev); 2289 if (r) { 2290 DRM_ERROR("Failed to load gfx firmware!\n"); 2291 return r; 2292 } 2293 2294 r = adev->gfx.rlc.funcs->init(adev); 2295 if (r) { 2296 DRM_ERROR("Failed to init rlc BOs!\n"); 2297 return r; 2298 } 2299 2300 r = gfx_v9_0_mec_init(adev); 2301 if (r) { 2302 DRM_ERROR("Failed to init MEC BOs!\n"); 2303 return r; 2304 } 2305 2306 /* set up the gfx ring */ 2307 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 2308 ring = &adev->gfx.gfx_ring[i]; 2309 ring->ring_obj = NULL; 2310 if (!i) 2311 sprintf(ring->name, "gfx"); 2312 else 2313 sprintf(ring->name, "gfx_%d", i); 2314 ring->use_doorbell = true; 2315 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 2316 r = amdgpu_ring_init(adev, ring, 1024, 2317 &adev->gfx.eop_irq, 2318 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, 2319 AMDGPU_RING_PRIO_DEFAULT); 2320 if (r) 2321 return r; 2322 } 2323 2324 /* set up the compute queues - allocate horizontally across pipes */ 2325 ring_id = 0; 2326 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 2327 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 2328 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 2329 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j)) 2330 continue; 2331 2332 r = gfx_v9_0_compute_ring_init(adev, 2333 ring_id, 2334 i, k, j); 2335 if (r) 2336 return r; 2337 2338 ring_id++; 2339 } 2340 } 2341 } 2342 2343 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE); 2344 if (r) { 2345 DRM_ERROR("Failed to init KIQ BOs!\n"); 2346 return r; 2347 } 2348 2349 kiq = &adev->gfx.kiq; 2350 r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq); 2351 if (r) 2352 return r; 2353 2354 /* create MQD for all compute queues as wel as KIQ for SRIOV case */ 2355 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation)); 2356 if (r) 2357 return r; 2358 2359 adev->gfx.ce_ram_size = 0x8000; 2360 2361 r = gfx_v9_0_gpu_early_init(adev); 2362 if (r) 2363 return r; 2364 2365 return 0; 2366 } 2367 2368 2369 static int gfx_v9_0_sw_fini(void *handle) 2370 { 2371 int i; 2372 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2373 2374 amdgpu_gfx_ras_fini(adev); 2375 2376 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 2377 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 2378 for (i = 0; i < adev->gfx.num_compute_rings; i++) 2379 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 2380 2381 amdgpu_gfx_mqd_sw_fini(adev); 2382 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring); 2383 amdgpu_gfx_kiq_fini(adev); 2384 2385 gfx_v9_0_mec_fini(adev); 2386 amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj); 2387 if (adev->flags & AMD_IS_APU) { 2388 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 2389 &adev->gfx.rlc.cp_table_gpu_addr, 2390 (void **)&adev->gfx.rlc.cp_table_ptr); 2391 } 2392 gfx_v9_0_free_microcode(adev); 2393 2394 return 0; 2395 } 2396 2397 2398 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev) 2399 { 2400 /* TODO */ 2401 } 2402 2403 static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance) 2404 { 2405 u32 data; 2406 2407 if (instance == 0xffffffff) 2408 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); 2409 else 2410 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance); 2411 2412 if (se_num == 0xffffffff) 2413 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); 2414 else 2415 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 2416 2417 if (sh_num == 0xffffffff) 2418 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1); 2419 else 2420 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num); 2421 2422 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data); 2423 } 2424 2425 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev) 2426 { 2427 u32 data, mask; 2428 2429 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE); 2430 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE); 2431 2432 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK; 2433 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT; 2434 2435 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / 2436 adev->gfx.config.max_sh_per_se); 2437 2438 return (~data) & mask; 2439 } 2440 2441 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev) 2442 { 2443 int i, j; 2444 u32 data; 2445 u32 active_rbs = 0; 2446 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / 2447 adev->gfx.config.max_sh_per_se; 2448 2449 mutex_lock(&adev->grbm_idx_mutex); 2450 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2451 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2452 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2453 data = gfx_v9_0_get_rb_active_bitmap(adev); 2454 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * 2455 rb_bitmap_width_per_sh); 2456 } 2457 } 2458 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2459 mutex_unlock(&adev->grbm_idx_mutex); 2460 2461 adev->gfx.config.backend_enable_mask = active_rbs; 2462 adev->gfx.config.num_rbs = hweight32(active_rbs); 2463 } 2464 2465 #define DEFAULT_SH_MEM_BASES (0x6000) 2466 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev) 2467 { 2468 int i; 2469 uint32_t sh_mem_config; 2470 uint32_t sh_mem_bases; 2471 2472 /* 2473 * Configure apertures: 2474 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 2475 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 2476 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 2477 */ 2478 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16); 2479 2480 sh_mem_config = SH_MEM_ADDRESS_MODE_64 | 2481 SH_MEM_ALIGNMENT_MODE_UNALIGNED << 2482 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT; 2483 2484 mutex_lock(&adev->srbm_mutex); 2485 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2486 soc15_grbm_select(adev, 0, 0, 0, i); 2487 /* CP and shaders */ 2488 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config); 2489 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases); 2490 } 2491 soc15_grbm_select(adev, 0, 0, 0, 0); 2492 mutex_unlock(&adev->srbm_mutex); 2493 2494 /* Initialize all compute VMIDs to have no GDS, GWS, or OA 2495 acccess. These should be enabled by FW for target VMIDs. */ 2496 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2497 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0); 2498 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0); 2499 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0); 2500 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0); 2501 } 2502 } 2503 2504 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev) 2505 { 2506 int vmid; 2507 2508 /* 2509 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 2510 * access. Compute VMIDs should be enabled by FW for target VMIDs, 2511 * the driver can enable them for graphics. VMID0 should maintain 2512 * access so that HWS firmware can save/restore entries. 2513 */ 2514 for (vmid = 1; vmid < 16; vmid++) { 2515 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0); 2516 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0); 2517 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0); 2518 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0); 2519 } 2520 } 2521 2522 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev) 2523 { 2524 uint32_t tmp; 2525 2526 switch (adev->asic_type) { 2527 case CHIP_ARCTURUS: 2528 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG); 2529 tmp = REG_SET_FIELD(tmp, SQ_CONFIG, 2530 DISABLE_BARRIER_WAITCNT, 1); 2531 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp); 2532 break; 2533 default: 2534 break; 2535 } 2536 } 2537 2538 static void gfx_v9_0_constants_init(struct amdgpu_device *adev) 2539 { 2540 u32 tmp; 2541 int i; 2542 2543 WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); 2544 2545 gfx_v9_0_tiling_mode_table_init(adev); 2546 2547 gfx_v9_0_setup_rb(adev); 2548 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info); 2549 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2); 2550 2551 /* XXX SH_MEM regs */ 2552 /* where to put LDS, scratch, GPUVM in FSA64 space */ 2553 mutex_lock(&adev->srbm_mutex); 2554 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) { 2555 soc15_grbm_select(adev, 0, 0, 0, i); 2556 /* CP and shaders */ 2557 if (i == 0) { 2558 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2559 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2560 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2561 !!amdgpu_noretry); 2562 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2563 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0); 2564 } else { 2565 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2566 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2567 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2568 !!amdgpu_noretry); 2569 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2570 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 2571 (adev->gmc.private_aperture_start >> 48)); 2572 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE, 2573 (adev->gmc.shared_aperture_start >> 48)); 2574 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp); 2575 } 2576 } 2577 soc15_grbm_select(adev, 0, 0, 0, 0); 2578 2579 mutex_unlock(&adev->srbm_mutex); 2580 2581 gfx_v9_0_init_compute_vmid(adev); 2582 gfx_v9_0_init_gds_vmid(adev); 2583 gfx_v9_0_init_sq_config(adev); 2584 } 2585 2586 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev) 2587 { 2588 u32 i, j, k; 2589 u32 mask; 2590 2591 mutex_lock(&adev->grbm_idx_mutex); 2592 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2593 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2594 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2595 for (k = 0; k < adev->usec_timeout; k++) { 2596 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0) 2597 break; 2598 udelay(1); 2599 } 2600 if (k == adev->usec_timeout) { 2601 gfx_v9_0_select_se_sh(adev, 0xffffffff, 2602 0xffffffff, 0xffffffff); 2603 mutex_unlock(&adev->grbm_idx_mutex); 2604 DRM_INFO("Timeout wait for RLC serdes %u,%u\n", 2605 i, j); 2606 return; 2607 } 2608 } 2609 } 2610 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2611 mutex_unlock(&adev->grbm_idx_mutex); 2612 2613 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | 2614 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK | 2615 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK | 2616 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK; 2617 for (k = 0; k < adev->usec_timeout; k++) { 2618 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 2619 break; 2620 udelay(1); 2621 } 2622 } 2623 2624 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 2625 bool enable) 2626 { 2627 u32 tmp = RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0); 2628 2629 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0); 2630 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0); 2631 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0); 2632 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0); 2633 2634 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp); 2635 } 2636 2637 static void gfx_v9_0_init_csb(struct amdgpu_device *adev) 2638 { 2639 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 2640 /* csib */ 2641 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI), 2642 adev->gfx.rlc.clear_state_gpu_addr >> 32); 2643 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO), 2644 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 2645 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH), 2646 adev->gfx.rlc.clear_state_size); 2647 } 2648 2649 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format, 2650 int indirect_offset, 2651 int list_size, 2652 int *unique_indirect_regs, 2653 int unique_indirect_reg_count, 2654 int *indirect_start_offsets, 2655 int *indirect_start_offsets_count, 2656 int max_start_offsets_count) 2657 { 2658 int idx; 2659 2660 for (; indirect_offset < list_size; indirect_offset++) { 2661 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count); 2662 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset; 2663 *indirect_start_offsets_count = *indirect_start_offsets_count + 1; 2664 2665 while (register_list_format[indirect_offset] != 0xFFFFFFFF) { 2666 indirect_offset += 2; 2667 2668 /* look for the matching indice */ 2669 for (idx = 0; idx < unique_indirect_reg_count; idx++) { 2670 if (unique_indirect_regs[idx] == 2671 register_list_format[indirect_offset] || 2672 !unique_indirect_regs[idx]) 2673 break; 2674 } 2675 2676 BUG_ON(idx >= unique_indirect_reg_count); 2677 2678 if (!unique_indirect_regs[idx]) 2679 unique_indirect_regs[idx] = register_list_format[indirect_offset]; 2680 2681 indirect_offset++; 2682 } 2683 } 2684 } 2685 2686 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev) 2687 { 2688 int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2689 int unique_indirect_reg_count = 0; 2690 2691 int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2692 int indirect_start_offsets_count = 0; 2693 2694 int list_size = 0; 2695 int i = 0, j = 0; 2696 u32 tmp = 0; 2697 2698 u32 *register_list_format = 2699 kmemdup(adev->gfx.rlc.register_list_format, 2700 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); 2701 if (!register_list_format) 2702 return -ENOMEM; 2703 2704 /* setup unique_indirect_regs array and indirect_start_offsets array */ 2705 unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs); 2706 gfx_v9_1_parse_ind_reg_list(register_list_format, 2707 adev->gfx.rlc.reg_list_format_direct_reg_list_length, 2708 adev->gfx.rlc.reg_list_format_size_bytes >> 2, 2709 unique_indirect_regs, 2710 unique_indirect_reg_count, 2711 indirect_start_offsets, 2712 &indirect_start_offsets_count, 2713 ARRAY_SIZE(indirect_start_offsets)); 2714 2715 /* enable auto inc in case it is disabled */ 2716 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL)); 2717 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK; 2718 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp); 2719 2720 /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */ 2721 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR), 2722 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET); 2723 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) 2724 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA), 2725 adev->gfx.rlc.register_restore[i]); 2726 2727 /* load indirect register */ 2728 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2729 adev->gfx.rlc.reg_list_format_start); 2730 2731 /* direct register portion */ 2732 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++) 2733 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2734 register_list_format[i]); 2735 2736 /* indirect register portion */ 2737 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) { 2738 if (register_list_format[i] == 0xFFFFFFFF) { 2739 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2740 continue; 2741 } 2742 2743 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2744 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2745 2746 for (j = 0; j < unique_indirect_reg_count; j++) { 2747 if (register_list_format[i] == unique_indirect_regs[j]) { 2748 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j); 2749 break; 2750 } 2751 } 2752 2753 BUG_ON(j >= unique_indirect_reg_count); 2754 2755 i++; 2756 } 2757 2758 /* set save/restore list size */ 2759 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; 2760 list_size = list_size >> 1; 2761 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2762 adev->gfx.rlc.reg_restore_list_size); 2763 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size); 2764 2765 /* write the starting offsets to RLC scratch ram */ 2766 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2767 adev->gfx.rlc.starting_offsets_start); 2768 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++) 2769 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2770 indirect_start_offsets[i]); 2771 2772 /* load unique indirect regs*/ 2773 for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) { 2774 if (unique_indirect_regs[i] != 0) { 2775 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0) 2776 + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i], 2777 unique_indirect_regs[i] & 0x3FFFF); 2778 2779 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0) 2780 + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i], 2781 unique_indirect_regs[i] >> 20); 2782 } 2783 } 2784 2785 kfree(register_list_format); 2786 return 0; 2787 } 2788 2789 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev) 2790 { 2791 WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1); 2792 } 2793 2794 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev, 2795 bool enable) 2796 { 2797 uint32_t data = 0; 2798 uint32_t default_data = 0; 2799 2800 default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS)); 2801 if (enable == true) { 2802 /* enable GFXIP control over CGPG */ 2803 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2804 if(default_data != data) 2805 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2806 2807 /* update status */ 2808 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK; 2809 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT); 2810 if(default_data != data) 2811 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2812 } else { 2813 /* restore GFXIP control over GCPG */ 2814 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2815 if(default_data != data) 2816 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2817 } 2818 } 2819 2820 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev) 2821 { 2822 uint32_t data = 0; 2823 2824 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2825 AMD_PG_SUPPORT_GFX_SMG | 2826 AMD_PG_SUPPORT_GFX_DMG)) { 2827 /* init IDLE_POLL_COUNT = 60 */ 2828 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL)); 2829 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK; 2830 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 2831 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data); 2832 2833 /* init RLC PG Delay */ 2834 data = 0; 2835 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT); 2836 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT); 2837 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT); 2838 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT); 2839 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data); 2840 2841 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2)); 2842 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK; 2843 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT); 2844 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data); 2845 2846 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3)); 2847 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK; 2848 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT); 2849 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data); 2850 2851 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL)); 2852 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK; 2853 2854 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */ 2855 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT); 2856 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data); 2857 if (adev->asic_type != CHIP_RENOIR) 2858 pwr_10_0_gfxip_control_over_cgpg(adev, true); 2859 } 2860 } 2861 2862 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev, 2863 bool enable) 2864 { 2865 uint32_t data = 0; 2866 uint32_t default_data = 0; 2867 2868 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2869 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2870 SMU_CLK_SLOWDOWN_ON_PU_ENABLE, 2871 enable ? 1 : 0); 2872 if (default_data != data) 2873 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2874 } 2875 2876 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev, 2877 bool enable) 2878 { 2879 uint32_t data = 0; 2880 uint32_t default_data = 0; 2881 2882 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2883 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2884 SMU_CLK_SLOWDOWN_ON_PD_ENABLE, 2885 enable ? 1 : 0); 2886 if(default_data != data) 2887 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2888 } 2889 2890 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev, 2891 bool enable) 2892 { 2893 uint32_t data = 0; 2894 uint32_t default_data = 0; 2895 2896 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2897 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2898 CP_PG_DISABLE, 2899 enable ? 0 : 1); 2900 if(default_data != data) 2901 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2902 } 2903 2904 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev, 2905 bool enable) 2906 { 2907 uint32_t data, default_data; 2908 2909 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2910 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2911 GFX_POWER_GATING_ENABLE, 2912 enable ? 1 : 0); 2913 if(default_data != data) 2914 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2915 } 2916 2917 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev, 2918 bool enable) 2919 { 2920 uint32_t data, default_data; 2921 2922 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2923 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2924 GFX_PIPELINE_PG_ENABLE, 2925 enable ? 1 : 0); 2926 if(default_data != data) 2927 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2928 2929 if (!enable) 2930 /* read any GFX register to wake up GFX */ 2931 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL)); 2932 } 2933 2934 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, 2935 bool enable) 2936 { 2937 uint32_t data, default_data; 2938 2939 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2940 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2941 STATIC_PER_CU_PG_ENABLE, 2942 enable ? 1 : 0); 2943 if(default_data != data) 2944 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2945 } 2946 2947 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev, 2948 bool enable) 2949 { 2950 uint32_t data, default_data; 2951 2952 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2953 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2954 DYN_PER_CU_PG_ENABLE, 2955 enable ? 1 : 0); 2956 if(default_data != data) 2957 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2958 } 2959 2960 static void gfx_v9_0_init_pg(struct amdgpu_device *adev) 2961 { 2962 gfx_v9_0_init_csb(adev); 2963 2964 /* 2965 * Rlc save restore list is workable since v2_1. 2966 * And it's needed by gfxoff feature. 2967 */ 2968 if (adev->gfx.rlc.is_rlc_v2_1) { 2969 if (adev->asic_type == CHIP_VEGA12 || 2970 (adev->apu_flags & AMD_APU_IS_RAVEN2)) 2971 gfx_v9_1_init_rlc_save_restore_list(adev); 2972 gfx_v9_0_enable_save_restore_machine(adev); 2973 } 2974 2975 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2976 AMD_PG_SUPPORT_GFX_SMG | 2977 AMD_PG_SUPPORT_GFX_DMG | 2978 AMD_PG_SUPPORT_CP | 2979 AMD_PG_SUPPORT_GDS | 2980 AMD_PG_SUPPORT_RLC_SMU_HS)) { 2981 WREG32(mmRLC_JUMP_TABLE_RESTORE, 2982 adev->gfx.rlc.cp_table_gpu_addr >> 8); 2983 gfx_v9_0_init_gfx_power_gating(adev); 2984 } 2985 } 2986 2987 void gfx_v9_0_rlc_stop(struct amdgpu_device *adev) 2988 { 2989 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0); 2990 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 2991 gfx_v9_0_wait_for_rlc_serdes(adev); 2992 } 2993 2994 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev) 2995 { 2996 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 2997 udelay(50); 2998 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 2999 udelay(50); 3000 } 3001 3002 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev) 3003 { 3004 #ifdef AMDGPU_RLC_DEBUG_RETRY 3005 u32 rlc_ucode_ver; 3006 #endif 3007 3008 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1); 3009 udelay(50); 3010 3011 /* carrizo do enable cp interrupt after cp inited */ 3012 if (!(adev->flags & AMD_IS_APU)) { 3013 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3014 udelay(50); 3015 } 3016 3017 #ifdef AMDGPU_RLC_DEBUG_RETRY 3018 /* RLC_GPM_GENERAL_6 : RLC Ucode version */ 3019 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6); 3020 if(rlc_ucode_ver == 0x108) { 3021 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n", 3022 rlc_ucode_ver, adev->gfx.rlc_fw_version); 3023 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles, 3024 * default is 0x9C4 to create a 100us interval */ 3025 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4); 3026 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr 3027 * to disable the page fault retry interrupts, default is 3028 * 0x100 (256) */ 3029 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100); 3030 } 3031 #endif 3032 } 3033 3034 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev) 3035 { 3036 const struct rlc_firmware_header_v2_0 *hdr; 3037 const __le32 *fw_data; 3038 unsigned i, fw_size; 3039 3040 if (!adev->gfx.rlc_fw) 3041 return -EINVAL; 3042 3043 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 3044 amdgpu_ucode_print_rlc_hdr(&hdr->header); 3045 3046 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 3047 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 3048 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 3049 3050 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, 3051 RLCG_UCODE_LOADING_START_ADDRESS); 3052 for (i = 0; i < fw_size; i++) 3053 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); 3054 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); 3055 3056 return 0; 3057 } 3058 3059 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev) 3060 { 3061 int r; 3062 3063 if (amdgpu_sriov_vf(adev)) { 3064 gfx_v9_0_init_csb(adev); 3065 return 0; 3066 } 3067 3068 adev->gfx.rlc.funcs->stop(adev); 3069 3070 /* disable CG */ 3071 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0); 3072 3073 gfx_v9_0_init_pg(adev); 3074 3075 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3076 /* legacy rlc firmware loading */ 3077 r = gfx_v9_0_rlc_load_microcode(adev); 3078 if (r) 3079 return r; 3080 } 3081 3082 switch (adev->asic_type) { 3083 case CHIP_RAVEN: 3084 if (amdgpu_lbpw == 0) 3085 gfx_v9_0_enable_lbpw(adev, false); 3086 else 3087 gfx_v9_0_enable_lbpw(adev, true); 3088 break; 3089 case CHIP_VEGA20: 3090 if (amdgpu_lbpw > 0) 3091 gfx_v9_0_enable_lbpw(adev, true); 3092 else 3093 gfx_v9_0_enable_lbpw(adev, false); 3094 break; 3095 default: 3096 break; 3097 } 3098 3099 adev->gfx.rlc.funcs->start(adev); 3100 3101 return 0; 3102 } 3103 3104 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 3105 { 3106 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL); 3107 3108 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); 3109 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); 3110 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1); 3111 WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp); 3112 udelay(50); 3113 } 3114 3115 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev) 3116 { 3117 const struct gfx_firmware_header_v1_0 *pfp_hdr; 3118 const struct gfx_firmware_header_v1_0 *ce_hdr; 3119 const struct gfx_firmware_header_v1_0 *me_hdr; 3120 const __le32 *fw_data; 3121 unsigned i, fw_size; 3122 3123 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw) 3124 return -EINVAL; 3125 3126 pfp_hdr = (const struct gfx_firmware_header_v1_0 *) 3127 adev->gfx.pfp_fw->data; 3128 ce_hdr = (const struct gfx_firmware_header_v1_0 *) 3129 adev->gfx.ce_fw->data; 3130 me_hdr = (const struct gfx_firmware_header_v1_0 *) 3131 adev->gfx.me_fw->data; 3132 3133 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3134 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header); 3135 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3136 3137 gfx_v9_0_cp_gfx_enable(adev, false); 3138 3139 /* PFP */ 3140 fw_data = (const __le32 *) 3141 (adev->gfx.pfp_fw->data + 3142 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 3143 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 3144 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0); 3145 for (i = 0; i < fw_size; i++) 3146 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 3147 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); 3148 3149 /* CE */ 3150 fw_data = (const __le32 *) 3151 (adev->gfx.ce_fw->data + 3152 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 3153 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 3154 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0); 3155 for (i = 0; i < fw_size; i++) 3156 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 3157 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version); 3158 3159 /* ME */ 3160 fw_data = (const __le32 *) 3161 (adev->gfx.me_fw->data + 3162 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 3163 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 3164 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0); 3165 for (i = 0; i < fw_size; i++) 3166 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 3167 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version); 3168 3169 return 0; 3170 } 3171 3172 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev) 3173 { 3174 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; 3175 const struct cs_section_def *sect = NULL; 3176 const struct cs_extent_def *ext = NULL; 3177 int r, i, tmp; 3178 3179 /* init the CP */ 3180 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); 3181 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1); 3182 3183 gfx_v9_0_cp_gfx_enable(adev, true); 3184 3185 r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3); 3186 if (r) { 3187 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r); 3188 return r; 3189 } 3190 3191 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3192 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3193 3194 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 3195 amdgpu_ring_write(ring, 0x80000000); 3196 amdgpu_ring_write(ring, 0x80000000); 3197 3198 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 3199 for (ext = sect->section; ext->extent != NULL; ++ext) { 3200 if (sect->id == SECT_CONTEXT) { 3201 amdgpu_ring_write(ring, 3202 PACKET3(PACKET3_SET_CONTEXT_REG, 3203 ext->reg_count)); 3204 amdgpu_ring_write(ring, 3205 ext->reg_index - PACKET3_SET_CONTEXT_REG_START); 3206 for (i = 0; i < ext->reg_count; i++) 3207 amdgpu_ring_write(ring, ext->extent[i]); 3208 } 3209 } 3210 } 3211 3212 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3213 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3214 3215 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3216 amdgpu_ring_write(ring, 0); 3217 3218 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 3219 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 3220 amdgpu_ring_write(ring, 0x8000); 3221 amdgpu_ring_write(ring, 0x8000); 3222 3223 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1)); 3224 tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE | 3225 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START)); 3226 amdgpu_ring_write(ring, tmp); 3227 amdgpu_ring_write(ring, 0); 3228 3229 amdgpu_ring_commit(ring); 3230 3231 return 0; 3232 } 3233 3234 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev) 3235 { 3236 struct amdgpu_ring *ring; 3237 u32 tmp; 3238 u32 rb_bufsz; 3239 u64 rb_addr, rptr_addr, wptr_gpu_addr; 3240 3241 /* Set the write pointer delay */ 3242 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0); 3243 3244 /* set the RB to use vmid 0 */ 3245 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0); 3246 3247 /* Set ring buffer size */ 3248 ring = &adev->gfx.gfx_ring[0]; 3249 rb_bufsz = order_base_2(ring->ring_size / 8); 3250 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 3251 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 3252 #ifdef __BIG_ENDIAN 3253 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1); 3254 #endif 3255 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3256 3257 /* Initialize the ring buffer's write pointers */ 3258 ring->wptr = 0; 3259 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 3260 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 3261 3262 /* set the wb address wether it's enabled or not */ 3263 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3264 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 3265 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3266 3267 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3268 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr)); 3269 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr)); 3270 3271 mdelay(1); 3272 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3273 3274 rb_addr = ring->gpu_addr >> 8; 3275 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr); 3276 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 3277 3278 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL); 3279 if (ring->use_doorbell) { 3280 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3281 DOORBELL_OFFSET, ring->doorbell_index); 3282 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3283 DOORBELL_EN, 1); 3284 } else { 3285 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0); 3286 } 3287 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp); 3288 3289 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 3290 DOORBELL_RANGE_LOWER, ring->doorbell_index); 3291 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp); 3292 3293 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER, 3294 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 3295 3296 3297 /* start the ring */ 3298 gfx_v9_0_cp_gfx_start(adev); 3299 ring->sched.ready = true; 3300 3301 return 0; 3302 } 3303 3304 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 3305 { 3306 if (enable) { 3307 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0); 3308 } else { 3309 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 3310 (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK)); 3311 adev->gfx.kiq.ring.sched.ready = false; 3312 } 3313 udelay(50); 3314 } 3315 3316 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev) 3317 { 3318 const struct gfx_firmware_header_v1_0 *mec_hdr; 3319 const __le32 *fw_data; 3320 unsigned i; 3321 u32 tmp; 3322 3323 if (!adev->gfx.mec_fw) 3324 return -EINVAL; 3325 3326 gfx_v9_0_cp_compute_enable(adev, false); 3327 3328 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 3329 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3330 3331 fw_data = (const __le32 *) 3332 (adev->gfx.mec_fw->data + 3333 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 3334 tmp = 0; 3335 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 3336 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 3337 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp); 3338 3339 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO, 3340 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000); 3341 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI, 3342 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 3343 3344 /* MEC1 */ 3345 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3346 mec_hdr->jt_offset); 3347 for (i = 0; i < mec_hdr->jt_size; i++) 3348 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA, 3349 le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); 3350 3351 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3352 adev->gfx.mec_fw_version); 3353 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */ 3354 3355 return 0; 3356 } 3357 3358 /* KIQ functions */ 3359 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring) 3360 { 3361 uint32_t tmp; 3362 struct amdgpu_device *adev = ring->adev; 3363 3364 /* tell RLC which is KIQ queue */ 3365 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS); 3366 tmp &= 0xffffff00; 3367 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 3368 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3369 tmp |= 0x80; 3370 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3371 } 3372 3373 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd) 3374 { 3375 struct amdgpu_device *adev = ring->adev; 3376 3377 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 3378 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring->queue)) { 3379 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; 3380 mqd->cp_hqd_queue_priority = 3381 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; 3382 } 3383 } 3384 } 3385 3386 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring) 3387 { 3388 struct amdgpu_device *adev = ring->adev; 3389 struct v9_mqd *mqd = ring->mqd_ptr; 3390 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 3391 uint32_t tmp; 3392 3393 mqd->header = 0xC0310800; 3394 mqd->compute_pipelinestat_enable = 0x00000001; 3395 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 3396 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 3397 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 3398 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 3399 mqd->compute_static_thread_mgmt_se4 = 0xffffffff; 3400 mqd->compute_static_thread_mgmt_se5 = 0xffffffff; 3401 mqd->compute_static_thread_mgmt_se6 = 0xffffffff; 3402 mqd->compute_static_thread_mgmt_se7 = 0xffffffff; 3403 mqd->compute_misc_reserved = 0x00000003; 3404 3405 mqd->dynamic_cu_mask_addr_lo = 3406 lower_32_bits(ring->mqd_gpu_addr 3407 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3408 mqd->dynamic_cu_mask_addr_hi = 3409 upper_32_bits(ring->mqd_gpu_addr 3410 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3411 3412 eop_base_addr = ring->eop_gpu_addr >> 8; 3413 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 3414 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 3415 3416 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3417 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL); 3418 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 3419 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1)); 3420 3421 mqd->cp_hqd_eop_control = tmp; 3422 3423 /* enable doorbell? */ 3424 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3425 3426 if (ring->use_doorbell) { 3427 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3428 DOORBELL_OFFSET, ring->doorbell_index); 3429 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3430 DOORBELL_EN, 1); 3431 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3432 DOORBELL_SOURCE, 0); 3433 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3434 DOORBELL_HIT, 0); 3435 } else { 3436 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3437 DOORBELL_EN, 0); 3438 } 3439 3440 mqd->cp_hqd_pq_doorbell_control = tmp; 3441 3442 /* disable the queue if it's active */ 3443 ring->wptr = 0; 3444 mqd->cp_hqd_dequeue_request = 0; 3445 mqd->cp_hqd_pq_rptr = 0; 3446 mqd->cp_hqd_pq_wptr_lo = 0; 3447 mqd->cp_hqd_pq_wptr_hi = 0; 3448 3449 /* set the pointer to the MQD */ 3450 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; 3451 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); 3452 3453 /* set MQD vmid to 0 */ 3454 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL); 3455 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 3456 mqd->cp_mqd_control = tmp; 3457 3458 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3459 hqd_gpu_addr = ring->gpu_addr >> 8; 3460 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 3461 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 3462 3463 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3464 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL); 3465 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 3466 (order_base_2(ring->ring_size / 4) - 1)); 3467 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 3468 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8)); 3469 #ifdef __BIG_ENDIAN 3470 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1); 3471 #endif 3472 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0); 3473 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0); 3474 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 3475 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 3476 mqd->cp_hqd_pq_control = tmp; 3477 3478 /* set the wb address whether it's enabled or not */ 3479 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3480 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 3481 mqd->cp_hqd_pq_rptr_report_addr_hi = 3482 upper_32_bits(wb_gpu_addr) & 0xffff; 3483 3484 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3485 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3486 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 3487 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 3488 3489 tmp = 0; 3490 /* enable the doorbell if requested */ 3491 if (ring->use_doorbell) { 3492 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3493 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3494 DOORBELL_OFFSET, ring->doorbell_index); 3495 3496 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3497 DOORBELL_EN, 1); 3498 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3499 DOORBELL_SOURCE, 0); 3500 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3501 DOORBELL_HIT, 0); 3502 } 3503 3504 mqd->cp_hqd_pq_doorbell_control = tmp; 3505 3506 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3507 ring->wptr = 0; 3508 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR); 3509 3510 /* set the vmid for the queue */ 3511 mqd->cp_hqd_vmid = 0; 3512 3513 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE); 3514 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53); 3515 mqd->cp_hqd_persistent_state = tmp; 3516 3517 /* set MIN_IB_AVAIL_SIZE */ 3518 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL); 3519 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 3520 mqd->cp_hqd_ib_control = tmp; 3521 3522 /* set static priority for a queue/ring */ 3523 gfx_v9_0_mqd_set_priority(ring, mqd); 3524 mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM); 3525 3526 /* map_queues packet doesn't need activate the queue, 3527 * so only kiq need set this field. 3528 */ 3529 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 3530 mqd->cp_hqd_active = 1; 3531 3532 return 0; 3533 } 3534 3535 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring) 3536 { 3537 struct amdgpu_device *adev = ring->adev; 3538 struct v9_mqd *mqd = ring->mqd_ptr; 3539 int j; 3540 3541 /* disable wptr polling */ 3542 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3543 3544 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR, 3545 mqd->cp_hqd_eop_base_addr_lo); 3546 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI, 3547 mqd->cp_hqd_eop_base_addr_hi); 3548 3549 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3550 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL, 3551 mqd->cp_hqd_eop_control); 3552 3553 /* enable doorbell? */ 3554 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3555 mqd->cp_hqd_pq_doorbell_control); 3556 3557 /* disable the queue if it's active */ 3558 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3559 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3560 for (j = 0; j < adev->usec_timeout; j++) { 3561 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3562 break; 3563 udelay(1); 3564 } 3565 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3566 mqd->cp_hqd_dequeue_request); 3567 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 3568 mqd->cp_hqd_pq_rptr); 3569 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3570 mqd->cp_hqd_pq_wptr_lo); 3571 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3572 mqd->cp_hqd_pq_wptr_hi); 3573 } 3574 3575 /* set the pointer to the MQD */ 3576 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR, 3577 mqd->cp_mqd_base_addr_lo); 3578 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI, 3579 mqd->cp_mqd_base_addr_hi); 3580 3581 /* set MQD vmid to 0 */ 3582 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL, 3583 mqd->cp_mqd_control); 3584 3585 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3586 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE, 3587 mqd->cp_hqd_pq_base_lo); 3588 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI, 3589 mqd->cp_hqd_pq_base_hi); 3590 3591 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3592 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL, 3593 mqd->cp_hqd_pq_control); 3594 3595 /* set the wb address whether it's enabled or not */ 3596 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR, 3597 mqd->cp_hqd_pq_rptr_report_addr_lo); 3598 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 3599 mqd->cp_hqd_pq_rptr_report_addr_hi); 3600 3601 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3602 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR, 3603 mqd->cp_hqd_pq_wptr_poll_addr_lo); 3604 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI, 3605 mqd->cp_hqd_pq_wptr_poll_addr_hi); 3606 3607 /* enable the doorbell if requested */ 3608 if (ring->use_doorbell) { 3609 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER, 3610 (adev->doorbell_index.kiq * 2) << 2); 3611 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3612 (adev->doorbell_index.userqueue_end * 2) << 2); 3613 } 3614 3615 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3616 mqd->cp_hqd_pq_doorbell_control); 3617 3618 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3619 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3620 mqd->cp_hqd_pq_wptr_lo); 3621 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3622 mqd->cp_hqd_pq_wptr_hi); 3623 3624 /* set the vmid for the queue */ 3625 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid); 3626 3627 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 3628 mqd->cp_hqd_persistent_state); 3629 3630 /* activate the queue */ 3631 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 3632 mqd->cp_hqd_active); 3633 3634 if (ring->use_doorbell) 3635 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1); 3636 3637 return 0; 3638 } 3639 3640 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring) 3641 { 3642 struct amdgpu_device *adev = ring->adev; 3643 int j; 3644 3645 /* disable the queue if it's active */ 3646 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3647 3648 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3649 3650 for (j = 0; j < adev->usec_timeout; j++) { 3651 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3652 break; 3653 udelay(1); 3654 } 3655 3656 if (j == AMDGPU_MAX_USEC_TIMEOUT) { 3657 DRM_DEBUG("KIQ dequeue request failed.\n"); 3658 3659 /* Manual disable if dequeue request times out */ 3660 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0); 3661 } 3662 3663 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3664 0); 3665 } 3666 3667 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0); 3668 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0); 3669 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0); 3670 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000); 3671 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0); 3672 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0); 3673 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0); 3674 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0); 3675 3676 return 0; 3677 } 3678 3679 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring) 3680 { 3681 struct amdgpu_device *adev = ring->adev; 3682 struct v9_mqd *mqd = ring->mqd_ptr; 3683 int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS; 3684 3685 gfx_v9_0_kiq_setting(ring); 3686 3687 if (adev->in_gpu_reset) { /* for GPU_RESET case */ 3688 /* reset MQD to a clean status */ 3689 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3690 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3691 3692 /* reset ring buffer */ 3693 ring->wptr = 0; 3694 amdgpu_ring_clear_ring(ring); 3695 3696 mutex_lock(&adev->srbm_mutex); 3697 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3698 gfx_v9_0_kiq_init_register(ring); 3699 soc15_grbm_select(adev, 0, 0, 0, 0); 3700 mutex_unlock(&adev->srbm_mutex); 3701 } else { 3702 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3703 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3704 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3705 mutex_lock(&adev->srbm_mutex); 3706 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3707 gfx_v9_0_mqd_init(ring); 3708 gfx_v9_0_kiq_init_register(ring); 3709 soc15_grbm_select(adev, 0, 0, 0, 0); 3710 mutex_unlock(&adev->srbm_mutex); 3711 3712 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3713 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3714 } 3715 3716 return 0; 3717 } 3718 3719 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring) 3720 { 3721 struct amdgpu_device *adev = ring->adev; 3722 struct v9_mqd *mqd = ring->mqd_ptr; 3723 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 3724 3725 if (!adev->in_gpu_reset && !adev->in_suspend) { 3726 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3727 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3728 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3729 mutex_lock(&adev->srbm_mutex); 3730 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3731 gfx_v9_0_mqd_init(ring); 3732 soc15_grbm_select(adev, 0, 0, 0, 0); 3733 mutex_unlock(&adev->srbm_mutex); 3734 3735 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3736 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3737 } else if (adev->in_gpu_reset) { /* for GPU_RESET case */ 3738 /* reset MQD to a clean status */ 3739 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3740 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3741 3742 /* reset ring buffer */ 3743 ring->wptr = 0; 3744 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0); 3745 amdgpu_ring_clear_ring(ring); 3746 } else { 3747 amdgpu_ring_clear_ring(ring); 3748 } 3749 3750 return 0; 3751 } 3752 3753 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev) 3754 { 3755 struct amdgpu_ring *ring; 3756 int r; 3757 3758 ring = &adev->gfx.kiq.ring; 3759 3760 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3761 if (unlikely(r != 0)) 3762 return r; 3763 3764 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3765 if (unlikely(r != 0)) 3766 return r; 3767 3768 gfx_v9_0_kiq_init_queue(ring); 3769 amdgpu_bo_kunmap(ring->mqd_obj); 3770 ring->mqd_ptr = NULL; 3771 amdgpu_bo_unreserve(ring->mqd_obj); 3772 ring->sched.ready = true; 3773 return 0; 3774 } 3775 3776 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev) 3777 { 3778 struct amdgpu_ring *ring = NULL; 3779 int r = 0, i; 3780 3781 gfx_v9_0_cp_compute_enable(adev, true); 3782 3783 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3784 ring = &adev->gfx.compute_ring[i]; 3785 3786 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3787 if (unlikely(r != 0)) 3788 goto done; 3789 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3790 if (!r) { 3791 r = gfx_v9_0_kcq_init_queue(ring); 3792 amdgpu_bo_kunmap(ring->mqd_obj); 3793 ring->mqd_ptr = NULL; 3794 } 3795 amdgpu_bo_unreserve(ring->mqd_obj); 3796 if (r) 3797 goto done; 3798 } 3799 3800 r = amdgpu_gfx_enable_kcq(adev); 3801 done: 3802 return r; 3803 } 3804 3805 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev) 3806 { 3807 int r, i; 3808 struct amdgpu_ring *ring; 3809 3810 if (!(adev->flags & AMD_IS_APU)) 3811 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3812 3813 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3814 if (adev->asic_type != CHIP_ARCTURUS) { 3815 /* legacy firmware loading */ 3816 r = gfx_v9_0_cp_gfx_load_microcode(adev); 3817 if (r) 3818 return r; 3819 } 3820 3821 r = gfx_v9_0_cp_compute_load_microcode(adev); 3822 if (r) 3823 return r; 3824 } 3825 3826 r = gfx_v9_0_kiq_resume(adev); 3827 if (r) 3828 return r; 3829 3830 if (adev->asic_type != CHIP_ARCTURUS) { 3831 r = gfx_v9_0_cp_gfx_resume(adev); 3832 if (r) 3833 return r; 3834 } 3835 3836 r = gfx_v9_0_kcq_resume(adev); 3837 if (r) 3838 return r; 3839 3840 if (adev->asic_type != CHIP_ARCTURUS) { 3841 ring = &adev->gfx.gfx_ring[0]; 3842 r = amdgpu_ring_test_helper(ring); 3843 if (r) 3844 return r; 3845 } 3846 3847 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3848 ring = &adev->gfx.compute_ring[i]; 3849 amdgpu_ring_test_helper(ring); 3850 } 3851 3852 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3853 3854 return 0; 3855 } 3856 3857 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev) 3858 { 3859 u32 tmp; 3860 3861 if (adev->asic_type != CHIP_ARCTURUS) 3862 return; 3863 3864 tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG); 3865 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH, 3866 adev->df.hash_status.hash_64k); 3867 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH, 3868 adev->df.hash_status.hash_2m); 3869 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH, 3870 adev->df.hash_status.hash_1g); 3871 WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp); 3872 } 3873 3874 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable) 3875 { 3876 if (adev->asic_type != CHIP_ARCTURUS) 3877 gfx_v9_0_cp_gfx_enable(adev, enable); 3878 gfx_v9_0_cp_compute_enable(adev, enable); 3879 } 3880 3881 static int gfx_v9_0_hw_init(void *handle) 3882 { 3883 int r; 3884 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3885 3886 if (!amdgpu_sriov_vf(adev)) 3887 gfx_v9_0_init_golden_registers(adev); 3888 3889 gfx_v9_0_constants_init(adev); 3890 3891 gfx_v9_0_init_tcp_config(adev); 3892 3893 r = adev->gfx.rlc.funcs->resume(adev); 3894 if (r) 3895 return r; 3896 3897 r = gfx_v9_0_cp_resume(adev); 3898 if (r) 3899 return r; 3900 3901 return r; 3902 } 3903 3904 static int gfx_v9_0_hw_fini(void *handle) 3905 { 3906 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3907 3908 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); 3909 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 3910 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 3911 3912 /* DF freeze and kcq disable will fail */ 3913 if (!amdgpu_ras_intr_triggered()) 3914 /* disable KCQ to avoid CPC touch memory not valid anymore */ 3915 amdgpu_gfx_disable_kcq(adev); 3916 3917 if (amdgpu_sriov_vf(adev)) { 3918 gfx_v9_0_cp_gfx_enable(adev, false); 3919 /* must disable polling for SRIOV when hw finished, otherwise 3920 * CPC engine may still keep fetching WB address which is already 3921 * invalid after sw finished and trigger DMAR reading error in 3922 * hypervisor side. 3923 */ 3924 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3925 return 0; 3926 } 3927 3928 /* Use deinitialize sequence from CAIL when unbinding device from driver, 3929 * otherwise KIQ is hanging when binding back 3930 */ 3931 if (!adev->in_gpu_reset && !adev->in_suspend) { 3932 mutex_lock(&adev->srbm_mutex); 3933 soc15_grbm_select(adev, adev->gfx.kiq.ring.me, 3934 adev->gfx.kiq.ring.pipe, 3935 adev->gfx.kiq.ring.queue, 0); 3936 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring); 3937 soc15_grbm_select(adev, 0, 0, 0, 0); 3938 mutex_unlock(&adev->srbm_mutex); 3939 } 3940 3941 gfx_v9_0_cp_enable(adev, false); 3942 adev->gfx.rlc.funcs->stop(adev); 3943 3944 return 0; 3945 } 3946 3947 static int gfx_v9_0_suspend(void *handle) 3948 { 3949 return gfx_v9_0_hw_fini(handle); 3950 } 3951 3952 static int gfx_v9_0_resume(void *handle) 3953 { 3954 return gfx_v9_0_hw_init(handle); 3955 } 3956 3957 static bool gfx_v9_0_is_idle(void *handle) 3958 { 3959 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3960 3961 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS), 3962 GRBM_STATUS, GUI_ACTIVE)) 3963 return false; 3964 else 3965 return true; 3966 } 3967 3968 static int gfx_v9_0_wait_for_idle(void *handle) 3969 { 3970 unsigned i; 3971 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3972 3973 for (i = 0; i < adev->usec_timeout; i++) { 3974 if (gfx_v9_0_is_idle(handle)) 3975 return 0; 3976 udelay(1); 3977 } 3978 return -ETIMEDOUT; 3979 } 3980 3981 static int gfx_v9_0_soft_reset(void *handle) 3982 { 3983 u32 grbm_soft_reset = 0; 3984 u32 tmp; 3985 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3986 3987 /* GRBM_STATUS */ 3988 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS); 3989 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK | 3990 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK | 3991 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK | 3992 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK | 3993 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK | 3994 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) { 3995 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 3996 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 3997 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 3998 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1); 3999 } 4000 4001 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) { 4002 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4003 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4004 } 4005 4006 /* GRBM_STATUS2 */ 4007 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2); 4008 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY)) 4009 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4010 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 4011 4012 4013 if (grbm_soft_reset) { 4014 /* stop the rlc */ 4015 adev->gfx.rlc.funcs->stop(adev); 4016 4017 if (adev->asic_type != CHIP_ARCTURUS) 4018 /* Disable GFX parsing/prefetching */ 4019 gfx_v9_0_cp_gfx_enable(adev, false); 4020 4021 /* Disable MEC parsing/prefetching */ 4022 gfx_v9_0_cp_compute_enable(adev, false); 4023 4024 if (grbm_soft_reset) { 4025 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4026 tmp |= grbm_soft_reset; 4027 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 4028 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4029 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4030 4031 udelay(50); 4032 4033 tmp &= ~grbm_soft_reset; 4034 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4035 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4036 } 4037 4038 /* Wait a little for things to settle down */ 4039 udelay(50); 4040 } 4041 return 0; 4042 } 4043 4044 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev) 4045 { 4046 signed long r, cnt = 0; 4047 unsigned long flags; 4048 uint32_t seq, reg_val_offs = 0; 4049 uint64_t value = 0; 4050 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 4051 struct amdgpu_ring *ring = &kiq->ring; 4052 4053 BUG_ON(!ring->funcs->emit_rreg); 4054 4055 spin_lock_irqsave(&kiq->ring_lock, flags); 4056 if (amdgpu_device_wb_get(adev, ®_val_offs)) { 4057 pr_err("critical bug! too many kiq readers\n"); 4058 goto failed_unlock; 4059 } 4060 amdgpu_ring_alloc(ring, 32); 4061 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 4062 amdgpu_ring_write(ring, 9 | /* src: register*/ 4063 (5 << 8) | /* dst: memory */ 4064 (1 << 16) | /* count sel */ 4065 (1 << 20)); /* write confirm */ 4066 amdgpu_ring_write(ring, 0); 4067 amdgpu_ring_write(ring, 0); 4068 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 4069 reg_val_offs * 4)); 4070 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 4071 reg_val_offs * 4)); 4072 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 4073 if (r) 4074 goto failed_undo; 4075 4076 amdgpu_ring_commit(ring); 4077 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4078 4079 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4080 4081 /* don't wait anymore for gpu reset case because this way may 4082 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 4083 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 4084 * never return if we keep waiting in virt_kiq_rreg, which cause 4085 * gpu_recover() hang there. 4086 * 4087 * also don't wait anymore for IRQ context 4088 * */ 4089 if (r < 1 && (adev->in_gpu_reset || in_interrupt())) 4090 goto failed_kiq_read; 4091 4092 might_sleep(); 4093 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 4094 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 4095 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4096 } 4097 4098 if (cnt > MAX_KIQ_REG_TRY) 4099 goto failed_kiq_read; 4100 4101 mb(); 4102 value = (uint64_t)adev->wb.wb[reg_val_offs] | 4103 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL; 4104 amdgpu_device_wb_free(adev, reg_val_offs); 4105 return value; 4106 4107 failed_undo: 4108 amdgpu_ring_undo(ring); 4109 failed_unlock: 4110 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4111 failed_kiq_read: 4112 if (reg_val_offs) 4113 amdgpu_device_wb_free(adev, reg_val_offs); 4114 pr_err("failed to read gpu clock\n"); 4115 return ~0; 4116 } 4117 4118 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev) 4119 { 4120 uint64_t clock; 4121 4122 amdgpu_gfx_off_ctrl(adev, false); 4123 mutex_lock(&adev->gfx.gpu_clock_mutex); 4124 if (adev->asic_type == CHIP_VEGA10 && amdgpu_sriov_runtime(adev)) { 4125 clock = gfx_v9_0_kiq_read_clock(adev); 4126 } else { 4127 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1); 4128 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) | 4129 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 4130 } 4131 mutex_unlock(&adev->gfx.gpu_clock_mutex); 4132 amdgpu_gfx_off_ctrl(adev, true); 4133 return clock; 4134 } 4135 4136 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 4137 uint32_t vmid, 4138 uint32_t gds_base, uint32_t gds_size, 4139 uint32_t gws_base, uint32_t gws_size, 4140 uint32_t oa_base, uint32_t oa_size) 4141 { 4142 struct amdgpu_device *adev = ring->adev; 4143 4144 /* GDS Base */ 4145 gfx_v9_0_write_data_to_reg(ring, 0, false, 4146 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid, 4147 gds_base); 4148 4149 /* GDS Size */ 4150 gfx_v9_0_write_data_to_reg(ring, 0, false, 4151 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid, 4152 gds_size); 4153 4154 /* GWS */ 4155 gfx_v9_0_write_data_to_reg(ring, 0, false, 4156 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid, 4157 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 4158 4159 /* OA */ 4160 gfx_v9_0_write_data_to_reg(ring, 0, false, 4161 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid, 4162 (1 << (oa_size + oa_base)) - (1 << oa_base)); 4163 } 4164 4165 static const u32 vgpr_init_compute_shader[] = 4166 { 4167 0xb07c0000, 0xbe8000ff, 4168 0x000000f8, 0xbf110800, 4169 0x7e000280, 0x7e020280, 4170 0x7e040280, 0x7e060280, 4171 0x7e080280, 0x7e0a0280, 4172 0x7e0c0280, 0x7e0e0280, 4173 0x80808800, 0xbe803200, 4174 0xbf84fff5, 0xbf9c0000, 4175 0xd28c0001, 0x0001007f, 4176 0xd28d0001, 0x0002027e, 4177 0x10020288, 0xb8810904, 4178 0xb7814000, 0xd1196a01, 4179 0x00000301, 0xbe800087, 4180 0xbefc00c1, 0xd89c4000, 4181 0x00020201, 0xd89cc080, 4182 0x00040401, 0x320202ff, 4183 0x00000800, 0x80808100, 4184 0xbf84fff8, 0x7e020280, 4185 0xbf810000, 0x00000000, 4186 }; 4187 4188 static const u32 sgpr_init_compute_shader[] = 4189 { 4190 0xb07c0000, 0xbe8000ff, 4191 0x0000005f, 0xbee50080, 4192 0xbe812c65, 0xbe822c65, 4193 0xbe832c65, 0xbe842c65, 4194 0xbe852c65, 0xb77c0005, 4195 0x80808500, 0xbf84fff8, 4196 0xbe800080, 0xbf810000, 4197 }; 4198 4199 static const u32 vgpr_init_compute_shader_arcturus[] = { 4200 0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080, 4201 0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080, 4202 0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080, 4203 0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080, 4204 0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080, 4205 0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080, 4206 0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080, 4207 0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080, 4208 0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080, 4209 0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080, 4210 0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080, 4211 0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080, 4212 0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080, 4213 0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080, 4214 0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080, 4215 0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080, 4216 0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080, 4217 0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080, 4218 0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080, 4219 0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080, 4220 0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080, 4221 0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080, 4222 0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080, 4223 0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080, 4224 0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080, 4225 0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080, 4226 0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080, 4227 0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080, 4228 0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080, 4229 0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080, 4230 0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080, 4231 0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080, 4232 0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080, 4233 0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080, 4234 0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080, 4235 0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080, 4236 0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080, 4237 0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080, 4238 0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080, 4239 0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080, 4240 0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080, 4241 0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080, 4242 0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080, 4243 0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080, 4244 0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080, 4245 0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080, 4246 0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080, 4247 0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080, 4248 0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080, 4249 0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080, 4250 0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080, 4251 0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080, 4252 0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080, 4253 0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080, 4254 0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080, 4255 0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080, 4256 0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080, 4257 0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080, 4258 0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080, 4259 0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080, 4260 0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080, 4261 0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080, 4262 0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080, 4263 0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080, 4264 0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080, 4265 0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080, 4266 0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080, 4267 0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080, 4268 0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080, 4269 0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080, 4270 0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080, 4271 0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080, 4272 0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080, 4273 0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080, 4274 0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080, 4275 0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080, 4276 0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080, 4277 0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080, 4278 0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080, 4279 0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080, 4280 0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080, 4281 0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080, 4282 0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080, 4283 0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080, 4284 0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080, 4285 0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a, 4286 0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280, 4287 0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000, 4288 0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904, 4289 0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000, 4290 0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a, 4291 0xbf84fff8, 0xbf810000, 4292 }; 4293 4294 /* When below register arrays changed, please update gpr_reg_size, 4295 and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds, 4296 to cover all gfx9 ASICs */ 4297 static const struct soc15_reg_entry vgpr_init_regs[] = { 4298 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4299 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4300 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4301 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4302 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f }, 4303 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4304 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4305 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4306 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4307 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4308 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4309 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4310 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4311 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4312 }; 4313 4314 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = { 4315 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4316 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4317 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4318 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4319 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf }, 4320 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4321 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4322 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4323 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4324 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4325 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4326 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4327 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4328 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4329 }; 4330 4331 static const struct soc15_reg_entry sgpr1_init_regs[] = { 4332 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4333 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4334 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4335 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4336 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4337 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4338 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff }, 4339 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff }, 4340 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff }, 4341 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff }, 4342 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff }, 4343 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff }, 4344 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff }, 4345 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff }, 4346 }; 4347 4348 static const struct soc15_reg_entry sgpr2_init_regs[] = { 4349 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4350 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4351 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4352 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4353 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4354 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4355 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 }, 4356 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 }, 4357 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 }, 4358 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 }, 4359 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 }, 4360 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 }, 4361 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 }, 4362 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 }, 4363 }; 4364 4365 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = { 4366 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1}, 4367 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1}, 4368 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1}, 4369 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1}, 4370 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1}, 4371 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1}, 4372 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1}, 4373 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1}, 4374 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1}, 4375 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1}, 4376 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1}, 4377 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1}, 4378 { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1}, 4379 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6}, 4380 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16}, 4381 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16}, 4382 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16}, 4383 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16}, 4384 { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16}, 4385 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16}, 4386 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16}, 4387 { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16}, 4388 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6}, 4389 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16}, 4390 { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16}, 4391 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1}, 4392 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1}, 4393 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32}, 4394 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32}, 4395 { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72}, 4396 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16}, 4397 { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2}, 4398 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6}, 4399 }; 4400 4401 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev) 4402 { 4403 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4404 int i, r; 4405 4406 /* only support when RAS is enabled */ 4407 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4408 return 0; 4409 4410 r = amdgpu_ring_alloc(ring, 7); 4411 if (r) { 4412 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n", 4413 ring->name, r); 4414 return r; 4415 } 4416 4417 WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000); 4418 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size); 4419 4420 amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 4421 amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC | 4422 PACKET3_DMA_DATA_DST_SEL(1) | 4423 PACKET3_DMA_DATA_SRC_SEL(2) | 4424 PACKET3_DMA_DATA_ENGINE(0))); 4425 amdgpu_ring_write(ring, 0); 4426 amdgpu_ring_write(ring, 0); 4427 amdgpu_ring_write(ring, 0); 4428 amdgpu_ring_write(ring, 0); 4429 amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT | 4430 adev->gds.gds_size); 4431 4432 amdgpu_ring_commit(ring); 4433 4434 for (i = 0; i < adev->usec_timeout; i++) { 4435 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring)) 4436 break; 4437 udelay(1); 4438 } 4439 4440 if (i >= adev->usec_timeout) 4441 r = -ETIMEDOUT; 4442 4443 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000); 4444 4445 return r; 4446 } 4447 4448 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) 4449 { 4450 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4451 struct amdgpu_ib ib; 4452 struct dma_fence *f = NULL; 4453 int r, i; 4454 unsigned total_size, vgpr_offset, sgpr_offset; 4455 u64 gpu_addr; 4456 4457 int compute_dim_x = adev->gfx.config.max_shader_engines * 4458 adev->gfx.config.max_cu_per_sh * 4459 adev->gfx.config.max_sh_per_se; 4460 int sgpr_work_group_size = 5; 4461 int gpr_reg_size = adev->gfx.config.max_shader_engines + 6; 4462 int vgpr_init_shader_size; 4463 const u32 *vgpr_init_shader_ptr; 4464 const struct soc15_reg_entry *vgpr_init_regs_ptr; 4465 4466 /* only support when RAS is enabled */ 4467 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4468 return 0; 4469 4470 /* bail if the compute ring is not ready */ 4471 if (!ring->sched.ready) 4472 return 0; 4473 4474 if (adev->asic_type == CHIP_ARCTURUS) { 4475 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus; 4476 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus); 4477 vgpr_init_regs_ptr = vgpr_init_regs_arcturus; 4478 } else { 4479 vgpr_init_shader_ptr = vgpr_init_compute_shader; 4480 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader); 4481 vgpr_init_regs_ptr = vgpr_init_regs; 4482 } 4483 4484 total_size = 4485 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */ 4486 total_size += 4487 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */ 4488 total_size += 4489 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */ 4490 total_size = ALIGN(total_size, 256); 4491 vgpr_offset = total_size; 4492 total_size += ALIGN(vgpr_init_shader_size, 256); 4493 sgpr_offset = total_size; 4494 total_size += sizeof(sgpr_init_compute_shader); 4495 4496 /* allocate an indirect buffer to put the commands in */ 4497 memset(&ib, 0, sizeof(ib)); 4498 r = amdgpu_ib_get(adev, NULL, total_size, 4499 AMDGPU_IB_POOL_DIRECT, &ib); 4500 if (r) { 4501 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); 4502 return r; 4503 } 4504 4505 /* load the compute shaders */ 4506 for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++) 4507 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i]; 4508 4509 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++) 4510 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i]; 4511 4512 /* init the ib length to 0 */ 4513 ib.length_dw = 0; 4514 4515 /* VGPR */ 4516 /* write the register state for the compute dispatch */ 4517 for (i = 0; i < gpr_reg_size; i++) { 4518 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4519 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i]) 4520 - PACKET3_SET_SH_REG_START; 4521 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value; 4522 } 4523 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4524 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8; 4525 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4526 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4527 - PACKET3_SET_SH_REG_START; 4528 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4529 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4530 4531 /* write dispatch packet */ 4532 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4533 ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */ 4534 ib.ptr[ib.length_dw++] = 1; /* y */ 4535 ib.ptr[ib.length_dw++] = 1; /* z */ 4536 ib.ptr[ib.length_dw++] = 4537 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4538 4539 /* write CS partial flush packet */ 4540 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4541 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4542 4543 /* SGPR1 */ 4544 /* write the register state for the compute dispatch */ 4545 for (i = 0; i < gpr_reg_size; i++) { 4546 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4547 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i]) 4548 - PACKET3_SET_SH_REG_START; 4549 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value; 4550 } 4551 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4552 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4553 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4554 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4555 - PACKET3_SET_SH_REG_START; 4556 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4557 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4558 4559 /* write dispatch packet */ 4560 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4561 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4562 ib.ptr[ib.length_dw++] = 1; /* y */ 4563 ib.ptr[ib.length_dw++] = 1; /* z */ 4564 ib.ptr[ib.length_dw++] = 4565 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4566 4567 /* write CS partial flush packet */ 4568 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4569 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4570 4571 /* SGPR2 */ 4572 /* write the register state for the compute dispatch */ 4573 for (i = 0; i < gpr_reg_size; i++) { 4574 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4575 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i]) 4576 - PACKET3_SET_SH_REG_START; 4577 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value; 4578 } 4579 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4580 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4581 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4582 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4583 - PACKET3_SET_SH_REG_START; 4584 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4585 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4586 4587 /* write dispatch packet */ 4588 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4589 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4590 ib.ptr[ib.length_dw++] = 1; /* y */ 4591 ib.ptr[ib.length_dw++] = 1; /* z */ 4592 ib.ptr[ib.length_dw++] = 4593 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4594 4595 /* write CS partial flush packet */ 4596 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4597 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4598 4599 /* shedule the ib on the ring */ 4600 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 4601 if (r) { 4602 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r); 4603 goto fail; 4604 } 4605 4606 /* wait for the GPU to finish processing the IB */ 4607 r = dma_fence_wait(f, false); 4608 if (r) { 4609 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); 4610 goto fail; 4611 } 4612 4613 fail: 4614 amdgpu_ib_free(adev, &ib, NULL); 4615 dma_fence_put(f); 4616 4617 return r; 4618 } 4619 4620 static int gfx_v9_0_early_init(void *handle) 4621 { 4622 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4623 4624 if (adev->asic_type == CHIP_ARCTURUS) 4625 adev->gfx.num_gfx_rings = 0; 4626 else 4627 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS; 4628 adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS; 4629 gfx_v9_0_set_kiq_pm4_funcs(adev); 4630 gfx_v9_0_set_ring_funcs(adev); 4631 gfx_v9_0_set_irq_funcs(adev); 4632 gfx_v9_0_set_gds_init(adev); 4633 gfx_v9_0_set_rlc_funcs(adev); 4634 4635 return 0; 4636 } 4637 4638 static int gfx_v9_0_ecc_late_init(void *handle) 4639 { 4640 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4641 int r; 4642 4643 /* 4644 * Temp workaround to fix the issue that CP firmware fails to 4645 * update read pointer when CPDMA is writing clearing operation 4646 * to GDS in suspend/resume sequence on several cards. So just 4647 * limit this operation in cold boot sequence. 4648 */ 4649 if (!adev->in_suspend) { 4650 r = gfx_v9_0_do_edc_gds_workarounds(adev); 4651 if (r) 4652 return r; 4653 } 4654 4655 /* requires IBs so do in late init after IB pool is initialized */ 4656 r = gfx_v9_0_do_edc_gpr_workarounds(adev); 4657 if (r) 4658 return r; 4659 4660 if (adev->gfx.funcs && 4661 adev->gfx.funcs->reset_ras_error_count) 4662 adev->gfx.funcs->reset_ras_error_count(adev); 4663 4664 r = amdgpu_gfx_ras_late_init(adev); 4665 if (r) 4666 return r; 4667 4668 return 0; 4669 } 4670 4671 static int gfx_v9_0_late_init(void *handle) 4672 { 4673 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4674 int r; 4675 4676 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 4677 if (r) 4678 return r; 4679 4680 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 4681 if (r) 4682 return r; 4683 4684 r = gfx_v9_0_ecc_late_init(handle); 4685 if (r) 4686 return r; 4687 4688 return 0; 4689 } 4690 4691 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev) 4692 { 4693 uint32_t rlc_setting; 4694 4695 /* if RLC is not enabled, do nothing */ 4696 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL); 4697 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK)) 4698 return false; 4699 4700 return true; 4701 } 4702 4703 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev) 4704 { 4705 uint32_t data; 4706 unsigned i; 4707 4708 data = RLC_SAFE_MODE__CMD_MASK; 4709 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 4710 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4711 4712 /* wait for RLC_SAFE_MODE */ 4713 for (i = 0; i < adev->usec_timeout; i++) { 4714 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 4715 break; 4716 udelay(1); 4717 } 4718 } 4719 4720 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev) 4721 { 4722 uint32_t data; 4723 4724 data = RLC_SAFE_MODE__CMD_MASK; 4725 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4726 } 4727 4728 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev, 4729 bool enable) 4730 { 4731 amdgpu_gfx_rlc_enter_safe_mode(adev); 4732 4733 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { 4734 gfx_v9_0_enable_gfx_cg_power_gating(adev, true); 4735 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4736 gfx_v9_0_enable_gfx_pipeline_powergating(adev, true); 4737 } else { 4738 gfx_v9_0_enable_gfx_cg_power_gating(adev, false); 4739 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4740 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false); 4741 } 4742 4743 amdgpu_gfx_rlc_exit_safe_mode(adev); 4744 } 4745 4746 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev, 4747 bool enable) 4748 { 4749 /* TODO: double check if we need to perform under safe mode */ 4750 /* gfx_v9_0_enter_rlc_safe_mode(adev); */ 4751 4752 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 4753 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true); 4754 else 4755 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false); 4756 4757 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 4758 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true); 4759 else 4760 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false); 4761 4762 /* gfx_v9_0_exit_rlc_safe_mode(adev); */ 4763 } 4764 4765 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 4766 bool enable) 4767 { 4768 uint32_t data, def; 4769 4770 amdgpu_gfx_rlc_enter_safe_mode(adev); 4771 4772 /* It is disabled by HW by default */ 4773 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { 4774 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 4775 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4776 4777 if (adev->asic_type != CHIP_VEGA12) 4778 data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4779 4780 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4781 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4782 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4783 4784 /* only for Vega10 & Raven1 */ 4785 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK; 4786 4787 if (def != data) 4788 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4789 4790 /* MGLS is a global flag to control all MGLS in GFX */ 4791 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 4792 /* 2 - RLC memory Light sleep */ 4793 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { 4794 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4795 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4796 if (def != data) 4797 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4798 } 4799 /* 3 - CP memory Light sleep */ 4800 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { 4801 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4802 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4803 if (def != data) 4804 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4805 } 4806 } 4807 } else { 4808 /* 1 - MGCG_OVERRIDE */ 4809 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4810 4811 if (adev->asic_type != CHIP_VEGA12) 4812 data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4813 4814 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 4815 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4816 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4817 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4818 4819 if (def != data) 4820 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4821 4822 /* 2 - disable MGLS in RLC */ 4823 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4824 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) { 4825 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4826 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4827 } 4828 4829 /* 3 - disable MGLS in CP */ 4830 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4831 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) { 4832 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4833 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4834 } 4835 } 4836 4837 amdgpu_gfx_rlc_exit_safe_mode(adev); 4838 } 4839 4840 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev, 4841 bool enable) 4842 { 4843 uint32_t data, def; 4844 4845 if (adev->asic_type == CHIP_ARCTURUS) 4846 return; 4847 4848 amdgpu_gfx_rlc_enter_safe_mode(adev); 4849 4850 /* Enable 3D CGCG/CGLS */ 4851 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)) { 4852 /* write cmd to clear cgcg/cgls ov */ 4853 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4854 /* unset CGCG override */ 4855 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 4856 /* update CGCG and CGLS override bits */ 4857 if (def != data) 4858 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4859 4860 /* enable 3Dcgcg FSM(0x0000363f) */ 4861 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4862 4863 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4864 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 4865 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 4866 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 4867 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 4868 if (def != data) 4869 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4870 4871 /* set IDLE_POLL_COUNT(0x00900100) */ 4872 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 4873 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 4874 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 4875 if (def != data) 4876 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 4877 } else { 4878 /* Disable CGCG/CGLS */ 4879 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4880 /* disable cgcg, cgls should be disabled */ 4881 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK | 4882 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK); 4883 /* disable cgcg and cgls in FSM */ 4884 if (def != data) 4885 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4886 } 4887 4888 amdgpu_gfx_rlc_exit_safe_mode(adev); 4889 } 4890 4891 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 4892 bool enable) 4893 { 4894 uint32_t def, data; 4895 4896 amdgpu_gfx_rlc_enter_safe_mode(adev); 4897 4898 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { 4899 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4900 /* unset CGCG override */ 4901 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 4902 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 4903 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 4904 else 4905 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 4906 /* update CGCG and CGLS override bits */ 4907 if (def != data) 4908 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4909 4910 /* enable cgcg FSM(0x0000363F) */ 4911 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 4912 4913 if (adev->asic_type == CHIP_ARCTURUS) 4914 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4915 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 4916 else 4917 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4918 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 4919 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 4920 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 4921 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 4922 if (def != data) 4923 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 4924 4925 /* set IDLE_POLL_COUNT(0x00900100) */ 4926 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 4927 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 4928 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 4929 if (def != data) 4930 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 4931 } else { 4932 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 4933 /* reset CGCG/CGLS bits */ 4934 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK); 4935 /* disable cgcg and cgls in FSM */ 4936 if (def != data) 4937 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 4938 } 4939 4940 amdgpu_gfx_rlc_exit_safe_mode(adev); 4941 } 4942 4943 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev, 4944 bool enable) 4945 { 4946 if (enable) { 4947 /* CGCG/CGLS should be enabled after MGCG/MGLS 4948 * === MGCG + MGLS === 4949 */ 4950 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 4951 /* === CGCG /CGLS for GFX 3D Only === */ 4952 gfx_v9_0_update_3d_clock_gating(adev, enable); 4953 /* === CGCG + CGLS === */ 4954 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 4955 } else { 4956 /* CGCG/CGLS should be disabled before MGCG/MGLS 4957 * === CGCG + CGLS === 4958 */ 4959 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 4960 /* === CGCG /CGLS for GFX 3D Only === */ 4961 gfx_v9_0_update_3d_clock_gating(adev, enable); 4962 /* === MGCG + MGLS === */ 4963 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 4964 } 4965 return 0; 4966 } 4967 4968 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid) 4969 { 4970 u32 reg, data; 4971 4972 reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL); 4973 if (amdgpu_sriov_is_pp_one_vf(adev)) 4974 data = RREG32_NO_KIQ(reg); 4975 else 4976 data = RREG32(reg); 4977 4978 data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK; 4979 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; 4980 4981 if (amdgpu_sriov_is_pp_one_vf(adev)) 4982 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data); 4983 else 4984 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data); 4985 } 4986 4987 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev, 4988 uint32_t offset, 4989 struct soc15_reg_rlcg *entries, int arr_size) 4990 { 4991 int i; 4992 uint32_t reg; 4993 4994 if (!entries) 4995 return false; 4996 4997 for (i = 0; i < arr_size; i++) { 4998 const struct soc15_reg_rlcg *entry; 4999 5000 entry = &entries[i]; 5001 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg; 5002 if (offset == reg) 5003 return true; 5004 } 5005 5006 return false; 5007 } 5008 5009 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset) 5010 { 5011 return gfx_v9_0_check_rlcg_range(adev, offset, 5012 (void *)rlcg_access_gc_9_0, 5013 ARRAY_SIZE(rlcg_access_gc_9_0)); 5014 } 5015 5016 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = { 5017 .is_rlc_enabled = gfx_v9_0_is_rlc_enabled, 5018 .set_safe_mode = gfx_v9_0_set_safe_mode, 5019 .unset_safe_mode = gfx_v9_0_unset_safe_mode, 5020 .init = gfx_v9_0_rlc_init, 5021 .get_csb_size = gfx_v9_0_get_csb_size, 5022 .get_csb_buffer = gfx_v9_0_get_csb_buffer, 5023 .get_cp_table_num = gfx_v9_0_cp_jump_table_num, 5024 .resume = gfx_v9_0_rlc_resume, 5025 .stop = gfx_v9_0_rlc_stop, 5026 .reset = gfx_v9_0_rlc_reset, 5027 .start = gfx_v9_0_rlc_start, 5028 .update_spm_vmid = gfx_v9_0_update_spm_vmid, 5029 .rlcg_wreg = gfx_v9_0_rlcg_wreg, 5030 .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range, 5031 }; 5032 5033 static int gfx_v9_0_set_powergating_state(void *handle, 5034 enum amd_powergating_state state) 5035 { 5036 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5037 bool enable = (state == AMD_PG_STATE_GATE); 5038 5039 switch (adev->asic_type) { 5040 case CHIP_RAVEN: 5041 case CHIP_RENOIR: 5042 if (!enable) 5043 amdgpu_gfx_off_ctrl(adev, false); 5044 5045 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { 5046 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true); 5047 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true); 5048 } else { 5049 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false); 5050 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false); 5051 } 5052 5053 if (adev->pg_flags & AMD_PG_SUPPORT_CP) 5054 gfx_v9_0_enable_cp_power_gating(adev, true); 5055 else 5056 gfx_v9_0_enable_cp_power_gating(adev, false); 5057 5058 /* update gfx cgpg state */ 5059 gfx_v9_0_update_gfx_cg_power_gating(adev, enable); 5060 5061 /* update mgcg state */ 5062 gfx_v9_0_update_gfx_mg_power_gating(adev, enable); 5063 5064 if (enable) 5065 amdgpu_gfx_off_ctrl(adev, true); 5066 break; 5067 case CHIP_VEGA12: 5068 amdgpu_gfx_off_ctrl(adev, enable); 5069 break; 5070 default: 5071 break; 5072 } 5073 5074 return 0; 5075 } 5076 5077 static int gfx_v9_0_set_clockgating_state(void *handle, 5078 enum amd_clockgating_state state) 5079 { 5080 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5081 5082 if (amdgpu_sriov_vf(adev)) 5083 return 0; 5084 5085 switch (adev->asic_type) { 5086 case CHIP_VEGA10: 5087 case CHIP_VEGA12: 5088 case CHIP_VEGA20: 5089 case CHIP_RAVEN: 5090 case CHIP_ARCTURUS: 5091 case CHIP_RENOIR: 5092 gfx_v9_0_update_gfx_clock_gating(adev, 5093 state == AMD_CG_STATE_GATE); 5094 break; 5095 default: 5096 break; 5097 } 5098 return 0; 5099 } 5100 5101 static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags) 5102 { 5103 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5104 int data; 5105 5106 if (amdgpu_sriov_vf(adev)) 5107 *flags = 0; 5108 5109 /* AMD_CG_SUPPORT_GFX_MGCG */ 5110 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE)); 5111 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 5112 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5113 5114 /* AMD_CG_SUPPORT_GFX_CGCG */ 5115 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL)); 5116 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5117 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5118 5119 /* AMD_CG_SUPPORT_GFX_CGLS */ 5120 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5121 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5122 5123 /* AMD_CG_SUPPORT_GFX_RLC_LS */ 5124 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL)); 5125 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) 5126 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS; 5127 5128 /* AMD_CG_SUPPORT_GFX_CP_LS */ 5129 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL)); 5130 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) 5131 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS; 5132 5133 if (adev->asic_type != CHIP_ARCTURUS) { 5134 /* AMD_CG_SUPPORT_GFX_3D_CGCG */ 5135 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D)); 5136 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) 5137 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; 5138 5139 /* AMD_CG_SUPPORT_GFX_3D_CGLS */ 5140 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK) 5141 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS; 5142 } 5143 } 5144 5145 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) 5146 { 5147 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/ 5148 } 5149 5150 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5151 { 5152 struct amdgpu_device *adev = ring->adev; 5153 u64 wptr; 5154 5155 /* XXX check if swapping is necessary on BE */ 5156 if (ring->use_doorbell) { 5157 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]); 5158 } else { 5159 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR); 5160 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32; 5161 } 5162 5163 return wptr; 5164 } 5165 5166 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5167 { 5168 struct amdgpu_device *adev = ring->adev; 5169 5170 if (ring->use_doorbell) { 5171 /* XXX check if swapping is necessary on BE */ 5172 atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5173 WDOORBELL64(ring->doorbell_index, ring->wptr); 5174 } else { 5175 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 5176 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 5177 } 5178 } 5179 5180 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5181 { 5182 struct amdgpu_device *adev = ring->adev; 5183 u32 ref_and_mask, reg_mem_engine; 5184 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg; 5185 5186 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 5187 switch (ring->me) { 5188 case 1: 5189 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe; 5190 break; 5191 case 2: 5192 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe; 5193 break; 5194 default: 5195 return; 5196 } 5197 reg_mem_engine = 0; 5198 } else { 5199 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0; 5200 reg_mem_engine = 1; /* pfp */ 5201 } 5202 5203 gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1, 5204 adev->nbio.funcs->get_hdp_flush_req_offset(adev), 5205 adev->nbio.funcs->get_hdp_flush_done_offset(adev), 5206 ref_and_mask, ref_and_mask, 0x20); 5207 } 5208 5209 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 5210 struct amdgpu_job *job, 5211 struct amdgpu_ib *ib, 5212 uint32_t flags) 5213 { 5214 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5215 u32 header, control = 0; 5216 5217 if (ib->flags & AMDGPU_IB_FLAG_CE) 5218 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 5219 else 5220 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 5221 5222 control |= ib->length_dw | (vmid << 24); 5223 5224 if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { 5225 control |= INDIRECT_BUFFER_PRE_ENB(1); 5226 5227 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid) 5228 gfx_v9_0_ring_emit_de_meta(ring); 5229 } 5230 5231 amdgpu_ring_write(ring, header); 5232 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5233 amdgpu_ring_write(ring, 5234 #ifdef __BIG_ENDIAN 5235 (2 << 0) | 5236 #endif 5237 lower_32_bits(ib->gpu_addr)); 5238 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5239 amdgpu_ring_write(ring, control); 5240 } 5241 5242 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 5243 struct amdgpu_job *job, 5244 struct amdgpu_ib *ib, 5245 uint32_t flags) 5246 { 5247 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5248 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 5249 5250 /* Currently, there is a high possibility to get wave ID mismatch 5251 * between ME and GDS, leading to a hw deadlock, because ME generates 5252 * different wave IDs than the GDS expects. This situation happens 5253 * randomly when at least 5 compute pipes use GDS ordered append. 5254 * The wave IDs generated by ME are also wrong after suspend/resume. 5255 * Those are probably bugs somewhere else in the kernel driver. 5256 * 5257 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 5258 * GDS to 0 for this ring (me/pipe). 5259 */ 5260 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 5261 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 5262 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID); 5263 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 5264 } 5265 5266 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 5267 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5268 amdgpu_ring_write(ring, 5269 #ifdef __BIG_ENDIAN 5270 (2 << 0) | 5271 #endif 5272 lower_32_bits(ib->gpu_addr)); 5273 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5274 amdgpu_ring_write(ring, control); 5275 } 5276 5277 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 5278 u64 seq, unsigned flags) 5279 { 5280 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 5281 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 5282 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY; 5283 5284 /* RELEASE_MEM - flush caches, send int */ 5285 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); 5286 amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN | 5287 EOP_TC_NC_ACTION_EN) : 5288 (EOP_TCL1_ACTION_EN | 5289 EOP_TC_ACTION_EN | 5290 EOP_TC_WB_ACTION_EN | 5291 EOP_TC_MD_ACTION_EN)) | 5292 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 5293 EVENT_INDEX(5))); 5294 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 5295 5296 /* 5297 * the address should be Qword aligned if 64bit write, Dword 5298 * aligned if only send 32bit data low (discard data high) 5299 */ 5300 if (write64bit) 5301 BUG_ON(addr & 0x7); 5302 else 5303 BUG_ON(addr & 0x3); 5304 amdgpu_ring_write(ring, lower_32_bits(addr)); 5305 amdgpu_ring_write(ring, upper_32_bits(addr)); 5306 amdgpu_ring_write(ring, lower_32_bits(seq)); 5307 amdgpu_ring_write(ring, upper_32_bits(seq)); 5308 amdgpu_ring_write(ring, 0); 5309 } 5310 5311 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 5312 { 5313 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5314 uint32_t seq = ring->fence_drv.sync_seq; 5315 uint64_t addr = ring->fence_drv.gpu_addr; 5316 5317 gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0, 5318 lower_32_bits(addr), upper_32_bits(addr), 5319 seq, 0xffffffff, 4); 5320 } 5321 5322 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 5323 unsigned vmid, uint64_t pd_addr) 5324 { 5325 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 5326 5327 /* compute doesn't have PFP */ 5328 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 5329 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 5330 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 5331 amdgpu_ring_write(ring, 0x0); 5332 } 5333 } 5334 5335 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring) 5336 { 5337 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */ 5338 } 5339 5340 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 5341 { 5342 u64 wptr; 5343 5344 /* XXX check if swapping is necessary on BE */ 5345 if (ring->use_doorbell) 5346 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]); 5347 else 5348 BUG(); 5349 return wptr; 5350 } 5351 5352 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 5353 { 5354 struct amdgpu_device *adev = ring->adev; 5355 5356 /* XXX check if swapping is necessary on BE */ 5357 if (ring->use_doorbell) { 5358 atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5359 WDOORBELL64(ring->doorbell_index, ring->wptr); 5360 } else{ 5361 BUG(); /* only DOORBELL method supported on gfx9 now */ 5362 } 5363 } 5364 5365 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 5366 u64 seq, unsigned int flags) 5367 { 5368 struct amdgpu_device *adev = ring->adev; 5369 5370 /* we only allocate 32bit for each seq wb address */ 5371 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); 5372 5373 /* write fence seq to the "addr" */ 5374 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5375 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5376 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 5377 amdgpu_ring_write(ring, lower_32_bits(addr)); 5378 amdgpu_ring_write(ring, upper_32_bits(addr)); 5379 amdgpu_ring_write(ring, lower_32_bits(seq)); 5380 5381 if (flags & AMDGPU_FENCE_FLAG_INT) { 5382 /* set register to trigger INT */ 5383 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5384 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5385 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 5386 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS)); 5387 amdgpu_ring_write(ring, 0); 5388 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 5389 } 5390 } 5391 5392 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring) 5393 { 5394 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 5395 amdgpu_ring_write(ring, 0); 5396 } 5397 5398 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring) 5399 { 5400 struct v9_ce_ib_state ce_payload = {0}; 5401 uint64_t csa_addr; 5402 int cnt; 5403 5404 cnt = (sizeof(ce_payload) >> 2) + 4 - 2; 5405 csa_addr = amdgpu_csa_vaddr(ring->adev); 5406 5407 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5408 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) | 5409 WRITE_DATA_DST_SEL(8) | 5410 WR_CONFIRM) | 5411 WRITE_DATA_CACHE_POLICY(0)); 5412 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5413 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5414 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2); 5415 } 5416 5417 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring) 5418 { 5419 struct v9_de_ib_state de_payload = {0}; 5420 uint64_t csa_addr, gds_addr; 5421 int cnt; 5422 5423 csa_addr = amdgpu_csa_vaddr(ring->adev); 5424 gds_addr = csa_addr + 4096; 5425 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr); 5426 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr); 5427 5428 cnt = (sizeof(de_payload) >> 2) + 4 - 2; 5429 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5430 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 5431 WRITE_DATA_DST_SEL(8) | 5432 WR_CONFIRM) | 5433 WRITE_DATA_CACHE_POLICY(0)); 5434 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5435 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5436 amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2); 5437 } 5438 5439 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, 5440 bool secure) 5441 { 5442 uint32_t v = secure ? FRAME_TMZ : 0; 5443 5444 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); 5445 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1)); 5446 } 5447 5448 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) 5449 { 5450 uint32_t dw2 = 0; 5451 5452 if (amdgpu_sriov_vf(ring->adev)) 5453 gfx_v9_0_ring_emit_ce_meta(ring); 5454 5455 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 5456 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 5457 /* set load_global_config & load_global_uconfig */ 5458 dw2 |= 0x8001; 5459 /* set load_cs_sh_regs */ 5460 dw2 |= 0x01000000; 5461 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 5462 dw2 |= 0x10002; 5463 5464 /* set load_ce_ram if preamble presented */ 5465 if (AMDGPU_PREAMBLE_IB_PRESENT & flags) 5466 dw2 |= 0x10000000; 5467 } else { 5468 /* still load_ce_ram if this is the first time preamble presented 5469 * although there is no context switch happens. 5470 */ 5471 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags) 5472 dw2 |= 0x10000000; 5473 } 5474 5475 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 5476 amdgpu_ring_write(ring, dw2); 5477 amdgpu_ring_write(ring, 0); 5478 } 5479 5480 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring) 5481 { 5482 unsigned ret; 5483 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 5484 amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr)); 5485 amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr)); 5486 amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */ 5487 ret = ring->wptr & ring->buf_mask; 5488 amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */ 5489 return ret; 5490 } 5491 5492 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset) 5493 { 5494 unsigned cur; 5495 BUG_ON(offset > ring->buf_mask); 5496 BUG_ON(ring->ring[offset] != 0x55aa55aa); 5497 5498 cur = (ring->wptr & ring->buf_mask) - 1; 5499 if (likely(cur > offset)) 5500 ring->ring[offset] = cur - offset; 5501 else 5502 ring->ring[offset] = (ring->ring_size>>2) - offset + cur; 5503 } 5504 5505 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 5506 uint32_t reg_val_offs) 5507 { 5508 struct amdgpu_device *adev = ring->adev; 5509 5510 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 5511 amdgpu_ring_write(ring, 0 | /* src: register*/ 5512 (5 << 8) | /* dst: memory */ 5513 (1 << 20)); /* write confirm */ 5514 amdgpu_ring_write(ring, reg); 5515 amdgpu_ring_write(ring, 0); 5516 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 5517 reg_val_offs * 4)); 5518 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 5519 reg_val_offs * 4)); 5520 } 5521 5522 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 5523 uint32_t val) 5524 { 5525 uint32_t cmd = 0; 5526 5527 switch (ring->funcs->type) { 5528 case AMDGPU_RING_TYPE_GFX: 5529 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 5530 break; 5531 case AMDGPU_RING_TYPE_KIQ: 5532 cmd = (1 << 16); /* no inc addr */ 5533 break; 5534 default: 5535 cmd = WR_CONFIRM; 5536 break; 5537 } 5538 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5539 amdgpu_ring_write(ring, cmd); 5540 amdgpu_ring_write(ring, reg); 5541 amdgpu_ring_write(ring, 0); 5542 amdgpu_ring_write(ring, val); 5543 } 5544 5545 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 5546 uint32_t val, uint32_t mask) 5547 { 5548 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); 5549 } 5550 5551 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, 5552 uint32_t reg0, uint32_t reg1, 5553 uint32_t ref, uint32_t mask) 5554 { 5555 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5556 struct amdgpu_device *adev = ring->adev; 5557 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ? 5558 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait; 5559 5560 if (fw_version_ok) 5561 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, 5562 ref, mask, 0x20); 5563 else 5564 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1, 5565 ref, mask); 5566 } 5567 5568 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid) 5569 { 5570 struct amdgpu_device *adev = ring->adev; 5571 uint32_t value = 0; 5572 5573 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03); 5574 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01); 5575 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1); 5576 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid); 5577 WREG32_SOC15(GC, 0, mmSQ_CMD, value); 5578 } 5579 5580 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 5581 enum amdgpu_interrupt_state state) 5582 { 5583 switch (state) { 5584 case AMDGPU_IRQ_STATE_DISABLE: 5585 case AMDGPU_IRQ_STATE_ENABLE: 5586 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5587 TIME_STAMP_INT_ENABLE, 5588 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5589 break; 5590 default: 5591 break; 5592 } 5593 } 5594 5595 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 5596 int me, int pipe, 5597 enum amdgpu_interrupt_state state) 5598 { 5599 u32 mec_int_cntl, mec_int_cntl_reg; 5600 5601 /* 5602 * amdgpu controls only the first MEC. That's why this function only 5603 * handles the setting of interrupts for this specific MEC. All other 5604 * pipes' interrupts are set by amdkfd. 5605 */ 5606 5607 if (me == 1) { 5608 switch (pipe) { 5609 case 0: 5610 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL); 5611 break; 5612 case 1: 5613 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL); 5614 break; 5615 case 2: 5616 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL); 5617 break; 5618 case 3: 5619 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL); 5620 break; 5621 default: 5622 DRM_DEBUG("invalid pipe %d\n", pipe); 5623 return; 5624 } 5625 } else { 5626 DRM_DEBUG("invalid me %d\n", me); 5627 return; 5628 } 5629 5630 switch (state) { 5631 case AMDGPU_IRQ_STATE_DISABLE: 5632 mec_int_cntl = RREG32(mec_int_cntl_reg); 5633 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5634 TIME_STAMP_INT_ENABLE, 0); 5635 WREG32(mec_int_cntl_reg, mec_int_cntl); 5636 break; 5637 case AMDGPU_IRQ_STATE_ENABLE: 5638 mec_int_cntl = RREG32(mec_int_cntl_reg); 5639 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5640 TIME_STAMP_INT_ENABLE, 1); 5641 WREG32(mec_int_cntl_reg, mec_int_cntl); 5642 break; 5643 default: 5644 break; 5645 } 5646 } 5647 5648 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 5649 struct amdgpu_irq_src *source, 5650 unsigned type, 5651 enum amdgpu_interrupt_state state) 5652 { 5653 switch (state) { 5654 case AMDGPU_IRQ_STATE_DISABLE: 5655 case AMDGPU_IRQ_STATE_ENABLE: 5656 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5657 PRIV_REG_INT_ENABLE, 5658 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5659 break; 5660 default: 5661 break; 5662 } 5663 5664 return 0; 5665 } 5666 5667 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 5668 struct amdgpu_irq_src *source, 5669 unsigned type, 5670 enum amdgpu_interrupt_state state) 5671 { 5672 switch (state) { 5673 case AMDGPU_IRQ_STATE_DISABLE: 5674 case AMDGPU_IRQ_STATE_ENABLE: 5675 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5676 PRIV_INSTR_INT_ENABLE, 5677 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5678 default: 5679 break; 5680 } 5681 5682 return 0; 5683 } 5684 5685 #define ENABLE_ECC_ON_ME_PIPE(me, pipe) \ 5686 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5687 CP_ECC_ERROR_INT_ENABLE, 1) 5688 5689 #define DISABLE_ECC_ON_ME_PIPE(me, pipe) \ 5690 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5691 CP_ECC_ERROR_INT_ENABLE, 0) 5692 5693 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev, 5694 struct amdgpu_irq_src *source, 5695 unsigned type, 5696 enum amdgpu_interrupt_state state) 5697 { 5698 switch (state) { 5699 case AMDGPU_IRQ_STATE_DISABLE: 5700 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5701 CP_ECC_ERROR_INT_ENABLE, 0); 5702 DISABLE_ECC_ON_ME_PIPE(1, 0); 5703 DISABLE_ECC_ON_ME_PIPE(1, 1); 5704 DISABLE_ECC_ON_ME_PIPE(1, 2); 5705 DISABLE_ECC_ON_ME_PIPE(1, 3); 5706 break; 5707 5708 case AMDGPU_IRQ_STATE_ENABLE: 5709 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5710 CP_ECC_ERROR_INT_ENABLE, 1); 5711 ENABLE_ECC_ON_ME_PIPE(1, 0); 5712 ENABLE_ECC_ON_ME_PIPE(1, 1); 5713 ENABLE_ECC_ON_ME_PIPE(1, 2); 5714 ENABLE_ECC_ON_ME_PIPE(1, 3); 5715 break; 5716 default: 5717 break; 5718 } 5719 5720 return 0; 5721 } 5722 5723 5724 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev, 5725 struct amdgpu_irq_src *src, 5726 unsigned type, 5727 enum amdgpu_interrupt_state state) 5728 { 5729 switch (type) { 5730 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 5731 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state); 5732 break; 5733 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 5734 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 5735 break; 5736 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 5737 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 5738 break; 5739 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 5740 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 5741 break; 5742 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 5743 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 5744 break; 5745 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP: 5746 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state); 5747 break; 5748 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP: 5749 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state); 5750 break; 5751 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP: 5752 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state); 5753 break; 5754 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP: 5755 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state); 5756 break; 5757 default: 5758 break; 5759 } 5760 return 0; 5761 } 5762 5763 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev, 5764 struct amdgpu_irq_src *source, 5765 struct amdgpu_iv_entry *entry) 5766 { 5767 int i; 5768 u8 me_id, pipe_id, queue_id; 5769 struct amdgpu_ring *ring; 5770 5771 DRM_DEBUG("IH: CP EOP\n"); 5772 me_id = (entry->ring_id & 0x0c) >> 2; 5773 pipe_id = (entry->ring_id & 0x03) >> 0; 5774 queue_id = (entry->ring_id & 0x70) >> 4; 5775 5776 switch (me_id) { 5777 case 0: 5778 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); 5779 break; 5780 case 1: 5781 case 2: 5782 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5783 ring = &adev->gfx.compute_ring[i]; 5784 /* Per-queue interrupt is supported for MEC starting from VI. 5785 * The interrupt can only be enabled/disabled per pipe instead of per queue. 5786 */ 5787 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) 5788 amdgpu_fence_process(ring); 5789 } 5790 break; 5791 } 5792 return 0; 5793 } 5794 5795 static void gfx_v9_0_fault(struct amdgpu_device *adev, 5796 struct amdgpu_iv_entry *entry) 5797 { 5798 u8 me_id, pipe_id, queue_id; 5799 struct amdgpu_ring *ring; 5800 int i; 5801 5802 me_id = (entry->ring_id & 0x0c) >> 2; 5803 pipe_id = (entry->ring_id & 0x03) >> 0; 5804 queue_id = (entry->ring_id & 0x70) >> 4; 5805 5806 switch (me_id) { 5807 case 0: 5808 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); 5809 break; 5810 case 1: 5811 case 2: 5812 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5813 ring = &adev->gfx.compute_ring[i]; 5814 if (ring->me == me_id && ring->pipe == pipe_id && 5815 ring->queue == queue_id) 5816 drm_sched_fault(&ring->sched); 5817 } 5818 break; 5819 } 5820 } 5821 5822 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev, 5823 struct amdgpu_irq_src *source, 5824 struct amdgpu_iv_entry *entry) 5825 { 5826 DRM_ERROR("Illegal register access in command stream\n"); 5827 gfx_v9_0_fault(adev, entry); 5828 return 0; 5829 } 5830 5831 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev, 5832 struct amdgpu_irq_src *source, 5833 struct amdgpu_iv_entry *entry) 5834 { 5835 DRM_ERROR("Illegal instruction in command stream\n"); 5836 gfx_v9_0_fault(adev, entry); 5837 return 0; 5838 } 5839 5840 5841 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = { 5842 { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 5843 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT), 5844 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT) 5845 }, 5846 { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 5847 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT), 5848 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT) 5849 }, 5850 { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5851 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1), 5852 0, 0 5853 }, 5854 { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5855 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2), 5856 0, 0 5857 }, 5858 { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 5859 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT), 5860 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT) 5861 }, 5862 { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5863 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT), 5864 0, 0 5865 }, 5866 { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5867 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT), 5868 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT) 5869 }, 5870 { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 5871 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT), 5872 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT) 5873 }, 5874 { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 5875 SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1), 5876 0, 0 5877 }, 5878 { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 5879 SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1), 5880 0, 0 5881 }, 5882 { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 5883 SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1), 5884 0, 0 5885 }, 5886 { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 5887 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC), 5888 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED) 5889 }, 5890 { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 5891 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED), 5892 0, 0 5893 }, 5894 { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5895 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC), 5896 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED) 5897 }, 5898 { "GDS_OA_PHY_PHY_CMD_RAM_MEM", 5899 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5900 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC), 5901 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED) 5902 }, 5903 { "GDS_OA_PHY_PHY_DATA_RAM_MEM", 5904 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5905 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED), 5906 0, 0 5907 }, 5908 { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM", 5909 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5910 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC), 5911 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED) 5912 }, 5913 { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM", 5914 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5915 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC), 5916 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED) 5917 }, 5918 { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM", 5919 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5920 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC), 5921 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED) 5922 }, 5923 { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM", 5924 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5925 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC), 5926 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED) 5927 }, 5928 { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 5929 SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT), 5930 0, 0 5931 }, 5932 { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5933 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT), 5934 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT) 5935 }, 5936 { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5937 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT), 5938 0, 0 5939 }, 5940 { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5941 SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT), 5942 0, 0 5943 }, 5944 { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5945 SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT), 5946 0, 0 5947 }, 5948 { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5949 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT), 5950 0, 0 5951 }, 5952 { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 5953 SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT), 5954 0, 0 5955 }, 5956 { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 5957 SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT), 5958 0, 0 5959 }, 5960 { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5961 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT), 5962 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT) 5963 }, 5964 { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5965 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT), 5966 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT) 5967 }, 5968 { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5969 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT), 5970 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT) 5971 }, 5972 { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5973 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT), 5974 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT) 5975 }, 5976 { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5977 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT), 5978 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT) 5979 }, 5980 { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5981 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT), 5982 0, 0 5983 }, 5984 { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5985 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT), 5986 0, 0 5987 }, 5988 { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5989 SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT), 5990 0, 0 5991 }, 5992 { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5993 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT), 5994 0, 0 5995 }, 5996 { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5997 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT), 5998 0, 0 5999 }, 6000 { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6001 SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT), 6002 0, 0 6003 }, 6004 { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6005 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT), 6006 0, 0 6007 }, 6008 { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6009 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT), 6010 0, 0 6011 }, 6012 { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6013 SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT), 6014 0, 0 6015 }, 6016 { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6017 SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT), 6018 0, 0 6019 }, 6020 { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6021 SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT), 6022 0, 0 6023 }, 6024 { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6025 SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT), 6026 0, 0 6027 }, 6028 { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6029 SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT), 6030 0, 0 6031 }, 6032 { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 6033 SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT), 6034 0, 0 6035 }, 6036 { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6037 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT), 6038 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT) 6039 }, 6040 { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6041 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT), 6042 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT) 6043 }, 6044 { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6045 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT), 6046 0, 0 6047 }, 6048 { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6049 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT), 6050 0, 0 6051 }, 6052 { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6053 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT), 6054 0, 0 6055 }, 6056 { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6057 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT), 6058 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT) 6059 }, 6060 { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6061 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT), 6062 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT) 6063 }, 6064 { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6065 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT), 6066 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT) 6067 }, 6068 { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6069 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT), 6070 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT) 6071 }, 6072 { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6073 SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT), 6074 0, 0 6075 }, 6076 { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6077 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT), 6078 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT) 6079 }, 6080 { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6081 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT), 6082 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT) 6083 }, 6084 { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6085 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT), 6086 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT) 6087 }, 6088 { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6089 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT), 6090 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT) 6091 }, 6092 { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6093 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT), 6094 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT) 6095 }, 6096 { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6097 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT), 6098 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT) 6099 }, 6100 { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6101 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT), 6102 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT) 6103 }, 6104 { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6105 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT), 6106 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT) 6107 }, 6108 { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6109 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT), 6110 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT) 6111 }, 6112 { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6113 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT), 6114 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT) 6115 }, 6116 { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6117 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT), 6118 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT) 6119 }, 6120 { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6121 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT), 6122 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT) 6123 }, 6124 { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6125 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT), 6126 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT) 6127 }, 6128 { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6129 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT), 6130 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT) 6131 }, 6132 { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6133 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT), 6134 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT) 6135 }, 6136 { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6137 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT), 6138 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT) 6139 }, 6140 { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6141 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT), 6142 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT) 6143 }, 6144 { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6145 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT), 6146 0, 0 6147 }, 6148 { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6149 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT), 6150 0, 0 6151 }, 6152 { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6153 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT), 6154 0, 0 6155 }, 6156 { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6157 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT), 6158 0, 0 6159 }, 6160 { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6161 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT), 6162 0, 0 6163 }, 6164 { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6165 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT), 6166 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT) 6167 }, 6168 { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6169 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT), 6170 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT) 6171 }, 6172 { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6173 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT), 6174 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT) 6175 }, 6176 { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6177 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT), 6178 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT) 6179 }, 6180 { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6181 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT), 6182 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT) 6183 }, 6184 { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6185 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT), 6186 0, 0 6187 }, 6188 { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6189 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT), 6190 0, 0 6191 }, 6192 { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6193 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT), 6194 0, 0 6195 }, 6196 { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6197 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT), 6198 0, 0 6199 }, 6200 { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6201 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT), 6202 0, 0 6203 }, 6204 { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6205 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), 6206 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT) 6207 }, 6208 { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6209 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), 6210 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT) 6211 }, 6212 { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6213 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), 6214 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT) 6215 }, 6216 { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6217 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT), 6218 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT) 6219 }, 6220 { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6221 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT), 6222 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT) 6223 }, 6224 { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6225 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), 6226 0, 0 6227 }, 6228 { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6229 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), 6230 0, 0 6231 }, 6232 { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6233 SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT), 6234 0, 0 6235 }, 6236 { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6237 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT), 6238 0, 0 6239 }, 6240 { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6241 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT), 6242 0, 0 6243 }, 6244 { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6245 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), 6246 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT) 6247 }, 6248 { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6249 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), 6250 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT) 6251 }, 6252 { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6253 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), 6254 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT) 6255 }, 6256 { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6257 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), 6258 0, 0 6259 }, 6260 { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6261 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), 6262 0, 0 6263 }, 6264 { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6265 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT), 6266 0, 0 6267 }, 6268 { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6269 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT), 6270 0, 0 6271 }, 6272 { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6273 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT), 6274 0, 0 6275 }, 6276 { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6277 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT), 6278 0, 0 6279 } 6280 }; 6281 6282 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 6283 void *inject_if) 6284 { 6285 struct ras_inject_if *info = (struct ras_inject_if *)inject_if; 6286 int ret; 6287 struct ta_ras_trigger_error_input block_info = { 0 }; 6288 6289 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6290 return -EINVAL; 6291 6292 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks)) 6293 return -EINVAL; 6294 6295 if (!ras_gfx_subblocks[info->head.sub_block_index].name) 6296 return -EPERM; 6297 6298 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type & 6299 info->head.type)) { 6300 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n", 6301 ras_gfx_subblocks[info->head.sub_block_index].name, 6302 info->head.type); 6303 return -EPERM; 6304 } 6305 6306 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type & 6307 info->head.type)) { 6308 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n", 6309 ras_gfx_subblocks[info->head.sub_block_index].name, 6310 info->head.type); 6311 return -EPERM; 6312 } 6313 6314 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block); 6315 block_info.sub_block_index = 6316 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock; 6317 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type); 6318 block_info.address = info->address; 6319 block_info.value = info->value; 6320 6321 mutex_lock(&adev->grbm_idx_mutex); 6322 ret = psp_ras_trigger_error(&adev->psp, &block_info); 6323 mutex_unlock(&adev->grbm_idx_mutex); 6324 6325 return ret; 6326 } 6327 6328 static const char *vml2_mems[] = { 6329 "UTC_VML2_BANK_CACHE_0_BIGK_MEM0", 6330 "UTC_VML2_BANK_CACHE_0_BIGK_MEM1", 6331 "UTC_VML2_BANK_CACHE_0_4K_MEM0", 6332 "UTC_VML2_BANK_CACHE_0_4K_MEM1", 6333 "UTC_VML2_BANK_CACHE_1_BIGK_MEM0", 6334 "UTC_VML2_BANK_CACHE_1_BIGK_MEM1", 6335 "UTC_VML2_BANK_CACHE_1_4K_MEM0", 6336 "UTC_VML2_BANK_CACHE_1_4K_MEM1", 6337 "UTC_VML2_BANK_CACHE_2_BIGK_MEM0", 6338 "UTC_VML2_BANK_CACHE_2_BIGK_MEM1", 6339 "UTC_VML2_BANK_CACHE_2_4K_MEM0", 6340 "UTC_VML2_BANK_CACHE_2_4K_MEM1", 6341 "UTC_VML2_BANK_CACHE_3_BIGK_MEM0", 6342 "UTC_VML2_BANK_CACHE_3_BIGK_MEM1", 6343 "UTC_VML2_BANK_CACHE_3_4K_MEM0", 6344 "UTC_VML2_BANK_CACHE_3_4K_MEM1", 6345 }; 6346 6347 static const char *vml2_walker_mems[] = { 6348 "UTC_VML2_CACHE_PDE0_MEM0", 6349 "UTC_VML2_CACHE_PDE0_MEM1", 6350 "UTC_VML2_CACHE_PDE1_MEM0", 6351 "UTC_VML2_CACHE_PDE1_MEM1", 6352 "UTC_VML2_CACHE_PDE2_MEM0", 6353 "UTC_VML2_CACHE_PDE2_MEM1", 6354 "UTC_VML2_RDIF_LOG_FIFO", 6355 }; 6356 6357 static const char *atc_l2_cache_2m_mems[] = { 6358 "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM", 6359 "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM", 6360 "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM", 6361 "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM", 6362 }; 6363 6364 static const char *atc_l2_cache_4k_mems[] = { 6365 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0", 6366 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1", 6367 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2", 6368 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3", 6369 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4", 6370 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5", 6371 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6", 6372 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7", 6373 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0", 6374 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1", 6375 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2", 6376 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3", 6377 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4", 6378 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5", 6379 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6", 6380 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7", 6381 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0", 6382 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1", 6383 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2", 6384 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3", 6385 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4", 6386 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5", 6387 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6", 6388 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7", 6389 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0", 6390 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1", 6391 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2", 6392 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3", 6393 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4", 6394 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5", 6395 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6", 6396 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7", 6397 }; 6398 6399 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev, 6400 struct ras_err_data *err_data) 6401 { 6402 uint32_t i, data; 6403 uint32_t sec_count, ded_count; 6404 6405 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6406 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6407 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6408 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6409 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6410 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6411 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6412 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6413 6414 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6415 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6416 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6417 6418 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT); 6419 if (sec_count) { 6420 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6421 "SEC %d\n", i, vml2_mems[i], sec_count); 6422 err_data->ce_count += sec_count; 6423 } 6424 6425 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT); 6426 if (ded_count) { 6427 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6428 "DED %d\n", i, vml2_mems[i], ded_count); 6429 err_data->ue_count += ded_count; 6430 } 6431 } 6432 6433 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6434 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6435 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6436 6437 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6438 SEC_COUNT); 6439 if (sec_count) { 6440 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6441 "SEC %d\n", i, vml2_walker_mems[i], sec_count); 6442 err_data->ce_count += sec_count; 6443 } 6444 6445 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6446 DED_COUNT); 6447 if (ded_count) { 6448 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6449 "DED %d\n", i, vml2_walker_mems[i], ded_count); 6450 err_data->ue_count += ded_count; 6451 } 6452 } 6453 6454 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6455 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6456 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6457 6458 sec_count = (data & 0x00006000L) >> 0xd; 6459 if (sec_count) { 6460 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6461 "SEC %d\n", i, atc_l2_cache_2m_mems[i], 6462 sec_count); 6463 err_data->ce_count += sec_count; 6464 } 6465 } 6466 6467 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6468 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6469 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6470 6471 sec_count = (data & 0x00006000L) >> 0xd; 6472 if (sec_count) { 6473 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6474 "SEC %d\n", i, atc_l2_cache_4k_mems[i], 6475 sec_count); 6476 err_data->ce_count += sec_count; 6477 } 6478 6479 ded_count = (data & 0x00018000L) >> 0xf; 6480 if (ded_count) { 6481 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6482 "DED %d\n", i, atc_l2_cache_4k_mems[i], 6483 ded_count); 6484 err_data->ue_count += ded_count; 6485 } 6486 } 6487 6488 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6489 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6490 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6491 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6492 6493 return 0; 6494 } 6495 6496 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev, 6497 const struct soc15_reg_entry *reg, 6498 uint32_t se_id, uint32_t inst_id, uint32_t value, 6499 uint32_t *sec_count, uint32_t *ded_count) 6500 { 6501 uint32_t i; 6502 uint32_t sec_cnt, ded_cnt; 6503 6504 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) { 6505 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset || 6506 gfx_v9_0_ras_fields[i].seg != reg->seg || 6507 gfx_v9_0_ras_fields[i].inst != reg->inst) 6508 continue; 6509 6510 sec_cnt = (value & 6511 gfx_v9_0_ras_fields[i].sec_count_mask) >> 6512 gfx_v9_0_ras_fields[i].sec_count_shift; 6513 if (sec_cnt) { 6514 dev_info(adev->dev, "GFX SubBlock %s, " 6515 "Instance[%d][%d], SEC %d\n", 6516 gfx_v9_0_ras_fields[i].name, 6517 se_id, inst_id, 6518 sec_cnt); 6519 *sec_count += sec_cnt; 6520 } 6521 6522 ded_cnt = (value & 6523 gfx_v9_0_ras_fields[i].ded_count_mask) >> 6524 gfx_v9_0_ras_fields[i].ded_count_shift; 6525 if (ded_cnt) { 6526 dev_info(adev->dev, "GFX SubBlock %s, " 6527 "Instance[%d][%d], DED %d\n", 6528 gfx_v9_0_ras_fields[i].name, 6529 se_id, inst_id, 6530 ded_cnt); 6531 *ded_count += ded_cnt; 6532 } 6533 } 6534 6535 return 0; 6536 } 6537 6538 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev) 6539 { 6540 int i, j, k; 6541 6542 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6543 return; 6544 6545 /* read back registers to clear the counters */ 6546 mutex_lock(&adev->grbm_idx_mutex); 6547 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6548 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6549 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6550 gfx_v9_0_select_se_sh(adev, j, 0x0, k); 6551 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6552 } 6553 } 6554 } 6555 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000); 6556 mutex_unlock(&adev->grbm_idx_mutex); 6557 6558 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6559 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6560 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6561 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6562 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6563 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6564 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6565 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6566 6567 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6568 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6569 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6570 } 6571 6572 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6573 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6574 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6575 } 6576 6577 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6578 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6579 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6580 } 6581 6582 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6583 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6584 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6585 } 6586 6587 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6588 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6589 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6590 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6591 } 6592 6593 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 6594 void *ras_error_status) 6595 { 6596 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; 6597 uint32_t sec_count = 0, ded_count = 0; 6598 uint32_t i, j, k; 6599 uint32_t reg_value; 6600 6601 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6602 return -EINVAL; 6603 6604 err_data->ue_count = 0; 6605 err_data->ce_count = 0; 6606 6607 mutex_lock(&adev->grbm_idx_mutex); 6608 6609 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6610 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6611 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6612 gfx_v9_0_select_se_sh(adev, j, 0, k); 6613 reg_value = 6614 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6615 if (reg_value) 6616 gfx_v9_0_ras_error_count(adev, 6617 &gfx_v9_0_edc_counter_regs[i], 6618 j, k, reg_value, 6619 &sec_count, &ded_count); 6620 } 6621 } 6622 } 6623 6624 err_data->ce_count += sec_count; 6625 err_data->ue_count += ded_count; 6626 6627 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 6628 mutex_unlock(&adev->grbm_idx_mutex); 6629 6630 gfx_v9_0_query_utc_edc_status(adev, err_data); 6631 6632 return 0; 6633 } 6634 6635 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring) 6636 { 6637 const unsigned int cp_coher_cntl = 6638 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) | 6639 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) | 6640 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) | 6641 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) | 6642 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1); 6643 6644 /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */ 6645 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5)); 6646 amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */ 6647 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6648 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */ 6649 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6650 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 6651 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ 6652 } 6653 6654 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = { 6655 .name = "gfx_v9_0", 6656 .early_init = gfx_v9_0_early_init, 6657 .late_init = gfx_v9_0_late_init, 6658 .sw_init = gfx_v9_0_sw_init, 6659 .sw_fini = gfx_v9_0_sw_fini, 6660 .hw_init = gfx_v9_0_hw_init, 6661 .hw_fini = gfx_v9_0_hw_fini, 6662 .suspend = gfx_v9_0_suspend, 6663 .resume = gfx_v9_0_resume, 6664 .is_idle = gfx_v9_0_is_idle, 6665 .wait_for_idle = gfx_v9_0_wait_for_idle, 6666 .soft_reset = gfx_v9_0_soft_reset, 6667 .set_clockgating_state = gfx_v9_0_set_clockgating_state, 6668 .set_powergating_state = gfx_v9_0_set_powergating_state, 6669 .get_clockgating_state = gfx_v9_0_get_clockgating_state, 6670 }; 6671 6672 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = { 6673 .type = AMDGPU_RING_TYPE_GFX, 6674 .align_mask = 0xff, 6675 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6676 .support_64bit_ptrs = true, 6677 .vmhub = AMDGPU_GFXHUB_0, 6678 .get_rptr = gfx_v9_0_ring_get_rptr_gfx, 6679 .get_wptr = gfx_v9_0_ring_get_wptr_gfx, 6680 .set_wptr = gfx_v9_0_ring_set_wptr_gfx, 6681 .emit_frame_size = /* totally 242 maximum if 16 IBs */ 6682 5 + /* COND_EXEC */ 6683 7 + /* PIPELINE_SYNC */ 6684 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6685 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6686 2 + /* VM_FLUSH */ 6687 8 + /* FENCE for VM_FLUSH */ 6688 20 + /* GDS switch */ 6689 4 + /* double SWITCH_BUFFER, 6690 the first COND_EXEC jump to the place just 6691 prior to this double SWITCH_BUFFER */ 6692 5 + /* COND_EXEC */ 6693 7 + /* HDP_flush */ 6694 4 + /* VGT_flush */ 6695 14 + /* CE_META */ 6696 31 + /* DE_META */ 6697 3 + /* CNTX_CTRL */ 6698 5 + /* HDP_INVL */ 6699 8 + 8 + /* FENCE x2 */ 6700 2 + /* SWITCH_BUFFER */ 6701 7, /* gfx_v9_0_emit_mem_sync */ 6702 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */ 6703 .emit_ib = gfx_v9_0_ring_emit_ib_gfx, 6704 .emit_fence = gfx_v9_0_ring_emit_fence, 6705 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6706 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6707 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6708 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6709 .test_ring = gfx_v9_0_ring_test_ring, 6710 .test_ib = gfx_v9_0_ring_test_ib, 6711 .insert_nop = amdgpu_ring_insert_nop, 6712 .pad_ib = amdgpu_ring_generic_pad_ib, 6713 .emit_switch_buffer = gfx_v9_ring_emit_sb, 6714 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl, 6715 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec, 6716 .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec, 6717 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl, 6718 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6719 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6720 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6721 .soft_recovery = gfx_v9_0_ring_soft_recovery, 6722 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6723 }; 6724 6725 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { 6726 .type = AMDGPU_RING_TYPE_COMPUTE, 6727 .align_mask = 0xff, 6728 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6729 .support_64bit_ptrs = true, 6730 .vmhub = AMDGPU_GFXHUB_0, 6731 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6732 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6733 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6734 .emit_frame_size = 6735 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6736 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6737 5 + /* hdp invalidate */ 6738 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6739 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6740 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6741 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6742 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */ 6743 7, /* gfx_v9_0_emit_mem_sync */ 6744 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6745 .emit_ib = gfx_v9_0_ring_emit_ib_compute, 6746 .emit_fence = gfx_v9_0_ring_emit_fence, 6747 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6748 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6749 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6750 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6751 .test_ring = gfx_v9_0_ring_test_ring, 6752 .test_ib = gfx_v9_0_ring_test_ib, 6753 .insert_nop = amdgpu_ring_insert_nop, 6754 .pad_ib = amdgpu_ring_generic_pad_ib, 6755 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6756 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6757 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6758 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6759 }; 6760 6761 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = { 6762 .type = AMDGPU_RING_TYPE_KIQ, 6763 .align_mask = 0xff, 6764 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6765 .support_64bit_ptrs = true, 6766 .vmhub = AMDGPU_GFXHUB_0, 6767 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6768 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6769 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6770 .emit_frame_size = 6771 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6772 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6773 5 + /* hdp invalidate */ 6774 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6775 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6776 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6777 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6778 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 6779 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6780 .emit_fence = gfx_v9_0_ring_emit_fence_kiq, 6781 .test_ring = gfx_v9_0_ring_test_ring, 6782 .insert_nop = amdgpu_ring_insert_nop, 6783 .pad_ib = amdgpu_ring_generic_pad_ib, 6784 .emit_rreg = gfx_v9_0_ring_emit_rreg, 6785 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6786 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6787 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6788 }; 6789 6790 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev) 6791 { 6792 int i; 6793 6794 adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq; 6795 6796 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 6797 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx; 6798 6799 for (i = 0; i < adev->gfx.num_compute_rings; i++) 6800 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute; 6801 } 6802 6803 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = { 6804 .set = gfx_v9_0_set_eop_interrupt_state, 6805 .process = gfx_v9_0_eop_irq, 6806 }; 6807 6808 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = { 6809 .set = gfx_v9_0_set_priv_reg_fault_state, 6810 .process = gfx_v9_0_priv_reg_irq, 6811 }; 6812 6813 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = { 6814 .set = gfx_v9_0_set_priv_inst_fault_state, 6815 .process = gfx_v9_0_priv_inst_irq, 6816 }; 6817 6818 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = { 6819 .set = gfx_v9_0_set_cp_ecc_error_state, 6820 .process = amdgpu_gfx_cp_ecc_error_irq, 6821 }; 6822 6823 6824 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev) 6825 { 6826 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 6827 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs; 6828 6829 adev->gfx.priv_reg_irq.num_types = 1; 6830 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs; 6831 6832 adev->gfx.priv_inst_irq.num_types = 1; 6833 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs; 6834 6835 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/ 6836 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs; 6837 } 6838 6839 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev) 6840 { 6841 switch (adev->asic_type) { 6842 case CHIP_VEGA10: 6843 case CHIP_VEGA12: 6844 case CHIP_VEGA20: 6845 case CHIP_RAVEN: 6846 case CHIP_ARCTURUS: 6847 case CHIP_RENOIR: 6848 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs; 6849 break; 6850 default: 6851 break; 6852 } 6853 } 6854 6855 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev) 6856 { 6857 /* init asci gds info */ 6858 switch (adev->asic_type) { 6859 case CHIP_VEGA10: 6860 case CHIP_VEGA12: 6861 case CHIP_VEGA20: 6862 adev->gds.gds_size = 0x10000; 6863 break; 6864 case CHIP_RAVEN: 6865 case CHIP_ARCTURUS: 6866 adev->gds.gds_size = 0x1000; 6867 break; 6868 default: 6869 adev->gds.gds_size = 0x10000; 6870 break; 6871 } 6872 6873 switch (adev->asic_type) { 6874 case CHIP_VEGA10: 6875 case CHIP_VEGA20: 6876 adev->gds.gds_compute_max_wave_id = 0x7ff; 6877 break; 6878 case CHIP_VEGA12: 6879 adev->gds.gds_compute_max_wave_id = 0x27f; 6880 break; 6881 case CHIP_RAVEN: 6882 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 6883 adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */ 6884 else 6885 adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */ 6886 break; 6887 case CHIP_ARCTURUS: 6888 adev->gds.gds_compute_max_wave_id = 0xfff; 6889 break; 6890 default: 6891 /* this really depends on the chip */ 6892 adev->gds.gds_compute_max_wave_id = 0x7ff; 6893 break; 6894 } 6895 6896 adev->gds.gws_size = 64; 6897 adev->gds.oa_size = 16; 6898 } 6899 6900 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev, 6901 u32 bitmap) 6902 { 6903 u32 data; 6904 6905 if (!bitmap) 6906 return; 6907 6908 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 6909 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 6910 6911 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data); 6912 } 6913 6914 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev) 6915 { 6916 u32 data, mask; 6917 6918 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG); 6919 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG); 6920 6921 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 6922 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 6923 6924 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); 6925 6926 return (~data) & mask; 6927 } 6928 6929 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 6930 struct amdgpu_cu_info *cu_info) 6931 { 6932 int i, j, k, counter, active_cu_number = 0; 6933 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0; 6934 unsigned disable_masks[4 * 4]; 6935 6936 if (!adev || !cu_info) 6937 return -EINVAL; 6938 6939 /* 6940 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs 6941 */ 6942 if (adev->gfx.config.max_shader_engines * 6943 adev->gfx.config.max_sh_per_se > 16) 6944 return -EINVAL; 6945 6946 amdgpu_gfx_parse_disable_cu(disable_masks, 6947 adev->gfx.config.max_shader_engines, 6948 adev->gfx.config.max_sh_per_se); 6949 6950 mutex_lock(&adev->grbm_idx_mutex); 6951 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 6952 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 6953 mask = 1; 6954 ao_bitmap = 0; 6955 counter = 0; 6956 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 6957 gfx_v9_0_set_user_cu_inactive_bitmap( 6958 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]); 6959 bitmap = gfx_v9_0_get_cu_active_bitmap(adev); 6960 6961 /* 6962 * The bitmap(and ao_cu_bitmap) in cu_info structure is 6963 * 4x4 size array, and it's usually suitable for Vega 6964 * ASICs which has 4*2 SE/SH layout. 6965 * But for Arcturus, SE/SH layout is changed to 8*1. 6966 * To mostly reduce the impact, we make it compatible 6967 * with current bitmap array as below: 6968 * SE4,SH0 --> bitmap[0][1] 6969 * SE5,SH0 --> bitmap[1][1] 6970 * SE6,SH0 --> bitmap[2][1] 6971 * SE7,SH0 --> bitmap[3][1] 6972 */ 6973 cu_info->bitmap[i % 4][j + i / 4] = bitmap; 6974 6975 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 6976 if (bitmap & mask) { 6977 if (counter < adev->gfx.config.max_cu_per_sh) 6978 ao_bitmap |= mask; 6979 counter ++; 6980 } 6981 mask <<= 1; 6982 } 6983 active_cu_number += counter; 6984 if (i < 2 && j < 2) 6985 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8)); 6986 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap; 6987 } 6988 } 6989 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 6990 mutex_unlock(&adev->grbm_idx_mutex); 6991 6992 cu_info->number = active_cu_number; 6993 cu_info->ao_cu_mask = ao_cu_mask; 6994 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 6995 6996 return 0; 6997 } 6998 6999 const struct amdgpu_ip_block_version gfx_v9_0_ip_block = 7000 { 7001 .type = AMD_IP_BLOCK_TYPE_GFX, 7002 .major = 9, 7003 .minor = 0, 7004 .rev = 0, 7005 .funcs = &gfx_v9_0_ip_funcs, 7006 }; 7007