1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/delay.h> 25 #include <linux/kernel.h> 26 #include <linux/firmware.h> 27 #include <linux/module.h> 28 #include <linux/pci.h> 29 30 #include "amdgpu.h" 31 #include "amdgpu_gfx.h" 32 #include "soc15.h" 33 #include "soc15d.h" 34 #include "amdgpu_atomfirmware.h" 35 #include "amdgpu_pm.h" 36 37 #include "gc/gc_9_0_offset.h" 38 #include "gc/gc_9_0_sh_mask.h" 39 40 #include "vega10_enum.h" 41 42 #include "soc15_common.h" 43 #include "clearstate_gfx9.h" 44 #include "v9_structs.h" 45 46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h" 47 48 #include "amdgpu_ras.h" 49 50 #include "gfx_v9_4.h" 51 #include "gfx_v9_0.h" 52 #include "gfx_v9_4_2.h" 53 54 #include "asic_reg/pwr/pwr_10_0_offset.h" 55 #include "asic_reg/pwr/pwr_10_0_sh_mask.h" 56 #include "asic_reg/gc/gc_9_0_default.h" 57 58 #define GFX9_NUM_GFX_RINGS 1 59 #define GFX9_MEC_HPD_SIZE 4096 60 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L 61 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L 62 63 #define mmGCEA_PROBE_MAP 0x070c 64 #define mmGCEA_PROBE_MAP_BASE_IDX 0 65 66 MODULE_FIRMWARE("amdgpu/vega10_ce.bin"); 67 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin"); 68 MODULE_FIRMWARE("amdgpu/vega10_me.bin"); 69 MODULE_FIRMWARE("amdgpu/vega10_mec.bin"); 70 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin"); 71 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin"); 72 73 MODULE_FIRMWARE("amdgpu/vega12_ce.bin"); 74 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin"); 75 MODULE_FIRMWARE("amdgpu/vega12_me.bin"); 76 MODULE_FIRMWARE("amdgpu/vega12_mec.bin"); 77 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin"); 78 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin"); 79 80 MODULE_FIRMWARE("amdgpu/vega20_ce.bin"); 81 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin"); 82 MODULE_FIRMWARE("amdgpu/vega20_me.bin"); 83 MODULE_FIRMWARE("amdgpu/vega20_mec.bin"); 84 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin"); 85 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin"); 86 87 MODULE_FIRMWARE("amdgpu/raven_ce.bin"); 88 MODULE_FIRMWARE("amdgpu/raven_pfp.bin"); 89 MODULE_FIRMWARE("amdgpu/raven_me.bin"); 90 MODULE_FIRMWARE("amdgpu/raven_mec.bin"); 91 MODULE_FIRMWARE("amdgpu/raven_mec2.bin"); 92 MODULE_FIRMWARE("amdgpu/raven_rlc.bin"); 93 94 MODULE_FIRMWARE("amdgpu/picasso_ce.bin"); 95 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin"); 96 MODULE_FIRMWARE("amdgpu/picasso_me.bin"); 97 MODULE_FIRMWARE("amdgpu/picasso_mec.bin"); 98 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin"); 99 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin"); 100 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin"); 101 102 MODULE_FIRMWARE("amdgpu/raven2_ce.bin"); 103 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin"); 104 MODULE_FIRMWARE("amdgpu/raven2_me.bin"); 105 MODULE_FIRMWARE("amdgpu/raven2_mec.bin"); 106 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin"); 107 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin"); 108 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin"); 109 110 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin"); 111 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin"); 112 113 MODULE_FIRMWARE("amdgpu/renoir_ce.bin"); 114 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin"); 115 MODULE_FIRMWARE("amdgpu/renoir_me.bin"); 116 MODULE_FIRMWARE("amdgpu/renoir_mec.bin"); 117 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin"); 118 119 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin"); 120 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin"); 121 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin"); 122 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin"); 123 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin"); 124 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin"); 125 126 MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin"); 127 MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin"); 128 MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin"); 129 130 #define mmTCP_CHAN_STEER_0_ARCT 0x0b03 131 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0 132 #define mmTCP_CHAN_STEER_1_ARCT 0x0b04 133 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0 134 #define mmTCP_CHAN_STEER_2_ARCT 0x0b09 135 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0 136 #define mmTCP_CHAN_STEER_3_ARCT 0x0b0a 137 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0 138 #define mmTCP_CHAN_STEER_4_ARCT 0x0b0b 139 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0 140 #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c 141 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0 142 143 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir 0x0025 144 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir_BASE_IDX 1 145 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir 0x0026 146 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir_BASE_IDX 1 147 148 enum ta_ras_gfx_subblock { 149 /*CPC*/ 150 TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0, 151 TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START, 152 TA_RAS_BLOCK__GFX_CPC_UCODE, 153 TA_RAS_BLOCK__GFX_DC_STATE_ME1, 154 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1, 155 TA_RAS_BLOCK__GFX_DC_RESTORE_ME1, 156 TA_RAS_BLOCK__GFX_DC_STATE_ME2, 157 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2, 158 TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 159 TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 160 /* CPF*/ 161 TA_RAS_BLOCK__GFX_CPF_INDEX_START, 162 TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START, 163 TA_RAS_BLOCK__GFX_CPF_ROQ_ME1, 164 TA_RAS_BLOCK__GFX_CPF_TAG, 165 TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG, 166 /* CPG*/ 167 TA_RAS_BLOCK__GFX_CPG_INDEX_START, 168 TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START, 169 TA_RAS_BLOCK__GFX_CPG_DMA_TAG, 170 TA_RAS_BLOCK__GFX_CPG_TAG, 171 TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG, 172 /* GDS*/ 173 TA_RAS_BLOCK__GFX_GDS_INDEX_START, 174 TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START, 175 TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE, 176 TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM, 177 TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM, 178 TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 179 TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 180 /* SPI*/ 181 TA_RAS_BLOCK__GFX_SPI_SR_MEM, 182 /* SQ*/ 183 TA_RAS_BLOCK__GFX_SQ_INDEX_START, 184 TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START, 185 TA_RAS_BLOCK__GFX_SQ_LDS_D, 186 TA_RAS_BLOCK__GFX_SQ_LDS_I, 187 TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/ 188 TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR, 189 /* SQC (3 ranges)*/ 190 TA_RAS_BLOCK__GFX_SQC_INDEX_START, 191 /* SQC range 0*/ 192 TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START, 193 TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO = 194 TA_RAS_BLOCK__GFX_SQC_INDEX0_START, 195 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 196 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO, 197 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 198 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO, 199 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 200 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 201 TA_RAS_BLOCK__GFX_SQC_INDEX0_END = 202 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 203 /* SQC range 1*/ 204 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 205 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM = 206 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 207 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 208 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO, 209 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM, 210 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM, 211 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO, 212 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO, 213 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 214 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 215 TA_RAS_BLOCK__GFX_SQC_INDEX1_END = 216 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 217 /* SQC range 2*/ 218 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 219 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM = 220 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 221 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 222 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO, 223 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM, 224 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM, 225 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO, 226 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO, 227 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 228 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 229 TA_RAS_BLOCK__GFX_SQC_INDEX2_END = 230 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 231 TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END, 232 /* TA*/ 233 TA_RAS_BLOCK__GFX_TA_INDEX_START, 234 TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START, 235 TA_RAS_BLOCK__GFX_TA_FS_AFIFO, 236 TA_RAS_BLOCK__GFX_TA_FL_LFIFO, 237 TA_RAS_BLOCK__GFX_TA_FX_LFIFO, 238 TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 239 TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 240 /* TCA*/ 241 TA_RAS_BLOCK__GFX_TCA_INDEX_START, 242 TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START, 243 TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 244 TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 245 /* TCC (5 sub-ranges)*/ 246 TA_RAS_BLOCK__GFX_TCC_INDEX_START, 247 /* TCC range 0*/ 248 TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START, 249 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START, 250 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1, 251 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0, 252 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1, 253 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0, 254 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1, 255 TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG, 256 TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 257 TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 258 /* TCC range 1*/ 259 TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 260 TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 261 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 262 TA_RAS_BLOCK__GFX_TCC_INDEX1_END = 263 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 264 /* TCC range 2*/ 265 TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 266 TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 267 TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL, 268 TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO, 269 TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN, 270 TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ, 271 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO, 272 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM, 273 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 274 TA_RAS_BLOCK__GFX_TCC_INDEX2_END = 275 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 276 /* TCC range 3*/ 277 TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 278 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 279 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 280 TA_RAS_BLOCK__GFX_TCC_INDEX3_END = 281 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 282 /* TCC range 4*/ 283 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 284 TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN = 285 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 286 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 287 TA_RAS_BLOCK__GFX_TCC_INDEX4_END = 288 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 289 TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END, 290 /* TCI*/ 291 TA_RAS_BLOCK__GFX_TCI_WRITE_RAM, 292 /* TCP*/ 293 TA_RAS_BLOCK__GFX_TCP_INDEX_START, 294 TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START, 295 TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM, 296 TA_RAS_BLOCK__GFX_TCP_CMD_FIFO, 297 TA_RAS_BLOCK__GFX_TCP_VM_FIFO, 298 TA_RAS_BLOCK__GFX_TCP_DB_RAM, 299 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0, 300 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 301 TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 302 /* TD*/ 303 TA_RAS_BLOCK__GFX_TD_INDEX_START, 304 TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START, 305 TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI, 306 TA_RAS_BLOCK__GFX_TD_CS_FIFO, 307 TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO, 308 /* EA (3 sub-ranges)*/ 309 TA_RAS_BLOCK__GFX_EA_INDEX_START, 310 /* EA range 0*/ 311 TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START, 312 TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START, 313 TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM, 314 TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM, 315 TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM, 316 TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM, 317 TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM, 318 TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM, 319 TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 320 TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 321 /* EA range 1*/ 322 TA_RAS_BLOCK__GFX_EA_INDEX1_START, 323 TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START, 324 TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM, 325 TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM, 326 TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM, 327 TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM, 328 TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM, 329 TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 330 TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 331 /* EA range 2*/ 332 TA_RAS_BLOCK__GFX_EA_INDEX2_START, 333 TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START, 334 TA_RAS_BLOCK__GFX_EA_MAM_D1MEM, 335 TA_RAS_BLOCK__GFX_EA_MAM_D2MEM, 336 TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 337 TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 338 TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END, 339 /* UTC VM L2 bank*/ 340 TA_RAS_BLOCK__UTC_VML2_BANK_CACHE, 341 /* UTC VM walker*/ 342 TA_RAS_BLOCK__UTC_VML2_WALKER, 343 /* UTC ATC L2 2MB cache*/ 344 TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK, 345 /* UTC ATC L2 4KB cache*/ 346 TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK, 347 TA_RAS_BLOCK__GFX_MAX 348 }; 349 350 struct ras_gfx_subblock { 351 unsigned char *name; 352 int ta_subblock; 353 int hw_supported_error_type; 354 int sw_supported_error_type; 355 }; 356 357 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \ 358 [AMDGPU_RAS_BLOCK__##subblock] = { \ 359 #subblock, \ 360 TA_RAS_BLOCK__##subblock, \ 361 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \ 362 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \ 363 } 364 365 static const struct ras_gfx_subblock ras_gfx_subblocks[] = { 366 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1), 367 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1), 368 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 369 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 370 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 371 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 372 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 373 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 374 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 375 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 376 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1), 377 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0), 378 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1), 379 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1), 380 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 381 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0), 382 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0, 383 0), 384 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0, 385 0), 386 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 387 AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0), 388 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0), 389 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1), 390 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0), 391 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0), 392 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1), 393 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 394 0, 0), 395 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 396 0), 397 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 398 0, 0), 399 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0, 400 0), 401 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 402 0, 0), 403 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 404 0), 405 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 406 1), 407 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 408 0, 0, 0), 409 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 410 0), 411 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 412 0), 413 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 414 0), 415 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 416 0), 417 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 418 0), 419 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 420 0, 0), 421 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 422 0), 423 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 424 0), 425 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 426 0, 0, 0), 427 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 428 0), 429 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 430 0), 431 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 432 0), 433 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 434 0), 435 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 436 0), 437 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 438 0, 0), 439 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 440 0), 441 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1), 442 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 443 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 444 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 445 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 446 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0), 447 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 448 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1), 449 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0, 450 1), 451 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0, 452 1), 453 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0, 454 1), 455 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0, 456 0), 457 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0, 458 0), 459 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 460 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 461 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0), 462 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0), 463 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0), 464 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0), 465 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 466 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0), 467 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0), 468 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 469 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0), 470 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0, 471 0), 472 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 473 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0, 474 0), 475 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0, 476 0, 0), 477 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0, 478 0), 479 AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 480 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1), 481 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0), 482 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 483 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 484 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 485 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0), 486 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0), 487 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1), 488 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0), 489 AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 490 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1), 491 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 492 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 493 AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 494 AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 495 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 496 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 497 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 498 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 499 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 500 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 501 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 502 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0), 503 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 504 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 505 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0), 506 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0), 507 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0), 508 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0), 509 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0), 510 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0), 511 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0), 512 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0), 513 }; 514 515 static const struct soc15_reg_golden golden_settings_gc_9_0[] = 516 { 517 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 518 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000), 519 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 520 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 521 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 522 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 523 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 524 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 525 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 526 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87), 527 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f), 528 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 529 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 530 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 531 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 532 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 533 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff), 534 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 535 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 536 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 537 }; 538 539 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] = 540 { 541 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107), 542 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 543 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 544 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 545 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 546 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 547 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042), 548 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 549 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000), 550 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 551 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 552 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 553 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 554 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 555 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 556 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107), 557 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800), 558 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080) 559 }; 560 561 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] = 562 { 563 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080), 564 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 565 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 566 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042), 567 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042), 568 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400), 569 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000), 570 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000), 571 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107), 572 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000), 573 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000) 574 }; 575 576 static const struct soc15_reg_golden golden_settings_gc_9_1[] = 577 { 578 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 579 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 580 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 581 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 582 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 583 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 584 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 585 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 586 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 587 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 588 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 589 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 590 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 591 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 592 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 593 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 594 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 595 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 596 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 597 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff), 598 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080), 599 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 600 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 601 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 602 }; 603 604 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] = 605 { 606 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 607 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042), 608 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042), 609 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000), 610 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000), 611 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 612 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800) 613 }; 614 615 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] = 616 { 617 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000), 618 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 619 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 620 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080), 621 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080), 622 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080), 623 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041), 624 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041), 625 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 626 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 627 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080), 628 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080), 629 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080), 630 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080), 631 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080), 632 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 633 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010), 634 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 635 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 636 }; 637 638 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] = 639 { 640 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 641 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 642 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 643 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042), 644 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042), 645 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 646 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 647 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 648 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 649 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 650 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 651 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc), 652 }; 653 654 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] = 655 { 656 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff), 657 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000), 658 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382) 659 }; 660 661 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] = 662 { 663 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 664 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 665 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 666 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 667 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 668 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 669 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 670 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 671 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87), 672 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f), 673 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 674 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 675 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 676 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 677 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 678 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff) 679 }; 680 681 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] = 682 { 683 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080), 684 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 685 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 686 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041), 687 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041), 688 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 689 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107), 690 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 691 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410), 692 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 693 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 694 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 695 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 696 }; 697 698 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] = 699 { 700 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 701 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000), 702 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e), 703 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca), 704 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098), 705 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3), 706 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1), 707 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135), 708 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000), 709 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00), 710 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000) 711 }; 712 713 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = { 714 {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)}, 715 {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)}, 716 }; 717 718 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] = 719 { 720 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 721 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 722 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 723 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 724 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 725 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 726 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 727 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 728 }; 729 730 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] = 731 { 732 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0, 733 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0, 734 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0, 735 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0, 736 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0, 737 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0, 738 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0, 739 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0, 740 }; 741 742 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042 743 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041 744 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042 745 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041 746 747 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev); 748 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev); 749 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev); 750 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev); 751 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 752 struct amdgpu_cu_info *cu_info); 753 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev); 754 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring); 755 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring); 756 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 757 void *ras_error_status); 758 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 759 void *inject_if); 760 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev); 761 762 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring, 763 uint64_t queue_mask) 764 { 765 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 766 amdgpu_ring_write(kiq_ring, 767 PACKET3_SET_RESOURCES_VMID_MASK(0) | 768 /* vmid_mask:0* queue_type:0 (KIQ) */ 769 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); 770 amdgpu_ring_write(kiq_ring, 771 lower_32_bits(queue_mask)); /* queue mask lo */ 772 amdgpu_ring_write(kiq_ring, 773 upper_32_bits(queue_mask)); /* queue mask hi */ 774 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */ 775 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */ 776 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 777 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 778 } 779 780 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring, 781 struct amdgpu_ring *ring) 782 { 783 struct amdgpu_device *adev = kiq_ring->adev; 784 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 785 uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 786 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 787 788 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 789 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 790 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 791 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 792 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ 793 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 794 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 795 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) | 796 /*queue_type: normal compute queue */ 797 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | 798 /* alloc format: all_on_one_pipe */ 799 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | 800 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) | 801 /* num_queues: must be 1 */ 802 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); 803 amdgpu_ring_write(kiq_ring, 804 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); 805 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 806 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 807 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 808 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 809 } 810 811 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, 812 struct amdgpu_ring *ring, 813 enum amdgpu_unmap_queues_action action, 814 u64 gpu_addr, u64 seq) 815 { 816 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 817 818 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 819 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 820 PACKET3_UNMAP_QUEUES_ACTION(action) | 821 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 822 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) | 823 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 824 amdgpu_ring_write(kiq_ring, 825 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 826 827 if (action == PREEMPT_QUEUES_NO_UNMAP) { 828 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr)); 829 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr)); 830 amdgpu_ring_write(kiq_ring, seq); 831 } else { 832 amdgpu_ring_write(kiq_ring, 0); 833 amdgpu_ring_write(kiq_ring, 0); 834 amdgpu_ring_write(kiq_ring, 0); 835 } 836 } 837 838 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring, 839 struct amdgpu_ring *ring, 840 u64 addr, 841 u64 seq) 842 { 843 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 844 845 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5)); 846 amdgpu_ring_write(kiq_ring, 847 PACKET3_QUERY_STATUS_CONTEXT_ID(0) | 848 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) | 849 PACKET3_QUERY_STATUS_COMMAND(2)); 850 /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 851 amdgpu_ring_write(kiq_ring, 852 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) | 853 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel)); 854 amdgpu_ring_write(kiq_ring, lower_32_bits(addr)); 855 amdgpu_ring_write(kiq_ring, upper_32_bits(addr)); 856 amdgpu_ring_write(kiq_ring, lower_32_bits(seq)); 857 amdgpu_ring_write(kiq_ring, upper_32_bits(seq)); 858 } 859 860 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring, 861 uint16_t pasid, uint32_t flush_type, 862 bool all_hub) 863 { 864 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); 865 amdgpu_ring_write(kiq_ring, 866 PACKET3_INVALIDATE_TLBS_DST_SEL(1) | 867 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) | 868 PACKET3_INVALIDATE_TLBS_PASID(pasid) | 869 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type)); 870 } 871 872 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = { 873 .kiq_set_resources = gfx_v9_0_kiq_set_resources, 874 .kiq_map_queues = gfx_v9_0_kiq_map_queues, 875 .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues, 876 .kiq_query_status = gfx_v9_0_kiq_query_status, 877 .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs, 878 .set_resources_size = 8, 879 .map_queues_size = 7, 880 .unmap_queues_size = 6, 881 .query_status_size = 7, 882 .invalidate_tlbs_size = 2, 883 }; 884 885 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) 886 { 887 adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs; 888 } 889 890 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev) 891 { 892 switch (adev->ip_versions[GC_HWIP][0]) { 893 case IP_VERSION(9, 0, 1): 894 soc15_program_register_sequence(adev, 895 golden_settings_gc_9_0, 896 ARRAY_SIZE(golden_settings_gc_9_0)); 897 soc15_program_register_sequence(adev, 898 golden_settings_gc_9_0_vg10, 899 ARRAY_SIZE(golden_settings_gc_9_0_vg10)); 900 break; 901 case IP_VERSION(9, 2, 1): 902 soc15_program_register_sequence(adev, 903 golden_settings_gc_9_2_1, 904 ARRAY_SIZE(golden_settings_gc_9_2_1)); 905 soc15_program_register_sequence(adev, 906 golden_settings_gc_9_2_1_vg12, 907 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12)); 908 break; 909 case IP_VERSION(9, 4, 0): 910 soc15_program_register_sequence(adev, 911 golden_settings_gc_9_0, 912 ARRAY_SIZE(golden_settings_gc_9_0)); 913 soc15_program_register_sequence(adev, 914 golden_settings_gc_9_0_vg20, 915 ARRAY_SIZE(golden_settings_gc_9_0_vg20)); 916 break; 917 case IP_VERSION(9, 4, 1): 918 soc15_program_register_sequence(adev, 919 golden_settings_gc_9_4_1_arct, 920 ARRAY_SIZE(golden_settings_gc_9_4_1_arct)); 921 break; 922 case IP_VERSION(9, 2, 2): 923 case IP_VERSION(9, 1, 0): 924 soc15_program_register_sequence(adev, golden_settings_gc_9_1, 925 ARRAY_SIZE(golden_settings_gc_9_1)); 926 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 927 soc15_program_register_sequence(adev, 928 golden_settings_gc_9_1_rv2, 929 ARRAY_SIZE(golden_settings_gc_9_1_rv2)); 930 else 931 soc15_program_register_sequence(adev, 932 golden_settings_gc_9_1_rv1, 933 ARRAY_SIZE(golden_settings_gc_9_1_rv1)); 934 break; 935 case IP_VERSION(9, 3, 0): 936 soc15_program_register_sequence(adev, 937 golden_settings_gc_9_1_rn, 938 ARRAY_SIZE(golden_settings_gc_9_1_rn)); 939 return; /* for renoir, don't need common goldensetting */ 940 case IP_VERSION(9, 4, 2): 941 gfx_v9_4_2_init_golden_registers(adev, 942 adev->smuio.funcs->get_die_id(adev)); 943 break; 944 default: 945 break; 946 } 947 948 if ((adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 1)) && 949 (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 2))) 950 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common, 951 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common)); 952 } 953 954 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev) 955 { 956 adev->gfx.scratch.num_reg = 8; 957 adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0); 958 adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1; 959 } 960 961 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel, 962 bool wc, uint32_t reg, uint32_t val) 963 { 964 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 965 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) | 966 WRITE_DATA_DST_SEL(0) | 967 (wc ? WR_CONFIRM : 0)); 968 amdgpu_ring_write(ring, reg); 969 amdgpu_ring_write(ring, 0); 970 amdgpu_ring_write(ring, val); 971 } 972 973 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, 974 int mem_space, int opt, uint32_t addr0, 975 uint32_t addr1, uint32_t ref, uint32_t mask, 976 uint32_t inv) 977 { 978 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 979 amdgpu_ring_write(ring, 980 /* memory (1) or register (0) */ 981 (WAIT_REG_MEM_MEM_SPACE(mem_space) | 982 WAIT_REG_MEM_OPERATION(opt) | /* wait */ 983 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 984 WAIT_REG_MEM_ENGINE(eng_sel))); 985 986 if (mem_space) 987 BUG_ON(addr0 & 0x3); /* Dword align */ 988 amdgpu_ring_write(ring, addr0); 989 amdgpu_ring_write(ring, addr1); 990 amdgpu_ring_write(ring, ref); 991 amdgpu_ring_write(ring, mask); 992 amdgpu_ring_write(ring, inv); /* poll interval */ 993 } 994 995 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring) 996 { 997 struct amdgpu_device *adev = ring->adev; 998 uint32_t scratch; 999 uint32_t tmp = 0; 1000 unsigned i; 1001 int r; 1002 1003 r = amdgpu_gfx_scratch_get(adev, &scratch); 1004 if (r) 1005 return r; 1006 1007 WREG32(scratch, 0xCAFEDEAD); 1008 r = amdgpu_ring_alloc(ring, 3); 1009 if (r) 1010 goto error_free_scratch; 1011 1012 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 1013 amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START)); 1014 amdgpu_ring_write(ring, 0xDEADBEEF); 1015 amdgpu_ring_commit(ring); 1016 1017 for (i = 0; i < adev->usec_timeout; i++) { 1018 tmp = RREG32(scratch); 1019 if (tmp == 0xDEADBEEF) 1020 break; 1021 udelay(1); 1022 } 1023 1024 if (i >= adev->usec_timeout) 1025 r = -ETIMEDOUT; 1026 1027 error_free_scratch: 1028 amdgpu_gfx_scratch_free(adev, scratch); 1029 return r; 1030 } 1031 1032 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 1033 { 1034 struct amdgpu_device *adev = ring->adev; 1035 struct amdgpu_ib ib; 1036 struct dma_fence *f = NULL; 1037 1038 unsigned index; 1039 uint64_t gpu_addr; 1040 uint32_t tmp; 1041 long r; 1042 1043 r = amdgpu_device_wb_get(adev, &index); 1044 if (r) 1045 return r; 1046 1047 gpu_addr = adev->wb.gpu_addr + (index * 4); 1048 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 1049 memset(&ib, 0, sizeof(ib)); 1050 r = amdgpu_ib_get(adev, NULL, 16, 1051 AMDGPU_IB_POOL_DIRECT, &ib); 1052 if (r) 1053 goto err1; 1054 1055 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 1056 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 1057 ib.ptr[2] = lower_32_bits(gpu_addr); 1058 ib.ptr[3] = upper_32_bits(gpu_addr); 1059 ib.ptr[4] = 0xDEADBEEF; 1060 ib.length_dw = 5; 1061 1062 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 1063 if (r) 1064 goto err2; 1065 1066 r = dma_fence_wait_timeout(f, false, timeout); 1067 if (r == 0) { 1068 r = -ETIMEDOUT; 1069 goto err2; 1070 } else if (r < 0) { 1071 goto err2; 1072 } 1073 1074 tmp = adev->wb.wb[index]; 1075 if (tmp == 0xDEADBEEF) 1076 r = 0; 1077 else 1078 r = -EINVAL; 1079 1080 err2: 1081 amdgpu_ib_free(adev, &ib, NULL); 1082 dma_fence_put(f); 1083 err1: 1084 amdgpu_device_wb_free(adev, index); 1085 return r; 1086 } 1087 1088 1089 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev) 1090 { 1091 release_firmware(adev->gfx.pfp_fw); 1092 adev->gfx.pfp_fw = NULL; 1093 release_firmware(adev->gfx.me_fw); 1094 adev->gfx.me_fw = NULL; 1095 release_firmware(adev->gfx.ce_fw); 1096 adev->gfx.ce_fw = NULL; 1097 release_firmware(adev->gfx.rlc_fw); 1098 adev->gfx.rlc_fw = NULL; 1099 release_firmware(adev->gfx.mec_fw); 1100 adev->gfx.mec_fw = NULL; 1101 release_firmware(adev->gfx.mec2_fw); 1102 adev->gfx.mec2_fw = NULL; 1103 1104 kfree(adev->gfx.rlc.register_list_format); 1105 } 1106 1107 static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev) 1108 { 1109 const struct rlc_firmware_header_v2_1 *rlc_hdr; 1110 1111 rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data; 1112 adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver); 1113 adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver); 1114 adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes); 1115 adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes); 1116 adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver); 1117 adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver); 1118 adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes); 1119 adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes); 1120 adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver); 1121 adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver); 1122 adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes); 1123 adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes); 1124 adev->gfx.rlc.reg_list_format_direct_reg_list_length = 1125 le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length); 1126 } 1127 1128 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev) 1129 { 1130 adev->gfx.me_fw_write_wait = false; 1131 adev->gfx.mec_fw_write_wait = false; 1132 1133 if ((adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 1)) && 1134 ((adev->gfx.mec_fw_version < 0x000001a5) || 1135 (adev->gfx.mec_feature_version < 46) || 1136 (adev->gfx.pfp_fw_version < 0x000000b7) || 1137 (adev->gfx.pfp_feature_version < 46))) 1138 DRM_WARN_ONCE("CP firmware version too old, please update!"); 1139 1140 switch (adev->ip_versions[GC_HWIP][0]) { 1141 case IP_VERSION(9, 0, 1): 1142 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1143 (adev->gfx.me_feature_version >= 42) && 1144 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1145 (adev->gfx.pfp_feature_version >= 42)) 1146 adev->gfx.me_fw_write_wait = true; 1147 1148 if ((adev->gfx.mec_fw_version >= 0x00000193) && 1149 (adev->gfx.mec_feature_version >= 42)) 1150 adev->gfx.mec_fw_write_wait = true; 1151 break; 1152 case IP_VERSION(9, 2, 1): 1153 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1154 (adev->gfx.me_feature_version >= 44) && 1155 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1156 (adev->gfx.pfp_feature_version >= 44)) 1157 adev->gfx.me_fw_write_wait = true; 1158 1159 if ((adev->gfx.mec_fw_version >= 0x00000196) && 1160 (adev->gfx.mec_feature_version >= 44)) 1161 adev->gfx.mec_fw_write_wait = true; 1162 break; 1163 case IP_VERSION(9, 4, 0): 1164 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1165 (adev->gfx.me_feature_version >= 44) && 1166 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1167 (adev->gfx.pfp_feature_version >= 44)) 1168 adev->gfx.me_fw_write_wait = true; 1169 1170 if ((adev->gfx.mec_fw_version >= 0x00000197) && 1171 (adev->gfx.mec_feature_version >= 44)) 1172 adev->gfx.mec_fw_write_wait = true; 1173 break; 1174 case IP_VERSION(9, 1, 0): 1175 case IP_VERSION(9, 2, 2): 1176 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1177 (adev->gfx.me_feature_version >= 42) && 1178 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1179 (adev->gfx.pfp_feature_version >= 42)) 1180 adev->gfx.me_fw_write_wait = true; 1181 1182 if ((adev->gfx.mec_fw_version >= 0x00000192) && 1183 (adev->gfx.mec_feature_version >= 42)) 1184 adev->gfx.mec_fw_write_wait = true; 1185 break; 1186 default: 1187 adev->gfx.me_fw_write_wait = true; 1188 adev->gfx.mec_fw_write_wait = true; 1189 break; 1190 } 1191 } 1192 1193 struct amdgpu_gfxoff_quirk { 1194 u16 chip_vendor; 1195 u16 chip_device; 1196 u16 subsys_vendor; 1197 u16 subsys_device; 1198 u8 revision; 1199 }; 1200 1201 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = { 1202 /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */ 1203 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 }, 1204 /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */ 1205 { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 }, 1206 /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */ 1207 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 }, 1208 { 0, 0, 0, 0, 0 }, 1209 }; 1210 1211 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev) 1212 { 1213 const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list; 1214 1215 while (p && p->chip_device != 0) { 1216 if (pdev->vendor == p->chip_vendor && 1217 pdev->device == p->chip_device && 1218 pdev->subsystem_vendor == p->subsys_vendor && 1219 pdev->subsystem_device == p->subsys_device && 1220 pdev->revision == p->revision) { 1221 return true; 1222 } 1223 ++p; 1224 } 1225 return false; 1226 } 1227 1228 static bool is_raven_kicker(struct amdgpu_device *adev) 1229 { 1230 if (adev->pm.fw_version >= 0x41e2b) 1231 return true; 1232 else 1233 return false; 1234 } 1235 1236 static bool check_if_enlarge_doorbell_range(struct amdgpu_device *adev) 1237 { 1238 if ((adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 3, 0)) && 1239 (adev->gfx.me_fw_version >= 0x000000a5) && 1240 (adev->gfx.me_feature_version >= 52)) 1241 return true; 1242 else 1243 return false; 1244 } 1245 1246 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev) 1247 { 1248 if (gfx_v9_0_should_disable_gfxoff(adev->pdev)) 1249 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1250 1251 switch (adev->ip_versions[GC_HWIP][0]) { 1252 case IP_VERSION(9, 0, 1): 1253 case IP_VERSION(9, 2, 1): 1254 case IP_VERSION(9, 4, 0): 1255 break; 1256 case IP_VERSION(9, 2, 2): 1257 case IP_VERSION(9, 1, 0): 1258 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) || 1259 (adev->apu_flags & AMD_APU_IS_PICASSO)) && 1260 ((!is_raven_kicker(adev) && 1261 adev->gfx.rlc_fw_version < 531) || 1262 (adev->gfx.rlc_feature_version < 1) || 1263 !adev->gfx.rlc.is_rlc_v2_1)) 1264 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1265 1266 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1267 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1268 AMD_PG_SUPPORT_CP | 1269 AMD_PG_SUPPORT_RLC_SMU_HS; 1270 break; 1271 case IP_VERSION(9, 3, 0): 1272 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1273 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1274 AMD_PG_SUPPORT_CP | 1275 AMD_PG_SUPPORT_RLC_SMU_HS; 1276 break; 1277 default: 1278 break; 1279 } 1280 } 1281 1282 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev, 1283 const char *chip_name) 1284 { 1285 char fw_name[30]; 1286 int err; 1287 struct amdgpu_firmware_info *info = NULL; 1288 const struct common_firmware_header *header = NULL; 1289 const struct gfx_firmware_header_v1_0 *cp_hdr; 1290 1291 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name); 1292 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev); 1293 if (err) 1294 goto out; 1295 err = amdgpu_ucode_validate(adev->gfx.pfp_fw); 1296 if (err) 1297 goto out; 1298 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; 1299 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1300 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1301 1302 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name); 1303 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev); 1304 if (err) 1305 goto out; 1306 err = amdgpu_ucode_validate(adev->gfx.me_fw); 1307 if (err) 1308 goto out; 1309 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; 1310 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1311 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1312 1313 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name); 1314 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev); 1315 if (err) 1316 goto out; 1317 err = amdgpu_ucode_validate(adev->gfx.ce_fw); 1318 if (err) 1319 goto out; 1320 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; 1321 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1322 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1323 1324 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1325 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP]; 1326 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP; 1327 info->fw = adev->gfx.pfp_fw; 1328 header = (const struct common_firmware_header *)info->fw->data; 1329 adev->firmware.fw_size += 1330 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1331 1332 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME]; 1333 info->ucode_id = AMDGPU_UCODE_ID_CP_ME; 1334 info->fw = adev->gfx.me_fw; 1335 header = (const struct common_firmware_header *)info->fw->data; 1336 adev->firmware.fw_size += 1337 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1338 1339 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE]; 1340 info->ucode_id = AMDGPU_UCODE_ID_CP_CE; 1341 info->fw = adev->gfx.ce_fw; 1342 header = (const struct common_firmware_header *)info->fw->data; 1343 adev->firmware.fw_size += 1344 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1345 } 1346 1347 out: 1348 if (err) { 1349 dev_err(adev->dev, 1350 "gfx9: Failed to load firmware \"%s\"\n", 1351 fw_name); 1352 release_firmware(adev->gfx.pfp_fw); 1353 adev->gfx.pfp_fw = NULL; 1354 release_firmware(adev->gfx.me_fw); 1355 adev->gfx.me_fw = NULL; 1356 release_firmware(adev->gfx.ce_fw); 1357 adev->gfx.ce_fw = NULL; 1358 } 1359 return err; 1360 } 1361 1362 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev, 1363 const char *chip_name) 1364 { 1365 char fw_name[30]; 1366 int err; 1367 struct amdgpu_firmware_info *info = NULL; 1368 const struct common_firmware_header *header = NULL; 1369 const struct rlc_firmware_header_v2_0 *rlc_hdr; 1370 unsigned int *tmp = NULL; 1371 unsigned int i = 0; 1372 uint16_t version_major; 1373 uint16_t version_minor; 1374 uint32_t smu_version; 1375 1376 /* 1377 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin 1378 * instead of picasso_rlc.bin. 1379 * Judgment method: 1380 * PCO AM4: revision >= 0xC8 && revision <= 0xCF 1381 * or revision >= 0xD8 && revision <= 0xDF 1382 * otherwise is PCO FP5 1383 */ 1384 if (!strcmp(chip_name, "picasso") && 1385 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) || 1386 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF)))) 1387 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name); 1388 else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) && 1389 (smu_version >= 0x41e2b)) 1390 /** 1391 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly. 1392 */ 1393 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name); 1394 else 1395 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name); 1396 err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev); 1397 if (err) 1398 goto out; 1399 err = amdgpu_ucode_validate(adev->gfx.rlc_fw); 1400 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 1401 1402 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 1403 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 1404 if (version_major == 2 && version_minor == 1) 1405 adev->gfx.rlc.is_rlc_v2_1 = true; 1406 1407 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); 1408 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); 1409 adev->gfx.rlc.save_and_restore_offset = 1410 le32_to_cpu(rlc_hdr->save_and_restore_offset); 1411 adev->gfx.rlc.clear_state_descriptor_offset = 1412 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset); 1413 adev->gfx.rlc.avail_scratch_ram_locations = 1414 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations); 1415 adev->gfx.rlc.reg_restore_list_size = 1416 le32_to_cpu(rlc_hdr->reg_restore_list_size); 1417 adev->gfx.rlc.reg_list_format_start = 1418 le32_to_cpu(rlc_hdr->reg_list_format_start); 1419 adev->gfx.rlc.reg_list_format_separate_start = 1420 le32_to_cpu(rlc_hdr->reg_list_format_separate_start); 1421 adev->gfx.rlc.starting_offsets_start = 1422 le32_to_cpu(rlc_hdr->starting_offsets_start); 1423 adev->gfx.rlc.reg_list_format_size_bytes = 1424 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes); 1425 adev->gfx.rlc.reg_list_size_bytes = 1426 le32_to_cpu(rlc_hdr->reg_list_size_bytes); 1427 adev->gfx.rlc.register_list_format = 1428 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes + 1429 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL); 1430 if (!adev->gfx.rlc.register_list_format) { 1431 err = -ENOMEM; 1432 goto out; 1433 } 1434 1435 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1436 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); 1437 for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++) 1438 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]); 1439 1440 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i; 1441 1442 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1443 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); 1444 for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++) 1445 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); 1446 1447 if (adev->gfx.rlc.is_rlc_v2_1) 1448 gfx_v9_0_init_rlc_ext_microcode(adev); 1449 1450 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1451 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G]; 1452 info->ucode_id = AMDGPU_UCODE_ID_RLC_G; 1453 info->fw = adev->gfx.rlc_fw; 1454 header = (const struct common_firmware_header *)info->fw->data; 1455 adev->firmware.fw_size += 1456 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1457 1458 if (adev->gfx.rlc.is_rlc_v2_1 && 1459 adev->gfx.rlc.save_restore_list_cntl_size_bytes && 1460 adev->gfx.rlc.save_restore_list_gpm_size_bytes && 1461 adev->gfx.rlc.save_restore_list_srm_size_bytes) { 1462 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL]; 1463 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL; 1464 info->fw = adev->gfx.rlc_fw; 1465 adev->firmware.fw_size += 1466 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE); 1467 1468 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM]; 1469 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM; 1470 info->fw = adev->gfx.rlc_fw; 1471 adev->firmware.fw_size += 1472 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE); 1473 1474 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM]; 1475 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM; 1476 info->fw = adev->gfx.rlc_fw; 1477 adev->firmware.fw_size += 1478 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE); 1479 } 1480 } 1481 1482 out: 1483 if (err) { 1484 dev_err(adev->dev, 1485 "gfx9: Failed to load firmware \"%s\"\n", 1486 fw_name); 1487 release_firmware(adev->gfx.rlc_fw); 1488 adev->gfx.rlc_fw = NULL; 1489 } 1490 return err; 1491 } 1492 1493 static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev) 1494 { 1495 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) || 1496 adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1) || 1497 adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 3, 0)) 1498 return false; 1499 1500 return true; 1501 } 1502 1503 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev, 1504 const char *chip_name) 1505 { 1506 char fw_name[30]; 1507 int err; 1508 struct amdgpu_firmware_info *info = NULL; 1509 const struct common_firmware_header *header = NULL; 1510 const struct gfx_firmware_header_v1_0 *cp_hdr; 1511 1512 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name); 1513 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev); 1514 if (err) 1515 goto out; 1516 err = amdgpu_ucode_validate(adev->gfx.mec_fw); 1517 if (err) 1518 goto out; 1519 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1520 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1521 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1522 1523 1524 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) { 1525 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name); 1526 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev); 1527 if (!err) { 1528 err = amdgpu_ucode_validate(adev->gfx.mec2_fw); 1529 if (err) 1530 goto out; 1531 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1532 adev->gfx.mec2_fw->data; 1533 adev->gfx.mec2_fw_version = 1534 le32_to_cpu(cp_hdr->header.ucode_version); 1535 adev->gfx.mec2_feature_version = 1536 le32_to_cpu(cp_hdr->ucode_feature_version); 1537 } else { 1538 err = 0; 1539 adev->gfx.mec2_fw = NULL; 1540 } 1541 } else { 1542 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version; 1543 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version; 1544 } 1545 1546 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1547 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1]; 1548 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1; 1549 info->fw = adev->gfx.mec_fw; 1550 header = (const struct common_firmware_header *)info->fw->data; 1551 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1552 adev->firmware.fw_size += 1553 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1554 1555 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT]; 1556 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT; 1557 info->fw = adev->gfx.mec_fw; 1558 adev->firmware.fw_size += 1559 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1560 1561 if (adev->gfx.mec2_fw) { 1562 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2]; 1563 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2; 1564 info->fw = adev->gfx.mec2_fw; 1565 header = (const struct common_firmware_header *)info->fw->data; 1566 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1567 adev->firmware.fw_size += 1568 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1569 1570 /* TODO: Determine if MEC2 JT FW loading can be removed 1571 for all GFX V9 asic and above */ 1572 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) { 1573 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT]; 1574 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT; 1575 info->fw = adev->gfx.mec2_fw; 1576 adev->firmware.fw_size += 1577 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, 1578 PAGE_SIZE); 1579 } 1580 } 1581 } 1582 1583 out: 1584 gfx_v9_0_check_if_need_gfxoff(adev); 1585 gfx_v9_0_check_fw_write_wait(adev); 1586 if (err) { 1587 dev_err(adev->dev, 1588 "gfx9: Failed to load firmware \"%s\"\n", 1589 fw_name); 1590 release_firmware(adev->gfx.mec_fw); 1591 adev->gfx.mec_fw = NULL; 1592 release_firmware(adev->gfx.mec2_fw); 1593 adev->gfx.mec2_fw = NULL; 1594 } 1595 return err; 1596 } 1597 1598 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev) 1599 { 1600 const char *chip_name; 1601 int r; 1602 1603 DRM_DEBUG("\n"); 1604 1605 switch (adev->ip_versions[GC_HWIP][0]) { 1606 case IP_VERSION(9, 0, 1): 1607 chip_name = "vega10"; 1608 break; 1609 case IP_VERSION(9, 2, 1): 1610 chip_name = "vega12"; 1611 break; 1612 case IP_VERSION(9, 4, 0): 1613 chip_name = "vega20"; 1614 break; 1615 case IP_VERSION(9, 2, 2): 1616 case IP_VERSION(9, 1, 0): 1617 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 1618 chip_name = "raven2"; 1619 else if (adev->apu_flags & AMD_APU_IS_PICASSO) 1620 chip_name = "picasso"; 1621 else 1622 chip_name = "raven"; 1623 break; 1624 case IP_VERSION(9, 4, 1): 1625 chip_name = "arcturus"; 1626 break; 1627 case IP_VERSION(9, 3, 0): 1628 if (adev->apu_flags & AMD_APU_IS_RENOIR) 1629 chip_name = "renoir"; 1630 else 1631 chip_name = "green_sardine"; 1632 break; 1633 case IP_VERSION(9, 4, 2): 1634 chip_name = "aldebaran"; 1635 break; 1636 default: 1637 BUG(); 1638 } 1639 1640 /* No CPG in Arcturus */ 1641 if (adev->gfx.num_gfx_rings) { 1642 r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name); 1643 if (r) 1644 return r; 1645 } 1646 1647 r = gfx_v9_0_init_rlc_microcode(adev, chip_name); 1648 if (r) 1649 return r; 1650 1651 r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name); 1652 if (r) 1653 return r; 1654 1655 return r; 1656 } 1657 1658 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev) 1659 { 1660 u32 count = 0; 1661 const struct cs_section_def *sect = NULL; 1662 const struct cs_extent_def *ext = NULL; 1663 1664 /* begin clear state */ 1665 count += 2; 1666 /* context control state */ 1667 count += 3; 1668 1669 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 1670 for (ext = sect->section; ext->extent != NULL; ++ext) { 1671 if (sect->id == SECT_CONTEXT) 1672 count += 2 + ext->reg_count; 1673 else 1674 return 0; 1675 } 1676 } 1677 1678 /* end clear state */ 1679 count += 2; 1680 /* clear state */ 1681 count += 2; 1682 1683 return count; 1684 } 1685 1686 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev, 1687 volatile u32 *buffer) 1688 { 1689 u32 count = 0, i; 1690 const struct cs_section_def *sect = NULL; 1691 const struct cs_extent_def *ext = NULL; 1692 1693 if (adev->gfx.rlc.cs_data == NULL) 1694 return; 1695 if (buffer == NULL) 1696 return; 1697 1698 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1699 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 1700 1701 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 1702 buffer[count++] = cpu_to_le32(0x80000000); 1703 buffer[count++] = cpu_to_le32(0x80000000); 1704 1705 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { 1706 for (ext = sect->section; ext->extent != NULL; ++ext) { 1707 if (sect->id == SECT_CONTEXT) { 1708 buffer[count++] = 1709 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 1710 buffer[count++] = cpu_to_le32(ext->reg_index - 1711 PACKET3_SET_CONTEXT_REG_START); 1712 for (i = 0; i < ext->reg_count; i++) 1713 buffer[count++] = cpu_to_le32(ext->extent[i]); 1714 } else { 1715 return; 1716 } 1717 } 1718 } 1719 1720 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1721 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 1722 1723 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 1724 buffer[count++] = cpu_to_le32(0); 1725 } 1726 1727 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev) 1728 { 1729 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; 1730 uint32_t pg_always_on_cu_num = 2; 1731 uint32_t always_on_cu_num; 1732 uint32_t i, j, k; 1733 uint32_t mask, cu_bitmap, counter; 1734 1735 if (adev->flags & AMD_IS_APU) 1736 always_on_cu_num = 4; 1737 else if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 2, 1)) 1738 always_on_cu_num = 8; 1739 else 1740 always_on_cu_num = 12; 1741 1742 mutex_lock(&adev->grbm_idx_mutex); 1743 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 1744 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 1745 mask = 1; 1746 cu_bitmap = 0; 1747 counter = 0; 1748 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 1749 1750 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 1751 if (cu_info->bitmap[i][j] & mask) { 1752 if (counter == pg_always_on_cu_num) 1753 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap); 1754 if (counter < always_on_cu_num) 1755 cu_bitmap |= mask; 1756 else 1757 break; 1758 counter++; 1759 } 1760 mask <<= 1; 1761 } 1762 1763 WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap); 1764 cu_info->ao_cu_bitmap[i][j] = cu_bitmap; 1765 } 1766 } 1767 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1768 mutex_unlock(&adev->grbm_idx_mutex); 1769 } 1770 1771 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev) 1772 { 1773 uint32_t data; 1774 1775 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1776 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1777 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7); 1778 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1779 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16)); 1780 1781 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1782 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1783 1784 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1785 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500); 1786 1787 mutex_lock(&adev->grbm_idx_mutex); 1788 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1789 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1790 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1791 1792 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1793 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1794 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1795 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1796 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1797 1798 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1799 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1800 data &= 0x0000FFFF; 1801 data |= 0x00C00000; 1802 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1803 1804 /* 1805 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven), 1806 * programmed in gfx_v9_0_init_always_on_cu_mask() 1807 */ 1808 1809 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1810 * but used for RLC_LB_CNTL configuration */ 1811 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1812 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1813 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1814 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1815 mutex_unlock(&adev->grbm_idx_mutex); 1816 1817 gfx_v9_0_init_always_on_cu_mask(adev); 1818 } 1819 1820 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev) 1821 { 1822 uint32_t data; 1823 1824 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1825 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1826 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8); 1827 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1828 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16)); 1829 1830 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1831 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1832 1833 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1834 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800); 1835 1836 mutex_lock(&adev->grbm_idx_mutex); 1837 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1838 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1839 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1840 1841 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1842 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1843 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1844 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1845 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1846 1847 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1848 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1849 data &= 0x0000FFFF; 1850 data |= 0x00C00000; 1851 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1852 1853 /* 1854 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON), 1855 * programmed in gfx_v9_0_init_always_on_cu_mask() 1856 */ 1857 1858 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1859 * but used for RLC_LB_CNTL configuration */ 1860 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1861 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1862 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1863 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1864 mutex_unlock(&adev->grbm_idx_mutex); 1865 1866 gfx_v9_0_init_always_on_cu_mask(adev); 1867 } 1868 1869 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable) 1870 { 1871 WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0); 1872 } 1873 1874 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev) 1875 { 1876 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) 1877 return 5; 1878 else 1879 return 4; 1880 } 1881 1882 static void gfx_v9_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev) 1883 { 1884 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl; 1885 1886 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl; 1887 reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0); 1888 reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG1); 1889 reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG2); 1890 reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG3); 1891 reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL); 1892 reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX); 1893 reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, mmRLC_SPARE_INT); 1894 adev->gfx.rlc.rlcg_reg_access_supported = true; 1895 } 1896 1897 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev) 1898 { 1899 const struct cs_section_def *cs_data; 1900 int r; 1901 1902 adev->gfx.rlc.cs_data = gfx9_cs_data; 1903 1904 cs_data = adev->gfx.rlc.cs_data; 1905 1906 if (cs_data) { 1907 /* init clear state block */ 1908 r = amdgpu_gfx_rlc_init_csb(adev); 1909 if (r) 1910 return r; 1911 } 1912 1913 if (adev->flags & AMD_IS_APU) { 1914 /* TODO: double check the cp_table_size for RV */ 1915 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ 1916 r = amdgpu_gfx_rlc_init_cpt(adev); 1917 if (r) 1918 return r; 1919 } 1920 1921 switch (adev->ip_versions[GC_HWIP][0]) { 1922 case IP_VERSION(9, 2, 2): 1923 case IP_VERSION(9, 1, 0): 1924 gfx_v9_0_init_lbpw(adev); 1925 break; 1926 case IP_VERSION(9, 4, 0): 1927 gfx_v9_4_init_lbpw(adev); 1928 break; 1929 default: 1930 break; 1931 } 1932 1933 /* init spm vmid with 0xf */ 1934 if (adev->gfx.rlc.funcs->update_spm_vmid) 1935 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf); 1936 1937 return 0; 1938 } 1939 1940 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev) 1941 { 1942 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 1943 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); 1944 } 1945 1946 static int gfx_v9_0_mec_init(struct amdgpu_device *adev) 1947 { 1948 int r; 1949 u32 *hpd; 1950 const __le32 *fw_data; 1951 unsigned fw_size; 1952 u32 *fw; 1953 size_t mec_hpd_size; 1954 1955 const struct gfx_firmware_header_v1_0 *mec_hdr; 1956 1957 bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 1958 1959 /* take ownership of the relevant compute queues */ 1960 amdgpu_gfx_compute_queue_acquire(adev); 1961 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE; 1962 if (mec_hpd_size) { 1963 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 1964 AMDGPU_GEM_DOMAIN_VRAM, 1965 &adev->gfx.mec.hpd_eop_obj, 1966 &adev->gfx.mec.hpd_eop_gpu_addr, 1967 (void **)&hpd); 1968 if (r) { 1969 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 1970 gfx_v9_0_mec_fini(adev); 1971 return r; 1972 } 1973 1974 memset(hpd, 0, mec_hpd_size); 1975 1976 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 1977 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 1978 } 1979 1980 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1981 1982 fw_data = (const __le32 *) 1983 (adev->gfx.mec_fw->data + 1984 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 1985 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes); 1986 1987 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes, 1988 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 1989 &adev->gfx.mec.mec_fw_obj, 1990 &adev->gfx.mec.mec_fw_gpu_addr, 1991 (void **)&fw); 1992 if (r) { 1993 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r); 1994 gfx_v9_0_mec_fini(adev); 1995 return r; 1996 } 1997 1998 memcpy(fw, fw_data, fw_size); 1999 2000 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 2001 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 2002 2003 return 0; 2004 } 2005 2006 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address) 2007 { 2008 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 2009 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 2010 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 2011 (address << SQ_IND_INDEX__INDEX__SHIFT) | 2012 (SQ_IND_INDEX__FORCE_READ_MASK)); 2013 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 2014 } 2015 2016 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd, 2017 uint32_t wave, uint32_t thread, 2018 uint32_t regno, uint32_t num, uint32_t *out) 2019 { 2020 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 2021 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 2022 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 2023 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 2024 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) | 2025 (SQ_IND_INDEX__FORCE_READ_MASK) | 2026 (SQ_IND_INDEX__AUTO_INCR_MASK)); 2027 while (num--) 2028 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 2029 } 2030 2031 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 2032 { 2033 /* type 1 wave data */ 2034 dst[(*no_fields)++] = 1; 2035 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS); 2036 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO); 2037 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI); 2038 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO); 2039 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI); 2040 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID); 2041 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0); 2042 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1); 2043 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC); 2044 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC); 2045 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS); 2046 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS); 2047 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); 2048 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); 2049 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE); 2050 } 2051 2052 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd, 2053 uint32_t wave, uint32_t start, 2054 uint32_t size, uint32_t *dst) 2055 { 2056 wave_read_regs( 2057 adev, simd, wave, 0, 2058 start + SQIND_WAVE_SGPRS_OFFSET, size, dst); 2059 } 2060 2061 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd, 2062 uint32_t wave, uint32_t thread, 2063 uint32_t start, uint32_t size, 2064 uint32_t *dst) 2065 { 2066 wave_read_regs( 2067 adev, simd, wave, thread, 2068 start + SQIND_WAVE_VGPRS_OFFSET, size, dst); 2069 } 2070 2071 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev, 2072 u32 me, u32 pipe, u32 q, u32 vm) 2073 { 2074 soc15_grbm_select(adev, me, pipe, q, vm); 2075 } 2076 2077 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = { 2078 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 2079 .select_se_sh = &gfx_v9_0_select_se_sh, 2080 .read_wave_data = &gfx_v9_0_read_wave_data, 2081 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 2082 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 2083 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 2084 }; 2085 2086 const struct amdgpu_ras_block_hw_ops gfx_v9_0_ras_ops = { 2087 .ras_error_inject = &gfx_v9_0_ras_error_inject, 2088 .query_ras_error_count = &gfx_v9_0_query_ras_error_count, 2089 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count, 2090 }; 2091 2092 static struct amdgpu_gfx_ras gfx_v9_0_ras = { 2093 .ras_block = { 2094 .hw_ops = &gfx_v9_0_ras_ops, 2095 }, 2096 }; 2097 2098 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev) 2099 { 2100 u32 gb_addr_config; 2101 int err; 2102 2103 adev->gfx.funcs = &gfx_v9_0_gfx_funcs; 2104 2105 switch (adev->ip_versions[GC_HWIP][0]) { 2106 case IP_VERSION(9, 0, 1): 2107 adev->gfx.config.max_hw_contexts = 8; 2108 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2109 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2110 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2111 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2112 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN; 2113 break; 2114 case IP_VERSION(9, 2, 1): 2115 adev->gfx.config.max_hw_contexts = 8; 2116 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2117 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2118 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2119 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2120 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN; 2121 DRM_INFO("fix gfx.config for vega12\n"); 2122 break; 2123 case IP_VERSION(9, 4, 0): 2124 adev->gfx.ras = &gfx_v9_0_ras; 2125 adev->gfx.config.max_hw_contexts = 8; 2126 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2127 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2128 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2129 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2130 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2131 gb_addr_config &= ~0xf3e777ff; 2132 gb_addr_config |= 0x22014042; 2133 /* check vbios table if gpu info is not available */ 2134 err = amdgpu_atomfirmware_get_gfx_info(adev); 2135 if (err) 2136 return err; 2137 break; 2138 case IP_VERSION(9, 2, 2): 2139 case IP_VERSION(9, 1, 0): 2140 adev->gfx.config.max_hw_contexts = 8; 2141 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2142 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2143 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2144 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2145 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 2146 gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN; 2147 else 2148 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN; 2149 break; 2150 case IP_VERSION(9, 4, 1): 2151 adev->gfx.ras = &gfx_v9_4_ras; 2152 adev->gfx.config.max_hw_contexts = 8; 2153 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2154 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2155 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2156 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2157 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2158 gb_addr_config &= ~0xf3e777ff; 2159 gb_addr_config |= 0x22014042; 2160 break; 2161 case IP_VERSION(9, 3, 0): 2162 adev->gfx.config.max_hw_contexts = 8; 2163 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2164 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2165 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; 2166 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2167 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2168 gb_addr_config &= ~0xf3e777ff; 2169 gb_addr_config |= 0x22010042; 2170 break; 2171 case IP_VERSION(9, 4, 2): 2172 adev->gfx.ras = &gfx_v9_4_2_ras; 2173 adev->gfx.config.max_hw_contexts = 8; 2174 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2175 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2176 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2177 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2178 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2179 gb_addr_config &= ~0xf3e777ff; 2180 gb_addr_config |= 0x22014042; 2181 /* check vbios table if gpu info is not available */ 2182 err = amdgpu_atomfirmware_get_gfx_info(adev); 2183 if (err) 2184 return err; 2185 break; 2186 default: 2187 BUG(); 2188 break; 2189 } 2190 2191 if (adev->gfx.ras) { 2192 err = amdgpu_ras_register_ras_block(adev, &adev->gfx.ras->ras_block); 2193 if (err) { 2194 DRM_ERROR("Failed to register gfx ras block!\n"); 2195 return err; 2196 } 2197 2198 strcpy(adev->gfx.ras->ras_block.ras_comm.name, "gfx"); 2199 adev->gfx.ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__GFX; 2200 adev->gfx.ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; 2201 adev->gfx.ras_if = &adev->gfx.ras->ras_block.ras_comm; 2202 2203 /* If not define special ras_late_init function, use gfx default ras_late_init */ 2204 if (!adev->gfx.ras->ras_block.ras_late_init) 2205 adev->gfx.ras->ras_block.ras_late_init = amdgpu_gfx_ras_late_init; 2206 2207 /* If not defined special ras_cb function, use default ras_cb */ 2208 if (!adev->gfx.ras->ras_block.ras_cb) 2209 adev->gfx.ras->ras_block.ras_cb = amdgpu_gfx_process_ras_data_cb; 2210 } 2211 2212 adev->gfx.config.gb_addr_config = gb_addr_config; 2213 2214 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << 2215 REG_GET_FIELD( 2216 adev->gfx.config.gb_addr_config, 2217 GB_ADDR_CONFIG, 2218 NUM_PIPES); 2219 2220 adev->gfx.config.max_tile_pipes = 2221 adev->gfx.config.gb_addr_config_fields.num_pipes; 2222 2223 adev->gfx.config.gb_addr_config_fields.num_banks = 1 << 2224 REG_GET_FIELD( 2225 adev->gfx.config.gb_addr_config, 2226 GB_ADDR_CONFIG, 2227 NUM_BANKS); 2228 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << 2229 REG_GET_FIELD( 2230 adev->gfx.config.gb_addr_config, 2231 GB_ADDR_CONFIG, 2232 MAX_COMPRESSED_FRAGS); 2233 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << 2234 REG_GET_FIELD( 2235 adev->gfx.config.gb_addr_config, 2236 GB_ADDR_CONFIG, 2237 NUM_RB_PER_SE); 2238 adev->gfx.config.gb_addr_config_fields.num_se = 1 << 2239 REG_GET_FIELD( 2240 adev->gfx.config.gb_addr_config, 2241 GB_ADDR_CONFIG, 2242 NUM_SHADER_ENGINES); 2243 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + 2244 REG_GET_FIELD( 2245 adev->gfx.config.gb_addr_config, 2246 GB_ADDR_CONFIG, 2247 PIPE_INTERLEAVE_SIZE)); 2248 2249 return 0; 2250 } 2251 2252 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 2253 int mec, int pipe, int queue) 2254 { 2255 unsigned irq_type; 2256 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; 2257 unsigned int hw_prio; 2258 2259 ring = &adev->gfx.compute_ring[ring_id]; 2260 2261 /* mec0 is me1 */ 2262 ring->me = mec + 1; 2263 ring->pipe = pipe; 2264 ring->queue = queue; 2265 2266 ring->ring_obj = NULL; 2267 ring->use_doorbell = true; 2268 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1; 2269 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 2270 + (ring_id * GFX9_MEC_HPD_SIZE); 2271 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 2272 2273 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 2274 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 2275 + ring->pipe; 2276 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? 2277 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 2278 /* type-2 packets are deprecated on MEC, use type-3 instead */ 2279 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 2280 hw_prio, NULL); 2281 } 2282 2283 static int gfx_v9_0_sw_init(void *handle) 2284 { 2285 int i, j, k, r, ring_id; 2286 struct amdgpu_ring *ring; 2287 struct amdgpu_kiq *kiq; 2288 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2289 2290 switch (adev->ip_versions[GC_HWIP][0]) { 2291 case IP_VERSION(9, 0, 1): 2292 case IP_VERSION(9, 2, 1): 2293 case IP_VERSION(9, 4, 0): 2294 case IP_VERSION(9, 2, 2): 2295 case IP_VERSION(9, 1, 0): 2296 case IP_VERSION(9, 4, 1): 2297 case IP_VERSION(9, 3, 0): 2298 case IP_VERSION(9, 4, 2): 2299 adev->gfx.mec.num_mec = 2; 2300 break; 2301 default: 2302 adev->gfx.mec.num_mec = 1; 2303 break; 2304 } 2305 2306 adev->gfx.mec.num_pipe_per_mec = 4; 2307 adev->gfx.mec.num_queue_per_pipe = 8; 2308 2309 /* EOP Event */ 2310 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq); 2311 if (r) 2312 return r; 2313 2314 /* Privileged reg */ 2315 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT, 2316 &adev->gfx.priv_reg_irq); 2317 if (r) 2318 return r; 2319 2320 /* Privileged inst */ 2321 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT, 2322 &adev->gfx.priv_inst_irq); 2323 if (r) 2324 return r; 2325 2326 /* ECC error */ 2327 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR, 2328 &adev->gfx.cp_ecc_error_irq); 2329 if (r) 2330 return r; 2331 2332 /* FUE error */ 2333 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR, 2334 &adev->gfx.cp_ecc_error_irq); 2335 if (r) 2336 return r; 2337 2338 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 2339 2340 gfx_v9_0_scratch_init(adev); 2341 2342 r = gfx_v9_0_init_microcode(adev); 2343 if (r) { 2344 DRM_ERROR("Failed to load gfx firmware!\n"); 2345 return r; 2346 } 2347 2348 if (adev->gfx.rlc.funcs) { 2349 if (adev->gfx.rlc.funcs->init) { 2350 r = adev->gfx.rlc.funcs->init(adev); 2351 if (r) { 2352 dev_err(adev->dev, "Failed to init rlc BOs!\n"); 2353 return r; 2354 } 2355 } 2356 } 2357 2358 r = gfx_v9_0_mec_init(adev); 2359 if (r) { 2360 DRM_ERROR("Failed to init MEC BOs!\n"); 2361 return r; 2362 } 2363 2364 /* set up the gfx ring */ 2365 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 2366 ring = &adev->gfx.gfx_ring[i]; 2367 ring->ring_obj = NULL; 2368 if (!i) 2369 sprintf(ring->name, "gfx"); 2370 else 2371 sprintf(ring->name, "gfx_%d", i); 2372 ring->use_doorbell = true; 2373 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 2374 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, 2375 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, 2376 AMDGPU_RING_PRIO_DEFAULT, NULL); 2377 if (r) 2378 return r; 2379 } 2380 2381 /* set up the compute queues - allocate horizontally across pipes */ 2382 ring_id = 0; 2383 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 2384 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 2385 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 2386 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j)) 2387 continue; 2388 2389 r = gfx_v9_0_compute_ring_init(adev, 2390 ring_id, 2391 i, k, j); 2392 if (r) 2393 return r; 2394 2395 ring_id++; 2396 } 2397 } 2398 } 2399 2400 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE); 2401 if (r) { 2402 DRM_ERROR("Failed to init KIQ BOs!\n"); 2403 return r; 2404 } 2405 2406 kiq = &adev->gfx.kiq; 2407 r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq); 2408 if (r) 2409 return r; 2410 2411 /* create MQD for all compute queues as wel as KIQ for SRIOV case */ 2412 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation)); 2413 if (r) 2414 return r; 2415 2416 adev->gfx.ce_ram_size = 0x8000; 2417 2418 r = gfx_v9_0_gpu_early_init(adev); 2419 if (r) 2420 return r; 2421 2422 return 0; 2423 } 2424 2425 2426 static int gfx_v9_0_sw_fini(void *handle) 2427 { 2428 int i; 2429 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2430 2431 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 2432 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 2433 for (i = 0; i < adev->gfx.num_compute_rings; i++) 2434 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 2435 2436 amdgpu_gfx_mqd_sw_fini(adev); 2437 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring); 2438 amdgpu_gfx_kiq_fini(adev); 2439 2440 gfx_v9_0_mec_fini(adev); 2441 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, 2442 &adev->gfx.rlc.clear_state_gpu_addr, 2443 (void **)&adev->gfx.rlc.cs_ptr); 2444 if (adev->flags & AMD_IS_APU) { 2445 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 2446 &adev->gfx.rlc.cp_table_gpu_addr, 2447 (void **)&adev->gfx.rlc.cp_table_ptr); 2448 } 2449 gfx_v9_0_free_microcode(adev); 2450 2451 return 0; 2452 } 2453 2454 2455 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev) 2456 { 2457 /* TODO */ 2458 } 2459 2460 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, 2461 u32 instance) 2462 { 2463 u32 data; 2464 2465 if (instance == 0xffffffff) 2466 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); 2467 else 2468 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance); 2469 2470 if (se_num == 0xffffffff) 2471 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); 2472 else 2473 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 2474 2475 if (sh_num == 0xffffffff) 2476 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1); 2477 else 2478 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num); 2479 2480 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data); 2481 } 2482 2483 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev) 2484 { 2485 u32 data, mask; 2486 2487 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE); 2488 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE); 2489 2490 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK; 2491 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT; 2492 2493 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / 2494 adev->gfx.config.max_sh_per_se); 2495 2496 return (~data) & mask; 2497 } 2498 2499 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev) 2500 { 2501 int i, j; 2502 u32 data; 2503 u32 active_rbs = 0; 2504 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / 2505 adev->gfx.config.max_sh_per_se; 2506 2507 mutex_lock(&adev->grbm_idx_mutex); 2508 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2509 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2510 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2511 data = gfx_v9_0_get_rb_active_bitmap(adev); 2512 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * 2513 rb_bitmap_width_per_sh); 2514 } 2515 } 2516 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2517 mutex_unlock(&adev->grbm_idx_mutex); 2518 2519 adev->gfx.config.backend_enable_mask = active_rbs; 2520 adev->gfx.config.num_rbs = hweight32(active_rbs); 2521 } 2522 2523 #define DEFAULT_SH_MEM_BASES (0x6000) 2524 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev) 2525 { 2526 int i; 2527 uint32_t sh_mem_config; 2528 uint32_t sh_mem_bases; 2529 2530 /* 2531 * Configure apertures: 2532 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 2533 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 2534 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 2535 */ 2536 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16); 2537 2538 sh_mem_config = SH_MEM_ADDRESS_MODE_64 | 2539 SH_MEM_ALIGNMENT_MODE_UNALIGNED << 2540 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT; 2541 2542 mutex_lock(&adev->srbm_mutex); 2543 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2544 soc15_grbm_select(adev, 0, 0, 0, i); 2545 /* CP and shaders */ 2546 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config); 2547 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases); 2548 } 2549 soc15_grbm_select(adev, 0, 0, 0, 0); 2550 mutex_unlock(&adev->srbm_mutex); 2551 2552 /* Initialize all compute VMIDs to have no GDS, GWS, or OA 2553 acccess. These should be enabled by FW for target VMIDs. */ 2554 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2555 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0); 2556 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0); 2557 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0); 2558 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0); 2559 } 2560 } 2561 2562 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev) 2563 { 2564 int vmid; 2565 2566 /* 2567 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 2568 * access. Compute VMIDs should be enabled by FW for target VMIDs, 2569 * the driver can enable them for graphics. VMID0 should maintain 2570 * access so that HWS firmware can save/restore entries. 2571 */ 2572 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { 2573 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0); 2574 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0); 2575 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0); 2576 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0); 2577 } 2578 } 2579 2580 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev) 2581 { 2582 uint32_t tmp; 2583 2584 switch (adev->ip_versions[GC_HWIP][0]) { 2585 case IP_VERSION(9, 4, 1): 2586 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG); 2587 tmp = REG_SET_FIELD(tmp, SQ_CONFIG, 2588 DISABLE_BARRIER_WAITCNT, 1); 2589 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp); 2590 break; 2591 default: 2592 break; 2593 } 2594 } 2595 2596 static void gfx_v9_0_constants_init(struct amdgpu_device *adev) 2597 { 2598 u32 tmp; 2599 int i; 2600 2601 WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); 2602 2603 gfx_v9_0_tiling_mode_table_init(adev); 2604 2605 gfx_v9_0_setup_rb(adev); 2606 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info); 2607 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2); 2608 2609 /* XXX SH_MEM regs */ 2610 /* where to put LDS, scratch, GPUVM in FSA64 space */ 2611 mutex_lock(&adev->srbm_mutex); 2612 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) { 2613 soc15_grbm_select(adev, 0, 0, 0, i); 2614 /* CP and shaders */ 2615 if (i == 0) { 2616 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2617 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2618 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2619 !!adev->gmc.noretry); 2620 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2621 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0); 2622 } else { 2623 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2624 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2625 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2626 !!adev->gmc.noretry); 2627 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2628 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 2629 (adev->gmc.private_aperture_start >> 48)); 2630 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE, 2631 (adev->gmc.shared_aperture_start >> 48)); 2632 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp); 2633 } 2634 } 2635 soc15_grbm_select(adev, 0, 0, 0, 0); 2636 2637 mutex_unlock(&adev->srbm_mutex); 2638 2639 gfx_v9_0_init_compute_vmid(adev); 2640 gfx_v9_0_init_gds_vmid(adev); 2641 gfx_v9_0_init_sq_config(adev); 2642 } 2643 2644 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev) 2645 { 2646 u32 i, j, k; 2647 u32 mask; 2648 2649 mutex_lock(&adev->grbm_idx_mutex); 2650 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2651 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2652 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2653 for (k = 0; k < adev->usec_timeout; k++) { 2654 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0) 2655 break; 2656 udelay(1); 2657 } 2658 if (k == adev->usec_timeout) { 2659 gfx_v9_0_select_se_sh(adev, 0xffffffff, 2660 0xffffffff, 0xffffffff); 2661 mutex_unlock(&adev->grbm_idx_mutex); 2662 DRM_INFO("Timeout wait for RLC serdes %u,%u\n", 2663 i, j); 2664 return; 2665 } 2666 } 2667 } 2668 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2669 mutex_unlock(&adev->grbm_idx_mutex); 2670 2671 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | 2672 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK | 2673 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK | 2674 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK; 2675 for (k = 0; k < adev->usec_timeout; k++) { 2676 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 2677 break; 2678 udelay(1); 2679 } 2680 } 2681 2682 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 2683 bool enable) 2684 { 2685 u32 tmp; 2686 2687 /* These interrupts should be enabled to drive DS clock */ 2688 2689 tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0); 2690 2691 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0); 2692 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0); 2693 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0); 2694 if(adev->gfx.num_gfx_rings) 2695 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0); 2696 2697 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp); 2698 } 2699 2700 static void gfx_v9_0_init_csb(struct amdgpu_device *adev) 2701 { 2702 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 2703 /* csib */ 2704 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI), 2705 adev->gfx.rlc.clear_state_gpu_addr >> 32); 2706 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO), 2707 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 2708 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH), 2709 adev->gfx.rlc.clear_state_size); 2710 } 2711 2712 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format, 2713 int indirect_offset, 2714 int list_size, 2715 int *unique_indirect_regs, 2716 int unique_indirect_reg_count, 2717 int *indirect_start_offsets, 2718 int *indirect_start_offsets_count, 2719 int max_start_offsets_count) 2720 { 2721 int idx; 2722 2723 for (; indirect_offset < list_size; indirect_offset++) { 2724 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count); 2725 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset; 2726 *indirect_start_offsets_count = *indirect_start_offsets_count + 1; 2727 2728 while (register_list_format[indirect_offset] != 0xFFFFFFFF) { 2729 indirect_offset += 2; 2730 2731 /* look for the matching indice */ 2732 for (idx = 0; idx < unique_indirect_reg_count; idx++) { 2733 if (unique_indirect_regs[idx] == 2734 register_list_format[indirect_offset] || 2735 !unique_indirect_regs[idx]) 2736 break; 2737 } 2738 2739 BUG_ON(idx >= unique_indirect_reg_count); 2740 2741 if (!unique_indirect_regs[idx]) 2742 unique_indirect_regs[idx] = register_list_format[indirect_offset]; 2743 2744 indirect_offset++; 2745 } 2746 } 2747 } 2748 2749 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev) 2750 { 2751 int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2752 int unique_indirect_reg_count = 0; 2753 2754 int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2755 int indirect_start_offsets_count = 0; 2756 2757 int list_size = 0; 2758 int i = 0, j = 0; 2759 u32 tmp = 0; 2760 2761 u32 *register_list_format = 2762 kmemdup(adev->gfx.rlc.register_list_format, 2763 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); 2764 if (!register_list_format) 2765 return -ENOMEM; 2766 2767 /* setup unique_indirect_regs array and indirect_start_offsets array */ 2768 unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs); 2769 gfx_v9_1_parse_ind_reg_list(register_list_format, 2770 adev->gfx.rlc.reg_list_format_direct_reg_list_length, 2771 adev->gfx.rlc.reg_list_format_size_bytes >> 2, 2772 unique_indirect_regs, 2773 unique_indirect_reg_count, 2774 indirect_start_offsets, 2775 &indirect_start_offsets_count, 2776 ARRAY_SIZE(indirect_start_offsets)); 2777 2778 /* enable auto inc in case it is disabled */ 2779 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL)); 2780 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK; 2781 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp); 2782 2783 /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */ 2784 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR), 2785 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET); 2786 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) 2787 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA), 2788 adev->gfx.rlc.register_restore[i]); 2789 2790 /* load indirect register */ 2791 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2792 adev->gfx.rlc.reg_list_format_start); 2793 2794 /* direct register portion */ 2795 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++) 2796 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2797 register_list_format[i]); 2798 2799 /* indirect register portion */ 2800 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) { 2801 if (register_list_format[i] == 0xFFFFFFFF) { 2802 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2803 continue; 2804 } 2805 2806 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2807 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2808 2809 for (j = 0; j < unique_indirect_reg_count; j++) { 2810 if (register_list_format[i] == unique_indirect_regs[j]) { 2811 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j); 2812 break; 2813 } 2814 } 2815 2816 BUG_ON(j >= unique_indirect_reg_count); 2817 2818 i++; 2819 } 2820 2821 /* set save/restore list size */ 2822 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; 2823 list_size = list_size >> 1; 2824 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2825 adev->gfx.rlc.reg_restore_list_size); 2826 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size); 2827 2828 /* write the starting offsets to RLC scratch ram */ 2829 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2830 adev->gfx.rlc.starting_offsets_start); 2831 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++) 2832 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2833 indirect_start_offsets[i]); 2834 2835 /* load unique indirect regs*/ 2836 for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) { 2837 if (unique_indirect_regs[i] != 0) { 2838 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0) 2839 + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i], 2840 unique_indirect_regs[i] & 0x3FFFF); 2841 2842 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0) 2843 + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i], 2844 unique_indirect_regs[i] >> 20); 2845 } 2846 } 2847 2848 kfree(register_list_format); 2849 return 0; 2850 } 2851 2852 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev) 2853 { 2854 WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1); 2855 } 2856 2857 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev, 2858 bool enable) 2859 { 2860 uint32_t data = 0; 2861 uint32_t default_data = 0; 2862 2863 default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS)); 2864 if (enable) { 2865 /* enable GFXIP control over CGPG */ 2866 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2867 if(default_data != data) 2868 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2869 2870 /* update status */ 2871 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK; 2872 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT); 2873 if(default_data != data) 2874 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2875 } else { 2876 /* restore GFXIP control over GCPG */ 2877 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2878 if(default_data != data) 2879 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2880 } 2881 } 2882 2883 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev) 2884 { 2885 uint32_t data = 0; 2886 2887 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2888 AMD_PG_SUPPORT_GFX_SMG | 2889 AMD_PG_SUPPORT_GFX_DMG)) { 2890 /* init IDLE_POLL_COUNT = 60 */ 2891 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL)); 2892 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK; 2893 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 2894 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data); 2895 2896 /* init RLC PG Delay */ 2897 data = 0; 2898 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT); 2899 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT); 2900 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT); 2901 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT); 2902 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data); 2903 2904 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2)); 2905 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK; 2906 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT); 2907 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data); 2908 2909 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3)); 2910 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK; 2911 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT); 2912 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data); 2913 2914 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL)); 2915 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK; 2916 2917 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */ 2918 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT); 2919 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data); 2920 if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 3, 0)) 2921 pwr_10_0_gfxip_control_over_cgpg(adev, true); 2922 } 2923 } 2924 2925 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev, 2926 bool enable) 2927 { 2928 uint32_t data = 0; 2929 uint32_t default_data = 0; 2930 2931 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2932 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2933 SMU_CLK_SLOWDOWN_ON_PU_ENABLE, 2934 enable ? 1 : 0); 2935 if (default_data != data) 2936 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2937 } 2938 2939 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev, 2940 bool enable) 2941 { 2942 uint32_t data = 0; 2943 uint32_t default_data = 0; 2944 2945 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2946 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2947 SMU_CLK_SLOWDOWN_ON_PD_ENABLE, 2948 enable ? 1 : 0); 2949 if(default_data != data) 2950 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2951 } 2952 2953 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev, 2954 bool enable) 2955 { 2956 uint32_t data = 0; 2957 uint32_t default_data = 0; 2958 2959 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2960 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2961 CP_PG_DISABLE, 2962 enable ? 0 : 1); 2963 if(default_data != data) 2964 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2965 } 2966 2967 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev, 2968 bool enable) 2969 { 2970 uint32_t data, default_data; 2971 2972 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2973 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2974 GFX_POWER_GATING_ENABLE, 2975 enable ? 1 : 0); 2976 if(default_data != data) 2977 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2978 } 2979 2980 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev, 2981 bool enable) 2982 { 2983 uint32_t data, default_data; 2984 2985 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2986 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2987 GFX_PIPELINE_PG_ENABLE, 2988 enable ? 1 : 0); 2989 if(default_data != data) 2990 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2991 2992 if (!enable) 2993 /* read any GFX register to wake up GFX */ 2994 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL)); 2995 } 2996 2997 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, 2998 bool enable) 2999 { 3000 uint32_t data, default_data; 3001 3002 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 3003 data = REG_SET_FIELD(data, RLC_PG_CNTL, 3004 STATIC_PER_CU_PG_ENABLE, 3005 enable ? 1 : 0); 3006 if(default_data != data) 3007 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3008 } 3009 3010 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev, 3011 bool enable) 3012 { 3013 uint32_t data, default_data; 3014 3015 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 3016 data = REG_SET_FIELD(data, RLC_PG_CNTL, 3017 DYN_PER_CU_PG_ENABLE, 3018 enable ? 1 : 0); 3019 if(default_data != data) 3020 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3021 } 3022 3023 static void gfx_v9_0_init_pg(struct amdgpu_device *adev) 3024 { 3025 gfx_v9_0_init_csb(adev); 3026 3027 /* 3028 * Rlc save restore list is workable since v2_1. 3029 * And it's needed by gfxoff feature. 3030 */ 3031 if (adev->gfx.rlc.is_rlc_v2_1) { 3032 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 2, 1) || 3033 (adev->apu_flags & AMD_APU_IS_RAVEN2)) 3034 gfx_v9_1_init_rlc_save_restore_list(adev); 3035 gfx_v9_0_enable_save_restore_machine(adev); 3036 } 3037 3038 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 3039 AMD_PG_SUPPORT_GFX_SMG | 3040 AMD_PG_SUPPORT_GFX_DMG | 3041 AMD_PG_SUPPORT_CP | 3042 AMD_PG_SUPPORT_GDS | 3043 AMD_PG_SUPPORT_RLC_SMU_HS)) { 3044 WREG32_SOC15(GC, 0, mmRLC_JUMP_TABLE_RESTORE, 3045 adev->gfx.rlc.cp_table_gpu_addr >> 8); 3046 gfx_v9_0_init_gfx_power_gating(adev); 3047 } 3048 } 3049 3050 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev) 3051 { 3052 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0); 3053 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3054 gfx_v9_0_wait_for_rlc_serdes(adev); 3055 } 3056 3057 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev) 3058 { 3059 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 3060 udelay(50); 3061 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 3062 udelay(50); 3063 } 3064 3065 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev) 3066 { 3067 #ifdef AMDGPU_RLC_DEBUG_RETRY 3068 u32 rlc_ucode_ver; 3069 #endif 3070 3071 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1); 3072 udelay(50); 3073 3074 /* carrizo do enable cp interrupt after cp inited */ 3075 if (!(adev->flags & AMD_IS_APU)) { 3076 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3077 udelay(50); 3078 } 3079 3080 #ifdef AMDGPU_RLC_DEBUG_RETRY 3081 /* RLC_GPM_GENERAL_6 : RLC Ucode version */ 3082 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6); 3083 if(rlc_ucode_ver == 0x108) { 3084 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n", 3085 rlc_ucode_ver, adev->gfx.rlc_fw_version); 3086 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles, 3087 * default is 0x9C4 to create a 100us interval */ 3088 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4); 3089 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr 3090 * to disable the page fault retry interrupts, default is 3091 * 0x100 (256) */ 3092 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100); 3093 } 3094 #endif 3095 } 3096 3097 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev) 3098 { 3099 const struct rlc_firmware_header_v2_0 *hdr; 3100 const __le32 *fw_data; 3101 unsigned i, fw_size; 3102 3103 if (!adev->gfx.rlc_fw) 3104 return -EINVAL; 3105 3106 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 3107 amdgpu_ucode_print_rlc_hdr(&hdr->header); 3108 3109 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 3110 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 3111 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 3112 3113 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, 3114 RLCG_UCODE_LOADING_START_ADDRESS); 3115 for (i = 0; i < fw_size; i++) 3116 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); 3117 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); 3118 3119 return 0; 3120 } 3121 3122 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev) 3123 { 3124 int r; 3125 3126 if (amdgpu_sriov_vf(adev)) { 3127 gfx_v9_0_init_csb(adev); 3128 return 0; 3129 } 3130 3131 adev->gfx.rlc.funcs->stop(adev); 3132 3133 /* disable CG */ 3134 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0); 3135 3136 gfx_v9_0_init_pg(adev); 3137 3138 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3139 /* legacy rlc firmware loading */ 3140 r = gfx_v9_0_rlc_load_microcode(adev); 3141 if (r) 3142 return r; 3143 } 3144 3145 switch (adev->ip_versions[GC_HWIP][0]) { 3146 case IP_VERSION(9, 2, 2): 3147 case IP_VERSION(9, 1, 0): 3148 if (amdgpu_lbpw == 0) 3149 gfx_v9_0_enable_lbpw(adev, false); 3150 else 3151 gfx_v9_0_enable_lbpw(adev, true); 3152 break; 3153 case IP_VERSION(9, 4, 0): 3154 if (amdgpu_lbpw > 0) 3155 gfx_v9_0_enable_lbpw(adev, true); 3156 else 3157 gfx_v9_0_enable_lbpw(adev, false); 3158 break; 3159 default: 3160 break; 3161 } 3162 3163 adev->gfx.rlc.funcs->start(adev); 3164 3165 return 0; 3166 } 3167 3168 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 3169 { 3170 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL); 3171 3172 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); 3173 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); 3174 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1); 3175 WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp); 3176 udelay(50); 3177 } 3178 3179 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev) 3180 { 3181 const struct gfx_firmware_header_v1_0 *pfp_hdr; 3182 const struct gfx_firmware_header_v1_0 *ce_hdr; 3183 const struct gfx_firmware_header_v1_0 *me_hdr; 3184 const __le32 *fw_data; 3185 unsigned i, fw_size; 3186 3187 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw) 3188 return -EINVAL; 3189 3190 pfp_hdr = (const struct gfx_firmware_header_v1_0 *) 3191 adev->gfx.pfp_fw->data; 3192 ce_hdr = (const struct gfx_firmware_header_v1_0 *) 3193 adev->gfx.ce_fw->data; 3194 me_hdr = (const struct gfx_firmware_header_v1_0 *) 3195 adev->gfx.me_fw->data; 3196 3197 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3198 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header); 3199 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3200 3201 gfx_v9_0_cp_gfx_enable(adev, false); 3202 3203 /* PFP */ 3204 fw_data = (const __le32 *) 3205 (adev->gfx.pfp_fw->data + 3206 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 3207 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 3208 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0); 3209 for (i = 0; i < fw_size; i++) 3210 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 3211 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); 3212 3213 /* CE */ 3214 fw_data = (const __le32 *) 3215 (adev->gfx.ce_fw->data + 3216 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 3217 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 3218 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0); 3219 for (i = 0; i < fw_size; i++) 3220 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 3221 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version); 3222 3223 /* ME */ 3224 fw_data = (const __le32 *) 3225 (adev->gfx.me_fw->data + 3226 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 3227 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 3228 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0); 3229 for (i = 0; i < fw_size; i++) 3230 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 3231 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version); 3232 3233 return 0; 3234 } 3235 3236 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev) 3237 { 3238 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; 3239 const struct cs_section_def *sect = NULL; 3240 const struct cs_extent_def *ext = NULL; 3241 int r, i, tmp; 3242 3243 /* init the CP */ 3244 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); 3245 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1); 3246 3247 gfx_v9_0_cp_gfx_enable(adev, true); 3248 3249 r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3); 3250 if (r) { 3251 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r); 3252 return r; 3253 } 3254 3255 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3256 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3257 3258 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 3259 amdgpu_ring_write(ring, 0x80000000); 3260 amdgpu_ring_write(ring, 0x80000000); 3261 3262 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 3263 for (ext = sect->section; ext->extent != NULL; ++ext) { 3264 if (sect->id == SECT_CONTEXT) { 3265 amdgpu_ring_write(ring, 3266 PACKET3(PACKET3_SET_CONTEXT_REG, 3267 ext->reg_count)); 3268 amdgpu_ring_write(ring, 3269 ext->reg_index - PACKET3_SET_CONTEXT_REG_START); 3270 for (i = 0; i < ext->reg_count; i++) 3271 amdgpu_ring_write(ring, ext->extent[i]); 3272 } 3273 } 3274 } 3275 3276 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3277 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3278 3279 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3280 amdgpu_ring_write(ring, 0); 3281 3282 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 3283 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 3284 amdgpu_ring_write(ring, 0x8000); 3285 amdgpu_ring_write(ring, 0x8000); 3286 3287 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1)); 3288 tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE | 3289 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START)); 3290 amdgpu_ring_write(ring, tmp); 3291 amdgpu_ring_write(ring, 0); 3292 3293 amdgpu_ring_commit(ring); 3294 3295 return 0; 3296 } 3297 3298 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev) 3299 { 3300 struct amdgpu_ring *ring; 3301 u32 tmp; 3302 u32 rb_bufsz; 3303 u64 rb_addr, rptr_addr, wptr_gpu_addr; 3304 3305 /* Set the write pointer delay */ 3306 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0); 3307 3308 /* set the RB to use vmid 0 */ 3309 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0); 3310 3311 /* Set ring buffer size */ 3312 ring = &adev->gfx.gfx_ring[0]; 3313 rb_bufsz = order_base_2(ring->ring_size / 8); 3314 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 3315 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 3316 #ifdef __BIG_ENDIAN 3317 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1); 3318 #endif 3319 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3320 3321 /* Initialize the ring buffer's write pointers */ 3322 ring->wptr = 0; 3323 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 3324 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 3325 3326 /* set the wb address wether it's enabled or not */ 3327 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3328 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 3329 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3330 3331 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3332 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr)); 3333 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr)); 3334 3335 mdelay(1); 3336 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3337 3338 rb_addr = ring->gpu_addr >> 8; 3339 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr); 3340 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 3341 3342 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL); 3343 if (ring->use_doorbell) { 3344 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3345 DOORBELL_OFFSET, ring->doorbell_index); 3346 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3347 DOORBELL_EN, 1); 3348 } else { 3349 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0); 3350 } 3351 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp); 3352 3353 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 3354 DOORBELL_RANGE_LOWER, ring->doorbell_index); 3355 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp); 3356 3357 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER, 3358 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 3359 3360 3361 /* start the ring */ 3362 gfx_v9_0_cp_gfx_start(adev); 3363 ring->sched.ready = true; 3364 3365 return 0; 3366 } 3367 3368 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 3369 { 3370 if (enable) { 3371 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0); 3372 } else { 3373 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 3374 (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK)); 3375 adev->gfx.kiq.ring.sched.ready = false; 3376 } 3377 udelay(50); 3378 } 3379 3380 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev) 3381 { 3382 const struct gfx_firmware_header_v1_0 *mec_hdr; 3383 const __le32 *fw_data; 3384 unsigned i; 3385 u32 tmp; 3386 3387 if (!adev->gfx.mec_fw) 3388 return -EINVAL; 3389 3390 gfx_v9_0_cp_compute_enable(adev, false); 3391 3392 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 3393 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3394 3395 fw_data = (const __le32 *) 3396 (adev->gfx.mec_fw->data + 3397 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 3398 tmp = 0; 3399 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 3400 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 3401 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp); 3402 3403 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO, 3404 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000); 3405 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI, 3406 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 3407 3408 /* MEC1 */ 3409 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3410 mec_hdr->jt_offset); 3411 for (i = 0; i < mec_hdr->jt_size; i++) 3412 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA, 3413 le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); 3414 3415 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3416 adev->gfx.mec_fw_version); 3417 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */ 3418 3419 return 0; 3420 } 3421 3422 /* KIQ functions */ 3423 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring) 3424 { 3425 uint32_t tmp; 3426 struct amdgpu_device *adev = ring->adev; 3427 3428 /* tell RLC which is KIQ queue */ 3429 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS); 3430 tmp &= 0xffffff00; 3431 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 3432 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3433 tmp |= 0x80; 3434 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3435 } 3436 3437 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd) 3438 { 3439 struct amdgpu_device *adev = ring->adev; 3440 3441 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 3442 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) { 3443 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; 3444 mqd->cp_hqd_queue_priority = 3445 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; 3446 } 3447 } 3448 } 3449 3450 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring) 3451 { 3452 struct amdgpu_device *adev = ring->adev; 3453 struct v9_mqd *mqd = ring->mqd_ptr; 3454 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 3455 uint32_t tmp; 3456 3457 mqd->header = 0xC0310800; 3458 mqd->compute_pipelinestat_enable = 0x00000001; 3459 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 3460 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 3461 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 3462 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 3463 mqd->compute_static_thread_mgmt_se4 = 0xffffffff; 3464 mqd->compute_static_thread_mgmt_se5 = 0xffffffff; 3465 mqd->compute_static_thread_mgmt_se6 = 0xffffffff; 3466 mqd->compute_static_thread_mgmt_se7 = 0xffffffff; 3467 mqd->compute_misc_reserved = 0x00000003; 3468 3469 mqd->dynamic_cu_mask_addr_lo = 3470 lower_32_bits(ring->mqd_gpu_addr 3471 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3472 mqd->dynamic_cu_mask_addr_hi = 3473 upper_32_bits(ring->mqd_gpu_addr 3474 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3475 3476 eop_base_addr = ring->eop_gpu_addr >> 8; 3477 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 3478 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 3479 3480 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3481 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL); 3482 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 3483 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1)); 3484 3485 mqd->cp_hqd_eop_control = tmp; 3486 3487 /* enable doorbell? */ 3488 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3489 3490 if (ring->use_doorbell) { 3491 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3492 DOORBELL_OFFSET, ring->doorbell_index); 3493 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3494 DOORBELL_EN, 1); 3495 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3496 DOORBELL_SOURCE, 0); 3497 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3498 DOORBELL_HIT, 0); 3499 } else { 3500 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3501 DOORBELL_EN, 0); 3502 } 3503 3504 mqd->cp_hqd_pq_doorbell_control = tmp; 3505 3506 /* disable the queue if it's active */ 3507 ring->wptr = 0; 3508 mqd->cp_hqd_dequeue_request = 0; 3509 mqd->cp_hqd_pq_rptr = 0; 3510 mqd->cp_hqd_pq_wptr_lo = 0; 3511 mqd->cp_hqd_pq_wptr_hi = 0; 3512 3513 /* set the pointer to the MQD */ 3514 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; 3515 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); 3516 3517 /* set MQD vmid to 0 */ 3518 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL); 3519 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 3520 mqd->cp_mqd_control = tmp; 3521 3522 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3523 hqd_gpu_addr = ring->gpu_addr >> 8; 3524 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 3525 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 3526 3527 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3528 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL); 3529 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 3530 (order_base_2(ring->ring_size / 4) - 1)); 3531 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 3532 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8)); 3533 #ifdef __BIG_ENDIAN 3534 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1); 3535 #endif 3536 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0); 3537 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0); 3538 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 3539 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 3540 mqd->cp_hqd_pq_control = tmp; 3541 3542 /* set the wb address whether it's enabled or not */ 3543 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3544 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 3545 mqd->cp_hqd_pq_rptr_report_addr_hi = 3546 upper_32_bits(wb_gpu_addr) & 0xffff; 3547 3548 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3549 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3550 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 3551 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 3552 3553 tmp = 0; 3554 /* enable the doorbell if requested */ 3555 if (ring->use_doorbell) { 3556 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3557 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3558 DOORBELL_OFFSET, ring->doorbell_index); 3559 3560 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3561 DOORBELL_EN, 1); 3562 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3563 DOORBELL_SOURCE, 0); 3564 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3565 DOORBELL_HIT, 0); 3566 } 3567 3568 mqd->cp_hqd_pq_doorbell_control = tmp; 3569 3570 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3571 ring->wptr = 0; 3572 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR); 3573 3574 /* set the vmid for the queue */ 3575 mqd->cp_hqd_vmid = 0; 3576 3577 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE); 3578 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53); 3579 mqd->cp_hqd_persistent_state = tmp; 3580 3581 /* set MIN_IB_AVAIL_SIZE */ 3582 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL); 3583 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 3584 mqd->cp_hqd_ib_control = tmp; 3585 3586 /* set static priority for a queue/ring */ 3587 gfx_v9_0_mqd_set_priority(ring, mqd); 3588 mqd->cp_hqd_quantum = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM); 3589 3590 /* map_queues packet doesn't need activate the queue, 3591 * so only kiq need set this field. 3592 */ 3593 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 3594 mqd->cp_hqd_active = 1; 3595 3596 return 0; 3597 } 3598 3599 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring) 3600 { 3601 struct amdgpu_device *adev = ring->adev; 3602 struct v9_mqd *mqd = ring->mqd_ptr; 3603 int j; 3604 3605 /* disable wptr polling */ 3606 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3607 3608 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR, 3609 mqd->cp_hqd_eop_base_addr_lo); 3610 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI, 3611 mqd->cp_hqd_eop_base_addr_hi); 3612 3613 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3614 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL, 3615 mqd->cp_hqd_eop_control); 3616 3617 /* enable doorbell? */ 3618 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3619 mqd->cp_hqd_pq_doorbell_control); 3620 3621 /* disable the queue if it's active */ 3622 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3623 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3624 for (j = 0; j < adev->usec_timeout; j++) { 3625 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3626 break; 3627 udelay(1); 3628 } 3629 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3630 mqd->cp_hqd_dequeue_request); 3631 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 3632 mqd->cp_hqd_pq_rptr); 3633 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3634 mqd->cp_hqd_pq_wptr_lo); 3635 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3636 mqd->cp_hqd_pq_wptr_hi); 3637 } 3638 3639 /* set the pointer to the MQD */ 3640 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR, 3641 mqd->cp_mqd_base_addr_lo); 3642 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI, 3643 mqd->cp_mqd_base_addr_hi); 3644 3645 /* set MQD vmid to 0 */ 3646 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL, 3647 mqd->cp_mqd_control); 3648 3649 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3650 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE, 3651 mqd->cp_hqd_pq_base_lo); 3652 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI, 3653 mqd->cp_hqd_pq_base_hi); 3654 3655 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3656 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL, 3657 mqd->cp_hqd_pq_control); 3658 3659 /* set the wb address whether it's enabled or not */ 3660 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR, 3661 mqd->cp_hqd_pq_rptr_report_addr_lo); 3662 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 3663 mqd->cp_hqd_pq_rptr_report_addr_hi); 3664 3665 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3666 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR, 3667 mqd->cp_hqd_pq_wptr_poll_addr_lo); 3668 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI, 3669 mqd->cp_hqd_pq_wptr_poll_addr_hi); 3670 3671 /* enable the doorbell if requested */ 3672 if (ring->use_doorbell) { 3673 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER, 3674 (adev->doorbell_index.kiq * 2) << 2); 3675 /* If GC has entered CGPG, ringing doorbell > first page 3676 * doesn't wakeup GC. Enlarge CP_MEC_DOORBELL_RANGE_UPPER to 3677 * workaround this issue. And this change has to align with firmware 3678 * update. 3679 */ 3680 if (check_if_enlarge_doorbell_range(adev)) 3681 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3682 (adev->doorbell.size - 4)); 3683 else 3684 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3685 (adev->doorbell_index.userqueue_end * 2) << 2); 3686 } 3687 3688 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3689 mqd->cp_hqd_pq_doorbell_control); 3690 3691 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3692 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3693 mqd->cp_hqd_pq_wptr_lo); 3694 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3695 mqd->cp_hqd_pq_wptr_hi); 3696 3697 /* set the vmid for the queue */ 3698 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid); 3699 3700 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 3701 mqd->cp_hqd_persistent_state); 3702 3703 /* activate the queue */ 3704 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 3705 mqd->cp_hqd_active); 3706 3707 if (ring->use_doorbell) 3708 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1); 3709 3710 return 0; 3711 } 3712 3713 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring) 3714 { 3715 struct amdgpu_device *adev = ring->adev; 3716 int j; 3717 3718 /* disable the queue if it's active */ 3719 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3720 3721 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3722 3723 for (j = 0; j < adev->usec_timeout; j++) { 3724 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3725 break; 3726 udelay(1); 3727 } 3728 3729 if (j == AMDGPU_MAX_USEC_TIMEOUT) { 3730 DRM_DEBUG("KIQ dequeue request failed.\n"); 3731 3732 /* Manual disable if dequeue request times out */ 3733 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0); 3734 } 3735 3736 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3737 0); 3738 } 3739 3740 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0); 3741 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0); 3742 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0); 3743 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000); 3744 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0); 3745 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0); 3746 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0); 3747 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0); 3748 3749 return 0; 3750 } 3751 3752 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring) 3753 { 3754 struct amdgpu_device *adev = ring->adev; 3755 struct v9_mqd *mqd = ring->mqd_ptr; 3756 int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS; 3757 struct v9_mqd *tmp_mqd; 3758 3759 gfx_v9_0_kiq_setting(ring); 3760 3761 /* GPU could be in bad state during probe, driver trigger the reset 3762 * after load the SMU, in this case , the mqd is not be initialized. 3763 * driver need to re-init the mqd. 3764 * check mqd->cp_hqd_pq_control since this value should not be 0 3765 */ 3766 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx]; 3767 if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){ 3768 /* for GPU_RESET case , reset MQD to a clean status */ 3769 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3770 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3771 3772 /* reset ring buffer */ 3773 ring->wptr = 0; 3774 amdgpu_ring_clear_ring(ring); 3775 3776 mutex_lock(&adev->srbm_mutex); 3777 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3778 gfx_v9_0_kiq_init_register(ring); 3779 soc15_grbm_select(adev, 0, 0, 0, 0); 3780 mutex_unlock(&adev->srbm_mutex); 3781 } else { 3782 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3783 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3784 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3785 mutex_lock(&adev->srbm_mutex); 3786 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3787 gfx_v9_0_mqd_init(ring); 3788 gfx_v9_0_kiq_init_register(ring); 3789 soc15_grbm_select(adev, 0, 0, 0, 0); 3790 mutex_unlock(&adev->srbm_mutex); 3791 3792 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3793 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3794 } 3795 3796 return 0; 3797 } 3798 3799 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring) 3800 { 3801 struct amdgpu_device *adev = ring->adev; 3802 struct v9_mqd *mqd = ring->mqd_ptr; 3803 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 3804 struct v9_mqd *tmp_mqd; 3805 3806 /* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control 3807 * is not be initialized before 3808 */ 3809 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx]; 3810 3811 if (!tmp_mqd->cp_hqd_pq_control || 3812 (!amdgpu_in_reset(adev) && !adev->in_suspend)) { 3813 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3814 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3815 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3816 mutex_lock(&adev->srbm_mutex); 3817 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3818 gfx_v9_0_mqd_init(ring); 3819 soc15_grbm_select(adev, 0, 0, 0, 0); 3820 mutex_unlock(&adev->srbm_mutex); 3821 3822 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3823 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3824 } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 3825 /* reset MQD to a clean status */ 3826 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3827 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3828 3829 /* reset ring buffer */ 3830 ring->wptr = 0; 3831 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0); 3832 amdgpu_ring_clear_ring(ring); 3833 } else { 3834 amdgpu_ring_clear_ring(ring); 3835 } 3836 3837 return 0; 3838 } 3839 3840 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev) 3841 { 3842 struct amdgpu_ring *ring; 3843 int r; 3844 3845 ring = &adev->gfx.kiq.ring; 3846 3847 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3848 if (unlikely(r != 0)) 3849 return r; 3850 3851 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3852 if (unlikely(r != 0)) 3853 return r; 3854 3855 gfx_v9_0_kiq_init_queue(ring); 3856 amdgpu_bo_kunmap(ring->mqd_obj); 3857 ring->mqd_ptr = NULL; 3858 amdgpu_bo_unreserve(ring->mqd_obj); 3859 ring->sched.ready = true; 3860 return 0; 3861 } 3862 3863 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev) 3864 { 3865 struct amdgpu_ring *ring = NULL; 3866 int r = 0, i; 3867 3868 gfx_v9_0_cp_compute_enable(adev, true); 3869 3870 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3871 ring = &adev->gfx.compute_ring[i]; 3872 3873 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3874 if (unlikely(r != 0)) 3875 goto done; 3876 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3877 if (!r) { 3878 r = gfx_v9_0_kcq_init_queue(ring); 3879 amdgpu_bo_kunmap(ring->mqd_obj); 3880 ring->mqd_ptr = NULL; 3881 } 3882 amdgpu_bo_unreserve(ring->mqd_obj); 3883 if (r) 3884 goto done; 3885 } 3886 3887 r = amdgpu_gfx_enable_kcq(adev); 3888 done: 3889 return r; 3890 } 3891 3892 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev) 3893 { 3894 int r, i; 3895 struct amdgpu_ring *ring; 3896 3897 if (!(adev->flags & AMD_IS_APU)) 3898 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3899 3900 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3901 if (adev->gfx.num_gfx_rings) { 3902 /* legacy firmware loading */ 3903 r = gfx_v9_0_cp_gfx_load_microcode(adev); 3904 if (r) 3905 return r; 3906 } 3907 3908 r = gfx_v9_0_cp_compute_load_microcode(adev); 3909 if (r) 3910 return r; 3911 } 3912 3913 r = gfx_v9_0_kiq_resume(adev); 3914 if (r) 3915 return r; 3916 3917 if (adev->gfx.num_gfx_rings) { 3918 r = gfx_v9_0_cp_gfx_resume(adev); 3919 if (r) 3920 return r; 3921 } 3922 3923 r = gfx_v9_0_kcq_resume(adev); 3924 if (r) 3925 return r; 3926 3927 if (adev->gfx.num_gfx_rings) { 3928 ring = &adev->gfx.gfx_ring[0]; 3929 r = amdgpu_ring_test_helper(ring); 3930 if (r) 3931 return r; 3932 } 3933 3934 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3935 ring = &adev->gfx.compute_ring[i]; 3936 amdgpu_ring_test_helper(ring); 3937 } 3938 3939 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3940 3941 return 0; 3942 } 3943 3944 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev) 3945 { 3946 u32 tmp; 3947 3948 if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 1) && 3949 adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 2)) 3950 return; 3951 3952 tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG); 3953 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH, 3954 adev->df.hash_status.hash_64k); 3955 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH, 3956 adev->df.hash_status.hash_2m); 3957 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH, 3958 adev->df.hash_status.hash_1g); 3959 WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp); 3960 } 3961 3962 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable) 3963 { 3964 if (adev->gfx.num_gfx_rings) 3965 gfx_v9_0_cp_gfx_enable(adev, enable); 3966 gfx_v9_0_cp_compute_enable(adev, enable); 3967 } 3968 3969 static int gfx_v9_0_hw_init(void *handle) 3970 { 3971 int r; 3972 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3973 3974 if (!amdgpu_sriov_vf(adev)) 3975 gfx_v9_0_init_golden_registers(adev); 3976 3977 gfx_v9_0_constants_init(adev); 3978 3979 gfx_v9_0_init_tcp_config(adev); 3980 3981 r = adev->gfx.rlc.funcs->resume(adev); 3982 if (r) 3983 return r; 3984 3985 r = gfx_v9_0_cp_resume(adev); 3986 if (r) 3987 return r; 3988 3989 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2)) 3990 gfx_v9_4_2_set_power_brake_sequence(adev); 3991 3992 return r; 3993 } 3994 3995 static int gfx_v9_0_hw_fini(void *handle) 3996 { 3997 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3998 3999 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); 4000 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 4001 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 4002 4003 /* DF freeze and kcq disable will fail */ 4004 if (!amdgpu_ras_intr_triggered()) 4005 /* disable KCQ to avoid CPC touch memory not valid anymore */ 4006 amdgpu_gfx_disable_kcq(adev); 4007 4008 if (amdgpu_sriov_vf(adev)) { 4009 gfx_v9_0_cp_gfx_enable(adev, false); 4010 /* must disable polling for SRIOV when hw finished, otherwise 4011 * CPC engine may still keep fetching WB address which is already 4012 * invalid after sw finished and trigger DMAR reading error in 4013 * hypervisor side. 4014 */ 4015 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 4016 return 0; 4017 } 4018 4019 /* Use deinitialize sequence from CAIL when unbinding device from driver, 4020 * otherwise KIQ is hanging when binding back 4021 */ 4022 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 4023 mutex_lock(&adev->srbm_mutex); 4024 soc15_grbm_select(adev, adev->gfx.kiq.ring.me, 4025 adev->gfx.kiq.ring.pipe, 4026 adev->gfx.kiq.ring.queue, 0); 4027 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring); 4028 soc15_grbm_select(adev, 0, 0, 0, 0); 4029 mutex_unlock(&adev->srbm_mutex); 4030 } 4031 4032 gfx_v9_0_cp_enable(adev, false); 4033 4034 /* Skip stopping RLC with A+A reset or when RLC controls GFX clock */ 4035 if ((adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) || 4036 (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(9, 4, 2))) { 4037 dev_dbg(adev->dev, "Skipping RLC halt\n"); 4038 return 0; 4039 } 4040 4041 adev->gfx.rlc.funcs->stop(adev); 4042 return 0; 4043 } 4044 4045 static int gfx_v9_0_suspend(void *handle) 4046 { 4047 return gfx_v9_0_hw_fini(handle); 4048 } 4049 4050 static int gfx_v9_0_resume(void *handle) 4051 { 4052 return gfx_v9_0_hw_init(handle); 4053 } 4054 4055 static bool gfx_v9_0_is_idle(void *handle) 4056 { 4057 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4058 4059 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS), 4060 GRBM_STATUS, GUI_ACTIVE)) 4061 return false; 4062 else 4063 return true; 4064 } 4065 4066 static int gfx_v9_0_wait_for_idle(void *handle) 4067 { 4068 unsigned i; 4069 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4070 4071 for (i = 0; i < adev->usec_timeout; i++) { 4072 if (gfx_v9_0_is_idle(handle)) 4073 return 0; 4074 udelay(1); 4075 } 4076 return -ETIMEDOUT; 4077 } 4078 4079 static int gfx_v9_0_soft_reset(void *handle) 4080 { 4081 u32 grbm_soft_reset = 0; 4082 u32 tmp; 4083 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4084 4085 /* GRBM_STATUS */ 4086 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS); 4087 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK | 4088 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK | 4089 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK | 4090 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK | 4091 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK | 4092 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) { 4093 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4094 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4095 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4096 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1); 4097 } 4098 4099 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) { 4100 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4101 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4102 } 4103 4104 /* GRBM_STATUS2 */ 4105 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2); 4106 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY)) 4107 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4108 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 4109 4110 4111 if (grbm_soft_reset) { 4112 /* stop the rlc */ 4113 adev->gfx.rlc.funcs->stop(adev); 4114 4115 if (adev->gfx.num_gfx_rings) 4116 /* Disable GFX parsing/prefetching */ 4117 gfx_v9_0_cp_gfx_enable(adev, false); 4118 4119 /* Disable MEC parsing/prefetching */ 4120 gfx_v9_0_cp_compute_enable(adev, false); 4121 4122 if (grbm_soft_reset) { 4123 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4124 tmp |= grbm_soft_reset; 4125 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 4126 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4127 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4128 4129 udelay(50); 4130 4131 tmp &= ~grbm_soft_reset; 4132 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4133 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4134 } 4135 4136 /* Wait a little for things to settle down */ 4137 udelay(50); 4138 } 4139 return 0; 4140 } 4141 4142 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev) 4143 { 4144 signed long r, cnt = 0; 4145 unsigned long flags; 4146 uint32_t seq, reg_val_offs = 0; 4147 uint64_t value = 0; 4148 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 4149 struct amdgpu_ring *ring = &kiq->ring; 4150 4151 BUG_ON(!ring->funcs->emit_rreg); 4152 4153 spin_lock_irqsave(&kiq->ring_lock, flags); 4154 if (amdgpu_device_wb_get(adev, ®_val_offs)) { 4155 pr_err("critical bug! too many kiq readers\n"); 4156 goto failed_unlock; 4157 } 4158 amdgpu_ring_alloc(ring, 32); 4159 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 4160 amdgpu_ring_write(ring, 9 | /* src: register*/ 4161 (5 << 8) | /* dst: memory */ 4162 (1 << 16) | /* count sel */ 4163 (1 << 20)); /* write confirm */ 4164 amdgpu_ring_write(ring, 0); 4165 amdgpu_ring_write(ring, 0); 4166 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 4167 reg_val_offs * 4)); 4168 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 4169 reg_val_offs * 4)); 4170 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 4171 if (r) 4172 goto failed_undo; 4173 4174 amdgpu_ring_commit(ring); 4175 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4176 4177 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4178 4179 /* don't wait anymore for gpu reset case because this way may 4180 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 4181 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 4182 * never return if we keep waiting in virt_kiq_rreg, which cause 4183 * gpu_recover() hang there. 4184 * 4185 * also don't wait anymore for IRQ context 4186 * */ 4187 if (r < 1 && (amdgpu_in_reset(adev))) 4188 goto failed_kiq_read; 4189 4190 might_sleep(); 4191 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 4192 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 4193 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4194 } 4195 4196 if (cnt > MAX_KIQ_REG_TRY) 4197 goto failed_kiq_read; 4198 4199 mb(); 4200 value = (uint64_t)adev->wb.wb[reg_val_offs] | 4201 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL; 4202 amdgpu_device_wb_free(adev, reg_val_offs); 4203 return value; 4204 4205 failed_undo: 4206 amdgpu_ring_undo(ring); 4207 failed_unlock: 4208 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4209 failed_kiq_read: 4210 if (reg_val_offs) 4211 amdgpu_device_wb_free(adev, reg_val_offs); 4212 pr_err("failed to read gpu clock\n"); 4213 return ~0; 4214 } 4215 4216 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev) 4217 { 4218 uint64_t clock, clock_lo, clock_hi, hi_check; 4219 4220 switch (adev->ip_versions[GC_HWIP][0]) { 4221 case IP_VERSION(9, 3, 0): 4222 preempt_disable(); 4223 clock_hi = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir); 4224 clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir); 4225 hi_check = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir); 4226 /* The SMUIO TSC clock frequency is 100MHz, which sets 32-bit carry over 4227 * roughly every 42 seconds. 4228 */ 4229 if (hi_check != clock_hi) { 4230 clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir); 4231 clock_hi = hi_check; 4232 } 4233 preempt_enable(); 4234 clock = clock_lo | (clock_hi << 32ULL); 4235 break; 4236 default: 4237 amdgpu_gfx_off_ctrl(adev, false); 4238 mutex_lock(&adev->gfx.gpu_clock_mutex); 4239 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 0, 1) && amdgpu_sriov_runtime(adev)) { 4240 clock = gfx_v9_0_kiq_read_clock(adev); 4241 } else { 4242 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1); 4243 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) | 4244 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 4245 } 4246 mutex_unlock(&adev->gfx.gpu_clock_mutex); 4247 amdgpu_gfx_off_ctrl(adev, true); 4248 break; 4249 } 4250 return clock; 4251 } 4252 4253 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 4254 uint32_t vmid, 4255 uint32_t gds_base, uint32_t gds_size, 4256 uint32_t gws_base, uint32_t gws_size, 4257 uint32_t oa_base, uint32_t oa_size) 4258 { 4259 struct amdgpu_device *adev = ring->adev; 4260 4261 /* GDS Base */ 4262 gfx_v9_0_write_data_to_reg(ring, 0, false, 4263 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid, 4264 gds_base); 4265 4266 /* GDS Size */ 4267 gfx_v9_0_write_data_to_reg(ring, 0, false, 4268 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid, 4269 gds_size); 4270 4271 /* GWS */ 4272 gfx_v9_0_write_data_to_reg(ring, 0, false, 4273 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid, 4274 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 4275 4276 /* OA */ 4277 gfx_v9_0_write_data_to_reg(ring, 0, false, 4278 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid, 4279 (1 << (oa_size + oa_base)) - (1 << oa_base)); 4280 } 4281 4282 static const u32 vgpr_init_compute_shader[] = 4283 { 4284 0xb07c0000, 0xbe8000ff, 4285 0x000000f8, 0xbf110800, 4286 0x7e000280, 0x7e020280, 4287 0x7e040280, 0x7e060280, 4288 0x7e080280, 0x7e0a0280, 4289 0x7e0c0280, 0x7e0e0280, 4290 0x80808800, 0xbe803200, 4291 0xbf84fff5, 0xbf9c0000, 4292 0xd28c0001, 0x0001007f, 4293 0xd28d0001, 0x0002027e, 4294 0x10020288, 0xb8810904, 4295 0xb7814000, 0xd1196a01, 4296 0x00000301, 0xbe800087, 4297 0xbefc00c1, 0xd89c4000, 4298 0x00020201, 0xd89cc080, 4299 0x00040401, 0x320202ff, 4300 0x00000800, 0x80808100, 4301 0xbf84fff8, 0x7e020280, 4302 0xbf810000, 0x00000000, 4303 }; 4304 4305 static const u32 sgpr_init_compute_shader[] = 4306 { 4307 0xb07c0000, 0xbe8000ff, 4308 0x0000005f, 0xbee50080, 4309 0xbe812c65, 0xbe822c65, 4310 0xbe832c65, 0xbe842c65, 4311 0xbe852c65, 0xb77c0005, 4312 0x80808500, 0xbf84fff8, 4313 0xbe800080, 0xbf810000, 4314 }; 4315 4316 static const u32 vgpr_init_compute_shader_arcturus[] = { 4317 0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080, 4318 0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080, 4319 0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080, 4320 0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080, 4321 0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080, 4322 0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080, 4323 0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080, 4324 0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080, 4325 0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080, 4326 0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080, 4327 0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080, 4328 0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080, 4329 0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080, 4330 0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080, 4331 0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080, 4332 0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080, 4333 0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080, 4334 0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080, 4335 0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080, 4336 0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080, 4337 0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080, 4338 0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080, 4339 0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080, 4340 0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080, 4341 0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080, 4342 0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080, 4343 0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080, 4344 0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080, 4345 0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080, 4346 0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080, 4347 0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080, 4348 0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080, 4349 0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080, 4350 0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080, 4351 0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080, 4352 0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080, 4353 0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080, 4354 0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080, 4355 0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080, 4356 0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080, 4357 0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080, 4358 0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080, 4359 0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080, 4360 0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080, 4361 0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080, 4362 0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080, 4363 0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080, 4364 0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080, 4365 0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080, 4366 0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080, 4367 0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080, 4368 0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080, 4369 0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080, 4370 0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080, 4371 0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080, 4372 0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080, 4373 0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080, 4374 0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080, 4375 0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080, 4376 0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080, 4377 0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080, 4378 0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080, 4379 0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080, 4380 0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080, 4381 0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080, 4382 0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080, 4383 0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080, 4384 0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080, 4385 0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080, 4386 0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080, 4387 0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080, 4388 0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080, 4389 0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080, 4390 0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080, 4391 0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080, 4392 0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080, 4393 0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080, 4394 0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080, 4395 0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080, 4396 0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080, 4397 0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080, 4398 0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080, 4399 0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080, 4400 0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080, 4401 0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080, 4402 0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a, 4403 0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280, 4404 0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000, 4405 0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904, 4406 0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000, 4407 0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a, 4408 0xbf84fff8, 0xbf810000, 4409 }; 4410 4411 /* When below register arrays changed, please update gpr_reg_size, 4412 and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds, 4413 to cover all gfx9 ASICs */ 4414 static const struct soc15_reg_entry vgpr_init_regs[] = { 4415 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4416 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4417 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4418 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4419 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f }, 4420 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4421 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4422 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4423 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4424 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4425 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4426 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4427 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4428 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4429 }; 4430 4431 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = { 4432 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4433 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4434 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4435 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4436 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf }, 4437 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4438 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4439 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4440 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4441 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4442 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4443 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4444 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4445 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4446 }; 4447 4448 static const struct soc15_reg_entry sgpr1_init_regs[] = { 4449 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4450 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4451 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4452 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4453 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4454 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4455 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff }, 4456 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff }, 4457 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff }, 4458 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff }, 4459 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff }, 4460 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff }, 4461 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff }, 4462 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff }, 4463 }; 4464 4465 static const struct soc15_reg_entry sgpr2_init_regs[] = { 4466 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4467 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4468 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4469 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4470 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4471 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4472 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 }, 4473 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 }, 4474 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 }, 4475 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 }, 4476 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 }, 4477 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 }, 4478 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 }, 4479 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 }, 4480 }; 4481 4482 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = { 4483 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1}, 4484 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1}, 4485 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1}, 4486 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1}, 4487 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1}, 4488 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1}, 4489 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1}, 4490 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1}, 4491 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1}, 4492 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1}, 4493 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1}, 4494 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1}, 4495 { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1}, 4496 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6}, 4497 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16}, 4498 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16}, 4499 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16}, 4500 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16}, 4501 { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16}, 4502 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16}, 4503 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16}, 4504 { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16}, 4505 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6}, 4506 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16}, 4507 { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16}, 4508 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1}, 4509 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1}, 4510 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32}, 4511 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32}, 4512 { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72}, 4513 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16}, 4514 { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2}, 4515 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6}, 4516 }; 4517 4518 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev) 4519 { 4520 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4521 int i, r; 4522 4523 /* only support when RAS is enabled */ 4524 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4525 return 0; 4526 4527 r = amdgpu_ring_alloc(ring, 7); 4528 if (r) { 4529 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n", 4530 ring->name, r); 4531 return r; 4532 } 4533 4534 WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000); 4535 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size); 4536 4537 amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 4538 amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC | 4539 PACKET3_DMA_DATA_DST_SEL(1) | 4540 PACKET3_DMA_DATA_SRC_SEL(2) | 4541 PACKET3_DMA_DATA_ENGINE(0))); 4542 amdgpu_ring_write(ring, 0); 4543 amdgpu_ring_write(ring, 0); 4544 amdgpu_ring_write(ring, 0); 4545 amdgpu_ring_write(ring, 0); 4546 amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT | 4547 adev->gds.gds_size); 4548 4549 amdgpu_ring_commit(ring); 4550 4551 for (i = 0; i < adev->usec_timeout; i++) { 4552 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring)) 4553 break; 4554 udelay(1); 4555 } 4556 4557 if (i >= adev->usec_timeout) 4558 r = -ETIMEDOUT; 4559 4560 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000); 4561 4562 return r; 4563 } 4564 4565 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) 4566 { 4567 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4568 struct amdgpu_ib ib; 4569 struct dma_fence *f = NULL; 4570 int r, i; 4571 unsigned total_size, vgpr_offset, sgpr_offset; 4572 u64 gpu_addr; 4573 4574 int compute_dim_x = adev->gfx.config.max_shader_engines * 4575 adev->gfx.config.max_cu_per_sh * 4576 adev->gfx.config.max_sh_per_se; 4577 int sgpr_work_group_size = 5; 4578 int gpr_reg_size = adev->gfx.config.max_shader_engines + 6; 4579 int vgpr_init_shader_size; 4580 const u32 *vgpr_init_shader_ptr; 4581 const struct soc15_reg_entry *vgpr_init_regs_ptr; 4582 4583 /* only support when RAS is enabled */ 4584 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4585 return 0; 4586 4587 /* bail if the compute ring is not ready */ 4588 if (!ring->sched.ready) 4589 return 0; 4590 4591 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1)) { 4592 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus; 4593 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus); 4594 vgpr_init_regs_ptr = vgpr_init_regs_arcturus; 4595 } else { 4596 vgpr_init_shader_ptr = vgpr_init_compute_shader; 4597 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader); 4598 vgpr_init_regs_ptr = vgpr_init_regs; 4599 } 4600 4601 total_size = 4602 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */ 4603 total_size += 4604 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */ 4605 total_size += 4606 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */ 4607 total_size = ALIGN(total_size, 256); 4608 vgpr_offset = total_size; 4609 total_size += ALIGN(vgpr_init_shader_size, 256); 4610 sgpr_offset = total_size; 4611 total_size += sizeof(sgpr_init_compute_shader); 4612 4613 /* allocate an indirect buffer to put the commands in */ 4614 memset(&ib, 0, sizeof(ib)); 4615 r = amdgpu_ib_get(adev, NULL, total_size, 4616 AMDGPU_IB_POOL_DIRECT, &ib); 4617 if (r) { 4618 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); 4619 return r; 4620 } 4621 4622 /* load the compute shaders */ 4623 for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++) 4624 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i]; 4625 4626 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++) 4627 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i]; 4628 4629 /* init the ib length to 0 */ 4630 ib.length_dw = 0; 4631 4632 /* VGPR */ 4633 /* write the register state for the compute dispatch */ 4634 for (i = 0; i < gpr_reg_size; i++) { 4635 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4636 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i]) 4637 - PACKET3_SET_SH_REG_START; 4638 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value; 4639 } 4640 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4641 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8; 4642 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4643 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4644 - PACKET3_SET_SH_REG_START; 4645 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4646 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4647 4648 /* write dispatch packet */ 4649 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4650 ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */ 4651 ib.ptr[ib.length_dw++] = 1; /* y */ 4652 ib.ptr[ib.length_dw++] = 1; /* z */ 4653 ib.ptr[ib.length_dw++] = 4654 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4655 4656 /* write CS partial flush packet */ 4657 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4658 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4659 4660 /* SGPR1 */ 4661 /* write the register state for the compute dispatch */ 4662 for (i = 0; i < gpr_reg_size; i++) { 4663 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4664 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i]) 4665 - PACKET3_SET_SH_REG_START; 4666 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value; 4667 } 4668 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4669 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4670 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4671 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4672 - PACKET3_SET_SH_REG_START; 4673 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4674 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4675 4676 /* write dispatch packet */ 4677 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4678 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4679 ib.ptr[ib.length_dw++] = 1; /* y */ 4680 ib.ptr[ib.length_dw++] = 1; /* z */ 4681 ib.ptr[ib.length_dw++] = 4682 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4683 4684 /* write CS partial flush packet */ 4685 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4686 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4687 4688 /* SGPR2 */ 4689 /* write the register state for the compute dispatch */ 4690 for (i = 0; i < gpr_reg_size; i++) { 4691 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4692 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i]) 4693 - PACKET3_SET_SH_REG_START; 4694 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value; 4695 } 4696 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4697 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4698 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4699 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4700 - PACKET3_SET_SH_REG_START; 4701 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4702 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4703 4704 /* write dispatch packet */ 4705 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4706 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4707 ib.ptr[ib.length_dw++] = 1; /* y */ 4708 ib.ptr[ib.length_dw++] = 1; /* z */ 4709 ib.ptr[ib.length_dw++] = 4710 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4711 4712 /* write CS partial flush packet */ 4713 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4714 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4715 4716 /* shedule the ib on the ring */ 4717 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 4718 if (r) { 4719 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r); 4720 goto fail; 4721 } 4722 4723 /* wait for the GPU to finish processing the IB */ 4724 r = dma_fence_wait(f, false); 4725 if (r) { 4726 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); 4727 goto fail; 4728 } 4729 4730 fail: 4731 amdgpu_ib_free(adev, &ib, NULL); 4732 dma_fence_put(f); 4733 4734 return r; 4735 } 4736 4737 static int gfx_v9_0_early_init(void *handle) 4738 { 4739 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4740 4741 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1) || 4742 adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2)) 4743 adev->gfx.num_gfx_rings = 0; 4744 else 4745 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS; 4746 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), 4747 AMDGPU_MAX_COMPUTE_RINGS); 4748 gfx_v9_0_set_kiq_pm4_funcs(adev); 4749 gfx_v9_0_set_ring_funcs(adev); 4750 gfx_v9_0_set_irq_funcs(adev); 4751 gfx_v9_0_set_gds_init(adev); 4752 gfx_v9_0_set_rlc_funcs(adev); 4753 4754 /* init rlcg reg access ctrl */ 4755 gfx_v9_0_init_rlcg_reg_access_ctrl(adev); 4756 4757 return 0; 4758 } 4759 4760 static int gfx_v9_0_ecc_late_init(void *handle) 4761 { 4762 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4763 int r; 4764 4765 /* 4766 * Temp workaround to fix the issue that CP firmware fails to 4767 * update read pointer when CPDMA is writing clearing operation 4768 * to GDS in suspend/resume sequence on several cards. So just 4769 * limit this operation in cold boot sequence. 4770 */ 4771 if ((!adev->in_suspend) && 4772 (adev->gds.gds_size)) { 4773 r = gfx_v9_0_do_edc_gds_workarounds(adev); 4774 if (r) 4775 return r; 4776 } 4777 4778 /* requires IBs so do in late init after IB pool is initialized */ 4779 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2)) 4780 r = gfx_v9_4_2_do_edc_gpr_workarounds(adev); 4781 else 4782 r = gfx_v9_0_do_edc_gpr_workarounds(adev); 4783 4784 if (r) 4785 return r; 4786 4787 if (adev->gfx.ras && 4788 adev->gfx.ras->enable_watchdog_timer) 4789 adev->gfx.ras->enable_watchdog_timer(adev); 4790 4791 return 0; 4792 } 4793 4794 static int gfx_v9_0_late_init(void *handle) 4795 { 4796 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4797 int r; 4798 4799 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 4800 if (r) 4801 return r; 4802 4803 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 4804 if (r) 4805 return r; 4806 4807 r = gfx_v9_0_ecc_late_init(handle); 4808 if (r) 4809 return r; 4810 4811 return 0; 4812 } 4813 4814 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev) 4815 { 4816 uint32_t rlc_setting; 4817 4818 /* if RLC is not enabled, do nothing */ 4819 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL); 4820 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK)) 4821 return false; 4822 4823 return true; 4824 } 4825 4826 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev) 4827 { 4828 uint32_t data; 4829 unsigned i; 4830 4831 data = RLC_SAFE_MODE__CMD_MASK; 4832 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 4833 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4834 4835 /* wait for RLC_SAFE_MODE */ 4836 for (i = 0; i < adev->usec_timeout; i++) { 4837 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 4838 break; 4839 udelay(1); 4840 } 4841 } 4842 4843 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev) 4844 { 4845 uint32_t data; 4846 4847 data = RLC_SAFE_MODE__CMD_MASK; 4848 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4849 } 4850 4851 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev, 4852 bool enable) 4853 { 4854 amdgpu_gfx_rlc_enter_safe_mode(adev); 4855 4856 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { 4857 gfx_v9_0_enable_gfx_cg_power_gating(adev, true); 4858 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4859 gfx_v9_0_enable_gfx_pipeline_powergating(adev, true); 4860 } else { 4861 gfx_v9_0_enable_gfx_cg_power_gating(adev, false); 4862 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4863 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false); 4864 } 4865 4866 amdgpu_gfx_rlc_exit_safe_mode(adev); 4867 } 4868 4869 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev, 4870 bool enable) 4871 { 4872 /* TODO: double check if we need to perform under safe mode */ 4873 /* gfx_v9_0_enter_rlc_safe_mode(adev); */ 4874 4875 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 4876 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true); 4877 else 4878 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false); 4879 4880 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 4881 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true); 4882 else 4883 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false); 4884 4885 /* gfx_v9_0_exit_rlc_safe_mode(adev); */ 4886 } 4887 4888 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 4889 bool enable) 4890 { 4891 uint32_t data, def; 4892 4893 amdgpu_gfx_rlc_enter_safe_mode(adev); 4894 4895 /* It is disabled by HW by default */ 4896 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { 4897 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 4898 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4899 4900 if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 2, 1)) 4901 data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4902 4903 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4904 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4905 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4906 4907 /* only for Vega10 & Raven1 */ 4908 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK; 4909 4910 if (def != data) 4911 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4912 4913 /* MGLS is a global flag to control all MGLS in GFX */ 4914 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 4915 /* 2 - RLC memory Light sleep */ 4916 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { 4917 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4918 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4919 if (def != data) 4920 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4921 } 4922 /* 3 - CP memory Light sleep */ 4923 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { 4924 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4925 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4926 if (def != data) 4927 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4928 } 4929 } 4930 } else { 4931 /* 1 - MGCG_OVERRIDE */ 4932 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4933 4934 if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 2, 1)) 4935 data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4936 4937 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 4938 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4939 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4940 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4941 4942 if (def != data) 4943 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4944 4945 /* 2 - disable MGLS in RLC */ 4946 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4947 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) { 4948 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4949 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4950 } 4951 4952 /* 3 - disable MGLS in CP */ 4953 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4954 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) { 4955 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4956 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4957 } 4958 } 4959 4960 amdgpu_gfx_rlc_exit_safe_mode(adev); 4961 } 4962 4963 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev, 4964 bool enable) 4965 { 4966 uint32_t data, def; 4967 4968 if (!adev->gfx.num_gfx_rings) 4969 return; 4970 4971 amdgpu_gfx_rlc_enter_safe_mode(adev); 4972 4973 /* Enable 3D CGCG/CGLS */ 4974 if (enable) { 4975 /* write cmd to clear cgcg/cgls ov */ 4976 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4977 /* unset CGCG override */ 4978 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 4979 /* update CGCG and CGLS override bits */ 4980 if (def != data) 4981 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4982 4983 /* enable 3Dcgcg FSM(0x0000363f) */ 4984 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4985 4986 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) 4987 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4988 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 4989 else 4990 data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT; 4991 4992 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 4993 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 4994 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 4995 if (def != data) 4996 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4997 4998 /* set IDLE_POLL_COUNT(0x00900100) */ 4999 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 5000 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 5001 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 5002 if (def != data) 5003 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 5004 } else { 5005 /* Disable CGCG/CGLS */ 5006 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 5007 /* disable cgcg, cgls should be disabled */ 5008 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK | 5009 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK); 5010 /* disable cgcg and cgls in FSM */ 5011 if (def != data) 5012 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 5013 } 5014 5015 amdgpu_gfx_rlc_exit_safe_mode(adev); 5016 } 5017 5018 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 5019 bool enable) 5020 { 5021 uint32_t def, data; 5022 5023 amdgpu_gfx_rlc_enter_safe_mode(adev); 5024 5025 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { 5026 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 5027 /* unset CGCG override */ 5028 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 5029 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5030 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 5031 else 5032 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 5033 /* update CGCG and CGLS override bits */ 5034 if (def != data) 5035 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 5036 5037 /* enable cgcg FSM(0x0000363F) */ 5038 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 5039 5040 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1)) 5041 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5042 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5043 else 5044 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5045 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5046 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5047 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 5048 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5049 if (def != data) 5050 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 5051 5052 /* set IDLE_POLL_COUNT(0x00900100) */ 5053 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 5054 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 5055 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 5056 if (def != data) 5057 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 5058 } else { 5059 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 5060 /* reset CGCG/CGLS bits */ 5061 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK); 5062 /* disable cgcg and cgls in FSM */ 5063 if (def != data) 5064 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 5065 } 5066 5067 amdgpu_gfx_rlc_exit_safe_mode(adev); 5068 } 5069 5070 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev, 5071 bool enable) 5072 { 5073 if (enable) { 5074 /* CGCG/CGLS should be enabled after MGCG/MGLS 5075 * === MGCG + MGLS === 5076 */ 5077 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 5078 /* === CGCG /CGLS for GFX 3D Only === */ 5079 gfx_v9_0_update_3d_clock_gating(adev, enable); 5080 /* === CGCG + CGLS === */ 5081 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 5082 } else { 5083 /* CGCG/CGLS should be disabled before MGCG/MGLS 5084 * === CGCG + CGLS === 5085 */ 5086 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 5087 /* === CGCG /CGLS for GFX 3D Only === */ 5088 gfx_v9_0_update_3d_clock_gating(adev, enable); 5089 /* === MGCG + MGLS === */ 5090 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 5091 } 5092 return 0; 5093 } 5094 5095 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid) 5096 { 5097 u32 reg, data; 5098 5099 amdgpu_gfx_off_ctrl(adev, false); 5100 5101 reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL); 5102 if (amdgpu_sriov_is_pp_one_vf(adev)) 5103 data = RREG32_NO_KIQ(reg); 5104 else 5105 data = RREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL); 5106 5107 data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK; 5108 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; 5109 5110 if (amdgpu_sriov_is_pp_one_vf(adev)) 5111 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data); 5112 else 5113 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data); 5114 5115 amdgpu_gfx_off_ctrl(adev, true); 5116 } 5117 5118 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev, 5119 uint32_t offset, 5120 struct soc15_reg_rlcg *entries, int arr_size) 5121 { 5122 int i; 5123 uint32_t reg; 5124 5125 if (!entries) 5126 return false; 5127 5128 for (i = 0; i < arr_size; i++) { 5129 const struct soc15_reg_rlcg *entry; 5130 5131 entry = &entries[i]; 5132 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg; 5133 if (offset == reg) 5134 return true; 5135 } 5136 5137 return false; 5138 } 5139 5140 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset) 5141 { 5142 return gfx_v9_0_check_rlcg_range(adev, offset, 5143 (void *)rlcg_access_gc_9_0, 5144 ARRAY_SIZE(rlcg_access_gc_9_0)); 5145 } 5146 5147 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = { 5148 .is_rlc_enabled = gfx_v9_0_is_rlc_enabled, 5149 .set_safe_mode = gfx_v9_0_set_safe_mode, 5150 .unset_safe_mode = gfx_v9_0_unset_safe_mode, 5151 .init = gfx_v9_0_rlc_init, 5152 .get_csb_size = gfx_v9_0_get_csb_size, 5153 .get_csb_buffer = gfx_v9_0_get_csb_buffer, 5154 .get_cp_table_num = gfx_v9_0_cp_jump_table_num, 5155 .resume = gfx_v9_0_rlc_resume, 5156 .stop = gfx_v9_0_rlc_stop, 5157 .reset = gfx_v9_0_rlc_reset, 5158 .start = gfx_v9_0_rlc_start, 5159 .update_spm_vmid = gfx_v9_0_update_spm_vmid, 5160 .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range, 5161 }; 5162 5163 static int gfx_v9_0_set_powergating_state(void *handle, 5164 enum amd_powergating_state state) 5165 { 5166 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5167 bool enable = (state == AMD_PG_STATE_GATE); 5168 5169 switch (adev->ip_versions[GC_HWIP][0]) { 5170 case IP_VERSION(9, 2, 2): 5171 case IP_VERSION(9, 1, 0): 5172 case IP_VERSION(9, 3, 0): 5173 if (!enable) 5174 amdgpu_gfx_off_ctrl(adev, false); 5175 5176 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { 5177 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true); 5178 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true); 5179 } else { 5180 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false); 5181 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false); 5182 } 5183 5184 if (adev->pg_flags & AMD_PG_SUPPORT_CP) 5185 gfx_v9_0_enable_cp_power_gating(adev, true); 5186 else 5187 gfx_v9_0_enable_cp_power_gating(adev, false); 5188 5189 /* update gfx cgpg state */ 5190 gfx_v9_0_update_gfx_cg_power_gating(adev, enable); 5191 5192 /* update mgcg state */ 5193 gfx_v9_0_update_gfx_mg_power_gating(adev, enable); 5194 5195 if (enable) 5196 amdgpu_gfx_off_ctrl(adev, true); 5197 break; 5198 case IP_VERSION(9, 2, 1): 5199 amdgpu_gfx_off_ctrl(adev, enable); 5200 break; 5201 default: 5202 break; 5203 } 5204 5205 return 0; 5206 } 5207 5208 static int gfx_v9_0_set_clockgating_state(void *handle, 5209 enum amd_clockgating_state state) 5210 { 5211 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5212 5213 if (amdgpu_sriov_vf(adev)) 5214 return 0; 5215 5216 switch (adev->ip_versions[GC_HWIP][0]) { 5217 case IP_VERSION(9, 0, 1): 5218 case IP_VERSION(9, 2, 1): 5219 case IP_VERSION(9, 4, 0): 5220 case IP_VERSION(9, 2, 2): 5221 case IP_VERSION(9, 1, 0): 5222 case IP_VERSION(9, 4, 1): 5223 case IP_VERSION(9, 3, 0): 5224 case IP_VERSION(9, 4, 2): 5225 gfx_v9_0_update_gfx_clock_gating(adev, 5226 state == AMD_CG_STATE_GATE); 5227 break; 5228 default: 5229 break; 5230 } 5231 return 0; 5232 } 5233 5234 static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags) 5235 { 5236 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5237 int data; 5238 5239 if (amdgpu_sriov_vf(adev)) 5240 *flags = 0; 5241 5242 /* AMD_CG_SUPPORT_GFX_MGCG */ 5243 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE)); 5244 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 5245 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5246 5247 /* AMD_CG_SUPPORT_GFX_CGCG */ 5248 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL)); 5249 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5250 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5251 5252 /* AMD_CG_SUPPORT_GFX_CGLS */ 5253 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5254 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5255 5256 /* AMD_CG_SUPPORT_GFX_RLC_LS */ 5257 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL)); 5258 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) 5259 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS; 5260 5261 /* AMD_CG_SUPPORT_GFX_CP_LS */ 5262 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL)); 5263 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) 5264 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS; 5265 5266 if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 1)) { 5267 /* AMD_CG_SUPPORT_GFX_3D_CGCG */ 5268 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D)); 5269 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) 5270 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; 5271 5272 /* AMD_CG_SUPPORT_GFX_3D_CGLS */ 5273 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK) 5274 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS; 5275 } 5276 } 5277 5278 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) 5279 { 5280 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/ 5281 } 5282 5283 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5284 { 5285 struct amdgpu_device *adev = ring->adev; 5286 u64 wptr; 5287 5288 /* XXX check if swapping is necessary on BE */ 5289 if (ring->use_doorbell) { 5290 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]); 5291 } else { 5292 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR); 5293 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32; 5294 } 5295 5296 return wptr; 5297 } 5298 5299 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5300 { 5301 struct amdgpu_device *adev = ring->adev; 5302 5303 if (ring->use_doorbell) { 5304 /* XXX check if swapping is necessary on BE */ 5305 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5306 WDOORBELL64(ring->doorbell_index, ring->wptr); 5307 } else { 5308 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 5309 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 5310 } 5311 } 5312 5313 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5314 { 5315 struct amdgpu_device *adev = ring->adev; 5316 u32 ref_and_mask, reg_mem_engine; 5317 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg; 5318 5319 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 5320 switch (ring->me) { 5321 case 1: 5322 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe; 5323 break; 5324 case 2: 5325 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe; 5326 break; 5327 default: 5328 return; 5329 } 5330 reg_mem_engine = 0; 5331 } else { 5332 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0; 5333 reg_mem_engine = 1; /* pfp */ 5334 } 5335 5336 gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1, 5337 adev->nbio.funcs->get_hdp_flush_req_offset(adev), 5338 adev->nbio.funcs->get_hdp_flush_done_offset(adev), 5339 ref_and_mask, ref_and_mask, 0x20); 5340 } 5341 5342 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 5343 struct amdgpu_job *job, 5344 struct amdgpu_ib *ib, 5345 uint32_t flags) 5346 { 5347 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5348 u32 header, control = 0; 5349 5350 if (ib->flags & AMDGPU_IB_FLAG_CE) 5351 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 5352 else 5353 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 5354 5355 control |= ib->length_dw | (vmid << 24); 5356 5357 if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { 5358 control |= INDIRECT_BUFFER_PRE_ENB(1); 5359 5360 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid) 5361 gfx_v9_0_ring_emit_de_meta(ring); 5362 } 5363 5364 amdgpu_ring_write(ring, header); 5365 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5366 amdgpu_ring_write(ring, 5367 #ifdef __BIG_ENDIAN 5368 (2 << 0) | 5369 #endif 5370 lower_32_bits(ib->gpu_addr)); 5371 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5372 amdgpu_ring_write(ring, control); 5373 } 5374 5375 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 5376 struct amdgpu_job *job, 5377 struct amdgpu_ib *ib, 5378 uint32_t flags) 5379 { 5380 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5381 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 5382 5383 /* Currently, there is a high possibility to get wave ID mismatch 5384 * between ME and GDS, leading to a hw deadlock, because ME generates 5385 * different wave IDs than the GDS expects. This situation happens 5386 * randomly when at least 5 compute pipes use GDS ordered append. 5387 * The wave IDs generated by ME are also wrong after suspend/resume. 5388 * Those are probably bugs somewhere else in the kernel driver. 5389 * 5390 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 5391 * GDS to 0 for this ring (me/pipe). 5392 */ 5393 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 5394 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 5395 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID); 5396 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 5397 } 5398 5399 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 5400 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5401 amdgpu_ring_write(ring, 5402 #ifdef __BIG_ENDIAN 5403 (2 << 0) | 5404 #endif 5405 lower_32_bits(ib->gpu_addr)); 5406 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5407 amdgpu_ring_write(ring, control); 5408 } 5409 5410 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 5411 u64 seq, unsigned flags) 5412 { 5413 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 5414 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 5415 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY; 5416 5417 /* RELEASE_MEM - flush caches, send int */ 5418 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); 5419 amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN | 5420 EOP_TC_NC_ACTION_EN) : 5421 (EOP_TCL1_ACTION_EN | 5422 EOP_TC_ACTION_EN | 5423 EOP_TC_WB_ACTION_EN | 5424 EOP_TC_MD_ACTION_EN)) | 5425 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 5426 EVENT_INDEX(5))); 5427 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 5428 5429 /* 5430 * the address should be Qword aligned if 64bit write, Dword 5431 * aligned if only send 32bit data low (discard data high) 5432 */ 5433 if (write64bit) 5434 BUG_ON(addr & 0x7); 5435 else 5436 BUG_ON(addr & 0x3); 5437 amdgpu_ring_write(ring, lower_32_bits(addr)); 5438 amdgpu_ring_write(ring, upper_32_bits(addr)); 5439 amdgpu_ring_write(ring, lower_32_bits(seq)); 5440 amdgpu_ring_write(ring, upper_32_bits(seq)); 5441 amdgpu_ring_write(ring, 0); 5442 } 5443 5444 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 5445 { 5446 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5447 uint32_t seq = ring->fence_drv.sync_seq; 5448 uint64_t addr = ring->fence_drv.gpu_addr; 5449 5450 gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0, 5451 lower_32_bits(addr), upper_32_bits(addr), 5452 seq, 0xffffffff, 4); 5453 } 5454 5455 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 5456 unsigned vmid, uint64_t pd_addr) 5457 { 5458 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 5459 5460 /* compute doesn't have PFP */ 5461 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 5462 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 5463 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 5464 amdgpu_ring_write(ring, 0x0); 5465 } 5466 } 5467 5468 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring) 5469 { 5470 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */ 5471 } 5472 5473 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 5474 { 5475 u64 wptr; 5476 5477 /* XXX check if swapping is necessary on BE */ 5478 if (ring->use_doorbell) 5479 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]); 5480 else 5481 BUG(); 5482 return wptr; 5483 } 5484 5485 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 5486 { 5487 struct amdgpu_device *adev = ring->adev; 5488 5489 /* XXX check if swapping is necessary on BE */ 5490 if (ring->use_doorbell) { 5491 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5492 WDOORBELL64(ring->doorbell_index, ring->wptr); 5493 } else{ 5494 BUG(); /* only DOORBELL method supported on gfx9 now */ 5495 } 5496 } 5497 5498 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 5499 u64 seq, unsigned int flags) 5500 { 5501 struct amdgpu_device *adev = ring->adev; 5502 5503 /* we only allocate 32bit for each seq wb address */ 5504 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); 5505 5506 /* write fence seq to the "addr" */ 5507 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5508 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5509 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 5510 amdgpu_ring_write(ring, lower_32_bits(addr)); 5511 amdgpu_ring_write(ring, upper_32_bits(addr)); 5512 amdgpu_ring_write(ring, lower_32_bits(seq)); 5513 5514 if (flags & AMDGPU_FENCE_FLAG_INT) { 5515 /* set register to trigger INT */ 5516 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5517 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5518 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 5519 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS)); 5520 amdgpu_ring_write(ring, 0); 5521 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 5522 } 5523 } 5524 5525 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring) 5526 { 5527 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 5528 amdgpu_ring_write(ring, 0); 5529 } 5530 5531 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring) 5532 { 5533 struct v9_ce_ib_state ce_payload = {0}; 5534 uint64_t csa_addr; 5535 int cnt; 5536 5537 cnt = (sizeof(ce_payload) >> 2) + 4 - 2; 5538 csa_addr = amdgpu_csa_vaddr(ring->adev); 5539 5540 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5541 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) | 5542 WRITE_DATA_DST_SEL(8) | 5543 WR_CONFIRM) | 5544 WRITE_DATA_CACHE_POLICY(0)); 5545 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5546 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5547 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2); 5548 } 5549 5550 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring) 5551 { 5552 struct v9_de_ib_state de_payload = {0}; 5553 uint64_t csa_addr, gds_addr; 5554 int cnt; 5555 5556 csa_addr = amdgpu_csa_vaddr(ring->adev); 5557 gds_addr = csa_addr + 4096; 5558 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr); 5559 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr); 5560 5561 cnt = (sizeof(de_payload) >> 2) + 4 - 2; 5562 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5563 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 5564 WRITE_DATA_DST_SEL(8) | 5565 WR_CONFIRM) | 5566 WRITE_DATA_CACHE_POLICY(0)); 5567 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5568 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5569 amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2); 5570 } 5571 5572 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, 5573 bool secure) 5574 { 5575 uint32_t v = secure ? FRAME_TMZ : 0; 5576 5577 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); 5578 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1)); 5579 } 5580 5581 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) 5582 { 5583 uint32_t dw2 = 0; 5584 5585 if (amdgpu_sriov_vf(ring->adev)) 5586 gfx_v9_0_ring_emit_ce_meta(ring); 5587 5588 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 5589 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 5590 /* set load_global_config & load_global_uconfig */ 5591 dw2 |= 0x8001; 5592 /* set load_cs_sh_regs */ 5593 dw2 |= 0x01000000; 5594 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 5595 dw2 |= 0x10002; 5596 5597 /* set load_ce_ram if preamble presented */ 5598 if (AMDGPU_PREAMBLE_IB_PRESENT & flags) 5599 dw2 |= 0x10000000; 5600 } else { 5601 /* still load_ce_ram if this is the first time preamble presented 5602 * although there is no context switch happens. 5603 */ 5604 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags) 5605 dw2 |= 0x10000000; 5606 } 5607 5608 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 5609 amdgpu_ring_write(ring, dw2); 5610 amdgpu_ring_write(ring, 0); 5611 } 5612 5613 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring) 5614 { 5615 unsigned ret; 5616 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 5617 amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr)); 5618 amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr)); 5619 amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */ 5620 ret = ring->wptr & ring->buf_mask; 5621 amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */ 5622 return ret; 5623 } 5624 5625 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset) 5626 { 5627 unsigned cur; 5628 BUG_ON(offset > ring->buf_mask); 5629 BUG_ON(ring->ring[offset] != 0x55aa55aa); 5630 5631 cur = (ring->wptr & ring->buf_mask) - 1; 5632 if (likely(cur > offset)) 5633 ring->ring[offset] = cur - offset; 5634 else 5635 ring->ring[offset] = (ring->ring_size>>2) - offset + cur; 5636 } 5637 5638 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 5639 uint32_t reg_val_offs) 5640 { 5641 struct amdgpu_device *adev = ring->adev; 5642 5643 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 5644 amdgpu_ring_write(ring, 0 | /* src: register*/ 5645 (5 << 8) | /* dst: memory */ 5646 (1 << 20)); /* write confirm */ 5647 amdgpu_ring_write(ring, reg); 5648 amdgpu_ring_write(ring, 0); 5649 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 5650 reg_val_offs * 4)); 5651 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 5652 reg_val_offs * 4)); 5653 } 5654 5655 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 5656 uint32_t val) 5657 { 5658 uint32_t cmd = 0; 5659 5660 switch (ring->funcs->type) { 5661 case AMDGPU_RING_TYPE_GFX: 5662 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 5663 break; 5664 case AMDGPU_RING_TYPE_KIQ: 5665 cmd = (1 << 16); /* no inc addr */ 5666 break; 5667 default: 5668 cmd = WR_CONFIRM; 5669 break; 5670 } 5671 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5672 amdgpu_ring_write(ring, cmd); 5673 amdgpu_ring_write(ring, reg); 5674 amdgpu_ring_write(ring, 0); 5675 amdgpu_ring_write(ring, val); 5676 } 5677 5678 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 5679 uint32_t val, uint32_t mask) 5680 { 5681 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); 5682 } 5683 5684 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, 5685 uint32_t reg0, uint32_t reg1, 5686 uint32_t ref, uint32_t mask) 5687 { 5688 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5689 struct amdgpu_device *adev = ring->adev; 5690 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ? 5691 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait; 5692 5693 if (fw_version_ok) 5694 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, 5695 ref, mask, 0x20); 5696 else 5697 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1, 5698 ref, mask); 5699 } 5700 5701 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid) 5702 { 5703 struct amdgpu_device *adev = ring->adev; 5704 uint32_t value = 0; 5705 5706 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03); 5707 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01); 5708 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1); 5709 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid); 5710 WREG32_SOC15(GC, 0, mmSQ_CMD, value); 5711 } 5712 5713 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 5714 enum amdgpu_interrupt_state state) 5715 { 5716 switch (state) { 5717 case AMDGPU_IRQ_STATE_DISABLE: 5718 case AMDGPU_IRQ_STATE_ENABLE: 5719 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5720 TIME_STAMP_INT_ENABLE, 5721 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5722 break; 5723 default: 5724 break; 5725 } 5726 } 5727 5728 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 5729 int me, int pipe, 5730 enum amdgpu_interrupt_state state) 5731 { 5732 u32 mec_int_cntl, mec_int_cntl_reg; 5733 5734 /* 5735 * amdgpu controls only the first MEC. That's why this function only 5736 * handles the setting of interrupts for this specific MEC. All other 5737 * pipes' interrupts are set by amdkfd. 5738 */ 5739 5740 if (me == 1) { 5741 switch (pipe) { 5742 case 0: 5743 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL); 5744 break; 5745 case 1: 5746 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL); 5747 break; 5748 case 2: 5749 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL); 5750 break; 5751 case 3: 5752 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL); 5753 break; 5754 default: 5755 DRM_DEBUG("invalid pipe %d\n", pipe); 5756 return; 5757 } 5758 } else { 5759 DRM_DEBUG("invalid me %d\n", me); 5760 return; 5761 } 5762 5763 switch (state) { 5764 case AMDGPU_IRQ_STATE_DISABLE: 5765 mec_int_cntl = RREG32_SOC15_IP(GC,mec_int_cntl_reg); 5766 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5767 TIME_STAMP_INT_ENABLE, 0); 5768 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl); 5769 break; 5770 case AMDGPU_IRQ_STATE_ENABLE: 5771 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg); 5772 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5773 TIME_STAMP_INT_ENABLE, 1); 5774 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl); 5775 break; 5776 default: 5777 break; 5778 } 5779 } 5780 5781 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 5782 struct amdgpu_irq_src *source, 5783 unsigned type, 5784 enum amdgpu_interrupt_state state) 5785 { 5786 switch (state) { 5787 case AMDGPU_IRQ_STATE_DISABLE: 5788 case AMDGPU_IRQ_STATE_ENABLE: 5789 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5790 PRIV_REG_INT_ENABLE, 5791 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5792 break; 5793 default: 5794 break; 5795 } 5796 5797 return 0; 5798 } 5799 5800 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 5801 struct amdgpu_irq_src *source, 5802 unsigned type, 5803 enum amdgpu_interrupt_state state) 5804 { 5805 switch (state) { 5806 case AMDGPU_IRQ_STATE_DISABLE: 5807 case AMDGPU_IRQ_STATE_ENABLE: 5808 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5809 PRIV_INSTR_INT_ENABLE, 5810 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5811 break; 5812 default: 5813 break; 5814 } 5815 5816 return 0; 5817 } 5818 5819 #define ENABLE_ECC_ON_ME_PIPE(me, pipe) \ 5820 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5821 CP_ECC_ERROR_INT_ENABLE, 1) 5822 5823 #define DISABLE_ECC_ON_ME_PIPE(me, pipe) \ 5824 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5825 CP_ECC_ERROR_INT_ENABLE, 0) 5826 5827 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev, 5828 struct amdgpu_irq_src *source, 5829 unsigned type, 5830 enum amdgpu_interrupt_state state) 5831 { 5832 switch (state) { 5833 case AMDGPU_IRQ_STATE_DISABLE: 5834 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5835 CP_ECC_ERROR_INT_ENABLE, 0); 5836 DISABLE_ECC_ON_ME_PIPE(1, 0); 5837 DISABLE_ECC_ON_ME_PIPE(1, 1); 5838 DISABLE_ECC_ON_ME_PIPE(1, 2); 5839 DISABLE_ECC_ON_ME_PIPE(1, 3); 5840 break; 5841 5842 case AMDGPU_IRQ_STATE_ENABLE: 5843 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5844 CP_ECC_ERROR_INT_ENABLE, 1); 5845 ENABLE_ECC_ON_ME_PIPE(1, 0); 5846 ENABLE_ECC_ON_ME_PIPE(1, 1); 5847 ENABLE_ECC_ON_ME_PIPE(1, 2); 5848 ENABLE_ECC_ON_ME_PIPE(1, 3); 5849 break; 5850 default: 5851 break; 5852 } 5853 5854 return 0; 5855 } 5856 5857 5858 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev, 5859 struct amdgpu_irq_src *src, 5860 unsigned type, 5861 enum amdgpu_interrupt_state state) 5862 { 5863 switch (type) { 5864 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 5865 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state); 5866 break; 5867 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 5868 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 5869 break; 5870 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 5871 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 5872 break; 5873 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 5874 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 5875 break; 5876 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 5877 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 5878 break; 5879 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP: 5880 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state); 5881 break; 5882 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP: 5883 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state); 5884 break; 5885 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP: 5886 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state); 5887 break; 5888 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP: 5889 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state); 5890 break; 5891 default: 5892 break; 5893 } 5894 return 0; 5895 } 5896 5897 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev, 5898 struct amdgpu_irq_src *source, 5899 struct amdgpu_iv_entry *entry) 5900 { 5901 int i; 5902 u8 me_id, pipe_id, queue_id; 5903 struct amdgpu_ring *ring; 5904 5905 DRM_DEBUG("IH: CP EOP\n"); 5906 me_id = (entry->ring_id & 0x0c) >> 2; 5907 pipe_id = (entry->ring_id & 0x03) >> 0; 5908 queue_id = (entry->ring_id & 0x70) >> 4; 5909 5910 switch (me_id) { 5911 case 0: 5912 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); 5913 break; 5914 case 1: 5915 case 2: 5916 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5917 ring = &adev->gfx.compute_ring[i]; 5918 /* Per-queue interrupt is supported for MEC starting from VI. 5919 * The interrupt can only be enabled/disabled per pipe instead of per queue. 5920 */ 5921 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) 5922 amdgpu_fence_process(ring); 5923 } 5924 break; 5925 } 5926 return 0; 5927 } 5928 5929 static void gfx_v9_0_fault(struct amdgpu_device *adev, 5930 struct amdgpu_iv_entry *entry) 5931 { 5932 u8 me_id, pipe_id, queue_id; 5933 struct amdgpu_ring *ring; 5934 int i; 5935 5936 me_id = (entry->ring_id & 0x0c) >> 2; 5937 pipe_id = (entry->ring_id & 0x03) >> 0; 5938 queue_id = (entry->ring_id & 0x70) >> 4; 5939 5940 switch (me_id) { 5941 case 0: 5942 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); 5943 break; 5944 case 1: 5945 case 2: 5946 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5947 ring = &adev->gfx.compute_ring[i]; 5948 if (ring->me == me_id && ring->pipe == pipe_id && 5949 ring->queue == queue_id) 5950 drm_sched_fault(&ring->sched); 5951 } 5952 break; 5953 } 5954 } 5955 5956 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev, 5957 struct amdgpu_irq_src *source, 5958 struct amdgpu_iv_entry *entry) 5959 { 5960 DRM_ERROR("Illegal register access in command stream\n"); 5961 gfx_v9_0_fault(adev, entry); 5962 return 0; 5963 } 5964 5965 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev, 5966 struct amdgpu_irq_src *source, 5967 struct amdgpu_iv_entry *entry) 5968 { 5969 DRM_ERROR("Illegal instruction in command stream\n"); 5970 gfx_v9_0_fault(adev, entry); 5971 return 0; 5972 } 5973 5974 5975 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = { 5976 { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 5977 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT), 5978 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT) 5979 }, 5980 { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 5981 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT), 5982 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT) 5983 }, 5984 { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5985 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1), 5986 0, 0 5987 }, 5988 { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5989 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2), 5990 0, 0 5991 }, 5992 { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 5993 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT), 5994 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT) 5995 }, 5996 { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5997 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT), 5998 0, 0 5999 }, 6000 { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 6001 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT), 6002 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT) 6003 }, 6004 { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 6005 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT), 6006 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT) 6007 }, 6008 { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 6009 SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1), 6010 0, 0 6011 }, 6012 { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 6013 SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1), 6014 0, 0 6015 }, 6016 { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 6017 SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1), 6018 0, 0 6019 }, 6020 { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 6021 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC), 6022 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED) 6023 }, 6024 { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 6025 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED), 6026 0, 0 6027 }, 6028 { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 6029 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC), 6030 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED) 6031 }, 6032 { "GDS_OA_PHY_PHY_CMD_RAM_MEM", 6033 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 6034 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC), 6035 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED) 6036 }, 6037 { "GDS_OA_PHY_PHY_DATA_RAM_MEM", 6038 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 6039 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED), 6040 0, 0 6041 }, 6042 { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM", 6043 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6044 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC), 6045 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED) 6046 }, 6047 { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM", 6048 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6049 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC), 6050 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED) 6051 }, 6052 { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM", 6053 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6054 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC), 6055 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED) 6056 }, 6057 { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM", 6058 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6059 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC), 6060 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED) 6061 }, 6062 { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 6063 SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT), 6064 0, 0 6065 }, 6066 { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6067 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT), 6068 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT) 6069 }, 6070 { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6071 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT), 6072 0, 0 6073 }, 6074 { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6075 SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT), 6076 0, 0 6077 }, 6078 { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6079 SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT), 6080 0, 0 6081 }, 6082 { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6083 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT), 6084 0, 0 6085 }, 6086 { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 6087 SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT), 6088 0, 0 6089 }, 6090 { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 6091 SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT), 6092 0, 0 6093 }, 6094 { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6095 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT), 6096 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT) 6097 }, 6098 { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6099 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT), 6100 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT) 6101 }, 6102 { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6103 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT), 6104 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT) 6105 }, 6106 { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6107 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT), 6108 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT) 6109 }, 6110 { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6111 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT), 6112 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT) 6113 }, 6114 { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6115 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT), 6116 0, 0 6117 }, 6118 { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6119 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT), 6120 0, 0 6121 }, 6122 { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6123 SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT), 6124 0, 0 6125 }, 6126 { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6127 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT), 6128 0, 0 6129 }, 6130 { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6131 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT), 6132 0, 0 6133 }, 6134 { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6135 SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT), 6136 0, 0 6137 }, 6138 { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6139 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT), 6140 0, 0 6141 }, 6142 { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6143 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT), 6144 0, 0 6145 }, 6146 { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6147 SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT), 6148 0, 0 6149 }, 6150 { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6151 SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT), 6152 0, 0 6153 }, 6154 { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6155 SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT), 6156 0, 0 6157 }, 6158 { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6159 SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT), 6160 0, 0 6161 }, 6162 { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6163 SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT), 6164 0, 0 6165 }, 6166 { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 6167 SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT), 6168 0, 0 6169 }, 6170 { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6171 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT), 6172 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT) 6173 }, 6174 { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6175 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT), 6176 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT) 6177 }, 6178 { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6179 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT), 6180 0, 0 6181 }, 6182 { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6183 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT), 6184 0, 0 6185 }, 6186 { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6187 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT), 6188 0, 0 6189 }, 6190 { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6191 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT), 6192 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT) 6193 }, 6194 { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6195 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT), 6196 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT) 6197 }, 6198 { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6199 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT), 6200 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT) 6201 }, 6202 { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6203 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT), 6204 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT) 6205 }, 6206 { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6207 SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT), 6208 0, 0 6209 }, 6210 { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6211 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT), 6212 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT) 6213 }, 6214 { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6215 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT), 6216 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT) 6217 }, 6218 { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6219 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT), 6220 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT) 6221 }, 6222 { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6223 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT), 6224 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT) 6225 }, 6226 { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6227 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT), 6228 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT) 6229 }, 6230 { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6231 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT), 6232 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT) 6233 }, 6234 { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6235 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT), 6236 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT) 6237 }, 6238 { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6239 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT), 6240 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT) 6241 }, 6242 { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6243 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT), 6244 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT) 6245 }, 6246 { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6247 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT), 6248 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT) 6249 }, 6250 { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6251 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT), 6252 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT) 6253 }, 6254 { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6255 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT), 6256 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT) 6257 }, 6258 { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6259 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT), 6260 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT) 6261 }, 6262 { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6263 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT), 6264 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT) 6265 }, 6266 { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6267 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT), 6268 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT) 6269 }, 6270 { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6271 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT), 6272 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT) 6273 }, 6274 { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6275 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT), 6276 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT) 6277 }, 6278 { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6279 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT), 6280 0, 0 6281 }, 6282 { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6283 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT), 6284 0, 0 6285 }, 6286 { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6287 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT), 6288 0, 0 6289 }, 6290 { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6291 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT), 6292 0, 0 6293 }, 6294 { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6295 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT), 6296 0, 0 6297 }, 6298 { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6299 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT), 6300 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT) 6301 }, 6302 { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6303 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT), 6304 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT) 6305 }, 6306 { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6307 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT), 6308 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT) 6309 }, 6310 { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6311 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT), 6312 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT) 6313 }, 6314 { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6315 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT), 6316 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT) 6317 }, 6318 { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6319 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT), 6320 0, 0 6321 }, 6322 { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6323 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT), 6324 0, 0 6325 }, 6326 { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6327 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT), 6328 0, 0 6329 }, 6330 { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6331 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT), 6332 0, 0 6333 }, 6334 { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6335 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT), 6336 0, 0 6337 }, 6338 { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6339 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), 6340 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT) 6341 }, 6342 { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6343 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), 6344 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT) 6345 }, 6346 { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6347 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), 6348 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT) 6349 }, 6350 { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6351 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT), 6352 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT) 6353 }, 6354 { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6355 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT), 6356 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT) 6357 }, 6358 { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6359 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), 6360 0, 0 6361 }, 6362 { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6363 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), 6364 0, 0 6365 }, 6366 { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6367 SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT), 6368 0, 0 6369 }, 6370 { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6371 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT), 6372 0, 0 6373 }, 6374 { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6375 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT), 6376 0, 0 6377 }, 6378 { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6379 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), 6380 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT) 6381 }, 6382 { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6383 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), 6384 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT) 6385 }, 6386 { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6387 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), 6388 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT) 6389 }, 6390 { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6391 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), 6392 0, 0 6393 }, 6394 { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6395 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), 6396 0, 0 6397 }, 6398 { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6399 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT), 6400 0, 0 6401 }, 6402 { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6403 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT), 6404 0, 0 6405 }, 6406 { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6407 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT), 6408 0, 0 6409 }, 6410 { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6411 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT), 6412 0, 0 6413 } 6414 }; 6415 6416 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 6417 void *inject_if) 6418 { 6419 struct ras_inject_if *info = (struct ras_inject_if *)inject_if; 6420 int ret; 6421 struct ta_ras_trigger_error_input block_info = { 0 }; 6422 6423 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6424 return -EINVAL; 6425 6426 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks)) 6427 return -EINVAL; 6428 6429 if (!ras_gfx_subblocks[info->head.sub_block_index].name) 6430 return -EPERM; 6431 6432 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type & 6433 info->head.type)) { 6434 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n", 6435 ras_gfx_subblocks[info->head.sub_block_index].name, 6436 info->head.type); 6437 return -EPERM; 6438 } 6439 6440 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type & 6441 info->head.type)) { 6442 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n", 6443 ras_gfx_subblocks[info->head.sub_block_index].name, 6444 info->head.type); 6445 return -EPERM; 6446 } 6447 6448 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block); 6449 block_info.sub_block_index = 6450 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock; 6451 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type); 6452 block_info.address = info->address; 6453 block_info.value = info->value; 6454 6455 mutex_lock(&adev->grbm_idx_mutex); 6456 ret = psp_ras_trigger_error(&adev->psp, &block_info); 6457 mutex_unlock(&adev->grbm_idx_mutex); 6458 6459 return ret; 6460 } 6461 6462 static const char *vml2_mems[] = { 6463 "UTC_VML2_BANK_CACHE_0_BIGK_MEM0", 6464 "UTC_VML2_BANK_CACHE_0_BIGK_MEM1", 6465 "UTC_VML2_BANK_CACHE_0_4K_MEM0", 6466 "UTC_VML2_BANK_CACHE_0_4K_MEM1", 6467 "UTC_VML2_BANK_CACHE_1_BIGK_MEM0", 6468 "UTC_VML2_BANK_CACHE_1_BIGK_MEM1", 6469 "UTC_VML2_BANK_CACHE_1_4K_MEM0", 6470 "UTC_VML2_BANK_CACHE_1_4K_MEM1", 6471 "UTC_VML2_BANK_CACHE_2_BIGK_MEM0", 6472 "UTC_VML2_BANK_CACHE_2_BIGK_MEM1", 6473 "UTC_VML2_BANK_CACHE_2_4K_MEM0", 6474 "UTC_VML2_BANK_CACHE_2_4K_MEM1", 6475 "UTC_VML2_BANK_CACHE_3_BIGK_MEM0", 6476 "UTC_VML2_BANK_CACHE_3_BIGK_MEM1", 6477 "UTC_VML2_BANK_CACHE_3_4K_MEM0", 6478 "UTC_VML2_BANK_CACHE_3_4K_MEM1", 6479 }; 6480 6481 static const char *vml2_walker_mems[] = { 6482 "UTC_VML2_CACHE_PDE0_MEM0", 6483 "UTC_VML2_CACHE_PDE0_MEM1", 6484 "UTC_VML2_CACHE_PDE1_MEM0", 6485 "UTC_VML2_CACHE_PDE1_MEM1", 6486 "UTC_VML2_CACHE_PDE2_MEM0", 6487 "UTC_VML2_CACHE_PDE2_MEM1", 6488 "UTC_VML2_RDIF_LOG_FIFO", 6489 }; 6490 6491 static const char *atc_l2_cache_2m_mems[] = { 6492 "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM", 6493 "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM", 6494 "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM", 6495 "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM", 6496 }; 6497 6498 static const char *atc_l2_cache_4k_mems[] = { 6499 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0", 6500 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1", 6501 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2", 6502 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3", 6503 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4", 6504 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5", 6505 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6", 6506 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7", 6507 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0", 6508 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1", 6509 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2", 6510 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3", 6511 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4", 6512 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5", 6513 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6", 6514 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7", 6515 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0", 6516 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1", 6517 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2", 6518 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3", 6519 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4", 6520 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5", 6521 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6", 6522 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7", 6523 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0", 6524 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1", 6525 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2", 6526 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3", 6527 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4", 6528 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5", 6529 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6", 6530 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7", 6531 }; 6532 6533 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev, 6534 struct ras_err_data *err_data) 6535 { 6536 uint32_t i, data; 6537 uint32_t sec_count, ded_count; 6538 6539 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6540 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6541 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6542 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6543 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6544 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6545 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6546 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6547 6548 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6549 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6550 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6551 6552 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT); 6553 if (sec_count) { 6554 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6555 "SEC %d\n", i, vml2_mems[i], sec_count); 6556 err_data->ce_count += sec_count; 6557 } 6558 6559 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT); 6560 if (ded_count) { 6561 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6562 "DED %d\n", i, vml2_mems[i], ded_count); 6563 err_data->ue_count += ded_count; 6564 } 6565 } 6566 6567 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6568 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6569 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6570 6571 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6572 SEC_COUNT); 6573 if (sec_count) { 6574 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6575 "SEC %d\n", i, vml2_walker_mems[i], sec_count); 6576 err_data->ce_count += sec_count; 6577 } 6578 6579 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6580 DED_COUNT); 6581 if (ded_count) { 6582 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6583 "DED %d\n", i, vml2_walker_mems[i], ded_count); 6584 err_data->ue_count += ded_count; 6585 } 6586 } 6587 6588 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6589 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6590 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6591 6592 sec_count = (data & 0x00006000L) >> 0xd; 6593 if (sec_count) { 6594 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6595 "SEC %d\n", i, atc_l2_cache_2m_mems[i], 6596 sec_count); 6597 err_data->ce_count += sec_count; 6598 } 6599 } 6600 6601 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6602 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6603 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6604 6605 sec_count = (data & 0x00006000L) >> 0xd; 6606 if (sec_count) { 6607 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6608 "SEC %d\n", i, atc_l2_cache_4k_mems[i], 6609 sec_count); 6610 err_data->ce_count += sec_count; 6611 } 6612 6613 ded_count = (data & 0x00018000L) >> 0xf; 6614 if (ded_count) { 6615 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6616 "DED %d\n", i, atc_l2_cache_4k_mems[i], 6617 ded_count); 6618 err_data->ue_count += ded_count; 6619 } 6620 } 6621 6622 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6623 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6624 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6625 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6626 6627 return 0; 6628 } 6629 6630 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev, 6631 const struct soc15_reg_entry *reg, 6632 uint32_t se_id, uint32_t inst_id, uint32_t value, 6633 uint32_t *sec_count, uint32_t *ded_count) 6634 { 6635 uint32_t i; 6636 uint32_t sec_cnt, ded_cnt; 6637 6638 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) { 6639 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset || 6640 gfx_v9_0_ras_fields[i].seg != reg->seg || 6641 gfx_v9_0_ras_fields[i].inst != reg->inst) 6642 continue; 6643 6644 sec_cnt = (value & 6645 gfx_v9_0_ras_fields[i].sec_count_mask) >> 6646 gfx_v9_0_ras_fields[i].sec_count_shift; 6647 if (sec_cnt) { 6648 dev_info(adev->dev, "GFX SubBlock %s, " 6649 "Instance[%d][%d], SEC %d\n", 6650 gfx_v9_0_ras_fields[i].name, 6651 se_id, inst_id, 6652 sec_cnt); 6653 *sec_count += sec_cnt; 6654 } 6655 6656 ded_cnt = (value & 6657 gfx_v9_0_ras_fields[i].ded_count_mask) >> 6658 gfx_v9_0_ras_fields[i].ded_count_shift; 6659 if (ded_cnt) { 6660 dev_info(adev->dev, "GFX SubBlock %s, " 6661 "Instance[%d][%d], DED %d\n", 6662 gfx_v9_0_ras_fields[i].name, 6663 se_id, inst_id, 6664 ded_cnt); 6665 *ded_count += ded_cnt; 6666 } 6667 } 6668 6669 return 0; 6670 } 6671 6672 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev) 6673 { 6674 int i, j, k; 6675 6676 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6677 return; 6678 6679 /* read back registers to clear the counters */ 6680 mutex_lock(&adev->grbm_idx_mutex); 6681 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6682 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6683 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6684 gfx_v9_0_select_se_sh(adev, j, 0x0, k); 6685 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6686 } 6687 } 6688 } 6689 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000); 6690 mutex_unlock(&adev->grbm_idx_mutex); 6691 6692 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6693 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6694 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6695 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6696 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6697 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6698 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6699 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6700 6701 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6702 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6703 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6704 } 6705 6706 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6707 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6708 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6709 } 6710 6711 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6712 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6713 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6714 } 6715 6716 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6717 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6718 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6719 } 6720 6721 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6722 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6723 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6724 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6725 } 6726 6727 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 6728 void *ras_error_status) 6729 { 6730 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; 6731 uint32_t sec_count = 0, ded_count = 0; 6732 uint32_t i, j, k; 6733 uint32_t reg_value; 6734 6735 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6736 return; 6737 6738 err_data->ue_count = 0; 6739 err_data->ce_count = 0; 6740 6741 mutex_lock(&adev->grbm_idx_mutex); 6742 6743 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6744 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6745 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6746 gfx_v9_0_select_se_sh(adev, j, 0, k); 6747 reg_value = 6748 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6749 if (reg_value) 6750 gfx_v9_0_ras_error_count(adev, 6751 &gfx_v9_0_edc_counter_regs[i], 6752 j, k, reg_value, 6753 &sec_count, &ded_count); 6754 } 6755 } 6756 } 6757 6758 err_data->ce_count += sec_count; 6759 err_data->ue_count += ded_count; 6760 6761 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 6762 mutex_unlock(&adev->grbm_idx_mutex); 6763 6764 gfx_v9_0_query_utc_edc_status(adev, err_data); 6765 } 6766 6767 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring) 6768 { 6769 const unsigned int cp_coher_cntl = 6770 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) | 6771 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) | 6772 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) | 6773 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) | 6774 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1); 6775 6776 /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */ 6777 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5)); 6778 amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */ 6779 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6780 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */ 6781 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6782 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 6783 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ 6784 } 6785 6786 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring, 6787 uint32_t pipe, bool enable) 6788 { 6789 struct amdgpu_device *adev = ring->adev; 6790 uint32_t val; 6791 uint32_t wcl_cs_reg; 6792 6793 /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */ 6794 val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT; 6795 6796 switch (pipe) { 6797 case 0: 6798 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0); 6799 break; 6800 case 1: 6801 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1); 6802 break; 6803 case 2: 6804 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2); 6805 break; 6806 case 3: 6807 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3); 6808 break; 6809 default: 6810 DRM_DEBUG("invalid pipe %d\n", pipe); 6811 return; 6812 } 6813 6814 amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val); 6815 6816 } 6817 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable) 6818 { 6819 struct amdgpu_device *adev = ring->adev; 6820 uint32_t val; 6821 int i; 6822 6823 6824 /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit 6825 * number of gfx waves. Setting 5 bit will make sure gfx only gets 6826 * around 25% of gpu resources. 6827 */ 6828 val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT; 6829 amdgpu_ring_emit_wreg(ring, 6830 SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX), 6831 val); 6832 6833 /* Restrict waves for normal/low priority compute queues as well 6834 * to get best QoS for high priority compute jobs. 6835 * 6836 * amdgpu controls only 1st ME(0-3 CS pipes). 6837 */ 6838 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 6839 if (i != ring->pipe) 6840 gfx_v9_0_emit_wave_limit_cs(ring, i, enable); 6841 6842 } 6843 } 6844 6845 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = { 6846 .name = "gfx_v9_0", 6847 .early_init = gfx_v9_0_early_init, 6848 .late_init = gfx_v9_0_late_init, 6849 .sw_init = gfx_v9_0_sw_init, 6850 .sw_fini = gfx_v9_0_sw_fini, 6851 .hw_init = gfx_v9_0_hw_init, 6852 .hw_fini = gfx_v9_0_hw_fini, 6853 .suspend = gfx_v9_0_suspend, 6854 .resume = gfx_v9_0_resume, 6855 .is_idle = gfx_v9_0_is_idle, 6856 .wait_for_idle = gfx_v9_0_wait_for_idle, 6857 .soft_reset = gfx_v9_0_soft_reset, 6858 .set_clockgating_state = gfx_v9_0_set_clockgating_state, 6859 .set_powergating_state = gfx_v9_0_set_powergating_state, 6860 .get_clockgating_state = gfx_v9_0_get_clockgating_state, 6861 }; 6862 6863 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = { 6864 .type = AMDGPU_RING_TYPE_GFX, 6865 .align_mask = 0xff, 6866 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6867 .support_64bit_ptrs = true, 6868 .secure_submission_supported = true, 6869 .vmhub = AMDGPU_GFXHUB_0, 6870 .get_rptr = gfx_v9_0_ring_get_rptr_gfx, 6871 .get_wptr = gfx_v9_0_ring_get_wptr_gfx, 6872 .set_wptr = gfx_v9_0_ring_set_wptr_gfx, 6873 .emit_frame_size = /* totally 242 maximum if 16 IBs */ 6874 5 + /* COND_EXEC */ 6875 7 + /* PIPELINE_SYNC */ 6876 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6877 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6878 2 + /* VM_FLUSH */ 6879 8 + /* FENCE for VM_FLUSH */ 6880 20 + /* GDS switch */ 6881 4 + /* double SWITCH_BUFFER, 6882 the first COND_EXEC jump to the place just 6883 prior to this double SWITCH_BUFFER */ 6884 5 + /* COND_EXEC */ 6885 7 + /* HDP_flush */ 6886 4 + /* VGT_flush */ 6887 14 + /* CE_META */ 6888 31 + /* DE_META */ 6889 3 + /* CNTX_CTRL */ 6890 5 + /* HDP_INVL */ 6891 8 + 8 + /* FENCE x2 */ 6892 2 + /* SWITCH_BUFFER */ 6893 7, /* gfx_v9_0_emit_mem_sync */ 6894 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */ 6895 .emit_ib = gfx_v9_0_ring_emit_ib_gfx, 6896 .emit_fence = gfx_v9_0_ring_emit_fence, 6897 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6898 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6899 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6900 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6901 .test_ring = gfx_v9_0_ring_test_ring, 6902 .test_ib = gfx_v9_0_ring_test_ib, 6903 .insert_nop = amdgpu_ring_insert_nop, 6904 .pad_ib = amdgpu_ring_generic_pad_ib, 6905 .emit_switch_buffer = gfx_v9_ring_emit_sb, 6906 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl, 6907 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec, 6908 .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec, 6909 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl, 6910 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6911 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6912 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6913 .soft_recovery = gfx_v9_0_ring_soft_recovery, 6914 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6915 }; 6916 6917 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { 6918 .type = AMDGPU_RING_TYPE_COMPUTE, 6919 .align_mask = 0xff, 6920 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6921 .support_64bit_ptrs = true, 6922 .vmhub = AMDGPU_GFXHUB_0, 6923 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6924 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6925 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6926 .emit_frame_size = 6927 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6928 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6929 5 + /* hdp invalidate */ 6930 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6931 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6932 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6933 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6934 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */ 6935 7 + /* gfx_v9_0_emit_mem_sync */ 6936 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */ 6937 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */ 6938 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6939 .emit_ib = gfx_v9_0_ring_emit_ib_compute, 6940 .emit_fence = gfx_v9_0_ring_emit_fence, 6941 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6942 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6943 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6944 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6945 .test_ring = gfx_v9_0_ring_test_ring, 6946 .test_ib = gfx_v9_0_ring_test_ib, 6947 .insert_nop = amdgpu_ring_insert_nop, 6948 .pad_ib = amdgpu_ring_generic_pad_ib, 6949 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6950 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6951 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6952 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6953 .emit_wave_limit = gfx_v9_0_emit_wave_limit, 6954 }; 6955 6956 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = { 6957 .type = AMDGPU_RING_TYPE_KIQ, 6958 .align_mask = 0xff, 6959 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6960 .support_64bit_ptrs = true, 6961 .vmhub = AMDGPU_GFXHUB_0, 6962 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6963 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6964 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6965 .emit_frame_size = 6966 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6967 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6968 5 + /* hdp invalidate */ 6969 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6970 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6971 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6972 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6973 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 6974 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6975 .emit_fence = gfx_v9_0_ring_emit_fence_kiq, 6976 .test_ring = gfx_v9_0_ring_test_ring, 6977 .insert_nop = amdgpu_ring_insert_nop, 6978 .pad_ib = amdgpu_ring_generic_pad_ib, 6979 .emit_rreg = gfx_v9_0_ring_emit_rreg, 6980 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6981 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6982 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6983 }; 6984 6985 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev) 6986 { 6987 int i; 6988 6989 adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq; 6990 6991 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 6992 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx; 6993 6994 for (i = 0; i < adev->gfx.num_compute_rings; i++) 6995 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute; 6996 } 6997 6998 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = { 6999 .set = gfx_v9_0_set_eop_interrupt_state, 7000 .process = gfx_v9_0_eop_irq, 7001 }; 7002 7003 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = { 7004 .set = gfx_v9_0_set_priv_reg_fault_state, 7005 .process = gfx_v9_0_priv_reg_irq, 7006 }; 7007 7008 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = { 7009 .set = gfx_v9_0_set_priv_inst_fault_state, 7010 .process = gfx_v9_0_priv_inst_irq, 7011 }; 7012 7013 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = { 7014 .set = gfx_v9_0_set_cp_ecc_error_state, 7015 .process = amdgpu_gfx_cp_ecc_error_irq, 7016 }; 7017 7018 7019 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev) 7020 { 7021 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 7022 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs; 7023 7024 adev->gfx.priv_reg_irq.num_types = 1; 7025 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs; 7026 7027 adev->gfx.priv_inst_irq.num_types = 1; 7028 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs; 7029 7030 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/ 7031 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs; 7032 } 7033 7034 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev) 7035 { 7036 switch (adev->ip_versions[GC_HWIP][0]) { 7037 case IP_VERSION(9, 0, 1): 7038 case IP_VERSION(9, 2, 1): 7039 case IP_VERSION(9, 4, 0): 7040 case IP_VERSION(9, 2, 2): 7041 case IP_VERSION(9, 1, 0): 7042 case IP_VERSION(9, 4, 1): 7043 case IP_VERSION(9, 3, 0): 7044 case IP_VERSION(9, 4, 2): 7045 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs; 7046 break; 7047 default: 7048 break; 7049 } 7050 } 7051 7052 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev) 7053 { 7054 /* init asci gds info */ 7055 switch (adev->ip_versions[GC_HWIP][0]) { 7056 case IP_VERSION(9, 0, 1): 7057 case IP_VERSION(9, 2, 1): 7058 case IP_VERSION(9, 4, 0): 7059 adev->gds.gds_size = 0x10000; 7060 break; 7061 case IP_VERSION(9, 2, 2): 7062 case IP_VERSION(9, 1, 0): 7063 case IP_VERSION(9, 4, 1): 7064 adev->gds.gds_size = 0x1000; 7065 break; 7066 case IP_VERSION(9, 4, 2): 7067 /* aldebaran removed all the GDS internal memory, 7068 * only support GWS opcode in kernel, like barrier 7069 * semaphore.etc */ 7070 adev->gds.gds_size = 0; 7071 break; 7072 default: 7073 adev->gds.gds_size = 0x10000; 7074 break; 7075 } 7076 7077 switch (adev->ip_versions[GC_HWIP][0]) { 7078 case IP_VERSION(9, 0, 1): 7079 case IP_VERSION(9, 4, 0): 7080 adev->gds.gds_compute_max_wave_id = 0x7ff; 7081 break; 7082 case IP_VERSION(9, 2, 1): 7083 adev->gds.gds_compute_max_wave_id = 0x27f; 7084 break; 7085 case IP_VERSION(9, 2, 2): 7086 case IP_VERSION(9, 1, 0): 7087 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 7088 adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */ 7089 else 7090 adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */ 7091 break; 7092 case IP_VERSION(9, 4, 1): 7093 adev->gds.gds_compute_max_wave_id = 0xfff; 7094 break; 7095 case IP_VERSION(9, 4, 2): 7096 /* deprecated for Aldebaran, no usage at all */ 7097 adev->gds.gds_compute_max_wave_id = 0; 7098 break; 7099 default: 7100 /* this really depends on the chip */ 7101 adev->gds.gds_compute_max_wave_id = 0x7ff; 7102 break; 7103 } 7104 7105 adev->gds.gws_size = 64; 7106 adev->gds.oa_size = 16; 7107 } 7108 7109 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev, 7110 u32 bitmap) 7111 { 7112 u32 data; 7113 7114 if (!bitmap) 7115 return; 7116 7117 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 7118 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 7119 7120 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data); 7121 } 7122 7123 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev) 7124 { 7125 u32 data, mask; 7126 7127 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG); 7128 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG); 7129 7130 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 7131 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 7132 7133 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); 7134 7135 return (~data) & mask; 7136 } 7137 7138 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 7139 struct amdgpu_cu_info *cu_info) 7140 { 7141 int i, j, k, counter, active_cu_number = 0; 7142 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0; 7143 unsigned disable_masks[4 * 4]; 7144 7145 if (!adev || !cu_info) 7146 return -EINVAL; 7147 7148 /* 7149 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs 7150 */ 7151 if (adev->gfx.config.max_shader_engines * 7152 adev->gfx.config.max_sh_per_se > 16) 7153 return -EINVAL; 7154 7155 amdgpu_gfx_parse_disable_cu(disable_masks, 7156 adev->gfx.config.max_shader_engines, 7157 adev->gfx.config.max_sh_per_se); 7158 7159 mutex_lock(&adev->grbm_idx_mutex); 7160 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 7161 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 7162 mask = 1; 7163 ao_bitmap = 0; 7164 counter = 0; 7165 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 7166 gfx_v9_0_set_user_cu_inactive_bitmap( 7167 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]); 7168 bitmap = gfx_v9_0_get_cu_active_bitmap(adev); 7169 7170 /* 7171 * The bitmap(and ao_cu_bitmap) in cu_info structure is 7172 * 4x4 size array, and it's usually suitable for Vega 7173 * ASICs which has 4*2 SE/SH layout. 7174 * But for Arcturus, SE/SH layout is changed to 8*1. 7175 * To mostly reduce the impact, we make it compatible 7176 * with current bitmap array as below: 7177 * SE4,SH0 --> bitmap[0][1] 7178 * SE5,SH0 --> bitmap[1][1] 7179 * SE6,SH0 --> bitmap[2][1] 7180 * SE7,SH0 --> bitmap[3][1] 7181 */ 7182 cu_info->bitmap[i % 4][j + i / 4] = bitmap; 7183 7184 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 7185 if (bitmap & mask) { 7186 if (counter < adev->gfx.config.max_cu_per_sh) 7187 ao_bitmap |= mask; 7188 counter ++; 7189 } 7190 mask <<= 1; 7191 } 7192 active_cu_number += counter; 7193 if (i < 2 && j < 2) 7194 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8)); 7195 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap; 7196 } 7197 } 7198 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 7199 mutex_unlock(&adev->grbm_idx_mutex); 7200 7201 cu_info->number = active_cu_number; 7202 cu_info->ao_cu_mask = ao_cu_mask; 7203 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 7204 7205 return 0; 7206 } 7207 7208 const struct amdgpu_ip_block_version gfx_v9_0_ip_block = 7209 { 7210 .type = AMD_IP_BLOCK_TYPE_GFX, 7211 .major = 9, 7212 .minor = 0, 7213 .rev = 0, 7214 .funcs = &gfx_v9_0_ip_funcs, 7215 }; 7216