1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/delay.h> 25 #include <linux/kernel.h> 26 #include <linux/firmware.h> 27 #include <linux/module.h> 28 #include <linux/pci.h> 29 30 #include "amdgpu.h" 31 #include "amdgpu_gfx.h" 32 #include "soc15.h" 33 #include "soc15d.h" 34 #include "amdgpu_atomfirmware.h" 35 #include "amdgpu_pm.h" 36 37 #include "gc/gc_9_0_offset.h" 38 #include "gc/gc_9_0_sh_mask.h" 39 40 #include "vega10_enum.h" 41 42 #include "soc15_common.h" 43 #include "clearstate_gfx9.h" 44 #include "v9_structs.h" 45 46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h" 47 48 #include "amdgpu_ras.h" 49 50 #include "gfx_v9_4.h" 51 #include "gfx_v9_0.h" 52 #include "gfx_v9_4_2.h" 53 54 #include "asic_reg/pwr/pwr_10_0_offset.h" 55 #include "asic_reg/pwr/pwr_10_0_sh_mask.h" 56 #include "asic_reg/gc/gc_9_0_default.h" 57 58 #define GFX9_NUM_GFX_RINGS 1 59 #define GFX9_MEC_HPD_SIZE 4096 60 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L 61 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L 62 63 #define mmGCEA_PROBE_MAP 0x070c 64 #define mmGCEA_PROBE_MAP_BASE_IDX 0 65 66 MODULE_FIRMWARE("amdgpu/vega10_ce.bin"); 67 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin"); 68 MODULE_FIRMWARE("amdgpu/vega10_me.bin"); 69 MODULE_FIRMWARE("amdgpu/vega10_mec.bin"); 70 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin"); 71 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin"); 72 73 MODULE_FIRMWARE("amdgpu/vega12_ce.bin"); 74 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin"); 75 MODULE_FIRMWARE("amdgpu/vega12_me.bin"); 76 MODULE_FIRMWARE("amdgpu/vega12_mec.bin"); 77 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin"); 78 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin"); 79 80 MODULE_FIRMWARE("amdgpu/vega20_ce.bin"); 81 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin"); 82 MODULE_FIRMWARE("amdgpu/vega20_me.bin"); 83 MODULE_FIRMWARE("amdgpu/vega20_mec.bin"); 84 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin"); 85 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin"); 86 87 MODULE_FIRMWARE("amdgpu/raven_ce.bin"); 88 MODULE_FIRMWARE("amdgpu/raven_pfp.bin"); 89 MODULE_FIRMWARE("amdgpu/raven_me.bin"); 90 MODULE_FIRMWARE("amdgpu/raven_mec.bin"); 91 MODULE_FIRMWARE("amdgpu/raven_mec2.bin"); 92 MODULE_FIRMWARE("amdgpu/raven_rlc.bin"); 93 94 MODULE_FIRMWARE("amdgpu/picasso_ce.bin"); 95 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin"); 96 MODULE_FIRMWARE("amdgpu/picasso_me.bin"); 97 MODULE_FIRMWARE("amdgpu/picasso_mec.bin"); 98 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin"); 99 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin"); 100 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin"); 101 102 MODULE_FIRMWARE("amdgpu/raven2_ce.bin"); 103 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin"); 104 MODULE_FIRMWARE("amdgpu/raven2_me.bin"); 105 MODULE_FIRMWARE("amdgpu/raven2_mec.bin"); 106 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin"); 107 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin"); 108 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin"); 109 110 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin"); 111 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin"); 112 113 MODULE_FIRMWARE("amdgpu/renoir_ce.bin"); 114 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin"); 115 MODULE_FIRMWARE("amdgpu/renoir_me.bin"); 116 MODULE_FIRMWARE("amdgpu/renoir_mec.bin"); 117 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin"); 118 119 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin"); 120 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin"); 121 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin"); 122 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin"); 123 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin"); 124 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin"); 125 126 MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin"); 127 MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin"); 128 MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin"); 129 130 #define mmTCP_CHAN_STEER_0_ARCT 0x0b03 131 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0 132 #define mmTCP_CHAN_STEER_1_ARCT 0x0b04 133 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0 134 #define mmTCP_CHAN_STEER_2_ARCT 0x0b09 135 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0 136 #define mmTCP_CHAN_STEER_3_ARCT 0x0b0a 137 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0 138 #define mmTCP_CHAN_STEER_4_ARCT 0x0b0b 139 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0 140 #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c 141 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0 142 143 enum ta_ras_gfx_subblock { 144 /*CPC*/ 145 TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0, 146 TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START, 147 TA_RAS_BLOCK__GFX_CPC_UCODE, 148 TA_RAS_BLOCK__GFX_DC_STATE_ME1, 149 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1, 150 TA_RAS_BLOCK__GFX_DC_RESTORE_ME1, 151 TA_RAS_BLOCK__GFX_DC_STATE_ME2, 152 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2, 153 TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 154 TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 155 /* CPF*/ 156 TA_RAS_BLOCK__GFX_CPF_INDEX_START, 157 TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START, 158 TA_RAS_BLOCK__GFX_CPF_ROQ_ME1, 159 TA_RAS_BLOCK__GFX_CPF_TAG, 160 TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG, 161 /* CPG*/ 162 TA_RAS_BLOCK__GFX_CPG_INDEX_START, 163 TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START, 164 TA_RAS_BLOCK__GFX_CPG_DMA_TAG, 165 TA_RAS_BLOCK__GFX_CPG_TAG, 166 TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG, 167 /* GDS*/ 168 TA_RAS_BLOCK__GFX_GDS_INDEX_START, 169 TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START, 170 TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE, 171 TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM, 172 TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM, 173 TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 174 TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 175 /* SPI*/ 176 TA_RAS_BLOCK__GFX_SPI_SR_MEM, 177 /* SQ*/ 178 TA_RAS_BLOCK__GFX_SQ_INDEX_START, 179 TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START, 180 TA_RAS_BLOCK__GFX_SQ_LDS_D, 181 TA_RAS_BLOCK__GFX_SQ_LDS_I, 182 TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/ 183 TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR, 184 /* SQC (3 ranges)*/ 185 TA_RAS_BLOCK__GFX_SQC_INDEX_START, 186 /* SQC range 0*/ 187 TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START, 188 TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO = 189 TA_RAS_BLOCK__GFX_SQC_INDEX0_START, 190 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 191 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO, 192 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 193 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO, 194 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 195 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 196 TA_RAS_BLOCK__GFX_SQC_INDEX0_END = 197 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 198 /* SQC range 1*/ 199 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 200 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM = 201 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 202 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 203 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO, 204 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM, 205 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM, 206 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO, 207 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO, 208 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 209 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 210 TA_RAS_BLOCK__GFX_SQC_INDEX1_END = 211 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 212 /* SQC range 2*/ 213 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 214 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM = 215 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 216 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 217 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO, 218 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM, 219 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM, 220 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO, 221 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO, 222 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 223 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 224 TA_RAS_BLOCK__GFX_SQC_INDEX2_END = 225 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 226 TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END, 227 /* TA*/ 228 TA_RAS_BLOCK__GFX_TA_INDEX_START, 229 TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START, 230 TA_RAS_BLOCK__GFX_TA_FS_AFIFO, 231 TA_RAS_BLOCK__GFX_TA_FL_LFIFO, 232 TA_RAS_BLOCK__GFX_TA_FX_LFIFO, 233 TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 234 TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 235 /* TCA*/ 236 TA_RAS_BLOCK__GFX_TCA_INDEX_START, 237 TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START, 238 TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 239 TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 240 /* TCC (5 sub-ranges)*/ 241 TA_RAS_BLOCK__GFX_TCC_INDEX_START, 242 /* TCC range 0*/ 243 TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START, 244 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START, 245 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1, 246 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0, 247 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1, 248 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0, 249 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1, 250 TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG, 251 TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 252 TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 253 /* TCC range 1*/ 254 TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 255 TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 256 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 257 TA_RAS_BLOCK__GFX_TCC_INDEX1_END = 258 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 259 /* TCC range 2*/ 260 TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 261 TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 262 TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL, 263 TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO, 264 TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN, 265 TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ, 266 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO, 267 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM, 268 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 269 TA_RAS_BLOCK__GFX_TCC_INDEX2_END = 270 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 271 /* TCC range 3*/ 272 TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 273 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 274 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 275 TA_RAS_BLOCK__GFX_TCC_INDEX3_END = 276 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 277 /* TCC range 4*/ 278 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 279 TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN = 280 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 281 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 282 TA_RAS_BLOCK__GFX_TCC_INDEX4_END = 283 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 284 TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END, 285 /* TCI*/ 286 TA_RAS_BLOCK__GFX_TCI_WRITE_RAM, 287 /* TCP*/ 288 TA_RAS_BLOCK__GFX_TCP_INDEX_START, 289 TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START, 290 TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM, 291 TA_RAS_BLOCK__GFX_TCP_CMD_FIFO, 292 TA_RAS_BLOCK__GFX_TCP_VM_FIFO, 293 TA_RAS_BLOCK__GFX_TCP_DB_RAM, 294 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0, 295 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 296 TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 297 /* TD*/ 298 TA_RAS_BLOCK__GFX_TD_INDEX_START, 299 TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START, 300 TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI, 301 TA_RAS_BLOCK__GFX_TD_CS_FIFO, 302 TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO, 303 /* EA (3 sub-ranges)*/ 304 TA_RAS_BLOCK__GFX_EA_INDEX_START, 305 /* EA range 0*/ 306 TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START, 307 TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START, 308 TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM, 309 TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM, 310 TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM, 311 TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM, 312 TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM, 313 TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM, 314 TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 315 TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 316 /* EA range 1*/ 317 TA_RAS_BLOCK__GFX_EA_INDEX1_START, 318 TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START, 319 TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM, 320 TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM, 321 TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM, 322 TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM, 323 TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM, 324 TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 325 TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 326 /* EA range 2*/ 327 TA_RAS_BLOCK__GFX_EA_INDEX2_START, 328 TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START, 329 TA_RAS_BLOCK__GFX_EA_MAM_D1MEM, 330 TA_RAS_BLOCK__GFX_EA_MAM_D2MEM, 331 TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 332 TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 333 TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END, 334 /* UTC VM L2 bank*/ 335 TA_RAS_BLOCK__UTC_VML2_BANK_CACHE, 336 /* UTC VM walker*/ 337 TA_RAS_BLOCK__UTC_VML2_WALKER, 338 /* UTC ATC L2 2MB cache*/ 339 TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK, 340 /* UTC ATC L2 4KB cache*/ 341 TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK, 342 TA_RAS_BLOCK__GFX_MAX 343 }; 344 345 struct ras_gfx_subblock { 346 unsigned char *name; 347 int ta_subblock; 348 int hw_supported_error_type; 349 int sw_supported_error_type; 350 }; 351 352 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \ 353 [AMDGPU_RAS_BLOCK__##subblock] = { \ 354 #subblock, \ 355 TA_RAS_BLOCK__##subblock, \ 356 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \ 357 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \ 358 } 359 360 static const struct ras_gfx_subblock ras_gfx_subblocks[] = { 361 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1), 362 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1), 363 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 364 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 365 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 366 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 367 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 368 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 369 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 370 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 371 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1), 372 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0), 373 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1), 374 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1), 375 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 376 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0), 377 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0, 378 0), 379 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0, 380 0), 381 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 382 AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0), 383 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0), 384 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1), 385 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0), 386 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0), 387 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1), 388 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 389 0, 0), 390 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 391 0), 392 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 393 0, 0), 394 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0, 395 0), 396 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 397 0, 0), 398 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 399 0), 400 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 401 1), 402 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 403 0, 0, 0), 404 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 405 0), 406 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 407 0), 408 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 409 0), 410 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 411 0), 412 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 413 0), 414 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 415 0, 0), 416 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 417 0), 418 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 419 0), 420 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 421 0, 0, 0), 422 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 423 0), 424 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 425 0), 426 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 427 0), 428 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 429 0), 430 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 431 0), 432 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 433 0, 0), 434 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 435 0), 436 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1), 437 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 438 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 439 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 440 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 441 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0), 442 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 443 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1), 444 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0, 445 1), 446 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0, 447 1), 448 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0, 449 1), 450 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0, 451 0), 452 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0, 453 0), 454 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 455 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 456 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0), 457 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0), 458 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0), 459 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0), 460 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 461 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0), 462 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0), 463 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 464 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0), 465 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0, 466 0), 467 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 468 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0, 469 0), 470 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0, 471 0, 0), 472 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0, 473 0), 474 AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 475 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1), 476 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0), 477 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 478 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 479 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 480 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0), 481 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0), 482 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1), 483 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0), 484 AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 485 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1), 486 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 487 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 488 AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 489 AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 490 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 491 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 492 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 493 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 494 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 495 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 496 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 497 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0), 498 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 499 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 500 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0), 501 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0), 502 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0), 503 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0), 504 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0), 505 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0), 506 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0), 507 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0), 508 }; 509 510 static const struct soc15_reg_golden golden_settings_gc_9_0[] = 511 { 512 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 513 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000), 514 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 515 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 516 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 517 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 518 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 519 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 520 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 521 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87), 522 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f), 523 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 524 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 525 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 526 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 527 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 528 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff), 529 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 530 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 531 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 532 }; 533 534 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] = 535 { 536 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107), 537 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 538 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 539 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 540 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 541 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 542 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042), 543 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 544 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000), 545 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 546 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 547 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 548 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 549 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 550 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 551 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107), 552 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800), 553 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080) 554 }; 555 556 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] = 557 { 558 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080), 559 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 560 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 561 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042), 562 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042), 563 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400), 564 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000), 565 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000), 566 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107), 567 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000), 568 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000) 569 }; 570 571 static const struct soc15_reg_golden golden_settings_gc_9_1[] = 572 { 573 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 574 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 575 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 576 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 577 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 578 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 579 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 580 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 581 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 582 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 583 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 584 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 585 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 586 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 587 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 588 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 589 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 590 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 591 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 592 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff), 593 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080), 594 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 595 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 596 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 597 }; 598 599 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] = 600 { 601 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 602 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042), 603 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042), 604 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000), 605 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000), 606 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 607 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800) 608 }; 609 610 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] = 611 { 612 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000), 613 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 614 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 615 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080), 616 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080), 617 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080), 618 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041), 619 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041), 620 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 621 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 622 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080), 623 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080), 624 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080), 625 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080), 626 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080), 627 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 628 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010), 629 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 630 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 631 }; 632 633 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] = 634 { 635 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 636 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 637 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 638 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042), 639 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042), 640 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 641 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 642 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 643 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 644 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 645 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 646 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc), 647 }; 648 649 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] = 650 { 651 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff), 652 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000), 653 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382) 654 }; 655 656 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] = 657 { 658 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 659 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 660 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 661 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 662 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 663 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 664 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 665 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 666 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87), 667 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f), 668 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 669 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 670 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 671 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 672 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 673 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff) 674 }; 675 676 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] = 677 { 678 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080), 679 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 680 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 681 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041), 682 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041), 683 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 684 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107), 685 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 686 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410), 687 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 688 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 689 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 690 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 691 }; 692 693 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] = 694 { 695 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 696 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000), 697 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e), 698 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca), 699 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098), 700 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3), 701 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1), 702 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135), 703 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000), 704 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00), 705 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000) 706 }; 707 708 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = { 709 {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)}, 710 {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)}, 711 }; 712 713 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] = 714 { 715 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 716 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 717 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 718 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 719 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 720 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 721 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 722 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 723 }; 724 725 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] = 726 { 727 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0, 728 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0, 729 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0, 730 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0, 731 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0, 732 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0, 733 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0, 734 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0, 735 }; 736 737 static void gfx_v9_0_rlcg_rw(struct amdgpu_device *adev, u32 offset, u32 v, u32 flag) 738 { 739 static void *scratch_reg0; 740 static void *scratch_reg1; 741 static void *scratch_reg2; 742 static void *scratch_reg3; 743 static void *spare_int; 744 static uint32_t grbm_cntl; 745 static uint32_t grbm_idx; 746 747 scratch_reg0 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0)*4; 748 scratch_reg1 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1)*4; 749 scratch_reg2 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG2)*4; 750 scratch_reg3 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3)*4; 751 spare_int = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT)*4; 752 753 grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL; 754 grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX; 755 756 if (amdgpu_sriov_runtime(adev)) { 757 pr_err("shouldn't call rlcg write register during runtime\n"); 758 return; 759 } 760 761 if (offset == grbm_cntl || offset == grbm_idx) { 762 if (offset == grbm_cntl) 763 writel(v, scratch_reg2); 764 else if (offset == grbm_idx) 765 writel(v, scratch_reg3); 766 767 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4)); 768 } else { 769 uint32_t i = 0; 770 uint32_t retries = 50000; 771 772 writel(v, scratch_reg0); 773 writel(offset | 0x80000000, scratch_reg1); 774 writel(1, spare_int); 775 for (i = 0; i < retries; i++) { 776 u32 tmp; 777 778 tmp = readl(scratch_reg1); 779 if (!(tmp & 0x80000000)) 780 break; 781 782 udelay(10); 783 } 784 if (i >= retries) 785 pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset); 786 } 787 788 } 789 790 static void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v, u32 flag) 791 { 792 if (amdgpu_sriov_fullaccess(adev)) { 793 gfx_v9_0_rlcg_rw(adev, offset, v, flag); 794 795 return; 796 } 797 798 if (flag & AMDGPU_REGS_NO_KIQ) 799 WREG32_NO_KIQ(offset, v); 800 else 801 WREG32(offset, v); 802 } 803 804 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042 805 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041 806 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042 807 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041 808 809 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev); 810 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev); 811 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev); 812 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev); 813 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 814 struct amdgpu_cu_info *cu_info); 815 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev); 816 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring); 817 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring); 818 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 819 void *ras_error_status); 820 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 821 void *inject_if); 822 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev); 823 824 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring, 825 uint64_t queue_mask) 826 { 827 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 828 amdgpu_ring_write(kiq_ring, 829 PACKET3_SET_RESOURCES_VMID_MASK(0) | 830 /* vmid_mask:0* queue_type:0 (KIQ) */ 831 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); 832 amdgpu_ring_write(kiq_ring, 833 lower_32_bits(queue_mask)); /* queue mask lo */ 834 amdgpu_ring_write(kiq_ring, 835 upper_32_bits(queue_mask)); /* queue mask hi */ 836 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */ 837 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */ 838 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 839 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 840 } 841 842 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring, 843 struct amdgpu_ring *ring) 844 { 845 struct amdgpu_device *adev = kiq_ring->adev; 846 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 847 uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 848 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 849 850 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 851 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 852 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 853 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 854 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ 855 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 856 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 857 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) | 858 /*queue_type: normal compute queue */ 859 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | 860 /* alloc format: all_on_one_pipe */ 861 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | 862 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) | 863 /* num_queues: must be 1 */ 864 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); 865 amdgpu_ring_write(kiq_ring, 866 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); 867 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 868 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 869 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 870 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 871 } 872 873 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, 874 struct amdgpu_ring *ring, 875 enum amdgpu_unmap_queues_action action, 876 u64 gpu_addr, u64 seq) 877 { 878 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 879 880 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 881 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 882 PACKET3_UNMAP_QUEUES_ACTION(action) | 883 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 884 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) | 885 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 886 amdgpu_ring_write(kiq_ring, 887 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 888 889 if (action == PREEMPT_QUEUES_NO_UNMAP) { 890 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr)); 891 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr)); 892 amdgpu_ring_write(kiq_ring, seq); 893 } else { 894 amdgpu_ring_write(kiq_ring, 0); 895 amdgpu_ring_write(kiq_ring, 0); 896 amdgpu_ring_write(kiq_ring, 0); 897 } 898 } 899 900 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring, 901 struct amdgpu_ring *ring, 902 u64 addr, 903 u64 seq) 904 { 905 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 906 907 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5)); 908 amdgpu_ring_write(kiq_ring, 909 PACKET3_QUERY_STATUS_CONTEXT_ID(0) | 910 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) | 911 PACKET3_QUERY_STATUS_COMMAND(2)); 912 /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 913 amdgpu_ring_write(kiq_ring, 914 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) | 915 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel)); 916 amdgpu_ring_write(kiq_ring, lower_32_bits(addr)); 917 amdgpu_ring_write(kiq_ring, upper_32_bits(addr)); 918 amdgpu_ring_write(kiq_ring, lower_32_bits(seq)); 919 amdgpu_ring_write(kiq_ring, upper_32_bits(seq)); 920 } 921 922 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring, 923 uint16_t pasid, uint32_t flush_type, 924 bool all_hub) 925 { 926 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); 927 amdgpu_ring_write(kiq_ring, 928 PACKET3_INVALIDATE_TLBS_DST_SEL(1) | 929 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) | 930 PACKET3_INVALIDATE_TLBS_PASID(pasid) | 931 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type)); 932 } 933 934 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = { 935 .kiq_set_resources = gfx_v9_0_kiq_set_resources, 936 .kiq_map_queues = gfx_v9_0_kiq_map_queues, 937 .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues, 938 .kiq_query_status = gfx_v9_0_kiq_query_status, 939 .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs, 940 .set_resources_size = 8, 941 .map_queues_size = 7, 942 .unmap_queues_size = 6, 943 .query_status_size = 7, 944 .invalidate_tlbs_size = 2, 945 }; 946 947 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) 948 { 949 adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs; 950 } 951 952 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev) 953 { 954 switch (adev->asic_type) { 955 case CHIP_VEGA10: 956 soc15_program_register_sequence(adev, 957 golden_settings_gc_9_0, 958 ARRAY_SIZE(golden_settings_gc_9_0)); 959 soc15_program_register_sequence(adev, 960 golden_settings_gc_9_0_vg10, 961 ARRAY_SIZE(golden_settings_gc_9_0_vg10)); 962 break; 963 case CHIP_VEGA12: 964 soc15_program_register_sequence(adev, 965 golden_settings_gc_9_2_1, 966 ARRAY_SIZE(golden_settings_gc_9_2_1)); 967 soc15_program_register_sequence(adev, 968 golden_settings_gc_9_2_1_vg12, 969 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12)); 970 break; 971 case CHIP_VEGA20: 972 soc15_program_register_sequence(adev, 973 golden_settings_gc_9_0, 974 ARRAY_SIZE(golden_settings_gc_9_0)); 975 soc15_program_register_sequence(adev, 976 golden_settings_gc_9_0_vg20, 977 ARRAY_SIZE(golden_settings_gc_9_0_vg20)); 978 break; 979 case CHIP_ARCTURUS: 980 soc15_program_register_sequence(adev, 981 golden_settings_gc_9_4_1_arct, 982 ARRAY_SIZE(golden_settings_gc_9_4_1_arct)); 983 break; 984 case CHIP_RAVEN: 985 soc15_program_register_sequence(adev, golden_settings_gc_9_1, 986 ARRAY_SIZE(golden_settings_gc_9_1)); 987 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 988 soc15_program_register_sequence(adev, 989 golden_settings_gc_9_1_rv2, 990 ARRAY_SIZE(golden_settings_gc_9_1_rv2)); 991 else 992 soc15_program_register_sequence(adev, 993 golden_settings_gc_9_1_rv1, 994 ARRAY_SIZE(golden_settings_gc_9_1_rv1)); 995 break; 996 case CHIP_RENOIR: 997 soc15_program_register_sequence(adev, 998 golden_settings_gc_9_1_rn, 999 ARRAY_SIZE(golden_settings_gc_9_1_rn)); 1000 return; /* for renoir, don't need common goldensetting */ 1001 case CHIP_ALDEBARAN: 1002 gfx_v9_4_2_init_golden_registers(adev, 1003 adev->smuio.funcs->get_die_id(adev)); 1004 break; 1005 default: 1006 break; 1007 } 1008 1009 if ((adev->asic_type != CHIP_ARCTURUS) && 1010 (adev->asic_type != CHIP_ALDEBARAN)) 1011 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common, 1012 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common)); 1013 } 1014 1015 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev) 1016 { 1017 adev->gfx.scratch.num_reg = 8; 1018 adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0); 1019 adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1; 1020 } 1021 1022 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel, 1023 bool wc, uint32_t reg, uint32_t val) 1024 { 1025 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 1026 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) | 1027 WRITE_DATA_DST_SEL(0) | 1028 (wc ? WR_CONFIRM : 0)); 1029 amdgpu_ring_write(ring, reg); 1030 amdgpu_ring_write(ring, 0); 1031 amdgpu_ring_write(ring, val); 1032 } 1033 1034 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, 1035 int mem_space, int opt, uint32_t addr0, 1036 uint32_t addr1, uint32_t ref, uint32_t mask, 1037 uint32_t inv) 1038 { 1039 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 1040 amdgpu_ring_write(ring, 1041 /* memory (1) or register (0) */ 1042 (WAIT_REG_MEM_MEM_SPACE(mem_space) | 1043 WAIT_REG_MEM_OPERATION(opt) | /* wait */ 1044 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 1045 WAIT_REG_MEM_ENGINE(eng_sel))); 1046 1047 if (mem_space) 1048 BUG_ON(addr0 & 0x3); /* Dword align */ 1049 amdgpu_ring_write(ring, addr0); 1050 amdgpu_ring_write(ring, addr1); 1051 amdgpu_ring_write(ring, ref); 1052 amdgpu_ring_write(ring, mask); 1053 amdgpu_ring_write(ring, inv); /* poll interval */ 1054 } 1055 1056 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring) 1057 { 1058 struct amdgpu_device *adev = ring->adev; 1059 uint32_t scratch; 1060 uint32_t tmp = 0; 1061 unsigned i; 1062 int r; 1063 1064 r = amdgpu_gfx_scratch_get(adev, &scratch); 1065 if (r) 1066 return r; 1067 1068 WREG32(scratch, 0xCAFEDEAD); 1069 r = amdgpu_ring_alloc(ring, 3); 1070 if (r) 1071 goto error_free_scratch; 1072 1073 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 1074 amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START)); 1075 amdgpu_ring_write(ring, 0xDEADBEEF); 1076 amdgpu_ring_commit(ring); 1077 1078 for (i = 0; i < adev->usec_timeout; i++) { 1079 tmp = RREG32(scratch); 1080 if (tmp == 0xDEADBEEF) 1081 break; 1082 udelay(1); 1083 } 1084 1085 if (i >= adev->usec_timeout) 1086 r = -ETIMEDOUT; 1087 1088 error_free_scratch: 1089 amdgpu_gfx_scratch_free(adev, scratch); 1090 return r; 1091 } 1092 1093 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 1094 { 1095 struct amdgpu_device *adev = ring->adev; 1096 struct amdgpu_ib ib; 1097 struct dma_fence *f = NULL; 1098 1099 unsigned index; 1100 uint64_t gpu_addr; 1101 uint32_t tmp; 1102 long r; 1103 1104 r = amdgpu_device_wb_get(adev, &index); 1105 if (r) 1106 return r; 1107 1108 gpu_addr = adev->wb.gpu_addr + (index * 4); 1109 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 1110 memset(&ib, 0, sizeof(ib)); 1111 r = amdgpu_ib_get(adev, NULL, 16, 1112 AMDGPU_IB_POOL_DIRECT, &ib); 1113 if (r) 1114 goto err1; 1115 1116 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 1117 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 1118 ib.ptr[2] = lower_32_bits(gpu_addr); 1119 ib.ptr[3] = upper_32_bits(gpu_addr); 1120 ib.ptr[4] = 0xDEADBEEF; 1121 ib.length_dw = 5; 1122 1123 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 1124 if (r) 1125 goto err2; 1126 1127 r = dma_fence_wait_timeout(f, false, timeout); 1128 if (r == 0) { 1129 r = -ETIMEDOUT; 1130 goto err2; 1131 } else if (r < 0) { 1132 goto err2; 1133 } 1134 1135 tmp = adev->wb.wb[index]; 1136 if (tmp == 0xDEADBEEF) 1137 r = 0; 1138 else 1139 r = -EINVAL; 1140 1141 err2: 1142 amdgpu_ib_free(adev, &ib, NULL); 1143 dma_fence_put(f); 1144 err1: 1145 amdgpu_device_wb_free(adev, index); 1146 return r; 1147 } 1148 1149 1150 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev) 1151 { 1152 release_firmware(adev->gfx.pfp_fw); 1153 adev->gfx.pfp_fw = NULL; 1154 release_firmware(adev->gfx.me_fw); 1155 adev->gfx.me_fw = NULL; 1156 release_firmware(adev->gfx.ce_fw); 1157 adev->gfx.ce_fw = NULL; 1158 release_firmware(adev->gfx.rlc_fw); 1159 adev->gfx.rlc_fw = NULL; 1160 release_firmware(adev->gfx.mec_fw); 1161 adev->gfx.mec_fw = NULL; 1162 release_firmware(adev->gfx.mec2_fw); 1163 adev->gfx.mec2_fw = NULL; 1164 1165 kfree(adev->gfx.rlc.register_list_format); 1166 } 1167 1168 static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev) 1169 { 1170 const struct rlc_firmware_header_v2_1 *rlc_hdr; 1171 1172 rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data; 1173 adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver); 1174 adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver); 1175 adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes); 1176 adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes); 1177 adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver); 1178 adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver); 1179 adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes); 1180 adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes); 1181 adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver); 1182 adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver); 1183 adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes); 1184 adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes); 1185 adev->gfx.rlc.reg_list_format_direct_reg_list_length = 1186 le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length); 1187 } 1188 1189 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev) 1190 { 1191 adev->gfx.me_fw_write_wait = false; 1192 adev->gfx.mec_fw_write_wait = false; 1193 1194 if ((adev->asic_type != CHIP_ARCTURUS) && 1195 ((adev->gfx.mec_fw_version < 0x000001a5) || 1196 (adev->gfx.mec_feature_version < 46) || 1197 (adev->gfx.pfp_fw_version < 0x000000b7) || 1198 (adev->gfx.pfp_feature_version < 46))) 1199 DRM_WARN_ONCE("CP firmware version too old, please update!"); 1200 1201 switch (adev->asic_type) { 1202 case CHIP_VEGA10: 1203 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1204 (adev->gfx.me_feature_version >= 42) && 1205 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1206 (adev->gfx.pfp_feature_version >= 42)) 1207 adev->gfx.me_fw_write_wait = true; 1208 1209 if ((adev->gfx.mec_fw_version >= 0x00000193) && 1210 (adev->gfx.mec_feature_version >= 42)) 1211 adev->gfx.mec_fw_write_wait = true; 1212 break; 1213 case CHIP_VEGA12: 1214 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1215 (adev->gfx.me_feature_version >= 44) && 1216 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1217 (adev->gfx.pfp_feature_version >= 44)) 1218 adev->gfx.me_fw_write_wait = true; 1219 1220 if ((adev->gfx.mec_fw_version >= 0x00000196) && 1221 (adev->gfx.mec_feature_version >= 44)) 1222 adev->gfx.mec_fw_write_wait = true; 1223 break; 1224 case CHIP_VEGA20: 1225 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1226 (adev->gfx.me_feature_version >= 44) && 1227 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1228 (adev->gfx.pfp_feature_version >= 44)) 1229 adev->gfx.me_fw_write_wait = true; 1230 1231 if ((adev->gfx.mec_fw_version >= 0x00000197) && 1232 (adev->gfx.mec_feature_version >= 44)) 1233 adev->gfx.mec_fw_write_wait = true; 1234 break; 1235 case CHIP_RAVEN: 1236 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1237 (adev->gfx.me_feature_version >= 42) && 1238 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1239 (adev->gfx.pfp_feature_version >= 42)) 1240 adev->gfx.me_fw_write_wait = true; 1241 1242 if ((adev->gfx.mec_fw_version >= 0x00000192) && 1243 (adev->gfx.mec_feature_version >= 42)) 1244 adev->gfx.mec_fw_write_wait = true; 1245 break; 1246 default: 1247 adev->gfx.me_fw_write_wait = true; 1248 adev->gfx.mec_fw_write_wait = true; 1249 break; 1250 } 1251 } 1252 1253 struct amdgpu_gfxoff_quirk { 1254 u16 chip_vendor; 1255 u16 chip_device; 1256 u16 subsys_vendor; 1257 u16 subsys_device; 1258 u8 revision; 1259 }; 1260 1261 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = { 1262 /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */ 1263 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 }, 1264 /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */ 1265 { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 }, 1266 /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */ 1267 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 }, 1268 { 0, 0, 0, 0, 0 }, 1269 }; 1270 1271 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev) 1272 { 1273 const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list; 1274 1275 while (p && p->chip_device != 0) { 1276 if (pdev->vendor == p->chip_vendor && 1277 pdev->device == p->chip_device && 1278 pdev->subsystem_vendor == p->subsys_vendor && 1279 pdev->subsystem_device == p->subsys_device && 1280 pdev->revision == p->revision) { 1281 return true; 1282 } 1283 ++p; 1284 } 1285 return false; 1286 } 1287 1288 static bool is_raven_kicker(struct amdgpu_device *adev) 1289 { 1290 if (adev->pm.fw_version >= 0x41e2b) 1291 return true; 1292 else 1293 return false; 1294 } 1295 1296 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev) 1297 { 1298 if (gfx_v9_0_should_disable_gfxoff(adev->pdev)) 1299 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1300 1301 switch (adev->asic_type) { 1302 case CHIP_VEGA10: 1303 case CHIP_VEGA12: 1304 case CHIP_VEGA20: 1305 break; 1306 case CHIP_RAVEN: 1307 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) || 1308 (adev->apu_flags & AMD_APU_IS_PICASSO)) && 1309 ((!is_raven_kicker(adev) && 1310 adev->gfx.rlc_fw_version < 531) || 1311 (adev->gfx.rlc_feature_version < 1) || 1312 !adev->gfx.rlc.is_rlc_v2_1)) 1313 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1314 1315 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1316 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1317 AMD_PG_SUPPORT_CP | 1318 AMD_PG_SUPPORT_RLC_SMU_HS; 1319 break; 1320 case CHIP_RENOIR: 1321 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1322 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1323 AMD_PG_SUPPORT_CP | 1324 AMD_PG_SUPPORT_RLC_SMU_HS; 1325 break; 1326 default: 1327 break; 1328 } 1329 } 1330 1331 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev, 1332 const char *chip_name) 1333 { 1334 char fw_name[30]; 1335 int err; 1336 struct amdgpu_firmware_info *info = NULL; 1337 const struct common_firmware_header *header = NULL; 1338 const struct gfx_firmware_header_v1_0 *cp_hdr; 1339 1340 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name); 1341 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev); 1342 if (err) 1343 goto out; 1344 err = amdgpu_ucode_validate(adev->gfx.pfp_fw); 1345 if (err) 1346 goto out; 1347 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; 1348 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1349 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1350 1351 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name); 1352 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev); 1353 if (err) 1354 goto out; 1355 err = amdgpu_ucode_validate(adev->gfx.me_fw); 1356 if (err) 1357 goto out; 1358 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; 1359 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1360 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1361 1362 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name); 1363 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev); 1364 if (err) 1365 goto out; 1366 err = amdgpu_ucode_validate(adev->gfx.ce_fw); 1367 if (err) 1368 goto out; 1369 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; 1370 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1371 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1372 1373 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1374 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP]; 1375 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP; 1376 info->fw = adev->gfx.pfp_fw; 1377 header = (const struct common_firmware_header *)info->fw->data; 1378 adev->firmware.fw_size += 1379 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1380 1381 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME]; 1382 info->ucode_id = AMDGPU_UCODE_ID_CP_ME; 1383 info->fw = adev->gfx.me_fw; 1384 header = (const struct common_firmware_header *)info->fw->data; 1385 adev->firmware.fw_size += 1386 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1387 1388 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE]; 1389 info->ucode_id = AMDGPU_UCODE_ID_CP_CE; 1390 info->fw = adev->gfx.ce_fw; 1391 header = (const struct common_firmware_header *)info->fw->data; 1392 adev->firmware.fw_size += 1393 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1394 } 1395 1396 out: 1397 if (err) { 1398 dev_err(adev->dev, 1399 "gfx9: Failed to load firmware \"%s\"\n", 1400 fw_name); 1401 release_firmware(adev->gfx.pfp_fw); 1402 adev->gfx.pfp_fw = NULL; 1403 release_firmware(adev->gfx.me_fw); 1404 adev->gfx.me_fw = NULL; 1405 release_firmware(adev->gfx.ce_fw); 1406 adev->gfx.ce_fw = NULL; 1407 } 1408 return err; 1409 } 1410 1411 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev, 1412 const char *chip_name) 1413 { 1414 char fw_name[30]; 1415 int err; 1416 struct amdgpu_firmware_info *info = NULL; 1417 const struct common_firmware_header *header = NULL; 1418 const struct rlc_firmware_header_v2_0 *rlc_hdr; 1419 unsigned int *tmp = NULL; 1420 unsigned int i = 0; 1421 uint16_t version_major; 1422 uint16_t version_minor; 1423 uint32_t smu_version; 1424 1425 /* 1426 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin 1427 * instead of picasso_rlc.bin. 1428 * Judgment method: 1429 * PCO AM4: revision >= 0xC8 && revision <= 0xCF 1430 * or revision >= 0xD8 && revision <= 0xDF 1431 * otherwise is PCO FP5 1432 */ 1433 if (!strcmp(chip_name, "picasso") && 1434 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) || 1435 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF)))) 1436 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name); 1437 else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) && 1438 (smu_version >= 0x41e2b)) 1439 /** 1440 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly. 1441 */ 1442 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name); 1443 else 1444 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name); 1445 err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev); 1446 if (err) 1447 goto out; 1448 err = amdgpu_ucode_validate(adev->gfx.rlc_fw); 1449 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 1450 1451 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 1452 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 1453 if (version_major == 2 && version_minor == 1) 1454 adev->gfx.rlc.is_rlc_v2_1 = true; 1455 1456 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); 1457 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); 1458 adev->gfx.rlc.save_and_restore_offset = 1459 le32_to_cpu(rlc_hdr->save_and_restore_offset); 1460 adev->gfx.rlc.clear_state_descriptor_offset = 1461 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset); 1462 adev->gfx.rlc.avail_scratch_ram_locations = 1463 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations); 1464 adev->gfx.rlc.reg_restore_list_size = 1465 le32_to_cpu(rlc_hdr->reg_restore_list_size); 1466 adev->gfx.rlc.reg_list_format_start = 1467 le32_to_cpu(rlc_hdr->reg_list_format_start); 1468 adev->gfx.rlc.reg_list_format_separate_start = 1469 le32_to_cpu(rlc_hdr->reg_list_format_separate_start); 1470 adev->gfx.rlc.starting_offsets_start = 1471 le32_to_cpu(rlc_hdr->starting_offsets_start); 1472 adev->gfx.rlc.reg_list_format_size_bytes = 1473 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes); 1474 adev->gfx.rlc.reg_list_size_bytes = 1475 le32_to_cpu(rlc_hdr->reg_list_size_bytes); 1476 adev->gfx.rlc.register_list_format = 1477 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes + 1478 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL); 1479 if (!adev->gfx.rlc.register_list_format) { 1480 err = -ENOMEM; 1481 goto out; 1482 } 1483 1484 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1485 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); 1486 for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++) 1487 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]); 1488 1489 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i; 1490 1491 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1492 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); 1493 for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++) 1494 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); 1495 1496 if (adev->gfx.rlc.is_rlc_v2_1) 1497 gfx_v9_0_init_rlc_ext_microcode(adev); 1498 1499 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1500 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G]; 1501 info->ucode_id = AMDGPU_UCODE_ID_RLC_G; 1502 info->fw = adev->gfx.rlc_fw; 1503 header = (const struct common_firmware_header *)info->fw->data; 1504 adev->firmware.fw_size += 1505 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1506 1507 if (adev->gfx.rlc.is_rlc_v2_1 && 1508 adev->gfx.rlc.save_restore_list_cntl_size_bytes && 1509 adev->gfx.rlc.save_restore_list_gpm_size_bytes && 1510 adev->gfx.rlc.save_restore_list_srm_size_bytes) { 1511 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL]; 1512 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL; 1513 info->fw = adev->gfx.rlc_fw; 1514 adev->firmware.fw_size += 1515 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE); 1516 1517 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM]; 1518 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM; 1519 info->fw = adev->gfx.rlc_fw; 1520 adev->firmware.fw_size += 1521 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE); 1522 1523 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM]; 1524 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM; 1525 info->fw = adev->gfx.rlc_fw; 1526 adev->firmware.fw_size += 1527 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE); 1528 } 1529 } 1530 1531 out: 1532 if (err) { 1533 dev_err(adev->dev, 1534 "gfx9: Failed to load firmware \"%s\"\n", 1535 fw_name); 1536 release_firmware(adev->gfx.rlc_fw); 1537 adev->gfx.rlc_fw = NULL; 1538 } 1539 return err; 1540 } 1541 1542 static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev) 1543 { 1544 if (adev->asic_type == CHIP_ALDEBARAN || 1545 adev->asic_type == CHIP_ARCTURUS || 1546 adev->asic_type == CHIP_RENOIR) 1547 return false; 1548 1549 return true; 1550 } 1551 1552 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev, 1553 const char *chip_name) 1554 { 1555 char fw_name[30]; 1556 int err; 1557 struct amdgpu_firmware_info *info = NULL; 1558 const struct common_firmware_header *header = NULL; 1559 const struct gfx_firmware_header_v1_0 *cp_hdr; 1560 1561 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name); 1562 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev); 1563 if (err) 1564 goto out; 1565 err = amdgpu_ucode_validate(adev->gfx.mec_fw); 1566 if (err) 1567 goto out; 1568 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1569 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1570 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1571 1572 1573 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) { 1574 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name); 1575 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev); 1576 if (!err) { 1577 err = amdgpu_ucode_validate(adev->gfx.mec2_fw); 1578 if (err) 1579 goto out; 1580 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1581 adev->gfx.mec2_fw->data; 1582 adev->gfx.mec2_fw_version = 1583 le32_to_cpu(cp_hdr->header.ucode_version); 1584 adev->gfx.mec2_feature_version = 1585 le32_to_cpu(cp_hdr->ucode_feature_version); 1586 } else { 1587 err = 0; 1588 adev->gfx.mec2_fw = NULL; 1589 } 1590 } else { 1591 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version; 1592 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version; 1593 } 1594 1595 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1596 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1]; 1597 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1; 1598 info->fw = adev->gfx.mec_fw; 1599 header = (const struct common_firmware_header *)info->fw->data; 1600 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1601 adev->firmware.fw_size += 1602 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1603 1604 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT]; 1605 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT; 1606 info->fw = adev->gfx.mec_fw; 1607 adev->firmware.fw_size += 1608 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1609 1610 if (adev->gfx.mec2_fw) { 1611 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2]; 1612 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2; 1613 info->fw = adev->gfx.mec2_fw; 1614 header = (const struct common_firmware_header *)info->fw->data; 1615 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1616 adev->firmware.fw_size += 1617 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1618 1619 /* TODO: Determine if MEC2 JT FW loading can be removed 1620 for all GFX V9 asic and above */ 1621 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) { 1622 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT]; 1623 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT; 1624 info->fw = adev->gfx.mec2_fw; 1625 adev->firmware.fw_size += 1626 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, 1627 PAGE_SIZE); 1628 } 1629 } 1630 } 1631 1632 out: 1633 gfx_v9_0_check_if_need_gfxoff(adev); 1634 gfx_v9_0_check_fw_write_wait(adev); 1635 if (err) { 1636 dev_err(adev->dev, 1637 "gfx9: Failed to load firmware \"%s\"\n", 1638 fw_name); 1639 release_firmware(adev->gfx.mec_fw); 1640 adev->gfx.mec_fw = NULL; 1641 release_firmware(adev->gfx.mec2_fw); 1642 adev->gfx.mec2_fw = NULL; 1643 } 1644 return err; 1645 } 1646 1647 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev) 1648 { 1649 const char *chip_name; 1650 int r; 1651 1652 DRM_DEBUG("\n"); 1653 1654 switch (adev->asic_type) { 1655 case CHIP_VEGA10: 1656 chip_name = "vega10"; 1657 break; 1658 case CHIP_VEGA12: 1659 chip_name = "vega12"; 1660 break; 1661 case CHIP_VEGA20: 1662 chip_name = "vega20"; 1663 break; 1664 case CHIP_RAVEN: 1665 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 1666 chip_name = "raven2"; 1667 else if (adev->apu_flags & AMD_APU_IS_PICASSO) 1668 chip_name = "picasso"; 1669 else 1670 chip_name = "raven"; 1671 break; 1672 case CHIP_ARCTURUS: 1673 chip_name = "arcturus"; 1674 break; 1675 case CHIP_RENOIR: 1676 if (adev->apu_flags & AMD_APU_IS_RENOIR) 1677 chip_name = "renoir"; 1678 else 1679 chip_name = "green_sardine"; 1680 break; 1681 case CHIP_ALDEBARAN: 1682 chip_name = "aldebaran"; 1683 break; 1684 default: 1685 BUG(); 1686 } 1687 1688 /* No CPG in Arcturus */ 1689 if (adev->gfx.num_gfx_rings) { 1690 r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name); 1691 if (r) 1692 return r; 1693 } 1694 1695 r = gfx_v9_0_init_rlc_microcode(adev, chip_name); 1696 if (r) 1697 return r; 1698 1699 r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name); 1700 if (r) 1701 return r; 1702 1703 return r; 1704 } 1705 1706 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev) 1707 { 1708 u32 count = 0; 1709 const struct cs_section_def *sect = NULL; 1710 const struct cs_extent_def *ext = NULL; 1711 1712 /* begin clear state */ 1713 count += 2; 1714 /* context control state */ 1715 count += 3; 1716 1717 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 1718 for (ext = sect->section; ext->extent != NULL; ++ext) { 1719 if (sect->id == SECT_CONTEXT) 1720 count += 2 + ext->reg_count; 1721 else 1722 return 0; 1723 } 1724 } 1725 1726 /* end clear state */ 1727 count += 2; 1728 /* clear state */ 1729 count += 2; 1730 1731 return count; 1732 } 1733 1734 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev, 1735 volatile u32 *buffer) 1736 { 1737 u32 count = 0, i; 1738 const struct cs_section_def *sect = NULL; 1739 const struct cs_extent_def *ext = NULL; 1740 1741 if (adev->gfx.rlc.cs_data == NULL) 1742 return; 1743 if (buffer == NULL) 1744 return; 1745 1746 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1747 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 1748 1749 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 1750 buffer[count++] = cpu_to_le32(0x80000000); 1751 buffer[count++] = cpu_to_le32(0x80000000); 1752 1753 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { 1754 for (ext = sect->section; ext->extent != NULL; ++ext) { 1755 if (sect->id == SECT_CONTEXT) { 1756 buffer[count++] = 1757 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 1758 buffer[count++] = cpu_to_le32(ext->reg_index - 1759 PACKET3_SET_CONTEXT_REG_START); 1760 for (i = 0; i < ext->reg_count; i++) 1761 buffer[count++] = cpu_to_le32(ext->extent[i]); 1762 } else { 1763 return; 1764 } 1765 } 1766 } 1767 1768 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1769 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 1770 1771 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 1772 buffer[count++] = cpu_to_le32(0); 1773 } 1774 1775 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev) 1776 { 1777 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; 1778 uint32_t pg_always_on_cu_num = 2; 1779 uint32_t always_on_cu_num; 1780 uint32_t i, j, k; 1781 uint32_t mask, cu_bitmap, counter; 1782 1783 if (adev->flags & AMD_IS_APU) 1784 always_on_cu_num = 4; 1785 else if (adev->asic_type == CHIP_VEGA12) 1786 always_on_cu_num = 8; 1787 else 1788 always_on_cu_num = 12; 1789 1790 mutex_lock(&adev->grbm_idx_mutex); 1791 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 1792 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 1793 mask = 1; 1794 cu_bitmap = 0; 1795 counter = 0; 1796 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 1797 1798 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 1799 if (cu_info->bitmap[i][j] & mask) { 1800 if (counter == pg_always_on_cu_num) 1801 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap); 1802 if (counter < always_on_cu_num) 1803 cu_bitmap |= mask; 1804 else 1805 break; 1806 counter++; 1807 } 1808 mask <<= 1; 1809 } 1810 1811 WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap); 1812 cu_info->ao_cu_bitmap[i][j] = cu_bitmap; 1813 } 1814 } 1815 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1816 mutex_unlock(&adev->grbm_idx_mutex); 1817 } 1818 1819 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev) 1820 { 1821 uint32_t data; 1822 1823 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1824 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1825 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7); 1826 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1827 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16)); 1828 1829 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1830 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1831 1832 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1833 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500); 1834 1835 mutex_lock(&adev->grbm_idx_mutex); 1836 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1837 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1838 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1839 1840 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1841 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1842 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1843 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1844 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1845 1846 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1847 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1848 data &= 0x0000FFFF; 1849 data |= 0x00C00000; 1850 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1851 1852 /* 1853 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven), 1854 * programmed in gfx_v9_0_init_always_on_cu_mask() 1855 */ 1856 1857 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1858 * but used for RLC_LB_CNTL configuration */ 1859 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1860 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1861 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1862 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1863 mutex_unlock(&adev->grbm_idx_mutex); 1864 1865 gfx_v9_0_init_always_on_cu_mask(adev); 1866 } 1867 1868 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev) 1869 { 1870 uint32_t data; 1871 1872 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1873 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1874 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8); 1875 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1876 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16)); 1877 1878 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1879 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1880 1881 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1882 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800); 1883 1884 mutex_lock(&adev->grbm_idx_mutex); 1885 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1886 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1887 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1888 1889 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1890 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1891 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1892 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1893 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1894 1895 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1896 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1897 data &= 0x0000FFFF; 1898 data |= 0x00C00000; 1899 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1900 1901 /* 1902 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON), 1903 * programmed in gfx_v9_0_init_always_on_cu_mask() 1904 */ 1905 1906 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1907 * but used for RLC_LB_CNTL configuration */ 1908 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1909 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1910 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1911 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1912 mutex_unlock(&adev->grbm_idx_mutex); 1913 1914 gfx_v9_0_init_always_on_cu_mask(adev); 1915 } 1916 1917 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable) 1918 { 1919 WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0); 1920 } 1921 1922 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev) 1923 { 1924 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) 1925 return 5; 1926 else 1927 return 4; 1928 } 1929 1930 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev) 1931 { 1932 const struct cs_section_def *cs_data; 1933 int r; 1934 1935 adev->gfx.rlc.cs_data = gfx9_cs_data; 1936 1937 cs_data = adev->gfx.rlc.cs_data; 1938 1939 if (cs_data) { 1940 /* init clear state block */ 1941 r = amdgpu_gfx_rlc_init_csb(adev); 1942 if (r) 1943 return r; 1944 } 1945 1946 if (adev->flags & AMD_IS_APU) { 1947 /* TODO: double check the cp_table_size for RV */ 1948 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ 1949 r = amdgpu_gfx_rlc_init_cpt(adev); 1950 if (r) 1951 return r; 1952 } 1953 1954 switch (adev->asic_type) { 1955 case CHIP_RAVEN: 1956 gfx_v9_0_init_lbpw(adev); 1957 break; 1958 case CHIP_VEGA20: 1959 gfx_v9_4_init_lbpw(adev); 1960 break; 1961 default: 1962 break; 1963 } 1964 1965 /* init spm vmid with 0xf */ 1966 if (adev->gfx.rlc.funcs->update_spm_vmid) 1967 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf); 1968 1969 return 0; 1970 } 1971 1972 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev) 1973 { 1974 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 1975 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); 1976 } 1977 1978 static int gfx_v9_0_mec_init(struct amdgpu_device *adev) 1979 { 1980 int r; 1981 u32 *hpd; 1982 const __le32 *fw_data; 1983 unsigned fw_size; 1984 u32 *fw; 1985 size_t mec_hpd_size; 1986 1987 const struct gfx_firmware_header_v1_0 *mec_hdr; 1988 1989 bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 1990 1991 /* take ownership of the relevant compute queues */ 1992 amdgpu_gfx_compute_queue_acquire(adev); 1993 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE; 1994 if (mec_hpd_size) { 1995 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 1996 AMDGPU_GEM_DOMAIN_VRAM, 1997 &adev->gfx.mec.hpd_eop_obj, 1998 &adev->gfx.mec.hpd_eop_gpu_addr, 1999 (void **)&hpd); 2000 if (r) { 2001 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 2002 gfx_v9_0_mec_fini(adev); 2003 return r; 2004 } 2005 2006 memset(hpd, 0, mec_hpd_size); 2007 2008 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 2009 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 2010 } 2011 2012 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 2013 2014 fw_data = (const __le32 *) 2015 (adev->gfx.mec_fw->data + 2016 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 2017 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes); 2018 2019 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes, 2020 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 2021 &adev->gfx.mec.mec_fw_obj, 2022 &adev->gfx.mec.mec_fw_gpu_addr, 2023 (void **)&fw); 2024 if (r) { 2025 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r); 2026 gfx_v9_0_mec_fini(adev); 2027 return r; 2028 } 2029 2030 memcpy(fw, fw_data, fw_size); 2031 2032 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 2033 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 2034 2035 return 0; 2036 } 2037 2038 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address) 2039 { 2040 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 2041 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 2042 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 2043 (address << SQ_IND_INDEX__INDEX__SHIFT) | 2044 (SQ_IND_INDEX__FORCE_READ_MASK)); 2045 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 2046 } 2047 2048 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd, 2049 uint32_t wave, uint32_t thread, 2050 uint32_t regno, uint32_t num, uint32_t *out) 2051 { 2052 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 2053 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 2054 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 2055 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 2056 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) | 2057 (SQ_IND_INDEX__FORCE_READ_MASK) | 2058 (SQ_IND_INDEX__AUTO_INCR_MASK)); 2059 while (num--) 2060 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 2061 } 2062 2063 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 2064 { 2065 /* type 1 wave data */ 2066 dst[(*no_fields)++] = 1; 2067 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS); 2068 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO); 2069 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI); 2070 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO); 2071 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI); 2072 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID); 2073 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0); 2074 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1); 2075 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC); 2076 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC); 2077 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS); 2078 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS); 2079 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); 2080 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); 2081 } 2082 2083 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd, 2084 uint32_t wave, uint32_t start, 2085 uint32_t size, uint32_t *dst) 2086 { 2087 wave_read_regs( 2088 adev, simd, wave, 0, 2089 start + SQIND_WAVE_SGPRS_OFFSET, size, dst); 2090 } 2091 2092 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd, 2093 uint32_t wave, uint32_t thread, 2094 uint32_t start, uint32_t size, 2095 uint32_t *dst) 2096 { 2097 wave_read_regs( 2098 adev, simd, wave, thread, 2099 start + SQIND_WAVE_VGPRS_OFFSET, size, dst); 2100 } 2101 2102 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev, 2103 u32 me, u32 pipe, u32 q, u32 vm) 2104 { 2105 soc15_grbm_select(adev, me, pipe, q, vm); 2106 } 2107 2108 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = { 2109 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 2110 .select_se_sh = &gfx_v9_0_select_se_sh, 2111 .read_wave_data = &gfx_v9_0_read_wave_data, 2112 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 2113 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 2114 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 2115 }; 2116 2117 static const struct amdgpu_gfx_ras_funcs gfx_v9_0_ras_funcs = { 2118 .ras_late_init = amdgpu_gfx_ras_late_init, 2119 .ras_fini = amdgpu_gfx_ras_fini, 2120 .ras_error_inject = &gfx_v9_0_ras_error_inject, 2121 .query_ras_error_count = &gfx_v9_0_query_ras_error_count, 2122 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count, 2123 }; 2124 2125 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev) 2126 { 2127 u32 gb_addr_config; 2128 int err; 2129 2130 adev->gfx.funcs = &gfx_v9_0_gfx_funcs; 2131 2132 switch (adev->asic_type) { 2133 case CHIP_VEGA10: 2134 adev->gfx.config.max_hw_contexts = 8; 2135 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2136 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2137 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2138 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2139 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN; 2140 break; 2141 case CHIP_VEGA12: 2142 adev->gfx.config.max_hw_contexts = 8; 2143 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2144 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2145 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2146 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2147 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN; 2148 DRM_INFO("fix gfx.config for vega12\n"); 2149 break; 2150 case CHIP_VEGA20: 2151 adev->gfx.ras_funcs = &gfx_v9_0_ras_funcs; 2152 adev->gfx.config.max_hw_contexts = 8; 2153 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2154 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2155 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2156 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2157 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2158 gb_addr_config &= ~0xf3e777ff; 2159 gb_addr_config |= 0x22014042; 2160 /* check vbios table if gpu info is not available */ 2161 err = amdgpu_atomfirmware_get_gfx_info(adev); 2162 if (err) 2163 return err; 2164 break; 2165 case CHIP_RAVEN: 2166 adev->gfx.config.max_hw_contexts = 8; 2167 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2168 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2169 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2170 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2171 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 2172 gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN; 2173 else 2174 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN; 2175 break; 2176 case CHIP_ARCTURUS: 2177 adev->gfx.ras_funcs = &gfx_v9_4_ras_funcs; 2178 adev->gfx.config.max_hw_contexts = 8; 2179 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2180 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2181 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2182 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2183 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2184 gb_addr_config &= ~0xf3e777ff; 2185 gb_addr_config |= 0x22014042; 2186 break; 2187 case CHIP_RENOIR: 2188 adev->gfx.config.max_hw_contexts = 8; 2189 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2190 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2191 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; 2192 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2193 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2194 gb_addr_config &= ~0xf3e777ff; 2195 gb_addr_config |= 0x22010042; 2196 break; 2197 case CHIP_ALDEBARAN: 2198 adev->gfx.ras_funcs = &gfx_v9_4_2_ras_funcs; 2199 adev->gfx.config.max_hw_contexts = 8; 2200 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2201 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2202 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2203 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2204 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2205 gb_addr_config &= ~0xf3e777ff; 2206 gb_addr_config |= 0x22014042; 2207 /* check vbios table if gpu info is not available */ 2208 err = amdgpu_atomfirmware_get_gfx_info(adev); 2209 if (err) 2210 return err; 2211 break; 2212 default: 2213 BUG(); 2214 break; 2215 } 2216 2217 adev->gfx.config.gb_addr_config = gb_addr_config; 2218 2219 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << 2220 REG_GET_FIELD( 2221 adev->gfx.config.gb_addr_config, 2222 GB_ADDR_CONFIG, 2223 NUM_PIPES); 2224 2225 adev->gfx.config.max_tile_pipes = 2226 adev->gfx.config.gb_addr_config_fields.num_pipes; 2227 2228 adev->gfx.config.gb_addr_config_fields.num_banks = 1 << 2229 REG_GET_FIELD( 2230 adev->gfx.config.gb_addr_config, 2231 GB_ADDR_CONFIG, 2232 NUM_BANKS); 2233 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << 2234 REG_GET_FIELD( 2235 adev->gfx.config.gb_addr_config, 2236 GB_ADDR_CONFIG, 2237 MAX_COMPRESSED_FRAGS); 2238 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << 2239 REG_GET_FIELD( 2240 adev->gfx.config.gb_addr_config, 2241 GB_ADDR_CONFIG, 2242 NUM_RB_PER_SE); 2243 adev->gfx.config.gb_addr_config_fields.num_se = 1 << 2244 REG_GET_FIELD( 2245 adev->gfx.config.gb_addr_config, 2246 GB_ADDR_CONFIG, 2247 NUM_SHADER_ENGINES); 2248 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + 2249 REG_GET_FIELD( 2250 adev->gfx.config.gb_addr_config, 2251 GB_ADDR_CONFIG, 2252 PIPE_INTERLEAVE_SIZE)); 2253 2254 return 0; 2255 } 2256 2257 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 2258 int mec, int pipe, int queue) 2259 { 2260 unsigned irq_type; 2261 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; 2262 unsigned int hw_prio; 2263 2264 ring = &adev->gfx.compute_ring[ring_id]; 2265 2266 /* mec0 is me1 */ 2267 ring->me = mec + 1; 2268 ring->pipe = pipe; 2269 ring->queue = queue; 2270 2271 ring->ring_obj = NULL; 2272 ring->use_doorbell = true; 2273 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1; 2274 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 2275 + (ring_id * GFX9_MEC_HPD_SIZE); 2276 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 2277 2278 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 2279 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 2280 + ring->pipe; 2281 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? 2282 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 2283 /* type-2 packets are deprecated on MEC, use type-3 instead */ 2284 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 2285 hw_prio, NULL); 2286 } 2287 2288 static int gfx_v9_0_sw_init(void *handle) 2289 { 2290 int i, j, k, r, ring_id; 2291 struct amdgpu_ring *ring; 2292 struct amdgpu_kiq *kiq; 2293 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2294 2295 switch (adev->asic_type) { 2296 case CHIP_VEGA10: 2297 case CHIP_VEGA12: 2298 case CHIP_VEGA20: 2299 case CHIP_RAVEN: 2300 case CHIP_ARCTURUS: 2301 case CHIP_RENOIR: 2302 case CHIP_ALDEBARAN: 2303 adev->gfx.mec.num_mec = 2; 2304 break; 2305 default: 2306 adev->gfx.mec.num_mec = 1; 2307 break; 2308 } 2309 2310 adev->gfx.mec.num_pipe_per_mec = 4; 2311 adev->gfx.mec.num_queue_per_pipe = 8; 2312 2313 /* EOP Event */ 2314 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq); 2315 if (r) 2316 return r; 2317 2318 /* Privileged reg */ 2319 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT, 2320 &adev->gfx.priv_reg_irq); 2321 if (r) 2322 return r; 2323 2324 /* Privileged inst */ 2325 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT, 2326 &adev->gfx.priv_inst_irq); 2327 if (r) 2328 return r; 2329 2330 /* ECC error */ 2331 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR, 2332 &adev->gfx.cp_ecc_error_irq); 2333 if (r) 2334 return r; 2335 2336 /* FUE error */ 2337 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR, 2338 &adev->gfx.cp_ecc_error_irq); 2339 if (r) 2340 return r; 2341 2342 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 2343 2344 gfx_v9_0_scratch_init(adev); 2345 2346 r = gfx_v9_0_init_microcode(adev); 2347 if (r) { 2348 DRM_ERROR("Failed to load gfx firmware!\n"); 2349 return r; 2350 } 2351 2352 r = adev->gfx.rlc.funcs->init(adev); 2353 if (r) { 2354 DRM_ERROR("Failed to init rlc BOs!\n"); 2355 return r; 2356 } 2357 2358 r = gfx_v9_0_mec_init(adev); 2359 if (r) { 2360 DRM_ERROR("Failed to init MEC BOs!\n"); 2361 return r; 2362 } 2363 2364 /* set up the gfx ring */ 2365 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 2366 ring = &adev->gfx.gfx_ring[i]; 2367 ring->ring_obj = NULL; 2368 if (!i) 2369 sprintf(ring->name, "gfx"); 2370 else 2371 sprintf(ring->name, "gfx_%d", i); 2372 ring->use_doorbell = true; 2373 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 2374 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, 2375 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, 2376 AMDGPU_RING_PRIO_DEFAULT, NULL); 2377 if (r) 2378 return r; 2379 } 2380 2381 /* set up the compute queues - allocate horizontally across pipes */ 2382 ring_id = 0; 2383 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 2384 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 2385 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 2386 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j)) 2387 continue; 2388 2389 r = gfx_v9_0_compute_ring_init(adev, 2390 ring_id, 2391 i, k, j); 2392 if (r) 2393 return r; 2394 2395 ring_id++; 2396 } 2397 } 2398 } 2399 2400 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE); 2401 if (r) { 2402 DRM_ERROR("Failed to init KIQ BOs!\n"); 2403 return r; 2404 } 2405 2406 kiq = &adev->gfx.kiq; 2407 r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq); 2408 if (r) 2409 return r; 2410 2411 /* create MQD for all compute queues as wel as KIQ for SRIOV case */ 2412 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation)); 2413 if (r) 2414 return r; 2415 2416 adev->gfx.ce_ram_size = 0x8000; 2417 2418 r = gfx_v9_0_gpu_early_init(adev); 2419 if (r) 2420 return r; 2421 2422 return 0; 2423 } 2424 2425 2426 static int gfx_v9_0_sw_fini(void *handle) 2427 { 2428 int i; 2429 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2430 2431 if (adev->gfx.ras_funcs && 2432 adev->gfx.ras_funcs->ras_fini) 2433 adev->gfx.ras_funcs->ras_fini(adev); 2434 2435 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 2436 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 2437 for (i = 0; i < adev->gfx.num_compute_rings; i++) 2438 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 2439 2440 amdgpu_gfx_mqd_sw_fini(adev); 2441 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring); 2442 amdgpu_gfx_kiq_fini(adev); 2443 2444 gfx_v9_0_mec_fini(adev); 2445 amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj); 2446 if (adev->flags & AMD_IS_APU) { 2447 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 2448 &adev->gfx.rlc.cp_table_gpu_addr, 2449 (void **)&adev->gfx.rlc.cp_table_ptr); 2450 } 2451 gfx_v9_0_free_microcode(adev); 2452 2453 return 0; 2454 } 2455 2456 2457 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev) 2458 { 2459 /* TODO */ 2460 } 2461 2462 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, 2463 u32 instance) 2464 { 2465 u32 data; 2466 2467 if (instance == 0xffffffff) 2468 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); 2469 else 2470 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance); 2471 2472 if (se_num == 0xffffffff) 2473 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); 2474 else 2475 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 2476 2477 if (sh_num == 0xffffffff) 2478 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1); 2479 else 2480 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num); 2481 2482 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data); 2483 } 2484 2485 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev) 2486 { 2487 u32 data, mask; 2488 2489 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE); 2490 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE); 2491 2492 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK; 2493 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT; 2494 2495 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / 2496 adev->gfx.config.max_sh_per_se); 2497 2498 return (~data) & mask; 2499 } 2500 2501 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev) 2502 { 2503 int i, j; 2504 u32 data; 2505 u32 active_rbs = 0; 2506 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / 2507 adev->gfx.config.max_sh_per_se; 2508 2509 mutex_lock(&adev->grbm_idx_mutex); 2510 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2511 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2512 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2513 data = gfx_v9_0_get_rb_active_bitmap(adev); 2514 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * 2515 rb_bitmap_width_per_sh); 2516 } 2517 } 2518 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2519 mutex_unlock(&adev->grbm_idx_mutex); 2520 2521 adev->gfx.config.backend_enable_mask = active_rbs; 2522 adev->gfx.config.num_rbs = hweight32(active_rbs); 2523 } 2524 2525 #define DEFAULT_SH_MEM_BASES (0x6000) 2526 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev) 2527 { 2528 int i; 2529 uint32_t sh_mem_config; 2530 uint32_t sh_mem_bases; 2531 2532 /* 2533 * Configure apertures: 2534 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 2535 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 2536 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 2537 */ 2538 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16); 2539 2540 sh_mem_config = SH_MEM_ADDRESS_MODE_64 | 2541 SH_MEM_ALIGNMENT_MODE_UNALIGNED << 2542 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT; 2543 2544 mutex_lock(&adev->srbm_mutex); 2545 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2546 soc15_grbm_select(adev, 0, 0, 0, i); 2547 /* CP and shaders */ 2548 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config); 2549 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases); 2550 } 2551 soc15_grbm_select(adev, 0, 0, 0, 0); 2552 mutex_unlock(&adev->srbm_mutex); 2553 2554 /* Initialize all compute VMIDs to have no GDS, GWS, or OA 2555 acccess. These should be enabled by FW for target VMIDs. */ 2556 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2557 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0); 2558 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0); 2559 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0); 2560 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0); 2561 } 2562 } 2563 2564 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev) 2565 { 2566 int vmid; 2567 2568 /* 2569 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 2570 * access. Compute VMIDs should be enabled by FW for target VMIDs, 2571 * the driver can enable them for graphics. VMID0 should maintain 2572 * access so that HWS firmware can save/restore entries. 2573 */ 2574 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { 2575 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0); 2576 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0); 2577 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0); 2578 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0); 2579 } 2580 } 2581 2582 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev) 2583 { 2584 uint32_t tmp; 2585 2586 switch (adev->asic_type) { 2587 case CHIP_ARCTURUS: 2588 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG); 2589 tmp = REG_SET_FIELD(tmp, SQ_CONFIG, 2590 DISABLE_BARRIER_WAITCNT, 1); 2591 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp); 2592 break; 2593 default: 2594 break; 2595 } 2596 } 2597 2598 static void gfx_v9_0_constants_init(struct amdgpu_device *adev) 2599 { 2600 u32 tmp; 2601 int i; 2602 2603 WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); 2604 2605 gfx_v9_0_tiling_mode_table_init(adev); 2606 2607 gfx_v9_0_setup_rb(adev); 2608 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info); 2609 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2); 2610 2611 /* XXX SH_MEM regs */ 2612 /* where to put LDS, scratch, GPUVM in FSA64 space */ 2613 mutex_lock(&adev->srbm_mutex); 2614 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) { 2615 soc15_grbm_select(adev, 0, 0, 0, i); 2616 /* CP and shaders */ 2617 if (i == 0) { 2618 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2619 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2620 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2621 !!adev->gmc.noretry); 2622 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2623 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0); 2624 } else { 2625 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2626 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2627 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2628 !!adev->gmc.noretry); 2629 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2630 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 2631 (adev->gmc.private_aperture_start >> 48)); 2632 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE, 2633 (adev->gmc.shared_aperture_start >> 48)); 2634 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp); 2635 } 2636 } 2637 soc15_grbm_select(adev, 0, 0, 0, 0); 2638 2639 mutex_unlock(&adev->srbm_mutex); 2640 2641 gfx_v9_0_init_compute_vmid(adev); 2642 gfx_v9_0_init_gds_vmid(adev); 2643 gfx_v9_0_init_sq_config(adev); 2644 } 2645 2646 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev) 2647 { 2648 u32 i, j, k; 2649 u32 mask; 2650 2651 mutex_lock(&adev->grbm_idx_mutex); 2652 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2653 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2654 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2655 for (k = 0; k < adev->usec_timeout; k++) { 2656 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0) 2657 break; 2658 udelay(1); 2659 } 2660 if (k == adev->usec_timeout) { 2661 gfx_v9_0_select_se_sh(adev, 0xffffffff, 2662 0xffffffff, 0xffffffff); 2663 mutex_unlock(&adev->grbm_idx_mutex); 2664 DRM_INFO("Timeout wait for RLC serdes %u,%u\n", 2665 i, j); 2666 return; 2667 } 2668 } 2669 } 2670 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2671 mutex_unlock(&adev->grbm_idx_mutex); 2672 2673 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | 2674 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK | 2675 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK | 2676 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK; 2677 for (k = 0; k < adev->usec_timeout; k++) { 2678 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 2679 break; 2680 udelay(1); 2681 } 2682 } 2683 2684 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 2685 bool enable) 2686 { 2687 u32 tmp; 2688 2689 /* These interrupts should be enabled to drive DS clock */ 2690 2691 tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0); 2692 2693 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0); 2694 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0); 2695 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0); 2696 if(adev->gfx.num_gfx_rings) 2697 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0); 2698 2699 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp); 2700 } 2701 2702 static void gfx_v9_0_init_csb(struct amdgpu_device *adev) 2703 { 2704 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 2705 /* csib */ 2706 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI), 2707 adev->gfx.rlc.clear_state_gpu_addr >> 32); 2708 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO), 2709 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 2710 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH), 2711 adev->gfx.rlc.clear_state_size); 2712 } 2713 2714 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format, 2715 int indirect_offset, 2716 int list_size, 2717 int *unique_indirect_regs, 2718 int unique_indirect_reg_count, 2719 int *indirect_start_offsets, 2720 int *indirect_start_offsets_count, 2721 int max_start_offsets_count) 2722 { 2723 int idx; 2724 2725 for (; indirect_offset < list_size; indirect_offset++) { 2726 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count); 2727 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset; 2728 *indirect_start_offsets_count = *indirect_start_offsets_count + 1; 2729 2730 while (register_list_format[indirect_offset] != 0xFFFFFFFF) { 2731 indirect_offset += 2; 2732 2733 /* look for the matching indice */ 2734 for (idx = 0; idx < unique_indirect_reg_count; idx++) { 2735 if (unique_indirect_regs[idx] == 2736 register_list_format[indirect_offset] || 2737 !unique_indirect_regs[idx]) 2738 break; 2739 } 2740 2741 BUG_ON(idx >= unique_indirect_reg_count); 2742 2743 if (!unique_indirect_regs[idx]) 2744 unique_indirect_regs[idx] = register_list_format[indirect_offset]; 2745 2746 indirect_offset++; 2747 } 2748 } 2749 } 2750 2751 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev) 2752 { 2753 int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2754 int unique_indirect_reg_count = 0; 2755 2756 int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2757 int indirect_start_offsets_count = 0; 2758 2759 int list_size = 0; 2760 int i = 0, j = 0; 2761 u32 tmp = 0; 2762 2763 u32 *register_list_format = 2764 kmemdup(adev->gfx.rlc.register_list_format, 2765 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); 2766 if (!register_list_format) 2767 return -ENOMEM; 2768 2769 /* setup unique_indirect_regs array and indirect_start_offsets array */ 2770 unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs); 2771 gfx_v9_1_parse_ind_reg_list(register_list_format, 2772 adev->gfx.rlc.reg_list_format_direct_reg_list_length, 2773 adev->gfx.rlc.reg_list_format_size_bytes >> 2, 2774 unique_indirect_regs, 2775 unique_indirect_reg_count, 2776 indirect_start_offsets, 2777 &indirect_start_offsets_count, 2778 ARRAY_SIZE(indirect_start_offsets)); 2779 2780 /* enable auto inc in case it is disabled */ 2781 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL)); 2782 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK; 2783 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp); 2784 2785 /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */ 2786 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR), 2787 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET); 2788 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) 2789 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA), 2790 adev->gfx.rlc.register_restore[i]); 2791 2792 /* load indirect register */ 2793 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2794 adev->gfx.rlc.reg_list_format_start); 2795 2796 /* direct register portion */ 2797 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++) 2798 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2799 register_list_format[i]); 2800 2801 /* indirect register portion */ 2802 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) { 2803 if (register_list_format[i] == 0xFFFFFFFF) { 2804 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2805 continue; 2806 } 2807 2808 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2809 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2810 2811 for (j = 0; j < unique_indirect_reg_count; j++) { 2812 if (register_list_format[i] == unique_indirect_regs[j]) { 2813 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j); 2814 break; 2815 } 2816 } 2817 2818 BUG_ON(j >= unique_indirect_reg_count); 2819 2820 i++; 2821 } 2822 2823 /* set save/restore list size */ 2824 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; 2825 list_size = list_size >> 1; 2826 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2827 adev->gfx.rlc.reg_restore_list_size); 2828 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size); 2829 2830 /* write the starting offsets to RLC scratch ram */ 2831 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2832 adev->gfx.rlc.starting_offsets_start); 2833 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++) 2834 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2835 indirect_start_offsets[i]); 2836 2837 /* load unique indirect regs*/ 2838 for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) { 2839 if (unique_indirect_regs[i] != 0) { 2840 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0) 2841 + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i], 2842 unique_indirect_regs[i] & 0x3FFFF); 2843 2844 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0) 2845 + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i], 2846 unique_indirect_regs[i] >> 20); 2847 } 2848 } 2849 2850 kfree(register_list_format); 2851 return 0; 2852 } 2853 2854 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev) 2855 { 2856 WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1); 2857 } 2858 2859 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev, 2860 bool enable) 2861 { 2862 uint32_t data = 0; 2863 uint32_t default_data = 0; 2864 2865 default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS)); 2866 if (enable) { 2867 /* enable GFXIP control over CGPG */ 2868 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2869 if(default_data != data) 2870 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2871 2872 /* update status */ 2873 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK; 2874 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT); 2875 if(default_data != data) 2876 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2877 } else { 2878 /* restore GFXIP control over GCPG */ 2879 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2880 if(default_data != data) 2881 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2882 } 2883 } 2884 2885 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev) 2886 { 2887 uint32_t data = 0; 2888 2889 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2890 AMD_PG_SUPPORT_GFX_SMG | 2891 AMD_PG_SUPPORT_GFX_DMG)) { 2892 /* init IDLE_POLL_COUNT = 60 */ 2893 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL)); 2894 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK; 2895 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 2896 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data); 2897 2898 /* init RLC PG Delay */ 2899 data = 0; 2900 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT); 2901 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT); 2902 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT); 2903 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT); 2904 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data); 2905 2906 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2)); 2907 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK; 2908 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT); 2909 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data); 2910 2911 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3)); 2912 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK; 2913 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT); 2914 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data); 2915 2916 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL)); 2917 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK; 2918 2919 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */ 2920 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT); 2921 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data); 2922 if (adev->asic_type != CHIP_RENOIR) 2923 pwr_10_0_gfxip_control_over_cgpg(adev, true); 2924 } 2925 } 2926 2927 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev, 2928 bool enable) 2929 { 2930 uint32_t data = 0; 2931 uint32_t default_data = 0; 2932 2933 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2934 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2935 SMU_CLK_SLOWDOWN_ON_PU_ENABLE, 2936 enable ? 1 : 0); 2937 if (default_data != data) 2938 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2939 } 2940 2941 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev, 2942 bool enable) 2943 { 2944 uint32_t data = 0; 2945 uint32_t default_data = 0; 2946 2947 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2948 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2949 SMU_CLK_SLOWDOWN_ON_PD_ENABLE, 2950 enable ? 1 : 0); 2951 if(default_data != data) 2952 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2953 } 2954 2955 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev, 2956 bool enable) 2957 { 2958 uint32_t data = 0; 2959 uint32_t default_data = 0; 2960 2961 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2962 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2963 CP_PG_DISABLE, 2964 enable ? 0 : 1); 2965 if(default_data != data) 2966 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2967 } 2968 2969 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev, 2970 bool enable) 2971 { 2972 uint32_t data, default_data; 2973 2974 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2975 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2976 GFX_POWER_GATING_ENABLE, 2977 enable ? 1 : 0); 2978 if(default_data != data) 2979 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2980 } 2981 2982 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev, 2983 bool enable) 2984 { 2985 uint32_t data, default_data; 2986 2987 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2988 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2989 GFX_PIPELINE_PG_ENABLE, 2990 enable ? 1 : 0); 2991 if(default_data != data) 2992 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2993 2994 if (!enable) 2995 /* read any GFX register to wake up GFX */ 2996 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL)); 2997 } 2998 2999 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, 3000 bool enable) 3001 { 3002 uint32_t data, default_data; 3003 3004 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 3005 data = REG_SET_FIELD(data, RLC_PG_CNTL, 3006 STATIC_PER_CU_PG_ENABLE, 3007 enable ? 1 : 0); 3008 if(default_data != data) 3009 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3010 } 3011 3012 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev, 3013 bool enable) 3014 { 3015 uint32_t data, default_data; 3016 3017 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 3018 data = REG_SET_FIELD(data, RLC_PG_CNTL, 3019 DYN_PER_CU_PG_ENABLE, 3020 enable ? 1 : 0); 3021 if(default_data != data) 3022 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3023 } 3024 3025 static void gfx_v9_0_init_pg(struct amdgpu_device *adev) 3026 { 3027 gfx_v9_0_init_csb(adev); 3028 3029 /* 3030 * Rlc save restore list is workable since v2_1. 3031 * And it's needed by gfxoff feature. 3032 */ 3033 if (adev->gfx.rlc.is_rlc_v2_1) { 3034 if (adev->asic_type == CHIP_VEGA12 || 3035 (adev->apu_flags & AMD_APU_IS_RAVEN2)) 3036 gfx_v9_1_init_rlc_save_restore_list(adev); 3037 gfx_v9_0_enable_save_restore_machine(adev); 3038 } 3039 3040 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 3041 AMD_PG_SUPPORT_GFX_SMG | 3042 AMD_PG_SUPPORT_GFX_DMG | 3043 AMD_PG_SUPPORT_CP | 3044 AMD_PG_SUPPORT_GDS | 3045 AMD_PG_SUPPORT_RLC_SMU_HS)) { 3046 WREG32(mmRLC_JUMP_TABLE_RESTORE, 3047 adev->gfx.rlc.cp_table_gpu_addr >> 8); 3048 gfx_v9_0_init_gfx_power_gating(adev); 3049 } 3050 } 3051 3052 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev) 3053 { 3054 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0); 3055 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3056 gfx_v9_0_wait_for_rlc_serdes(adev); 3057 } 3058 3059 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev) 3060 { 3061 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 3062 udelay(50); 3063 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 3064 udelay(50); 3065 } 3066 3067 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev) 3068 { 3069 #ifdef AMDGPU_RLC_DEBUG_RETRY 3070 u32 rlc_ucode_ver; 3071 #endif 3072 3073 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1); 3074 udelay(50); 3075 3076 /* carrizo do enable cp interrupt after cp inited */ 3077 if (!(adev->flags & AMD_IS_APU)) { 3078 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3079 udelay(50); 3080 } 3081 3082 #ifdef AMDGPU_RLC_DEBUG_RETRY 3083 /* RLC_GPM_GENERAL_6 : RLC Ucode version */ 3084 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6); 3085 if(rlc_ucode_ver == 0x108) { 3086 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n", 3087 rlc_ucode_ver, adev->gfx.rlc_fw_version); 3088 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles, 3089 * default is 0x9C4 to create a 100us interval */ 3090 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4); 3091 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr 3092 * to disable the page fault retry interrupts, default is 3093 * 0x100 (256) */ 3094 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100); 3095 } 3096 #endif 3097 } 3098 3099 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev) 3100 { 3101 const struct rlc_firmware_header_v2_0 *hdr; 3102 const __le32 *fw_data; 3103 unsigned i, fw_size; 3104 3105 if (!adev->gfx.rlc_fw) 3106 return -EINVAL; 3107 3108 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 3109 amdgpu_ucode_print_rlc_hdr(&hdr->header); 3110 3111 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 3112 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 3113 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 3114 3115 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, 3116 RLCG_UCODE_LOADING_START_ADDRESS); 3117 for (i = 0; i < fw_size; i++) 3118 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); 3119 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); 3120 3121 return 0; 3122 } 3123 3124 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev) 3125 { 3126 int r; 3127 3128 if (amdgpu_sriov_vf(adev)) { 3129 gfx_v9_0_init_csb(adev); 3130 return 0; 3131 } 3132 3133 adev->gfx.rlc.funcs->stop(adev); 3134 3135 /* disable CG */ 3136 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0); 3137 3138 gfx_v9_0_init_pg(adev); 3139 3140 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3141 /* legacy rlc firmware loading */ 3142 r = gfx_v9_0_rlc_load_microcode(adev); 3143 if (r) 3144 return r; 3145 } 3146 3147 switch (adev->asic_type) { 3148 case CHIP_RAVEN: 3149 if (amdgpu_lbpw == 0) 3150 gfx_v9_0_enable_lbpw(adev, false); 3151 else 3152 gfx_v9_0_enable_lbpw(adev, true); 3153 break; 3154 case CHIP_VEGA20: 3155 if (amdgpu_lbpw > 0) 3156 gfx_v9_0_enable_lbpw(adev, true); 3157 else 3158 gfx_v9_0_enable_lbpw(adev, false); 3159 break; 3160 default: 3161 break; 3162 } 3163 3164 adev->gfx.rlc.funcs->start(adev); 3165 3166 return 0; 3167 } 3168 3169 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 3170 { 3171 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL); 3172 3173 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); 3174 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); 3175 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1); 3176 WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp); 3177 udelay(50); 3178 } 3179 3180 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev) 3181 { 3182 const struct gfx_firmware_header_v1_0 *pfp_hdr; 3183 const struct gfx_firmware_header_v1_0 *ce_hdr; 3184 const struct gfx_firmware_header_v1_0 *me_hdr; 3185 const __le32 *fw_data; 3186 unsigned i, fw_size; 3187 3188 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw) 3189 return -EINVAL; 3190 3191 pfp_hdr = (const struct gfx_firmware_header_v1_0 *) 3192 adev->gfx.pfp_fw->data; 3193 ce_hdr = (const struct gfx_firmware_header_v1_0 *) 3194 adev->gfx.ce_fw->data; 3195 me_hdr = (const struct gfx_firmware_header_v1_0 *) 3196 adev->gfx.me_fw->data; 3197 3198 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3199 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header); 3200 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3201 3202 gfx_v9_0_cp_gfx_enable(adev, false); 3203 3204 /* PFP */ 3205 fw_data = (const __le32 *) 3206 (adev->gfx.pfp_fw->data + 3207 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 3208 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 3209 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0); 3210 for (i = 0; i < fw_size; i++) 3211 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 3212 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); 3213 3214 /* CE */ 3215 fw_data = (const __le32 *) 3216 (adev->gfx.ce_fw->data + 3217 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 3218 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 3219 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0); 3220 for (i = 0; i < fw_size; i++) 3221 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 3222 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version); 3223 3224 /* ME */ 3225 fw_data = (const __le32 *) 3226 (adev->gfx.me_fw->data + 3227 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 3228 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 3229 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0); 3230 for (i = 0; i < fw_size; i++) 3231 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 3232 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version); 3233 3234 return 0; 3235 } 3236 3237 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev) 3238 { 3239 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; 3240 const struct cs_section_def *sect = NULL; 3241 const struct cs_extent_def *ext = NULL; 3242 int r, i, tmp; 3243 3244 /* init the CP */ 3245 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); 3246 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1); 3247 3248 gfx_v9_0_cp_gfx_enable(adev, true); 3249 3250 r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3); 3251 if (r) { 3252 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r); 3253 return r; 3254 } 3255 3256 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3257 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3258 3259 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 3260 amdgpu_ring_write(ring, 0x80000000); 3261 amdgpu_ring_write(ring, 0x80000000); 3262 3263 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 3264 for (ext = sect->section; ext->extent != NULL; ++ext) { 3265 if (sect->id == SECT_CONTEXT) { 3266 amdgpu_ring_write(ring, 3267 PACKET3(PACKET3_SET_CONTEXT_REG, 3268 ext->reg_count)); 3269 amdgpu_ring_write(ring, 3270 ext->reg_index - PACKET3_SET_CONTEXT_REG_START); 3271 for (i = 0; i < ext->reg_count; i++) 3272 amdgpu_ring_write(ring, ext->extent[i]); 3273 } 3274 } 3275 } 3276 3277 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3278 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3279 3280 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3281 amdgpu_ring_write(ring, 0); 3282 3283 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 3284 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 3285 amdgpu_ring_write(ring, 0x8000); 3286 amdgpu_ring_write(ring, 0x8000); 3287 3288 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1)); 3289 tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE | 3290 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START)); 3291 amdgpu_ring_write(ring, tmp); 3292 amdgpu_ring_write(ring, 0); 3293 3294 amdgpu_ring_commit(ring); 3295 3296 return 0; 3297 } 3298 3299 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev) 3300 { 3301 struct amdgpu_ring *ring; 3302 u32 tmp; 3303 u32 rb_bufsz; 3304 u64 rb_addr, rptr_addr, wptr_gpu_addr; 3305 3306 /* Set the write pointer delay */ 3307 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0); 3308 3309 /* set the RB to use vmid 0 */ 3310 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0); 3311 3312 /* Set ring buffer size */ 3313 ring = &adev->gfx.gfx_ring[0]; 3314 rb_bufsz = order_base_2(ring->ring_size / 8); 3315 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 3316 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 3317 #ifdef __BIG_ENDIAN 3318 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1); 3319 #endif 3320 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3321 3322 /* Initialize the ring buffer's write pointers */ 3323 ring->wptr = 0; 3324 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 3325 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 3326 3327 /* set the wb address wether it's enabled or not */ 3328 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3329 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 3330 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3331 3332 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3333 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr)); 3334 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr)); 3335 3336 mdelay(1); 3337 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3338 3339 rb_addr = ring->gpu_addr >> 8; 3340 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr); 3341 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 3342 3343 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL); 3344 if (ring->use_doorbell) { 3345 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3346 DOORBELL_OFFSET, ring->doorbell_index); 3347 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3348 DOORBELL_EN, 1); 3349 } else { 3350 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0); 3351 } 3352 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp); 3353 3354 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 3355 DOORBELL_RANGE_LOWER, ring->doorbell_index); 3356 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp); 3357 3358 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER, 3359 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 3360 3361 3362 /* start the ring */ 3363 gfx_v9_0_cp_gfx_start(adev); 3364 ring->sched.ready = true; 3365 3366 return 0; 3367 } 3368 3369 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 3370 { 3371 if (enable) { 3372 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0); 3373 } else { 3374 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 3375 (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK)); 3376 adev->gfx.kiq.ring.sched.ready = false; 3377 } 3378 udelay(50); 3379 } 3380 3381 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev) 3382 { 3383 const struct gfx_firmware_header_v1_0 *mec_hdr; 3384 const __le32 *fw_data; 3385 unsigned i; 3386 u32 tmp; 3387 3388 if (!adev->gfx.mec_fw) 3389 return -EINVAL; 3390 3391 gfx_v9_0_cp_compute_enable(adev, false); 3392 3393 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 3394 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3395 3396 fw_data = (const __le32 *) 3397 (adev->gfx.mec_fw->data + 3398 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 3399 tmp = 0; 3400 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 3401 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 3402 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp); 3403 3404 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO, 3405 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000); 3406 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI, 3407 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 3408 3409 /* MEC1 */ 3410 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3411 mec_hdr->jt_offset); 3412 for (i = 0; i < mec_hdr->jt_size; i++) 3413 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA, 3414 le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); 3415 3416 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3417 adev->gfx.mec_fw_version); 3418 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */ 3419 3420 return 0; 3421 } 3422 3423 /* KIQ functions */ 3424 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring) 3425 { 3426 uint32_t tmp; 3427 struct amdgpu_device *adev = ring->adev; 3428 3429 /* tell RLC which is KIQ queue */ 3430 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS); 3431 tmp &= 0xffffff00; 3432 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 3433 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3434 tmp |= 0x80; 3435 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3436 } 3437 3438 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd) 3439 { 3440 struct amdgpu_device *adev = ring->adev; 3441 3442 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 3443 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) { 3444 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; 3445 mqd->cp_hqd_queue_priority = 3446 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; 3447 } 3448 } 3449 } 3450 3451 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring) 3452 { 3453 struct amdgpu_device *adev = ring->adev; 3454 struct v9_mqd *mqd = ring->mqd_ptr; 3455 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 3456 uint32_t tmp; 3457 3458 mqd->header = 0xC0310800; 3459 mqd->compute_pipelinestat_enable = 0x00000001; 3460 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 3461 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 3462 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 3463 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 3464 mqd->compute_static_thread_mgmt_se4 = 0xffffffff; 3465 mqd->compute_static_thread_mgmt_se5 = 0xffffffff; 3466 mqd->compute_static_thread_mgmt_se6 = 0xffffffff; 3467 mqd->compute_static_thread_mgmt_se7 = 0xffffffff; 3468 mqd->compute_misc_reserved = 0x00000003; 3469 3470 mqd->dynamic_cu_mask_addr_lo = 3471 lower_32_bits(ring->mqd_gpu_addr 3472 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3473 mqd->dynamic_cu_mask_addr_hi = 3474 upper_32_bits(ring->mqd_gpu_addr 3475 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3476 3477 eop_base_addr = ring->eop_gpu_addr >> 8; 3478 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 3479 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 3480 3481 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3482 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL); 3483 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 3484 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1)); 3485 3486 mqd->cp_hqd_eop_control = tmp; 3487 3488 /* enable doorbell? */ 3489 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3490 3491 if (ring->use_doorbell) { 3492 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3493 DOORBELL_OFFSET, ring->doorbell_index); 3494 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3495 DOORBELL_EN, 1); 3496 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3497 DOORBELL_SOURCE, 0); 3498 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3499 DOORBELL_HIT, 0); 3500 } else { 3501 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3502 DOORBELL_EN, 0); 3503 } 3504 3505 mqd->cp_hqd_pq_doorbell_control = tmp; 3506 3507 /* disable the queue if it's active */ 3508 ring->wptr = 0; 3509 mqd->cp_hqd_dequeue_request = 0; 3510 mqd->cp_hqd_pq_rptr = 0; 3511 mqd->cp_hqd_pq_wptr_lo = 0; 3512 mqd->cp_hqd_pq_wptr_hi = 0; 3513 3514 /* set the pointer to the MQD */ 3515 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; 3516 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); 3517 3518 /* set MQD vmid to 0 */ 3519 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL); 3520 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 3521 mqd->cp_mqd_control = tmp; 3522 3523 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3524 hqd_gpu_addr = ring->gpu_addr >> 8; 3525 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 3526 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 3527 3528 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3529 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL); 3530 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 3531 (order_base_2(ring->ring_size / 4) - 1)); 3532 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 3533 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8)); 3534 #ifdef __BIG_ENDIAN 3535 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1); 3536 #endif 3537 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0); 3538 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0); 3539 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 3540 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 3541 mqd->cp_hqd_pq_control = tmp; 3542 3543 /* set the wb address whether it's enabled or not */ 3544 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3545 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 3546 mqd->cp_hqd_pq_rptr_report_addr_hi = 3547 upper_32_bits(wb_gpu_addr) & 0xffff; 3548 3549 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3550 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3551 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 3552 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 3553 3554 tmp = 0; 3555 /* enable the doorbell if requested */ 3556 if (ring->use_doorbell) { 3557 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3558 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3559 DOORBELL_OFFSET, ring->doorbell_index); 3560 3561 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3562 DOORBELL_EN, 1); 3563 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3564 DOORBELL_SOURCE, 0); 3565 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3566 DOORBELL_HIT, 0); 3567 } 3568 3569 mqd->cp_hqd_pq_doorbell_control = tmp; 3570 3571 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3572 ring->wptr = 0; 3573 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR); 3574 3575 /* set the vmid for the queue */ 3576 mqd->cp_hqd_vmid = 0; 3577 3578 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE); 3579 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53); 3580 mqd->cp_hqd_persistent_state = tmp; 3581 3582 /* set MIN_IB_AVAIL_SIZE */ 3583 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL); 3584 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 3585 mqd->cp_hqd_ib_control = tmp; 3586 3587 /* set static priority for a queue/ring */ 3588 gfx_v9_0_mqd_set_priority(ring, mqd); 3589 mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM); 3590 3591 /* map_queues packet doesn't need activate the queue, 3592 * so only kiq need set this field. 3593 */ 3594 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 3595 mqd->cp_hqd_active = 1; 3596 3597 return 0; 3598 } 3599 3600 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring) 3601 { 3602 struct amdgpu_device *adev = ring->adev; 3603 struct v9_mqd *mqd = ring->mqd_ptr; 3604 int j; 3605 3606 /* disable wptr polling */ 3607 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3608 3609 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR, 3610 mqd->cp_hqd_eop_base_addr_lo); 3611 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI, 3612 mqd->cp_hqd_eop_base_addr_hi); 3613 3614 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3615 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL, 3616 mqd->cp_hqd_eop_control); 3617 3618 /* enable doorbell? */ 3619 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3620 mqd->cp_hqd_pq_doorbell_control); 3621 3622 /* disable the queue if it's active */ 3623 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3624 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3625 for (j = 0; j < adev->usec_timeout; j++) { 3626 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3627 break; 3628 udelay(1); 3629 } 3630 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3631 mqd->cp_hqd_dequeue_request); 3632 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 3633 mqd->cp_hqd_pq_rptr); 3634 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3635 mqd->cp_hqd_pq_wptr_lo); 3636 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3637 mqd->cp_hqd_pq_wptr_hi); 3638 } 3639 3640 /* set the pointer to the MQD */ 3641 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR, 3642 mqd->cp_mqd_base_addr_lo); 3643 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI, 3644 mqd->cp_mqd_base_addr_hi); 3645 3646 /* set MQD vmid to 0 */ 3647 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL, 3648 mqd->cp_mqd_control); 3649 3650 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3651 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE, 3652 mqd->cp_hqd_pq_base_lo); 3653 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI, 3654 mqd->cp_hqd_pq_base_hi); 3655 3656 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3657 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL, 3658 mqd->cp_hqd_pq_control); 3659 3660 /* set the wb address whether it's enabled or not */ 3661 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR, 3662 mqd->cp_hqd_pq_rptr_report_addr_lo); 3663 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 3664 mqd->cp_hqd_pq_rptr_report_addr_hi); 3665 3666 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3667 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR, 3668 mqd->cp_hqd_pq_wptr_poll_addr_lo); 3669 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI, 3670 mqd->cp_hqd_pq_wptr_poll_addr_hi); 3671 3672 /* enable the doorbell if requested */ 3673 if (ring->use_doorbell) { 3674 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER, 3675 (adev->doorbell_index.kiq * 2) << 2); 3676 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3677 (adev->doorbell_index.userqueue_end * 2) << 2); 3678 } 3679 3680 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3681 mqd->cp_hqd_pq_doorbell_control); 3682 3683 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3684 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3685 mqd->cp_hqd_pq_wptr_lo); 3686 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3687 mqd->cp_hqd_pq_wptr_hi); 3688 3689 /* set the vmid for the queue */ 3690 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid); 3691 3692 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 3693 mqd->cp_hqd_persistent_state); 3694 3695 /* activate the queue */ 3696 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 3697 mqd->cp_hqd_active); 3698 3699 if (ring->use_doorbell) 3700 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1); 3701 3702 return 0; 3703 } 3704 3705 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring) 3706 { 3707 struct amdgpu_device *adev = ring->adev; 3708 int j; 3709 3710 /* disable the queue if it's active */ 3711 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3712 3713 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3714 3715 for (j = 0; j < adev->usec_timeout; j++) { 3716 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3717 break; 3718 udelay(1); 3719 } 3720 3721 if (j == AMDGPU_MAX_USEC_TIMEOUT) { 3722 DRM_DEBUG("KIQ dequeue request failed.\n"); 3723 3724 /* Manual disable if dequeue request times out */ 3725 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0); 3726 } 3727 3728 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3729 0); 3730 } 3731 3732 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0); 3733 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0); 3734 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0); 3735 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000); 3736 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0); 3737 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0); 3738 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0); 3739 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0); 3740 3741 return 0; 3742 } 3743 3744 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring) 3745 { 3746 struct amdgpu_device *adev = ring->adev; 3747 struct v9_mqd *mqd = ring->mqd_ptr; 3748 int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS; 3749 struct v9_mqd *tmp_mqd; 3750 3751 gfx_v9_0_kiq_setting(ring); 3752 3753 /* GPU could be in bad state during probe, driver trigger the reset 3754 * after load the SMU, in this case , the mqd is not be initialized. 3755 * driver need to re-init the mqd. 3756 * check mqd->cp_hqd_pq_control since this value should not be 0 3757 */ 3758 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx]; 3759 if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){ 3760 /* for GPU_RESET case , reset MQD to a clean status */ 3761 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3762 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3763 3764 /* reset ring buffer */ 3765 ring->wptr = 0; 3766 amdgpu_ring_clear_ring(ring); 3767 3768 mutex_lock(&adev->srbm_mutex); 3769 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3770 gfx_v9_0_kiq_init_register(ring); 3771 soc15_grbm_select(adev, 0, 0, 0, 0); 3772 mutex_unlock(&adev->srbm_mutex); 3773 } else { 3774 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3775 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3776 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3777 mutex_lock(&adev->srbm_mutex); 3778 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3779 gfx_v9_0_mqd_init(ring); 3780 gfx_v9_0_kiq_init_register(ring); 3781 soc15_grbm_select(adev, 0, 0, 0, 0); 3782 mutex_unlock(&adev->srbm_mutex); 3783 3784 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3785 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3786 } 3787 3788 return 0; 3789 } 3790 3791 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring) 3792 { 3793 struct amdgpu_device *adev = ring->adev; 3794 struct v9_mqd *mqd = ring->mqd_ptr; 3795 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 3796 struct v9_mqd *tmp_mqd; 3797 3798 /* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control 3799 * is not be initialized before 3800 */ 3801 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx]; 3802 3803 if (!tmp_mqd->cp_hqd_pq_control || 3804 (!amdgpu_in_reset(adev) && !adev->in_suspend)) { 3805 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3806 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3807 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3808 mutex_lock(&adev->srbm_mutex); 3809 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3810 gfx_v9_0_mqd_init(ring); 3811 soc15_grbm_select(adev, 0, 0, 0, 0); 3812 mutex_unlock(&adev->srbm_mutex); 3813 3814 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3815 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3816 } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 3817 /* reset MQD to a clean status */ 3818 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3819 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3820 3821 /* reset ring buffer */ 3822 ring->wptr = 0; 3823 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0); 3824 amdgpu_ring_clear_ring(ring); 3825 } else { 3826 amdgpu_ring_clear_ring(ring); 3827 } 3828 3829 return 0; 3830 } 3831 3832 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev) 3833 { 3834 struct amdgpu_ring *ring; 3835 int r; 3836 3837 ring = &adev->gfx.kiq.ring; 3838 3839 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3840 if (unlikely(r != 0)) 3841 return r; 3842 3843 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3844 if (unlikely(r != 0)) 3845 return r; 3846 3847 gfx_v9_0_kiq_init_queue(ring); 3848 amdgpu_bo_kunmap(ring->mqd_obj); 3849 ring->mqd_ptr = NULL; 3850 amdgpu_bo_unreserve(ring->mqd_obj); 3851 ring->sched.ready = true; 3852 return 0; 3853 } 3854 3855 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev) 3856 { 3857 struct amdgpu_ring *ring = NULL; 3858 int r = 0, i; 3859 3860 gfx_v9_0_cp_compute_enable(adev, true); 3861 3862 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3863 ring = &adev->gfx.compute_ring[i]; 3864 3865 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3866 if (unlikely(r != 0)) 3867 goto done; 3868 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3869 if (!r) { 3870 r = gfx_v9_0_kcq_init_queue(ring); 3871 amdgpu_bo_kunmap(ring->mqd_obj); 3872 ring->mqd_ptr = NULL; 3873 } 3874 amdgpu_bo_unreserve(ring->mqd_obj); 3875 if (r) 3876 goto done; 3877 } 3878 3879 r = amdgpu_gfx_enable_kcq(adev); 3880 done: 3881 return r; 3882 } 3883 3884 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev) 3885 { 3886 int r, i; 3887 struct amdgpu_ring *ring; 3888 3889 if (!(adev->flags & AMD_IS_APU)) 3890 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3891 3892 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3893 if (adev->gfx.num_gfx_rings) { 3894 /* legacy firmware loading */ 3895 r = gfx_v9_0_cp_gfx_load_microcode(adev); 3896 if (r) 3897 return r; 3898 } 3899 3900 r = gfx_v9_0_cp_compute_load_microcode(adev); 3901 if (r) 3902 return r; 3903 } 3904 3905 r = gfx_v9_0_kiq_resume(adev); 3906 if (r) 3907 return r; 3908 3909 if (adev->gfx.num_gfx_rings) { 3910 r = gfx_v9_0_cp_gfx_resume(adev); 3911 if (r) 3912 return r; 3913 } 3914 3915 r = gfx_v9_0_kcq_resume(adev); 3916 if (r) 3917 return r; 3918 3919 if (adev->gfx.num_gfx_rings) { 3920 ring = &adev->gfx.gfx_ring[0]; 3921 r = amdgpu_ring_test_helper(ring); 3922 if (r) 3923 return r; 3924 } 3925 3926 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3927 ring = &adev->gfx.compute_ring[i]; 3928 amdgpu_ring_test_helper(ring); 3929 } 3930 3931 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3932 3933 return 0; 3934 } 3935 3936 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev) 3937 { 3938 u32 tmp; 3939 3940 if (adev->asic_type != CHIP_ARCTURUS) 3941 return; 3942 3943 tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG); 3944 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH, 3945 adev->df.hash_status.hash_64k); 3946 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH, 3947 adev->df.hash_status.hash_2m); 3948 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH, 3949 adev->df.hash_status.hash_1g); 3950 WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp); 3951 } 3952 3953 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable) 3954 { 3955 if (adev->gfx.num_gfx_rings) 3956 gfx_v9_0_cp_gfx_enable(adev, enable); 3957 gfx_v9_0_cp_compute_enable(adev, enable); 3958 } 3959 3960 static int gfx_v9_0_hw_init(void *handle) 3961 { 3962 int r; 3963 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3964 3965 if (!amdgpu_sriov_vf(adev)) 3966 gfx_v9_0_init_golden_registers(adev); 3967 3968 gfx_v9_0_constants_init(adev); 3969 3970 gfx_v9_0_init_tcp_config(adev); 3971 3972 r = adev->gfx.rlc.funcs->resume(adev); 3973 if (r) 3974 return r; 3975 3976 r = gfx_v9_0_cp_resume(adev); 3977 if (r) 3978 return r; 3979 3980 if (adev->asic_type == CHIP_ALDEBARAN) 3981 gfx_v9_4_2_set_power_brake_sequence(adev); 3982 3983 return r; 3984 } 3985 3986 static int gfx_v9_0_hw_fini(void *handle) 3987 { 3988 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3989 3990 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); 3991 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 3992 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 3993 3994 /* DF freeze and kcq disable will fail */ 3995 if (!amdgpu_ras_intr_triggered()) 3996 /* disable KCQ to avoid CPC touch memory not valid anymore */ 3997 amdgpu_gfx_disable_kcq(adev); 3998 3999 if (amdgpu_sriov_vf(adev)) { 4000 gfx_v9_0_cp_gfx_enable(adev, false); 4001 /* must disable polling for SRIOV when hw finished, otherwise 4002 * CPC engine may still keep fetching WB address which is already 4003 * invalid after sw finished and trigger DMAR reading error in 4004 * hypervisor side. 4005 */ 4006 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 4007 return 0; 4008 } 4009 4010 /* Use deinitialize sequence from CAIL when unbinding device from driver, 4011 * otherwise KIQ is hanging when binding back 4012 */ 4013 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 4014 mutex_lock(&adev->srbm_mutex); 4015 soc15_grbm_select(adev, adev->gfx.kiq.ring.me, 4016 adev->gfx.kiq.ring.pipe, 4017 adev->gfx.kiq.ring.queue, 0); 4018 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring); 4019 soc15_grbm_select(adev, 0, 0, 0, 0); 4020 mutex_unlock(&adev->srbm_mutex); 4021 } 4022 4023 gfx_v9_0_cp_enable(adev, false); 4024 4025 /* Skip suspend with A+A reset */ 4026 if (adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) { 4027 dev_dbg(adev->dev, "Device in reset. Skipping RLC halt\n"); 4028 return 0; 4029 } 4030 4031 adev->gfx.rlc.funcs->stop(adev); 4032 return 0; 4033 } 4034 4035 static int gfx_v9_0_suspend(void *handle) 4036 { 4037 return gfx_v9_0_hw_fini(handle); 4038 } 4039 4040 static int gfx_v9_0_resume(void *handle) 4041 { 4042 return gfx_v9_0_hw_init(handle); 4043 } 4044 4045 static bool gfx_v9_0_is_idle(void *handle) 4046 { 4047 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4048 4049 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS), 4050 GRBM_STATUS, GUI_ACTIVE)) 4051 return false; 4052 else 4053 return true; 4054 } 4055 4056 static int gfx_v9_0_wait_for_idle(void *handle) 4057 { 4058 unsigned i; 4059 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4060 4061 for (i = 0; i < adev->usec_timeout; i++) { 4062 if (gfx_v9_0_is_idle(handle)) 4063 return 0; 4064 udelay(1); 4065 } 4066 return -ETIMEDOUT; 4067 } 4068 4069 static int gfx_v9_0_soft_reset(void *handle) 4070 { 4071 u32 grbm_soft_reset = 0; 4072 u32 tmp; 4073 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4074 4075 /* GRBM_STATUS */ 4076 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS); 4077 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK | 4078 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK | 4079 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK | 4080 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK | 4081 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK | 4082 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) { 4083 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4084 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4085 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4086 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1); 4087 } 4088 4089 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) { 4090 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4091 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4092 } 4093 4094 /* GRBM_STATUS2 */ 4095 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2); 4096 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY)) 4097 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4098 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 4099 4100 4101 if (grbm_soft_reset) { 4102 /* stop the rlc */ 4103 adev->gfx.rlc.funcs->stop(adev); 4104 4105 if (adev->gfx.num_gfx_rings) 4106 /* Disable GFX parsing/prefetching */ 4107 gfx_v9_0_cp_gfx_enable(adev, false); 4108 4109 /* Disable MEC parsing/prefetching */ 4110 gfx_v9_0_cp_compute_enable(adev, false); 4111 4112 if (grbm_soft_reset) { 4113 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4114 tmp |= grbm_soft_reset; 4115 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 4116 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4117 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4118 4119 udelay(50); 4120 4121 tmp &= ~grbm_soft_reset; 4122 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4123 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4124 } 4125 4126 /* Wait a little for things to settle down */ 4127 udelay(50); 4128 } 4129 return 0; 4130 } 4131 4132 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev) 4133 { 4134 signed long r, cnt = 0; 4135 unsigned long flags; 4136 uint32_t seq, reg_val_offs = 0; 4137 uint64_t value = 0; 4138 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 4139 struct amdgpu_ring *ring = &kiq->ring; 4140 4141 BUG_ON(!ring->funcs->emit_rreg); 4142 4143 spin_lock_irqsave(&kiq->ring_lock, flags); 4144 if (amdgpu_device_wb_get(adev, ®_val_offs)) { 4145 pr_err("critical bug! too many kiq readers\n"); 4146 goto failed_unlock; 4147 } 4148 amdgpu_ring_alloc(ring, 32); 4149 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 4150 amdgpu_ring_write(ring, 9 | /* src: register*/ 4151 (5 << 8) | /* dst: memory */ 4152 (1 << 16) | /* count sel */ 4153 (1 << 20)); /* write confirm */ 4154 amdgpu_ring_write(ring, 0); 4155 amdgpu_ring_write(ring, 0); 4156 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 4157 reg_val_offs * 4)); 4158 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 4159 reg_val_offs * 4)); 4160 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 4161 if (r) 4162 goto failed_undo; 4163 4164 amdgpu_ring_commit(ring); 4165 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4166 4167 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4168 4169 /* don't wait anymore for gpu reset case because this way may 4170 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 4171 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 4172 * never return if we keep waiting in virt_kiq_rreg, which cause 4173 * gpu_recover() hang there. 4174 * 4175 * also don't wait anymore for IRQ context 4176 * */ 4177 if (r < 1 && (amdgpu_in_reset(adev))) 4178 goto failed_kiq_read; 4179 4180 might_sleep(); 4181 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 4182 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 4183 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4184 } 4185 4186 if (cnt > MAX_KIQ_REG_TRY) 4187 goto failed_kiq_read; 4188 4189 mb(); 4190 value = (uint64_t)adev->wb.wb[reg_val_offs] | 4191 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL; 4192 amdgpu_device_wb_free(adev, reg_val_offs); 4193 return value; 4194 4195 failed_undo: 4196 amdgpu_ring_undo(ring); 4197 failed_unlock: 4198 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4199 failed_kiq_read: 4200 if (reg_val_offs) 4201 amdgpu_device_wb_free(adev, reg_val_offs); 4202 pr_err("failed to read gpu clock\n"); 4203 return ~0; 4204 } 4205 4206 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev) 4207 { 4208 uint64_t clock; 4209 4210 amdgpu_gfx_off_ctrl(adev, false); 4211 mutex_lock(&adev->gfx.gpu_clock_mutex); 4212 if (adev->asic_type == CHIP_VEGA10 && amdgpu_sriov_runtime(adev)) { 4213 clock = gfx_v9_0_kiq_read_clock(adev); 4214 } else { 4215 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1); 4216 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) | 4217 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 4218 } 4219 mutex_unlock(&adev->gfx.gpu_clock_mutex); 4220 amdgpu_gfx_off_ctrl(adev, true); 4221 return clock; 4222 } 4223 4224 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 4225 uint32_t vmid, 4226 uint32_t gds_base, uint32_t gds_size, 4227 uint32_t gws_base, uint32_t gws_size, 4228 uint32_t oa_base, uint32_t oa_size) 4229 { 4230 struct amdgpu_device *adev = ring->adev; 4231 4232 /* GDS Base */ 4233 gfx_v9_0_write_data_to_reg(ring, 0, false, 4234 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid, 4235 gds_base); 4236 4237 /* GDS Size */ 4238 gfx_v9_0_write_data_to_reg(ring, 0, false, 4239 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid, 4240 gds_size); 4241 4242 /* GWS */ 4243 gfx_v9_0_write_data_to_reg(ring, 0, false, 4244 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid, 4245 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 4246 4247 /* OA */ 4248 gfx_v9_0_write_data_to_reg(ring, 0, false, 4249 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid, 4250 (1 << (oa_size + oa_base)) - (1 << oa_base)); 4251 } 4252 4253 static const u32 vgpr_init_compute_shader[] = 4254 { 4255 0xb07c0000, 0xbe8000ff, 4256 0x000000f8, 0xbf110800, 4257 0x7e000280, 0x7e020280, 4258 0x7e040280, 0x7e060280, 4259 0x7e080280, 0x7e0a0280, 4260 0x7e0c0280, 0x7e0e0280, 4261 0x80808800, 0xbe803200, 4262 0xbf84fff5, 0xbf9c0000, 4263 0xd28c0001, 0x0001007f, 4264 0xd28d0001, 0x0002027e, 4265 0x10020288, 0xb8810904, 4266 0xb7814000, 0xd1196a01, 4267 0x00000301, 0xbe800087, 4268 0xbefc00c1, 0xd89c4000, 4269 0x00020201, 0xd89cc080, 4270 0x00040401, 0x320202ff, 4271 0x00000800, 0x80808100, 4272 0xbf84fff8, 0x7e020280, 4273 0xbf810000, 0x00000000, 4274 }; 4275 4276 static const u32 sgpr_init_compute_shader[] = 4277 { 4278 0xb07c0000, 0xbe8000ff, 4279 0x0000005f, 0xbee50080, 4280 0xbe812c65, 0xbe822c65, 4281 0xbe832c65, 0xbe842c65, 4282 0xbe852c65, 0xb77c0005, 4283 0x80808500, 0xbf84fff8, 4284 0xbe800080, 0xbf810000, 4285 }; 4286 4287 static const u32 vgpr_init_compute_shader_arcturus[] = { 4288 0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080, 4289 0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080, 4290 0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080, 4291 0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080, 4292 0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080, 4293 0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080, 4294 0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080, 4295 0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080, 4296 0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080, 4297 0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080, 4298 0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080, 4299 0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080, 4300 0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080, 4301 0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080, 4302 0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080, 4303 0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080, 4304 0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080, 4305 0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080, 4306 0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080, 4307 0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080, 4308 0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080, 4309 0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080, 4310 0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080, 4311 0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080, 4312 0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080, 4313 0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080, 4314 0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080, 4315 0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080, 4316 0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080, 4317 0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080, 4318 0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080, 4319 0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080, 4320 0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080, 4321 0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080, 4322 0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080, 4323 0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080, 4324 0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080, 4325 0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080, 4326 0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080, 4327 0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080, 4328 0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080, 4329 0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080, 4330 0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080, 4331 0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080, 4332 0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080, 4333 0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080, 4334 0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080, 4335 0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080, 4336 0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080, 4337 0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080, 4338 0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080, 4339 0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080, 4340 0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080, 4341 0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080, 4342 0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080, 4343 0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080, 4344 0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080, 4345 0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080, 4346 0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080, 4347 0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080, 4348 0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080, 4349 0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080, 4350 0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080, 4351 0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080, 4352 0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080, 4353 0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080, 4354 0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080, 4355 0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080, 4356 0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080, 4357 0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080, 4358 0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080, 4359 0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080, 4360 0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080, 4361 0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080, 4362 0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080, 4363 0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080, 4364 0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080, 4365 0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080, 4366 0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080, 4367 0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080, 4368 0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080, 4369 0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080, 4370 0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080, 4371 0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080, 4372 0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080, 4373 0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a, 4374 0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280, 4375 0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000, 4376 0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904, 4377 0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000, 4378 0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a, 4379 0xbf84fff8, 0xbf810000, 4380 }; 4381 4382 /* When below register arrays changed, please update gpr_reg_size, 4383 and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds, 4384 to cover all gfx9 ASICs */ 4385 static const struct soc15_reg_entry vgpr_init_regs[] = { 4386 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4387 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4388 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4389 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4390 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f }, 4391 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4392 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4393 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4394 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4395 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4396 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4397 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4398 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4399 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4400 }; 4401 4402 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = { 4403 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4404 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4405 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4406 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4407 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf }, 4408 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4409 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4410 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4411 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4412 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4413 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4414 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4415 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4416 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4417 }; 4418 4419 static const struct soc15_reg_entry sgpr1_init_regs[] = { 4420 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4421 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4422 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4423 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4424 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4425 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4426 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff }, 4427 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff }, 4428 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff }, 4429 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff }, 4430 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff }, 4431 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff }, 4432 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff }, 4433 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff }, 4434 }; 4435 4436 static const struct soc15_reg_entry sgpr2_init_regs[] = { 4437 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4438 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4439 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4440 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4441 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4442 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4443 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 }, 4444 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 }, 4445 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 }, 4446 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 }, 4447 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 }, 4448 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 }, 4449 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 }, 4450 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 }, 4451 }; 4452 4453 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = { 4454 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1}, 4455 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1}, 4456 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1}, 4457 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1}, 4458 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1}, 4459 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1}, 4460 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1}, 4461 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1}, 4462 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1}, 4463 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1}, 4464 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1}, 4465 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1}, 4466 { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1}, 4467 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6}, 4468 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16}, 4469 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16}, 4470 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16}, 4471 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16}, 4472 { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16}, 4473 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16}, 4474 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16}, 4475 { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16}, 4476 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6}, 4477 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16}, 4478 { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16}, 4479 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1}, 4480 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1}, 4481 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32}, 4482 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32}, 4483 { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72}, 4484 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16}, 4485 { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2}, 4486 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6}, 4487 }; 4488 4489 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev) 4490 { 4491 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4492 int i, r; 4493 4494 /* only support when RAS is enabled */ 4495 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4496 return 0; 4497 4498 r = amdgpu_ring_alloc(ring, 7); 4499 if (r) { 4500 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n", 4501 ring->name, r); 4502 return r; 4503 } 4504 4505 WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000); 4506 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size); 4507 4508 amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 4509 amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC | 4510 PACKET3_DMA_DATA_DST_SEL(1) | 4511 PACKET3_DMA_DATA_SRC_SEL(2) | 4512 PACKET3_DMA_DATA_ENGINE(0))); 4513 amdgpu_ring_write(ring, 0); 4514 amdgpu_ring_write(ring, 0); 4515 amdgpu_ring_write(ring, 0); 4516 amdgpu_ring_write(ring, 0); 4517 amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT | 4518 adev->gds.gds_size); 4519 4520 amdgpu_ring_commit(ring); 4521 4522 for (i = 0; i < adev->usec_timeout; i++) { 4523 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring)) 4524 break; 4525 udelay(1); 4526 } 4527 4528 if (i >= adev->usec_timeout) 4529 r = -ETIMEDOUT; 4530 4531 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000); 4532 4533 return r; 4534 } 4535 4536 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) 4537 { 4538 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4539 struct amdgpu_ib ib; 4540 struct dma_fence *f = NULL; 4541 int r, i; 4542 unsigned total_size, vgpr_offset, sgpr_offset; 4543 u64 gpu_addr; 4544 4545 int compute_dim_x = adev->gfx.config.max_shader_engines * 4546 adev->gfx.config.max_cu_per_sh * 4547 adev->gfx.config.max_sh_per_se; 4548 int sgpr_work_group_size = 5; 4549 int gpr_reg_size = adev->gfx.config.max_shader_engines + 6; 4550 int vgpr_init_shader_size; 4551 const u32 *vgpr_init_shader_ptr; 4552 const struct soc15_reg_entry *vgpr_init_regs_ptr; 4553 4554 /* only support when RAS is enabled */ 4555 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4556 return 0; 4557 4558 /* bail if the compute ring is not ready */ 4559 if (!ring->sched.ready) 4560 return 0; 4561 4562 if (adev->asic_type == CHIP_ARCTURUS || 4563 adev->asic_type == CHIP_ALDEBARAN) { 4564 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus; 4565 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus); 4566 vgpr_init_regs_ptr = vgpr_init_regs_arcturus; 4567 } else { 4568 vgpr_init_shader_ptr = vgpr_init_compute_shader; 4569 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader); 4570 vgpr_init_regs_ptr = vgpr_init_regs; 4571 } 4572 4573 total_size = 4574 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */ 4575 total_size += 4576 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */ 4577 total_size += 4578 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */ 4579 total_size = ALIGN(total_size, 256); 4580 vgpr_offset = total_size; 4581 total_size += ALIGN(vgpr_init_shader_size, 256); 4582 sgpr_offset = total_size; 4583 total_size += sizeof(sgpr_init_compute_shader); 4584 4585 /* allocate an indirect buffer to put the commands in */ 4586 memset(&ib, 0, sizeof(ib)); 4587 r = amdgpu_ib_get(adev, NULL, total_size, 4588 AMDGPU_IB_POOL_DIRECT, &ib); 4589 if (r) { 4590 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); 4591 return r; 4592 } 4593 4594 /* load the compute shaders */ 4595 for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++) 4596 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i]; 4597 4598 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++) 4599 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i]; 4600 4601 /* init the ib length to 0 */ 4602 ib.length_dw = 0; 4603 4604 /* VGPR */ 4605 /* write the register state for the compute dispatch */ 4606 for (i = 0; i < gpr_reg_size; i++) { 4607 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4608 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i]) 4609 - PACKET3_SET_SH_REG_START; 4610 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value; 4611 } 4612 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4613 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8; 4614 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4615 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4616 - PACKET3_SET_SH_REG_START; 4617 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4618 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4619 4620 /* write dispatch packet */ 4621 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4622 ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */ 4623 ib.ptr[ib.length_dw++] = 1; /* y */ 4624 ib.ptr[ib.length_dw++] = 1; /* z */ 4625 ib.ptr[ib.length_dw++] = 4626 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4627 4628 /* write CS partial flush packet */ 4629 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4630 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4631 4632 /* SGPR1 */ 4633 /* write the register state for the compute dispatch */ 4634 for (i = 0; i < gpr_reg_size; i++) { 4635 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4636 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i]) 4637 - PACKET3_SET_SH_REG_START; 4638 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value; 4639 } 4640 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4641 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4642 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4643 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4644 - PACKET3_SET_SH_REG_START; 4645 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4646 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4647 4648 /* write dispatch packet */ 4649 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4650 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4651 ib.ptr[ib.length_dw++] = 1; /* y */ 4652 ib.ptr[ib.length_dw++] = 1; /* z */ 4653 ib.ptr[ib.length_dw++] = 4654 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4655 4656 /* write CS partial flush packet */ 4657 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4658 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4659 4660 /* SGPR2 */ 4661 /* write the register state for the compute dispatch */ 4662 for (i = 0; i < gpr_reg_size; i++) { 4663 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4664 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i]) 4665 - PACKET3_SET_SH_REG_START; 4666 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value; 4667 } 4668 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4669 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4670 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4671 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4672 - PACKET3_SET_SH_REG_START; 4673 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4674 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4675 4676 /* write dispatch packet */ 4677 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4678 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4679 ib.ptr[ib.length_dw++] = 1; /* y */ 4680 ib.ptr[ib.length_dw++] = 1; /* z */ 4681 ib.ptr[ib.length_dw++] = 4682 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4683 4684 /* write CS partial flush packet */ 4685 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4686 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4687 4688 /* shedule the ib on the ring */ 4689 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 4690 if (r) { 4691 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r); 4692 goto fail; 4693 } 4694 4695 /* wait for the GPU to finish processing the IB */ 4696 r = dma_fence_wait(f, false); 4697 if (r) { 4698 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); 4699 goto fail; 4700 } 4701 4702 fail: 4703 amdgpu_ib_free(adev, &ib, NULL); 4704 dma_fence_put(f); 4705 4706 return r; 4707 } 4708 4709 static int gfx_v9_0_early_init(void *handle) 4710 { 4711 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4712 4713 if (adev->asic_type == CHIP_ARCTURUS || 4714 adev->asic_type == CHIP_ALDEBARAN) 4715 adev->gfx.num_gfx_rings = 0; 4716 else 4717 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS; 4718 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), 4719 AMDGPU_MAX_COMPUTE_RINGS); 4720 gfx_v9_0_set_kiq_pm4_funcs(adev); 4721 gfx_v9_0_set_ring_funcs(adev); 4722 gfx_v9_0_set_irq_funcs(adev); 4723 gfx_v9_0_set_gds_init(adev); 4724 gfx_v9_0_set_rlc_funcs(adev); 4725 4726 return 0; 4727 } 4728 4729 static int gfx_v9_0_ecc_late_init(void *handle) 4730 { 4731 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4732 int r; 4733 4734 /* 4735 * Temp workaround to fix the issue that CP firmware fails to 4736 * update read pointer when CPDMA is writing clearing operation 4737 * to GDS in suspend/resume sequence on several cards. So just 4738 * limit this operation in cold boot sequence. 4739 */ 4740 if ((!adev->in_suspend) && 4741 (adev->gds.gds_size)) { 4742 r = gfx_v9_0_do_edc_gds_workarounds(adev); 4743 if (r) 4744 return r; 4745 } 4746 4747 /* requires IBs so do in late init after IB pool is initialized */ 4748 r = gfx_v9_0_do_edc_gpr_workarounds(adev); 4749 if (r) 4750 return r; 4751 4752 if (adev->gfx.ras_funcs && 4753 adev->gfx.ras_funcs->ras_late_init) { 4754 r = adev->gfx.ras_funcs->ras_late_init(adev); 4755 if (r) 4756 return r; 4757 } 4758 4759 if (adev->gfx.ras_funcs && 4760 adev->gfx.ras_funcs->enable_watchdog_timer) 4761 adev->gfx.ras_funcs->enable_watchdog_timer(adev); 4762 4763 return 0; 4764 } 4765 4766 static int gfx_v9_0_late_init(void *handle) 4767 { 4768 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4769 int r; 4770 4771 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 4772 if (r) 4773 return r; 4774 4775 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 4776 if (r) 4777 return r; 4778 4779 r = gfx_v9_0_ecc_late_init(handle); 4780 if (r) 4781 return r; 4782 4783 return 0; 4784 } 4785 4786 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev) 4787 { 4788 uint32_t rlc_setting; 4789 4790 /* if RLC is not enabled, do nothing */ 4791 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL); 4792 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK)) 4793 return false; 4794 4795 return true; 4796 } 4797 4798 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev) 4799 { 4800 uint32_t data; 4801 unsigned i; 4802 4803 data = RLC_SAFE_MODE__CMD_MASK; 4804 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 4805 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4806 4807 /* wait for RLC_SAFE_MODE */ 4808 for (i = 0; i < adev->usec_timeout; i++) { 4809 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 4810 break; 4811 udelay(1); 4812 } 4813 } 4814 4815 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev) 4816 { 4817 uint32_t data; 4818 4819 data = RLC_SAFE_MODE__CMD_MASK; 4820 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4821 } 4822 4823 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev, 4824 bool enable) 4825 { 4826 amdgpu_gfx_rlc_enter_safe_mode(adev); 4827 4828 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { 4829 gfx_v9_0_enable_gfx_cg_power_gating(adev, true); 4830 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4831 gfx_v9_0_enable_gfx_pipeline_powergating(adev, true); 4832 } else { 4833 gfx_v9_0_enable_gfx_cg_power_gating(adev, false); 4834 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4835 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false); 4836 } 4837 4838 amdgpu_gfx_rlc_exit_safe_mode(adev); 4839 } 4840 4841 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev, 4842 bool enable) 4843 { 4844 /* TODO: double check if we need to perform under safe mode */ 4845 /* gfx_v9_0_enter_rlc_safe_mode(adev); */ 4846 4847 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 4848 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true); 4849 else 4850 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false); 4851 4852 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 4853 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true); 4854 else 4855 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false); 4856 4857 /* gfx_v9_0_exit_rlc_safe_mode(adev); */ 4858 } 4859 4860 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 4861 bool enable) 4862 { 4863 uint32_t data, def; 4864 4865 amdgpu_gfx_rlc_enter_safe_mode(adev); 4866 4867 /* It is disabled by HW by default */ 4868 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { 4869 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 4870 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4871 4872 if (adev->asic_type != CHIP_VEGA12) 4873 data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4874 4875 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4876 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4877 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4878 4879 /* only for Vega10 & Raven1 */ 4880 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK; 4881 4882 if (def != data) 4883 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4884 4885 /* MGLS is a global flag to control all MGLS in GFX */ 4886 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 4887 /* 2 - RLC memory Light sleep */ 4888 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { 4889 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4890 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4891 if (def != data) 4892 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4893 } 4894 /* 3 - CP memory Light sleep */ 4895 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { 4896 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4897 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4898 if (def != data) 4899 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4900 } 4901 } 4902 } else { 4903 /* 1 - MGCG_OVERRIDE */ 4904 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4905 4906 if (adev->asic_type != CHIP_VEGA12) 4907 data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4908 4909 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 4910 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4911 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4912 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4913 4914 if (def != data) 4915 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4916 4917 /* 2 - disable MGLS in RLC */ 4918 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4919 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) { 4920 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4921 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4922 } 4923 4924 /* 3 - disable MGLS in CP */ 4925 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4926 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) { 4927 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4928 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4929 } 4930 } 4931 4932 amdgpu_gfx_rlc_exit_safe_mode(adev); 4933 } 4934 4935 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev, 4936 bool enable) 4937 { 4938 uint32_t data, def; 4939 4940 if (!adev->gfx.num_gfx_rings) 4941 return; 4942 4943 amdgpu_gfx_rlc_enter_safe_mode(adev); 4944 4945 /* Enable 3D CGCG/CGLS */ 4946 if (enable) { 4947 /* write cmd to clear cgcg/cgls ov */ 4948 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4949 /* unset CGCG override */ 4950 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 4951 /* update CGCG and CGLS override bits */ 4952 if (def != data) 4953 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4954 4955 /* enable 3Dcgcg FSM(0x0000363f) */ 4956 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4957 4958 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) 4959 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4960 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 4961 else 4962 data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT; 4963 4964 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 4965 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 4966 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 4967 if (def != data) 4968 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4969 4970 /* set IDLE_POLL_COUNT(0x00900100) */ 4971 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 4972 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 4973 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 4974 if (def != data) 4975 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 4976 } else { 4977 /* Disable CGCG/CGLS */ 4978 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4979 /* disable cgcg, cgls should be disabled */ 4980 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK | 4981 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK); 4982 /* disable cgcg and cgls in FSM */ 4983 if (def != data) 4984 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4985 } 4986 4987 amdgpu_gfx_rlc_exit_safe_mode(adev); 4988 } 4989 4990 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 4991 bool enable) 4992 { 4993 uint32_t def, data; 4994 4995 amdgpu_gfx_rlc_enter_safe_mode(adev); 4996 4997 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { 4998 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4999 /* unset CGCG override */ 5000 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 5001 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5002 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 5003 else 5004 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 5005 /* update CGCG and CGLS override bits */ 5006 if (def != data) 5007 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 5008 5009 /* enable cgcg FSM(0x0000363F) */ 5010 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 5011 5012 if (adev->asic_type == CHIP_ARCTURUS) 5013 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5014 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5015 else 5016 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5017 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5018 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5019 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 5020 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5021 if (def != data) 5022 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 5023 5024 /* set IDLE_POLL_COUNT(0x00900100) */ 5025 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 5026 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 5027 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 5028 if (def != data) 5029 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 5030 } else { 5031 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 5032 /* reset CGCG/CGLS bits */ 5033 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK); 5034 /* disable cgcg and cgls in FSM */ 5035 if (def != data) 5036 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 5037 } 5038 5039 amdgpu_gfx_rlc_exit_safe_mode(adev); 5040 } 5041 5042 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev, 5043 bool enable) 5044 { 5045 if (enable) { 5046 /* CGCG/CGLS should be enabled after MGCG/MGLS 5047 * === MGCG + MGLS === 5048 */ 5049 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 5050 /* === CGCG /CGLS for GFX 3D Only === */ 5051 gfx_v9_0_update_3d_clock_gating(adev, enable); 5052 /* === CGCG + CGLS === */ 5053 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 5054 } else { 5055 /* CGCG/CGLS should be disabled before MGCG/MGLS 5056 * === CGCG + CGLS === 5057 */ 5058 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 5059 /* === CGCG /CGLS for GFX 3D Only === */ 5060 gfx_v9_0_update_3d_clock_gating(adev, enable); 5061 /* === MGCG + MGLS === */ 5062 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 5063 } 5064 return 0; 5065 } 5066 5067 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid) 5068 { 5069 u32 reg, data; 5070 5071 reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL); 5072 if (amdgpu_sriov_is_pp_one_vf(adev)) 5073 data = RREG32_NO_KIQ(reg); 5074 else 5075 data = RREG32(reg); 5076 5077 data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK; 5078 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; 5079 5080 if (amdgpu_sriov_is_pp_one_vf(adev)) 5081 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data); 5082 else 5083 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data); 5084 } 5085 5086 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev, 5087 uint32_t offset, 5088 struct soc15_reg_rlcg *entries, int arr_size) 5089 { 5090 int i; 5091 uint32_t reg; 5092 5093 if (!entries) 5094 return false; 5095 5096 for (i = 0; i < arr_size; i++) { 5097 const struct soc15_reg_rlcg *entry; 5098 5099 entry = &entries[i]; 5100 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg; 5101 if (offset == reg) 5102 return true; 5103 } 5104 5105 return false; 5106 } 5107 5108 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset) 5109 { 5110 return gfx_v9_0_check_rlcg_range(adev, offset, 5111 (void *)rlcg_access_gc_9_0, 5112 ARRAY_SIZE(rlcg_access_gc_9_0)); 5113 } 5114 5115 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = { 5116 .is_rlc_enabled = gfx_v9_0_is_rlc_enabled, 5117 .set_safe_mode = gfx_v9_0_set_safe_mode, 5118 .unset_safe_mode = gfx_v9_0_unset_safe_mode, 5119 .init = gfx_v9_0_rlc_init, 5120 .get_csb_size = gfx_v9_0_get_csb_size, 5121 .get_csb_buffer = gfx_v9_0_get_csb_buffer, 5122 .get_cp_table_num = gfx_v9_0_cp_jump_table_num, 5123 .resume = gfx_v9_0_rlc_resume, 5124 .stop = gfx_v9_0_rlc_stop, 5125 .reset = gfx_v9_0_rlc_reset, 5126 .start = gfx_v9_0_rlc_start, 5127 .update_spm_vmid = gfx_v9_0_update_spm_vmid, 5128 .rlcg_wreg = gfx_v9_0_rlcg_wreg, 5129 .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range, 5130 }; 5131 5132 static int gfx_v9_0_set_powergating_state(void *handle, 5133 enum amd_powergating_state state) 5134 { 5135 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5136 bool enable = (state == AMD_PG_STATE_GATE); 5137 5138 switch (adev->asic_type) { 5139 case CHIP_RAVEN: 5140 case CHIP_RENOIR: 5141 if (!enable) 5142 amdgpu_gfx_off_ctrl(adev, false); 5143 5144 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { 5145 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true); 5146 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true); 5147 } else { 5148 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false); 5149 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false); 5150 } 5151 5152 if (adev->pg_flags & AMD_PG_SUPPORT_CP) 5153 gfx_v9_0_enable_cp_power_gating(adev, true); 5154 else 5155 gfx_v9_0_enable_cp_power_gating(adev, false); 5156 5157 /* update gfx cgpg state */ 5158 gfx_v9_0_update_gfx_cg_power_gating(adev, enable); 5159 5160 /* update mgcg state */ 5161 gfx_v9_0_update_gfx_mg_power_gating(adev, enable); 5162 5163 if (enable) 5164 amdgpu_gfx_off_ctrl(adev, true); 5165 break; 5166 case CHIP_VEGA12: 5167 amdgpu_gfx_off_ctrl(adev, enable); 5168 break; 5169 default: 5170 break; 5171 } 5172 5173 return 0; 5174 } 5175 5176 static int gfx_v9_0_set_clockgating_state(void *handle, 5177 enum amd_clockgating_state state) 5178 { 5179 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5180 5181 if (amdgpu_sriov_vf(adev)) 5182 return 0; 5183 5184 switch (adev->asic_type) { 5185 case CHIP_VEGA10: 5186 case CHIP_VEGA12: 5187 case CHIP_VEGA20: 5188 case CHIP_RAVEN: 5189 case CHIP_ARCTURUS: 5190 case CHIP_RENOIR: 5191 case CHIP_ALDEBARAN: 5192 gfx_v9_0_update_gfx_clock_gating(adev, 5193 state == AMD_CG_STATE_GATE); 5194 break; 5195 default: 5196 break; 5197 } 5198 return 0; 5199 } 5200 5201 static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags) 5202 { 5203 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5204 int data; 5205 5206 if (amdgpu_sriov_vf(adev)) 5207 *flags = 0; 5208 5209 /* AMD_CG_SUPPORT_GFX_MGCG */ 5210 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE)); 5211 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 5212 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5213 5214 /* AMD_CG_SUPPORT_GFX_CGCG */ 5215 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL)); 5216 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5217 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5218 5219 /* AMD_CG_SUPPORT_GFX_CGLS */ 5220 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5221 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5222 5223 /* AMD_CG_SUPPORT_GFX_RLC_LS */ 5224 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL)); 5225 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) 5226 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS; 5227 5228 /* AMD_CG_SUPPORT_GFX_CP_LS */ 5229 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL)); 5230 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) 5231 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS; 5232 5233 if (adev->asic_type != CHIP_ARCTURUS) { 5234 /* AMD_CG_SUPPORT_GFX_3D_CGCG */ 5235 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D)); 5236 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) 5237 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; 5238 5239 /* AMD_CG_SUPPORT_GFX_3D_CGLS */ 5240 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK) 5241 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS; 5242 } 5243 } 5244 5245 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) 5246 { 5247 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/ 5248 } 5249 5250 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5251 { 5252 struct amdgpu_device *adev = ring->adev; 5253 u64 wptr; 5254 5255 /* XXX check if swapping is necessary on BE */ 5256 if (ring->use_doorbell) { 5257 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]); 5258 } else { 5259 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR); 5260 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32; 5261 } 5262 5263 return wptr; 5264 } 5265 5266 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5267 { 5268 struct amdgpu_device *adev = ring->adev; 5269 5270 if (ring->use_doorbell) { 5271 /* XXX check if swapping is necessary on BE */ 5272 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5273 WDOORBELL64(ring->doorbell_index, ring->wptr); 5274 } else { 5275 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 5276 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 5277 } 5278 } 5279 5280 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5281 { 5282 struct amdgpu_device *adev = ring->adev; 5283 u32 ref_and_mask, reg_mem_engine; 5284 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg; 5285 5286 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 5287 switch (ring->me) { 5288 case 1: 5289 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe; 5290 break; 5291 case 2: 5292 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe; 5293 break; 5294 default: 5295 return; 5296 } 5297 reg_mem_engine = 0; 5298 } else { 5299 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0; 5300 reg_mem_engine = 1; /* pfp */ 5301 } 5302 5303 gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1, 5304 adev->nbio.funcs->get_hdp_flush_req_offset(adev), 5305 adev->nbio.funcs->get_hdp_flush_done_offset(adev), 5306 ref_and_mask, ref_and_mask, 0x20); 5307 } 5308 5309 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 5310 struct amdgpu_job *job, 5311 struct amdgpu_ib *ib, 5312 uint32_t flags) 5313 { 5314 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5315 u32 header, control = 0; 5316 5317 if (ib->flags & AMDGPU_IB_FLAG_CE) 5318 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 5319 else 5320 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 5321 5322 control |= ib->length_dw | (vmid << 24); 5323 5324 if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { 5325 control |= INDIRECT_BUFFER_PRE_ENB(1); 5326 5327 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid) 5328 gfx_v9_0_ring_emit_de_meta(ring); 5329 } 5330 5331 amdgpu_ring_write(ring, header); 5332 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5333 amdgpu_ring_write(ring, 5334 #ifdef __BIG_ENDIAN 5335 (2 << 0) | 5336 #endif 5337 lower_32_bits(ib->gpu_addr)); 5338 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5339 amdgpu_ring_write(ring, control); 5340 } 5341 5342 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 5343 struct amdgpu_job *job, 5344 struct amdgpu_ib *ib, 5345 uint32_t flags) 5346 { 5347 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5348 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 5349 5350 /* Currently, there is a high possibility to get wave ID mismatch 5351 * between ME and GDS, leading to a hw deadlock, because ME generates 5352 * different wave IDs than the GDS expects. This situation happens 5353 * randomly when at least 5 compute pipes use GDS ordered append. 5354 * The wave IDs generated by ME are also wrong after suspend/resume. 5355 * Those are probably bugs somewhere else in the kernel driver. 5356 * 5357 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 5358 * GDS to 0 for this ring (me/pipe). 5359 */ 5360 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 5361 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 5362 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID); 5363 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 5364 } 5365 5366 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 5367 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5368 amdgpu_ring_write(ring, 5369 #ifdef __BIG_ENDIAN 5370 (2 << 0) | 5371 #endif 5372 lower_32_bits(ib->gpu_addr)); 5373 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5374 amdgpu_ring_write(ring, control); 5375 } 5376 5377 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 5378 u64 seq, unsigned flags) 5379 { 5380 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 5381 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 5382 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY; 5383 5384 /* RELEASE_MEM - flush caches, send int */ 5385 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); 5386 amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN | 5387 EOP_TC_NC_ACTION_EN) : 5388 (EOP_TCL1_ACTION_EN | 5389 EOP_TC_ACTION_EN | 5390 EOP_TC_WB_ACTION_EN | 5391 EOP_TC_MD_ACTION_EN)) | 5392 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 5393 EVENT_INDEX(5))); 5394 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 5395 5396 /* 5397 * the address should be Qword aligned if 64bit write, Dword 5398 * aligned if only send 32bit data low (discard data high) 5399 */ 5400 if (write64bit) 5401 BUG_ON(addr & 0x7); 5402 else 5403 BUG_ON(addr & 0x3); 5404 amdgpu_ring_write(ring, lower_32_bits(addr)); 5405 amdgpu_ring_write(ring, upper_32_bits(addr)); 5406 amdgpu_ring_write(ring, lower_32_bits(seq)); 5407 amdgpu_ring_write(ring, upper_32_bits(seq)); 5408 amdgpu_ring_write(ring, 0); 5409 } 5410 5411 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 5412 { 5413 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5414 uint32_t seq = ring->fence_drv.sync_seq; 5415 uint64_t addr = ring->fence_drv.gpu_addr; 5416 5417 gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0, 5418 lower_32_bits(addr), upper_32_bits(addr), 5419 seq, 0xffffffff, 4); 5420 } 5421 5422 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 5423 unsigned vmid, uint64_t pd_addr) 5424 { 5425 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 5426 5427 /* compute doesn't have PFP */ 5428 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 5429 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 5430 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 5431 amdgpu_ring_write(ring, 0x0); 5432 } 5433 } 5434 5435 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring) 5436 { 5437 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */ 5438 } 5439 5440 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 5441 { 5442 u64 wptr; 5443 5444 /* XXX check if swapping is necessary on BE */ 5445 if (ring->use_doorbell) 5446 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]); 5447 else 5448 BUG(); 5449 return wptr; 5450 } 5451 5452 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 5453 { 5454 struct amdgpu_device *adev = ring->adev; 5455 5456 /* XXX check if swapping is necessary on BE */ 5457 if (ring->use_doorbell) { 5458 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5459 WDOORBELL64(ring->doorbell_index, ring->wptr); 5460 } else{ 5461 BUG(); /* only DOORBELL method supported on gfx9 now */ 5462 } 5463 } 5464 5465 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 5466 u64 seq, unsigned int flags) 5467 { 5468 struct amdgpu_device *adev = ring->adev; 5469 5470 /* we only allocate 32bit for each seq wb address */ 5471 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); 5472 5473 /* write fence seq to the "addr" */ 5474 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5475 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5476 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 5477 amdgpu_ring_write(ring, lower_32_bits(addr)); 5478 amdgpu_ring_write(ring, upper_32_bits(addr)); 5479 amdgpu_ring_write(ring, lower_32_bits(seq)); 5480 5481 if (flags & AMDGPU_FENCE_FLAG_INT) { 5482 /* set register to trigger INT */ 5483 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5484 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5485 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 5486 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS)); 5487 amdgpu_ring_write(ring, 0); 5488 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 5489 } 5490 } 5491 5492 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring) 5493 { 5494 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 5495 amdgpu_ring_write(ring, 0); 5496 } 5497 5498 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring) 5499 { 5500 struct v9_ce_ib_state ce_payload = {0}; 5501 uint64_t csa_addr; 5502 int cnt; 5503 5504 cnt = (sizeof(ce_payload) >> 2) + 4 - 2; 5505 csa_addr = amdgpu_csa_vaddr(ring->adev); 5506 5507 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5508 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) | 5509 WRITE_DATA_DST_SEL(8) | 5510 WR_CONFIRM) | 5511 WRITE_DATA_CACHE_POLICY(0)); 5512 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5513 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5514 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2); 5515 } 5516 5517 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring) 5518 { 5519 struct v9_de_ib_state de_payload = {0}; 5520 uint64_t csa_addr, gds_addr; 5521 int cnt; 5522 5523 csa_addr = amdgpu_csa_vaddr(ring->adev); 5524 gds_addr = csa_addr + 4096; 5525 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr); 5526 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr); 5527 5528 cnt = (sizeof(de_payload) >> 2) + 4 - 2; 5529 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5530 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 5531 WRITE_DATA_DST_SEL(8) | 5532 WR_CONFIRM) | 5533 WRITE_DATA_CACHE_POLICY(0)); 5534 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5535 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5536 amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2); 5537 } 5538 5539 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, 5540 bool secure) 5541 { 5542 uint32_t v = secure ? FRAME_TMZ : 0; 5543 5544 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); 5545 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1)); 5546 } 5547 5548 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) 5549 { 5550 uint32_t dw2 = 0; 5551 5552 if (amdgpu_sriov_vf(ring->adev)) 5553 gfx_v9_0_ring_emit_ce_meta(ring); 5554 5555 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 5556 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 5557 /* set load_global_config & load_global_uconfig */ 5558 dw2 |= 0x8001; 5559 /* set load_cs_sh_regs */ 5560 dw2 |= 0x01000000; 5561 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 5562 dw2 |= 0x10002; 5563 5564 /* set load_ce_ram if preamble presented */ 5565 if (AMDGPU_PREAMBLE_IB_PRESENT & flags) 5566 dw2 |= 0x10000000; 5567 } else { 5568 /* still load_ce_ram if this is the first time preamble presented 5569 * although there is no context switch happens. 5570 */ 5571 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags) 5572 dw2 |= 0x10000000; 5573 } 5574 5575 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 5576 amdgpu_ring_write(ring, dw2); 5577 amdgpu_ring_write(ring, 0); 5578 } 5579 5580 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring) 5581 { 5582 unsigned ret; 5583 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 5584 amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr)); 5585 amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr)); 5586 amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */ 5587 ret = ring->wptr & ring->buf_mask; 5588 amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */ 5589 return ret; 5590 } 5591 5592 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset) 5593 { 5594 unsigned cur; 5595 BUG_ON(offset > ring->buf_mask); 5596 BUG_ON(ring->ring[offset] != 0x55aa55aa); 5597 5598 cur = (ring->wptr & ring->buf_mask) - 1; 5599 if (likely(cur > offset)) 5600 ring->ring[offset] = cur - offset; 5601 else 5602 ring->ring[offset] = (ring->ring_size>>2) - offset + cur; 5603 } 5604 5605 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 5606 uint32_t reg_val_offs) 5607 { 5608 struct amdgpu_device *adev = ring->adev; 5609 5610 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 5611 amdgpu_ring_write(ring, 0 | /* src: register*/ 5612 (5 << 8) | /* dst: memory */ 5613 (1 << 20)); /* write confirm */ 5614 amdgpu_ring_write(ring, reg); 5615 amdgpu_ring_write(ring, 0); 5616 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 5617 reg_val_offs * 4)); 5618 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 5619 reg_val_offs * 4)); 5620 } 5621 5622 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 5623 uint32_t val) 5624 { 5625 uint32_t cmd = 0; 5626 5627 switch (ring->funcs->type) { 5628 case AMDGPU_RING_TYPE_GFX: 5629 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 5630 break; 5631 case AMDGPU_RING_TYPE_KIQ: 5632 cmd = (1 << 16); /* no inc addr */ 5633 break; 5634 default: 5635 cmd = WR_CONFIRM; 5636 break; 5637 } 5638 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5639 amdgpu_ring_write(ring, cmd); 5640 amdgpu_ring_write(ring, reg); 5641 amdgpu_ring_write(ring, 0); 5642 amdgpu_ring_write(ring, val); 5643 } 5644 5645 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 5646 uint32_t val, uint32_t mask) 5647 { 5648 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); 5649 } 5650 5651 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, 5652 uint32_t reg0, uint32_t reg1, 5653 uint32_t ref, uint32_t mask) 5654 { 5655 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5656 struct amdgpu_device *adev = ring->adev; 5657 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ? 5658 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait; 5659 5660 if (fw_version_ok) 5661 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, 5662 ref, mask, 0x20); 5663 else 5664 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1, 5665 ref, mask); 5666 } 5667 5668 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid) 5669 { 5670 struct amdgpu_device *adev = ring->adev; 5671 uint32_t value = 0; 5672 5673 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03); 5674 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01); 5675 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1); 5676 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid); 5677 WREG32_SOC15(GC, 0, mmSQ_CMD, value); 5678 } 5679 5680 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 5681 enum amdgpu_interrupt_state state) 5682 { 5683 switch (state) { 5684 case AMDGPU_IRQ_STATE_DISABLE: 5685 case AMDGPU_IRQ_STATE_ENABLE: 5686 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5687 TIME_STAMP_INT_ENABLE, 5688 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5689 break; 5690 default: 5691 break; 5692 } 5693 } 5694 5695 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 5696 int me, int pipe, 5697 enum amdgpu_interrupt_state state) 5698 { 5699 u32 mec_int_cntl, mec_int_cntl_reg; 5700 5701 /* 5702 * amdgpu controls only the first MEC. That's why this function only 5703 * handles the setting of interrupts for this specific MEC. All other 5704 * pipes' interrupts are set by amdkfd. 5705 */ 5706 5707 if (me == 1) { 5708 switch (pipe) { 5709 case 0: 5710 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL); 5711 break; 5712 case 1: 5713 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL); 5714 break; 5715 case 2: 5716 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL); 5717 break; 5718 case 3: 5719 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL); 5720 break; 5721 default: 5722 DRM_DEBUG("invalid pipe %d\n", pipe); 5723 return; 5724 } 5725 } else { 5726 DRM_DEBUG("invalid me %d\n", me); 5727 return; 5728 } 5729 5730 switch (state) { 5731 case AMDGPU_IRQ_STATE_DISABLE: 5732 mec_int_cntl = RREG32(mec_int_cntl_reg); 5733 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5734 TIME_STAMP_INT_ENABLE, 0); 5735 WREG32(mec_int_cntl_reg, mec_int_cntl); 5736 break; 5737 case AMDGPU_IRQ_STATE_ENABLE: 5738 mec_int_cntl = RREG32(mec_int_cntl_reg); 5739 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5740 TIME_STAMP_INT_ENABLE, 1); 5741 WREG32(mec_int_cntl_reg, mec_int_cntl); 5742 break; 5743 default: 5744 break; 5745 } 5746 } 5747 5748 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 5749 struct amdgpu_irq_src *source, 5750 unsigned type, 5751 enum amdgpu_interrupt_state state) 5752 { 5753 switch (state) { 5754 case AMDGPU_IRQ_STATE_DISABLE: 5755 case AMDGPU_IRQ_STATE_ENABLE: 5756 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5757 PRIV_REG_INT_ENABLE, 5758 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5759 break; 5760 default: 5761 break; 5762 } 5763 5764 return 0; 5765 } 5766 5767 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 5768 struct amdgpu_irq_src *source, 5769 unsigned type, 5770 enum amdgpu_interrupt_state state) 5771 { 5772 switch (state) { 5773 case AMDGPU_IRQ_STATE_DISABLE: 5774 case AMDGPU_IRQ_STATE_ENABLE: 5775 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5776 PRIV_INSTR_INT_ENABLE, 5777 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5778 break; 5779 default: 5780 break; 5781 } 5782 5783 return 0; 5784 } 5785 5786 #define ENABLE_ECC_ON_ME_PIPE(me, pipe) \ 5787 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5788 CP_ECC_ERROR_INT_ENABLE, 1) 5789 5790 #define DISABLE_ECC_ON_ME_PIPE(me, pipe) \ 5791 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5792 CP_ECC_ERROR_INT_ENABLE, 0) 5793 5794 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev, 5795 struct amdgpu_irq_src *source, 5796 unsigned type, 5797 enum amdgpu_interrupt_state state) 5798 { 5799 switch (state) { 5800 case AMDGPU_IRQ_STATE_DISABLE: 5801 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5802 CP_ECC_ERROR_INT_ENABLE, 0); 5803 DISABLE_ECC_ON_ME_PIPE(1, 0); 5804 DISABLE_ECC_ON_ME_PIPE(1, 1); 5805 DISABLE_ECC_ON_ME_PIPE(1, 2); 5806 DISABLE_ECC_ON_ME_PIPE(1, 3); 5807 break; 5808 5809 case AMDGPU_IRQ_STATE_ENABLE: 5810 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5811 CP_ECC_ERROR_INT_ENABLE, 1); 5812 ENABLE_ECC_ON_ME_PIPE(1, 0); 5813 ENABLE_ECC_ON_ME_PIPE(1, 1); 5814 ENABLE_ECC_ON_ME_PIPE(1, 2); 5815 ENABLE_ECC_ON_ME_PIPE(1, 3); 5816 break; 5817 default: 5818 break; 5819 } 5820 5821 return 0; 5822 } 5823 5824 5825 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev, 5826 struct amdgpu_irq_src *src, 5827 unsigned type, 5828 enum amdgpu_interrupt_state state) 5829 { 5830 switch (type) { 5831 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 5832 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state); 5833 break; 5834 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 5835 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 5836 break; 5837 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 5838 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 5839 break; 5840 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 5841 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 5842 break; 5843 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 5844 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 5845 break; 5846 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP: 5847 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state); 5848 break; 5849 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP: 5850 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state); 5851 break; 5852 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP: 5853 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state); 5854 break; 5855 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP: 5856 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state); 5857 break; 5858 default: 5859 break; 5860 } 5861 return 0; 5862 } 5863 5864 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev, 5865 struct amdgpu_irq_src *source, 5866 struct amdgpu_iv_entry *entry) 5867 { 5868 int i; 5869 u8 me_id, pipe_id, queue_id; 5870 struct amdgpu_ring *ring; 5871 5872 DRM_DEBUG("IH: CP EOP\n"); 5873 me_id = (entry->ring_id & 0x0c) >> 2; 5874 pipe_id = (entry->ring_id & 0x03) >> 0; 5875 queue_id = (entry->ring_id & 0x70) >> 4; 5876 5877 switch (me_id) { 5878 case 0: 5879 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); 5880 break; 5881 case 1: 5882 case 2: 5883 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5884 ring = &adev->gfx.compute_ring[i]; 5885 /* Per-queue interrupt is supported for MEC starting from VI. 5886 * The interrupt can only be enabled/disabled per pipe instead of per queue. 5887 */ 5888 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) 5889 amdgpu_fence_process(ring); 5890 } 5891 break; 5892 } 5893 return 0; 5894 } 5895 5896 static void gfx_v9_0_fault(struct amdgpu_device *adev, 5897 struct amdgpu_iv_entry *entry) 5898 { 5899 u8 me_id, pipe_id, queue_id; 5900 struct amdgpu_ring *ring; 5901 int i; 5902 5903 me_id = (entry->ring_id & 0x0c) >> 2; 5904 pipe_id = (entry->ring_id & 0x03) >> 0; 5905 queue_id = (entry->ring_id & 0x70) >> 4; 5906 5907 switch (me_id) { 5908 case 0: 5909 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); 5910 break; 5911 case 1: 5912 case 2: 5913 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5914 ring = &adev->gfx.compute_ring[i]; 5915 if (ring->me == me_id && ring->pipe == pipe_id && 5916 ring->queue == queue_id) 5917 drm_sched_fault(&ring->sched); 5918 } 5919 break; 5920 } 5921 } 5922 5923 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev, 5924 struct amdgpu_irq_src *source, 5925 struct amdgpu_iv_entry *entry) 5926 { 5927 DRM_ERROR("Illegal register access in command stream\n"); 5928 gfx_v9_0_fault(adev, entry); 5929 return 0; 5930 } 5931 5932 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev, 5933 struct amdgpu_irq_src *source, 5934 struct amdgpu_iv_entry *entry) 5935 { 5936 DRM_ERROR("Illegal instruction in command stream\n"); 5937 gfx_v9_0_fault(adev, entry); 5938 return 0; 5939 } 5940 5941 5942 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = { 5943 { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 5944 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT), 5945 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT) 5946 }, 5947 { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 5948 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT), 5949 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT) 5950 }, 5951 { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5952 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1), 5953 0, 0 5954 }, 5955 { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5956 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2), 5957 0, 0 5958 }, 5959 { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 5960 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT), 5961 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT) 5962 }, 5963 { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5964 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT), 5965 0, 0 5966 }, 5967 { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5968 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT), 5969 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT) 5970 }, 5971 { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 5972 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT), 5973 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT) 5974 }, 5975 { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 5976 SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1), 5977 0, 0 5978 }, 5979 { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 5980 SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1), 5981 0, 0 5982 }, 5983 { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 5984 SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1), 5985 0, 0 5986 }, 5987 { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 5988 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC), 5989 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED) 5990 }, 5991 { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 5992 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED), 5993 0, 0 5994 }, 5995 { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5996 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC), 5997 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED) 5998 }, 5999 { "GDS_OA_PHY_PHY_CMD_RAM_MEM", 6000 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 6001 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC), 6002 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED) 6003 }, 6004 { "GDS_OA_PHY_PHY_DATA_RAM_MEM", 6005 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 6006 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED), 6007 0, 0 6008 }, 6009 { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM", 6010 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6011 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC), 6012 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED) 6013 }, 6014 { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM", 6015 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6016 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC), 6017 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED) 6018 }, 6019 { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM", 6020 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6021 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC), 6022 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED) 6023 }, 6024 { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM", 6025 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6026 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC), 6027 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED) 6028 }, 6029 { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 6030 SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT), 6031 0, 0 6032 }, 6033 { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6034 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT), 6035 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT) 6036 }, 6037 { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6038 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT), 6039 0, 0 6040 }, 6041 { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6042 SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT), 6043 0, 0 6044 }, 6045 { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6046 SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT), 6047 0, 0 6048 }, 6049 { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6050 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT), 6051 0, 0 6052 }, 6053 { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 6054 SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT), 6055 0, 0 6056 }, 6057 { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 6058 SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT), 6059 0, 0 6060 }, 6061 { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6062 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT), 6063 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT) 6064 }, 6065 { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6066 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT), 6067 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT) 6068 }, 6069 { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6070 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT), 6071 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT) 6072 }, 6073 { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6074 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT), 6075 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT) 6076 }, 6077 { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6078 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT), 6079 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT) 6080 }, 6081 { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6082 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT), 6083 0, 0 6084 }, 6085 { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6086 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT), 6087 0, 0 6088 }, 6089 { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6090 SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT), 6091 0, 0 6092 }, 6093 { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6094 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT), 6095 0, 0 6096 }, 6097 { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6098 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT), 6099 0, 0 6100 }, 6101 { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6102 SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT), 6103 0, 0 6104 }, 6105 { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6106 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT), 6107 0, 0 6108 }, 6109 { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6110 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT), 6111 0, 0 6112 }, 6113 { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6114 SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT), 6115 0, 0 6116 }, 6117 { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6118 SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT), 6119 0, 0 6120 }, 6121 { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6122 SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT), 6123 0, 0 6124 }, 6125 { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6126 SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT), 6127 0, 0 6128 }, 6129 { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6130 SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT), 6131 0, 0 6132 }, 6133 { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 6134 SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT), 6135 0, 0 6136 }, 6137 { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6138 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT), 6139 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT) 6140 }, 6141 { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6142 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT), 6143 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT) 6144 }, 6145 { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6146 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT), 6147 0, 0 6148 }, 6149 { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6150 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT), 6151 0, 0 6152 }, 6153 { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6154 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT), 6155 0, 0 6156 }, 6157 { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6158 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT), 6159 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT) 6160 }, 6161 { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6162 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT), 6163 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT) 6164 }, 6165 { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6166 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT), 6167 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT) 6168 }, 6169 { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6170 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT), 6171 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT) 6172 }, 6173 { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6174 SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT), 6175 0, 0 6176 }, 6177 { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6178 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT), 6179 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT) 6180 }, 6181 { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6182 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT), 6183 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT) 6184 }, 6185 { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6186 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT), 6187 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT) 6188 }, 6189 { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6190 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT), 6191 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT) 6192 }, 6193 { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6194 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT), 6195 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT) 6196 }, 6197 { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6198 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT), 6199 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT) 6200 }, 6201 { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6202 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT), 6203 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT) 6204 }, 6205 { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6206 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT), 6207 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT) 6208 }, 6209 { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6210 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT), 6211 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT) 6212 }, 6213 { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6214 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT), 6215 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT) 6216 }, 6217 { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6218 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT), 6219 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT) 6220 }, 6221 { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6222 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT), 6223 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT) 6224 }, 6225 { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6226 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT), 6227 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT) 6228 }, 6229 { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6230 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT), 6231 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT) 6232 }, 6233 { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6234 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT), 6235 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT) 6236 }, 6237 { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6238 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT), 6239 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT) 6240 }, 6241 { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6242 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT), 6243 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT) 6244 }, 6245 { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6246 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT), 6247 0, 0 6248 }, 6249 { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6250 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT), 6251 0, 0 6252 }, 6253 { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6254 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT), 6255 0, 0 6256 }, 6257 { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6258 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT), 6259 0, 0 6260 }, 6261 { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6262 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT), 6263 0, 0 6264 }, 6265 { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6266 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT), 6267 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT) 6268 }, 6269 { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6270 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT), 6271 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT) 6272 }, 6273 { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6274 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT), 6275 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT) 6276 }, 6277 { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6278 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT), 6279 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT) 6280 }, 6281 { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6282 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT), 6283 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT) 6284 }, 6285 { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6286 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT), 6287 0, 0 6288 }, 6289 { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6290 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT), 6291 0, 0 6292 }, 6293 { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6294 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT), 6295 0, 0 6296 }, 6297 { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6298 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT), 6299 0, 0 6300 }, 6301 { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6302 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT), 6303 0, 0 6304 }, 6305 { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6306 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), 6307 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT) 6308 }, 6309 { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6310 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), 6311 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT) 6312 }, 6313 { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6314 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), 6315 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT) 6316 }, 6317 { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6318 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT), 6319 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT) 6320 }, 6321 { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6322 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT), 6323 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT) 6324 }, 6325 { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6326 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), 6327 0, 0 6328 }, 6329 { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6330 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), 6331 0, 0 6332 }, 6333 { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6334 SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT), 6335 0, 0 6336 }, 6337 { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6338 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT), 6339 0, 0 6340 }, 6341 { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6342 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT), 6343 0, 0 6344 }, 6345 { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6346 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), 6347 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT) 6348 }, 6349 { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6350 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), 6351 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT) 6352 }, 6353 { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6354 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), 6355 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT) 6356 }, 6357 { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6358 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), 6359 0, 0 6360 }, 6361 { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6362 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), 6363 0, 0 6364 }, 6365 { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6366 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT), 6367 0, 0 6368 }, 6369 { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6370 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT), 6371 0, 0 6372 }, 6373 { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6374 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT), 6375 0, 0 6376 }, 6377 { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6378 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT), 6379 0, 0 6380 } 6381 }; 6382 6383 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 6384 void *inject_if) 6385 { 6386 struct ras_inject_if *info = (struct ras_inject_if *)inject_if; 6387 int ret; 6388 struct ta_ras_trigger_error_input block_info = { 0 }; 6389 6390 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6391 return -EINVAL; 6392 6393 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks)) 6394 return -EINVAL; 6395 6396 if (!ras_gfx_subblocks[info->head.sub_block_index].name) 6397 return -EPERM; 6398 6399 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type & 6400 info->head.type)) { 6401 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n", 6402 ras_gfx_subblocks[info->head.sub_block_index].name, 6403 info->head.type); 6404 return -EPERM; 6405 } 6406 6407 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type & 6408 info->head.type)) { 6409 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n", 6410 ras_gfx_subblocks[info->head.sub_block_index].name, 6411 info->head.type); 6412 return -EPERM; 6413 } 6414 6415 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block); 6416 block_info.sub_block_index = 6417 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock; 6418 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type); 6419 block_info.address = info->address; 6420 block_info.value = info->value; 6421 6422 mutex_lock(&adev->grbm_idx_mutex); 6423 ret = psp_ras_trigger_error(&adev->psp, &block_info); 6424 mutex_unlock(&adev->grbm_idx_mutex); 6425 6426 return ret; 6427 } 6428 6429 static const char *vml2_mems[] = { 6430 "UTC_VML2_BANK_CACHE_0_BIGK_MEM0", 6431 "UTC_VML2_BANK_CACHE_0_BIGK_MEM1", 6432 "UTC_VML2_BANK_CACHE_0_4K_MEM0", 6433 "UTC_VML2_BANK_CACHE_0_4K_MEM1", 6434 "UTC_VML2_BANK_CACHE_1_BIGK_MEM0", 6435 "UTC_VML2_BANK_CACHE_1_BIGK_MEM1", 6436 "UTC_VML2_BANK_CACHE_1_4K_MEM0", 6437 "UTC_VML2_BANK_CACHE_1_4K_MEM1", 6438 "UTC_VML2_BANK_CACHE_2_BIGK_MEM0", 6439 "UTC_VML2_BANK_CACHE_2_BIGK_MEM1", 6440 "UTC_VML2_BANK_CACHE_2_4K_MEM0", 6441 "UTC_VML2_BANK_CACHE_2_4K_MEM1", 6442 "UTC_VML2_BANK_CACHE_3_BIGK_MEM0", 6443 "UTC_VML2_BANK_CACHE_3_BIGK_MEM1", 6444 "UTC_VML2_BANK_CACHE_3_4K_MEM0", 6445 "UTC_VML2_BANK_CACHE_3_4K_MEM1", 6446 }; 6447 6448 static const char *vml2_walker_mems[] = { 6449 "UTC_VML2_CACHE_PDE0_MEM0", 6450 "UTC_VML2_CACHE_PDE0_MEM1", 6451 "UTC_VML2_CACHE_PDE1_MEM0", 6452 "UTC_VML2_CACHE_PDE1_MEM1", 6453 "UTC_VML2_CACHE_PDE2_MEM0", 6454 "UTC_VML2_CACHE_PDE2_MEM1", 6455 "UTC_VML2_RDIF_LOG_FIFO", 6456 }; 6457 6458 static const char *atc_l2_cache_2m_mems[] = { 6459 "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM", 6460 "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM", 6461 "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM", 6462 "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM", 6463 }; 6464 6465 static const char *atc_l2_cache_4k_mems[] = { 6466 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0", 6467 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1", 6468 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2", 6469 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3", 6470 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4", 6471 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5", 6472 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6", 6473 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7", 6474 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0", 6475 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1", 6476 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2", 6477 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3", 6478 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4", 6479 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5", 6480 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6", 6481 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7", 6482 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0", 6483 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1", 6484 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2", 6485 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3", 6486 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4", 6487 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5", 6488 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6", 6489 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7", 6490 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0", 6491 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1", 6492 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2", 6493 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3", 6494 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4", 6495 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5", 6496 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6", 6497 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7", 6498 }; 6499 6500 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev, 6501 struct ras_err_data *err_data) 6502 { 6503 uint32_t i, data; 6504 uint32_t sec_count, ded_count; 6505 6506 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6507 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6508 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6509 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6510 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6511 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6512 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6513 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6514 6515 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6516 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6517 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6518 6519 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT); 6520 if (sec_count) { 6521 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6522 "SEC %d\n", i, vml2_mems[i], sec_count); 6523 err_data->ce_count += sec_count; 6524 } 6525 6526 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT); 6527 if (ded_count) { 6528 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6529 "DED %d\n", i, vml2_mems[i], ded_count); 6530 err_data->ue_count += ded_count; 6531 } 6532 } 6533 6534 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6535 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6536 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6537 6538 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6539 SEC_COUNT); 6540 if (sec_count) { 6541 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6542 "SEC %d\n", i, vml2_walker_mems[i], sec_count); 6543 err_data->ce_count += sec_count; 6544 } 6545 6546 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6547 DED_COUNT); 6548 if (ded_count) { 6549 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6550 "DED %d\n", i, vml2_walker_mems[i], ded_count); 6551 err_data->ue_count += ded_count; 6552 } 6553 } 6554 6555 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6556 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6557 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6558 6559 sec_count = (data & 0x00006000L) >> 0xd; 6560 if (sec_count) { 6561 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6562 "SEC %d\n", i, atc_l2_cache_2m_mems[i], 6563 sec_count); 6564 err_data->ce_count += sec_count; 6565 } 6566 } 6567 6568 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6569 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6570 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6571 6572 sec_count = (data & 0x00006000L) >> 0xd; 6573 if (sec_count) { 6574 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6575 "SEC %d\n", i, atc_l2_cache_4k_mems[i], 6576 sec_count); 6577 err_data->ce_count += sec_count; 6578 } 6579 6580 ded_count = (data & 0x00018000L) >> 0xf; 6581 if (ded_count) { 6582 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6583 "DED %d\n", i, atc_l2_cache_4k_mems[i], 6584 ded_count); 6585 err_data->ue_count += ded_count; 6586 } 6587 } 6588 6589 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6590 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6591 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6592 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6593 6594 return 0; 6595 } 6596 6597 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev, 6598 const struct soc15_reg_entry *reg, 6599 uint32_t se_id, uint32_t inst_id, uint32_t value, 6600 uint32_t *sec_count, uint32_t *ded_count) 6601 { 6602 uint32_t i; 6603 uint32_t sec_cnt, ded_cnt; 6604 6605 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) { 6606 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset || 6607 gfx_v9_0_ras_fields[i].seg != reg->seg || 6608 gfx_v9_0_ras_fields[i].inst != reg->inst) 6609 continue; 6610 6611 sec_cnt = (value & 6612 gfx_v9_0_ras_fields[i].sec_count_mask) >> 6613 gfx_v9_0_ras_fields[i].sec_count_shift; 6614 if (sec_cnt) { 6615 dev_info(adev->dev, "GFX SubBlock %s, " 6616 "Instance[%d][%d], SEC %d\n", 6617 gfx_v9_0_ras_fields[i].name, 6618 se_id, inst_id, 6619 sec_cnt); 6620 *sec_count += sec_cnt; 6621 } 6622 6623 ded_cnt = (value & 6624 gfx_v9_0_ras_fields[i].ded_count_mask) >> 6625 gfx_v9_0_ras_fields[i].ded_count_shift; 6626 if (ded_cnt) { 6627 dev_info(adev->dev, "GFX SubBlock %s, " 6628 "Instance[%d][%d], DED %d\n", 6629 gfx_v9_0_ras_fields[i].name, 6630 se_id, inst_id, 6631 ded_cnt); 6632 *ded_count += ded_cnt; 6633 } 6634 } 6635 6636 return 0; 6637 } 6638 6639 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev) 6640 { 6641 int i, j, k; 6642 6643 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6644 return; 6645 6646 /* read back registers to clear the counters */ 6647 mutex_lock(&adev->grbm_idx_mutex); 6648 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6649 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6650 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6651 gfx_v9_0_select_se_sh(adev, j, 0x0, k); 6652 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6653 } 6654 } 6655 } 6656 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000); 6657 mutex_unlock(&adev->grbm_idx_mutex); 6658 6659 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6660 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6661 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6662 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6663 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6664 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6665 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6666 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6667 6668 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6669 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6670 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6671 } 6672 6673 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6674 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6675 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6676 } 6677 6678 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6679 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6680 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6681 } 6682 6683 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6684 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6685 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6686 } 6687 6688 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6689 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6690 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6691 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6692 } 6693 6694 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 6695 void *ras_error_status) 6696 { 6697 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; 6698 uint32_t sec_count = 0, ded_count = 0; 6699 uint32_t i, j, k; 6700 uint32_t reg_value; 6701 6702 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6703 return -EINVAL; 6704 6705 err_data->ue_count = 0; 6706 err_data->ce_count = 0; 6707 6708 mutex_lock(&adev->grbm_idx_mutex); 6709 6710 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6711 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6712 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6713 gfx_v9_0_select_se_sh(adev, j, 0, k); 6714 reg_value = 6715 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6716 if (reg_value) 6717 gfx_v9_0_ras_error_count(adev, 6718 &gfx_v9_0_edc_counter_regs[i], 6719 j, k, reg_value, 6720 &sec_count, &ded_count); 6721 } 6722 } 6723 } 6724 6725 err_data->ce_count += sec_count; 6726 err_data->ue_count += ded_count; 6727 6728 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 6729 mutex_unlock(&adev->grbm_idx_mutex); 6730 6731 gfx_v9_0_query_utc_edc_status(adev, err_data); 6732 6733 return 0; 6734 } 6735 6736 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring) 6737 { 6738 const unsigned int cp_coher_cntl = 6739 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) | 6740 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) | 6741 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) | 6742 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) | 6743 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1); 6744 6745 /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */ 6746 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5)); 6747 amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */ 6748 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6749 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */ 6750 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6751 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 6752 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ 6753 } 6754 6755 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring, 6756 uint32_t pipe, bool enable) 6757 { 6758 struct amdgpu_device *adev = ring->adev; 6759 uint32_t val; 6760 uint32_t wcl_cs_reg; 6761 6762 /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */ 6763 val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT; 6764 6765 switch (pipe) { 6766 case 0: 6767 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0); 6768 break; 6769 case 1: 6770 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1); 6771 break; 6772 case 2: 6773 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2); 6774 break; 6775 case 3: 6776 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3); 6777 break; 6778 default: 6779 DRM_DEBUG("invalid pipe %d\n", pipe); 6780 return; 6781 } 6782 6783 amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val); 6784 6785 } 6786 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable) 6787 { 6788 struct amdgpu_device *adev = ring->adev; 6789 uint32_t val; 6790 int i; 6791 6792 6793 /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit 6794 * number of gfx waves. Setting 5 bit will make sure gfx only gets 6795 * around 25% of gpu resources. 6796 */ 6797 val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT; 6798 amdgpu_ring_emit_wreg(ring, 6799 SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX), 6800 val); 6801 6802 /* Restrict waves for normal/low priority compute queues as well 6803 * to get best QoS for high priority compute jobs. 6804 * 6805 * amdgpu controls only 1st ME(0-3 CS pipes). 6806 */ 6807 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 6808 if (i != ring->pipe) 6809 gfx_v9_0_emit_wave_limit_cs(ring, i, enable); 6810 6811 } 6812 } 6813 6814 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = { 6815 .name = "gfx_v9_0", 6816 .early_init = gfx_v9_0_early_init, 6817 .late_init = gfx_v9_0_late_init, 6818 .sw_init = gfx_v9_0_sw_init, 6819 .sw_fini = gfx_v9_0_sw_fini, 6820 .hw_init = gfx_v9_0_hw_init, 6821 .hw_fini = gfx_v9_0_hw_fini, 6822 .suspend = gfx_v9_0_suspend, 6823 .resume = gfx_v9_0_resume, 6824 .is_idle = gfx_v9_0_is_idle, 6825 .wait_for_idle = gfx_v9_0_wait_for_idle, 6826 .soft_reset = gfx_v9_0_soft_reset, 6827 .set_clockgating_state = gfx_v9_0_set_clockgating_state, 6828 .set_powergating_state = gfx_v9_0_set_powergating_state, 6829 .get_clockgating_state = gfx_v9_0_get_clockgating_state, 6830 }; 6831 6832 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = { 6833 .type = AMDGPU_RING_TYPE_GFX, 6834 .align_mask = 0xff, 6835 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6836 .support_64bit_ptrs = true, 6837 .vmhub = AMDGPU_GFXHUB_0, 6838 .get_rptr = gfx_v9_0_ring_get_rptr_gfx, 6839 .get_wptr = gfx_v9_0_ring_get_wptr_gfx, 6840 .set_wptr = gfx_v9_0_ring_set_wptr_gfx, 6841 .emit_frame_size = /* totally 242 maximum if 16 IBs */ 6842 5 + /* COND_EXEC */ 6843 7 + /* PIPELINE_SYNC */ 6844 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6845 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6846 2 + /* VM_FLUSH */ 6847 8 + /* FENCE for VM_FLUSH */ 6848 20 + /* GDS switch */ 6849 4 + /* double SWITCH_BUFFER, 6850 the first COND_EXEC jump to the place just 6851 prior to this double SWITCH_BUFFER */ 6852 5 + /* COND_EXEC */ 6853 7 + /* HDP_flush */ 6854 4 + /* VGT_flush */ 6855 14 + /* CE_META */ 6856 31 + /* DE_META */ 6857 3 + /* CNTX_CTRL */ 6858 5 + /* HDP_INVL */ 6859 8 + 8 + /* FENCE x2 */ 6860 2 + /* SWITCH_BUFFER */ 6861 7, /* gfx_v9_0_emit_mem_sync */ 6862 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */ 6863 .emit_ib = gfx_v9_0_ring_emit_ib_gfx, 6864 .emit_fence = gfx_v9_0_ring_emit_fence, 6865 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6866 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6867 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6868 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6869 .test_ring = gfx_v9_0_ring_test_ring, 6870 .test_ib = gfx_v9_0_ring_test_ib, 6871 .insert_nop = amdgpu_ring_insert_nop, 6872 .pad_ib = amdgpu_ring_generic_pad_ib, 6873 .emit_switch_buffer = gfx_v9_ring_emit_sb, 6874 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl, 6875 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec, 6876 .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec, 6877 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl, 6878 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6879 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6880 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6881 .soft_recovery = gfx_v9_0_ring_soft_recovery, 6882 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6883 }; 6884 6885 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { 6886 .type = AMDGPU_RING_TYPE_COMPUTE, 6887 .align_mask = 0xff, 6888 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6889 .support_64bit_ptrs = true, 6890 .vmhub = AMDGPU_GFXHUB_0, 6891 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6892 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6893 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6894 .emit_frame_size = 6895 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6896 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6897 5 + /* hdp invalidate */ 6898 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6899 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6900 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6901 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6902 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */ 6903 7 + /* gfx_v9_0_emit_mem_sync */ 6904 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */ 6905 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */ 6906 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6907 .emit_ib = gfx_v9_0_ring_emit_ib_compute, 6908 .emit_fence = gfx_v9_0_ring_emit_fence, 6909 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6910 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6911 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6912 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6913 .test_ring = gfx_v9_0_ring_test_ring, 6914 .test_ib = gfx_v9_0_ring_test_ib, 6915 .insert_nop = amdgpu_ring_insert_nop, 6916 .pad_ib = amdgpu_ring_generic_pad_ib, 6917 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6918 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6919 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6920 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6921 .emit_wave_limit = gfx_v9_0_emit_wave_limit, 6922 }; 6923 6924 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = { 6925 .type = AMDGPU_RING_TYPE_KIQ, 6926 .align_mask = 0xff, 6927 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6928 .support_64bit_ptrs = true, 6929 .vmhub = AMDGPU_GFXHUB_0, 6930 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6931 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6932 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6933 .emit_frame_size = 6934 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6935 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6936 5 + /* hdp invalidate */ 6937 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6938 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6939 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6940 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6941 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 6942 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6943 .emit_fence = gfx_v9_0_ring_emit_fence_kiq, 6944 .test_ring = gfx_v9_0_ring_test_ring, 6945 .insert_nop = amdgpu_ring_insert_nop, 6946 .pad_ib = amdgpu_ring_generic_pad_ib, 6947 .emit_rreg = gfx_v9_0_ring_emit_rreg, 6948 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6949 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6950 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6951 }; 6952 6953 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev) 6954 { 6955 int i; 6956 6957 adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq; 6958 6959 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 6960 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx; 6961 6962 for (i = 0; i < adev->gfx.num_compute_rings; i++) 6963 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute; 6964 } 6965 6966 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = { 6967 .set = gfx_v9_0_set_eop_interrupt_state, 6968 .process = gfx_v9_0_eop_irq, 6969 }; 6970 6971 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = { 6972 .set = gfx_v9_0_set_priv_reg_fault_state, 6973 .process = gfx_v9_0_priv_reg_irq, 6974 }; 6975 6976 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = { 6977 .set = gfx_v9_0_set_priv_inst_fault_state, 6978 .process = gfx_v9_0_priv_inst_irq, 6979 }; 6980 6981 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = { 6982 .set = gfx_v9_0_set_cp_ecc_error_state, 6983 .process = amdgpu_gfx_cp_ecc_error_irq, 6984 }; 6985 6986 6987 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev) 6988 { 6989 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 6990 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs; 6991 6992 adev->gfx.priv_reg_irq.num_types = 1; 6993 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs; 6994 6995 adev->gfx.priv_inst_irq.num_types = 1; 6996 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs; 6997 6998 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/ 6999 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs; 7000 } 7001 7002 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev) 7003 { 7004 switch (adev->asic_type) { 7005 case CHIP_VEGA10: 7006 case CHIP_VEGA12: 7007 case CHIP_VEGA20: 7008 case CHIP_RAVEN: 7009 case CHIP_ARCTURUS: 7010 case CHIP_RENOIR: 7011 case CHIP_ALDEBARAN: 7012 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs; 7013 break; 7014 default: 7015 break; 7016 } 7017 } 7018 7019 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev) 7020 { 7021 /* init asci gds info */ 7022 switch (adev->asic_type) { 7023 case CHIP_VEGA10: 7024 case CHIP_VEGA12: 7025 case CHIP_VEGA20: 7026 adev->gds.gds_size = 0x10000; 7027 break; 7028 case CHIP_RAVEN: 7029 case CHIP_ARCTURUS: 7030 adev->gds.gds_size = 0x1000; 7031 break; 7032 case CHIP_ALDEBARAN: 7033 /* aldebaran removed all the GDS internal memory, 7034 * only support GWS opcode in kernel, like barrier 7035 * semaphore.etc */ 7036 adev->gds.gds_size = 0; 7037 break; 7038 default: 7039 adev->gds.gds_size = 0x10000; 7040 break; 7041 } 7042 7043 switch (adev->asic_type) { 7044 case CHIP_VEGA10: 7045 case CHIP_VEGA20: 7046 adev->gds.gds_compute_max_wave_id = 0x7ff; 7047 break; 7048 case CHIP_VEGA12: 7049 adev->gds.gds_compute_max_wave_id = 0x27f; 7050 break; 7051 case CHIP_RAVEN: 7052 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 7053 adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */ 7054 else 7055 adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */ 7056 break; 7057 case CHIP_ARCTURUS: 7058 adev->gds.gds_compute_max_wave_id = 0xfff; 7059 break; 7060 case CHIP_ALDEBARAN: 7061 /* deprecated for Aldebaran, no usage at all */ 7062 adev->gds.gds_compute_max_wave_id = 0; 7063 break; 7064 default: 7065 /* this really depends on the chip */ 7066 adev->gds.gds_compute_max_wave_id = 0x7ff; 7067 break; 7068 } 7069 7070 adev->gds.gws_size = 64; 7071 adev->gds.oa_size = 16; 7072 } 7073 7074 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev, 7075 u32 bitmap) 7076 { 7077 u32 data; 7078 7079 if (!bitmap) 7080 return; 7081 7082 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 7083 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 7084 7085 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data); 7086 } 7087 7088 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev) 7089 { 7090 u32 data, mask; 7091 7092 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG); 7093 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG); 7094 7095 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 7096 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 7097 7098 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); 7099 7100 return (~data) & mask; 7101 } 7102 7103 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 7104 struct amdgpu_cu_info *cu_info) 7105 { 7106 int i, j, k, counter, active_cu_number = 0; 7107 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0; 7108 unsigned disable_masks[4 * 4]; 7109 7110 if (!adev || !cu_info) 7111 return -EINVAL; 7112 7113 /* 7114 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs 7115 */ 7116 if (adev->gfx.config.max_shader_engines * 7117 adev->gfx.config.max_sh_per_se > 16) 7118 return -EINVAL; 7119 7120 amdgpu_gfx_parse_disable_cu(disable_masks, 7121 adev->gfx.config.max_shader_engines, 7122 adev->gfx.config.max_sh_per_se); 7123 7124 mutex_lock(&adev->grbm_idx_mutex); 7125 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 7126 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 7127 mask = 1; 7128 ao_bitmap = 0; 7129 counter = 0; 7130 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 7131 gfx_v9_0_set_user_cu_inactive_bitmap( 7132 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]); 7133 bitmap = gfx_v9_0_get_cu_active_bitmap(adev); 7134 7135 /* 7136 * The bitmap(and ao_cu_bitmap) in cu_info structure is 7137 * 4x4 size array, and it's usually suitable for Vega 7138 * ASICs which has 4*2 SE/SH layout. 7139 * But for Arcturus, SE/SH layout is changed to 8*1. 7140 * To mostly reduce the impact, we make it compatible 7141 * with current bitmap array as below: 7142 * SE4,SH0 --> bitmap[0][1] 7143 * SE5,SH0 --> bitmap[1][1] 7144 * SE6,SH0 --> bitmap[2][1] 7145 * SE7,SH0 --> bitmap[3][1] 7146 */ 7147 cu_info->bitmap[i % 4][j + i / 4] = bitmap; 7148 7149 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 7150 if (bitmap & mask) { 7151 if (counter < adev->gfx.config.max_cu_per_sh) 7152 ao_bitmap |= mask; 7153 counter ++; 7154 } 7155 mask <<= 1; 7156 } 7157 active_cu_number += counter; 7158 if (i < 2 && j < 2) 7159 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8)); 7160 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap; 7161 } 7162 } 7163 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 7164 mutex_unlock(&adev->grbm_idx_mutex); 7165 7166 cu_info->number = active_cu_number; 7167 cu_info->ao_cu_mask = ao_cu_mask; 7168 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 7169 7170 return 0; 7171 } 7172 7173 const struct amdgpu_ip_block_version gfx_v9_0_ip_block = 7174 { 7175 .type = AMD_IP_BLOCK_TYPE_GFX, 7176 .major = 9, 7177 .minor = 0, 7178 .rev = 0, 7179 .funcs = &gfx_v9_0_ip_funcs, 7180 }; 7181