1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/delay.h> 25 #include <linux/kernel.h> 26 #include <linux/firmware.h> 27 #include <linux/module.h> 28 #include <linux/pci.h> 29 30 #include "amdgpu.h" 31 #include "amdgpu_gfx.h" 32 #include "soc15.h" 33 #include "soc15d.h" 34 #include "amdgpu_atomfirmware.h" 35 #include "amdgpu_pm.h" 36 37 #include "gc/gc_9_0_offset.h" 38 #include "gc/gc_9_0_sh_mask.h" 39 40 #include "vega10_enum.h" 41 42 #include "soc15_common.h" 43 #include "clearstate_gfx9.h" 44 #include "v9_structs.h" 45 46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h" 47 48 #include "amdgpu_ras.h" 49 50 #include "gfx_v9_4.h" 51 #include "gfx_v9_0.h" 52 #include "gfx_v9_4_2.h" 53 54 #include "asic_reg/pwr/pwr_10_0_offset.h" 55 #include "asic_reg/pwr/pwr_10_0_sh_mask.h" 56 #include "asic_reg/gc/gc_9_0_default.h" 57 58 #define GFX9_NUM_GFX_RINGS 1 59 #define GFX9_MEC_HPD_SIZE 4096 60 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L 61 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L 62 63 #define mmGCEA_PROBE_MAP 0x070c 64 #define mmGCEA_PROBE_MAP_BASE_IDX 0 65 66 MODULE_FIRMWARE("amdgpu/vega10_ce.bin"); 67 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin"); 68 MODULE_FIRMWARE("amdgpu/vega10_me.bin"); 69 MODULE_FIRMWARE("amdgpu/vega10_mec.bin"); 70 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin"); 71 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin"); 72 73 MODULE_FIRMWARE("amdgpu/vega12_ce.bin"); 74 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin"); 75 MODULE_FIRMWARE("amdgpu/vega12_me.bin"); 76 MODULE_FIRMWARE("amdgpu/vega12_mec.bin"); 77 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin"); 78 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin"); 79 80 MODULE_FIRMWARE("amdgpu/vega20_ce.bin"); 81 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin"); 82 MODULE_FIRMWARE("amdgpu/vega20_me.bin"); 83 MODULE_FIRMWARE("amdgpu/vega20_mec.bin"); 84 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin"); 85 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin"); 86 87 MODULE_FIRMWARE("amdgpu/raven_ce.bin"); 88 MODULE_FIRMWARE("amdgpu/raven_pfp.bin"); 89 MODULE_FIRMWARE("amdgpu/raven_me.bin"); 90 MODULE_FIRMWARE("amdgpu/raven_mec.bin"); 91 MODULE_FIRMWARE("amdgpu/raven_mec2.bin"); 92 MODULE_FIRMWARE("amdgpu/raven_rlc.bin"); 93 94 MODULE_FIRMWARE("amdgpu/picasso_ce.bin"); 95 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin"); 96 MODULE_FIRMWARE("amdgpu/picasso_me.bin"); 97 MODULE_FIRMWARE("amdgpu/picasso_mec.bin"); 98 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin"); 99 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin"); 100 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin"); 101 102 MODULE_FIRMWARE("amdgpu/raven2_ce.bin"); 103 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin"); 104 MODULE_FIRMWARE("amdgpu/raven2_me.bin"); 105 MODULE_FIRMWARE("amdgpu/raven2_mec.bin"); 106 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin"); 107 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin"); 108 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin"); 109 110 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin"); 111 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin"); 112 113 MODULE_FIRMWARE("amdgpu/renoir_ce.bin"); 114 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin"); 115 MODULE_FIRMWARE("amdgpu/renoir_me.bin"); 116 MODULE_FIRMWARE("amdgpu/renoir_mec.bin"); 117 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin"); 118 119 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin"); 120 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin"); 121 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin"); 122 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin"); 123 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin"); 124 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin"); 125 126 MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin"); 127 MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin"); 128 MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin"); 129 130 #define mmTCP_CHAN_STEER_0_ARCT 0x0b03 131 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0 132 #define mmTCP_CHAN_STEER_1_ARCT 0x0b04 133 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0 134 #define mmTCP_CHAN_STEER_2_ARCT 0x0b09 135 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0 136 #define mmTCP_CHAN_STEER_3_ARCT 0x0b0a 137 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0 138 #define mmTCP_CHAN_STEER_4_ARCT 0x0b0b 139 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0 140 #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c 141 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0 142 143 enum ta_ras_gfx_subblock { 144 /*CPC*/ 145 TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0, 146 TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START, 147 TA_RAS_BLOCK__GFX_CPC_UCODE, 148 TA_RAS_BLOCK__GFX_DC_STATE_ME1, 149 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1, 150 TA_RAS_BLOCK__GFX_DC_RESTORE_ME1, 151 TA_RAS_BLOCK__GFX_DC_STATE_ME2, 152 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2, 153 TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 154 TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 155 /* CPF*/ 156 TA_RAS_BLOCK__GFX_CPF_INDEX_START, 157 TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START, 158 TA_RAS_BLOCK__GFX_CPF_ROQ_ME1, 159 TA_RAS_BLOCK__GFX_CPF_TAG, 160 TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG, 161 /* CPG*/ 162 TA_RAS_BLOCK__GFX_CPG_INDEX_START, 163 TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START, 164 TA_RAS_BLOCK__GFX_CPG_DMA_TAG, 165 TA_RAS_BLOCK__GFX_CPG_TAG, 166 TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG, 167 /* GDS*/ 168 TA_RAS_BLOCK__GFX_GDS_INDEX_START, 169 TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START, 170 TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE, 171 TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM, 172 TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM, 173 TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 174 TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 175 /* SPI*/ 176 TA_RAS_BLOCK__GFX_SPI_SR_MEM, 177 /* SQ*/ 178 TA_RAS_BLOCK__GFX_SQ_INDEX_START, 179 TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START, 180 TA_RAS_BLOCK__GFX_SQ_LDS_D, 181 TA_RAS_BLOCK__GFX_SQ_LDS_I, 182 TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/ 183 TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR, 184 /* SQC (3 ranges)*/ 185 TA_RAS_BLOCK__GFX_SQC_INDEX_START, 186 /* SQC range 0*/ 187 TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START, 188 TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO = 189 TA_RAS_BLOCK__GFX_SQC_INDEX0_START, 190 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 191 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO, 192 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 193 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO, 194 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 195 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 196 TA_RAS_BLOCK__GFX_SQC_INDEX0_END = 197 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 198 /* SQC range 1*/ 199 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 200 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM = 201 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 202 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 203 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO, 204 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM, 205 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM, 206 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO, 207 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO, 208 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 209 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 210 TA_RAS_BLOCK__GFX_SQC_INDEX1_END = 211 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 212 /* SQC range 2*/ 213 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 214 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM = 215 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 216 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 217 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO, 218 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM, 219 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM, 220 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO, 221 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO, 222 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 223 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 224 TA_RAS_BLOCK__GFX_SQC_INDEX2_END = 225 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 226 TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END, 227 /* TA*/ 228 TA_RAS_BLOCK__GFX_TA_INDEX_START, 229 TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START, 230 TA_RAS_BLOCK__GFX_TA_FS_AFIFO, 231 TA_RAS_BLOCK__GFX_TA_FL_LFIFO, 232 TA_RAS_BLOCK__GFX_TA_FX_LFIFO, 233 TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 234 TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 235 /* TCA*/ 236 TA_RAS_BLOCK__GFX_TCA_INDEX_START, 237 TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START, 238 TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 239 TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 240 /* TCC (5 sub-ranges)*/ 241 TA_RAS_BLOCK__GFX_TCC_INDEX_START, 242 /* TCC range 0*/ 243 TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START, 244 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START, 245 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1, 246 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0, 247 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1, 248 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0, 249 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1, 250 TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG, 251 TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 252 TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 253 /* TCC range 1*/ 254 TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 255 TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 256 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 257 TA_RAS_BLOCK__GFX_TCC_INDEX1_END = 258 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 259 /* TCC range 2*/ 260 TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 261 TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 262 TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL, 263 TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO, 264 TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN, 265 TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ, 266 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO, 267 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM, 268 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 269 TA_RAS_BLOCK__GFX_TCC_INDEX2_END = 270 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 271 /* TCC range 3*/ 272 TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 273 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 274 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 275 TA_RAS_BLOCK__GFX_TCC_INDEX3_END = 276 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 277 /* TCC range 4*/ 278 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 279 TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN = 280 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 281 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 282 TA_RAS_BLOCK__GFX_TCC_INDEX4_END = 283 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 284 TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END, 285 /* TCI*/ 286 TA_RAS_BLOCK__GFX_TCI_WRITE_RAM, 287 /* TCP*/ 288 TA_RAS_BLOCK__GFX_TCP_INDEX_START, 289 TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START, 290 TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM, 291 TA_RAS_BLOCK__GFX_TCP_CMD_FIFO, 292 TA_RAS_BLOCK__GFX_TCP_VM_FIFO, 293 TA_RAS_BLOCK__GFX_TCP_DB_RAM, 294 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0, 295 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 296 TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 297 /* TD*/ 298 TA_RAS_BLOCK__GFX_TD_INDEX_START, 299 TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START, 300 TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI, 301 TA_RAS_BLOCK__GFX_TD_CS_FIFO, 302 TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO, 303 /* EA (3 sub-ranges)*/ 304 TA_RAS_BLOCK__GFX_EA_INDEX_START, 305 /* EA range 0*/ 306 TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START, 307 TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START, 308 TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM, 309 TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM, 310 TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM, 311 TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM, 312 TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM, 313 TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM, 314 TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 315 TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 316 /* EA range 1*/ 317 TA_RAS_BLOCK__GFX_EA_INDEX1_START, 318 TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START, 319 TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM, 320 TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM, 321 TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM, 322 TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM, 323 TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM, 324 TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 325 TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 326 /* EA range 2*/ 327 TA_RAS_BLOCK__GFX_EA_INDEX2_START, 328 TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START, 329 TA_RAS_BLOCK__GFX_EA_MAM_D1MEM, 330 TA_RAS_BLOCK__GFX_EA_MAM_D2MEM, 331 TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 332 TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 333 TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END, 334 /* UTC VM L2 bank*/ 335 TA_RAS_BLOCK__UTC_VML2_BANK_CACHE, 336 /* UTC VM walker*/ 337 TA_RAS_BLOCK__UTC_VML2_WALKER, 338 /* UTC ATC L2 2MB cache*/ 339 TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK, 340 /* UTC ATC L2 4KB cache*/ 341 TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK, 342 TA_RAS_BLOCK__GFX_MAX 343 }; 344 345 struct ras_gfx_subblock { 346 unsigned char *name; 347 int ta_subblock; 348 int hw_supported_error_type; 349 int sw_supported_error_type; 350 }; 351 352 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \ 353 [AMDGPU_RAS_BLOCK__##subblock] = { \ 354 #subblock, \ 355 TA_RAS_BLOCK__##subblock, \ 356 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \ 357 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \ 358 } 359 360 static const struct ras_gfx_subblock ras_gfx_subblocks[] = { 361 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1), 362 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1), 363 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 364 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 365 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 366 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 367 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 368 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 369 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 370 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 371 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1), 372 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0), 373 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1), 374 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1), 375 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 376 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0), 377 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0, 378 0), 379 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0, 380 0), 381 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 382 AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0), 383 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0), 384 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1), 385 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0), 386 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0), 387 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1), 388 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 389 0, 0), 390 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 391 0), 392 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 393 0, 0), 394 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0, 395 0), 396 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 397 0, 0), 398 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 399 0), 400 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 401 1), 402 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 403 0, 0, 0), 404 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 405 0), 406 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 407 0), 408 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 409 0), 410 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 411 0), 412 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 413 0), 414 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 415 0, 0), 416 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 417 0), 418 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 419 0), 420 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 421 0, 0, 0), 422 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 423 0), 424 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 425 0), 426 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 427 0), 428 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 429 0), 430 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 431 0), 432 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 433 0, 0), 434 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 435 0), 436 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1), 437 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 438 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 439 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 440 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 441 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0), 442 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 443 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1), 444 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0, 445 1), 446 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0, 447 1), 448 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0, 449 1), 450 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0, 451 0), 452 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0, 453 0), 454 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 455 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 456 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0), 457 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0), 458 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0), 459 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0), 460 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 461 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0), 462 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0), 463 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 464 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0), 465 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0, 466 0), 467 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 468 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0, 469 0), 470 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0, 471 0, 0), 472 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0, 473 0), 474 AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 475 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1), 476 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0), 477 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 478 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 479 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 480 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0), 481 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0), 482 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1), 483 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0), 484 AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 485 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1), 486 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 487 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 488 AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 489 AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 490 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 491 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 492 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 493 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 494 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 495 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 496 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 497 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0), 498 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 499 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 500 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0), 501 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0), 502 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0), 503 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0), 504 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0), 505 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0), 506 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0), 507 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0), 508 }; 509 510 static const struct soc15_reg_golden golden_settings_gc_9_0[] = 511 { 512 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 513 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000), 514 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 515 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 516 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 517 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 518 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 519 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 520 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 521 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87), 522 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f), 523 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 524 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 525 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 526 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 527 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 528 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff), 529 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 530 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 531 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 532 }; 533 534 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] = 535 { 536 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107), 537 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 538 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 539 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 540 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 541 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 542 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042), 543 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 544 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000), 545 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 546 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 547 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 548 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 549 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 550 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 551 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107), 552 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800), 553 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080) 554 }; 555 556 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] = 557 { 558 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080), 559 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 560 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 561 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042), 562 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042), 563 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400), 564 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000), 565 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000), 566 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107), 567 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000), 568 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000) 569 }; 570 571 static const struct soc15_reg_golden golden_settings_gc_9_1[] = 572 { 573 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 574 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 575 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 576 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 577 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 578 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 579 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 580 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 581 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 582 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 583 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 584 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 585 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 586 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 587 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 588 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 589 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 590 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 591 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 592 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff), 593 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080), 594 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 595 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 596 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 597 }; 598 599 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] = 600 { 601 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 602 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042), 603 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042), 604 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000), 605 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000), 606 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 607 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800) 608 }; 609 610 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] = 611 { 612 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000), 613 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 614 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 615 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080), 616 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080), 617 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080), 618 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041), 619 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041), 620 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 621 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 622 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080), 623 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080), 624 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080), 625 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080), 626 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080), 627 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 628 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010), 629 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 630 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 631 }; 632 633 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] = 634 { 635 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 636 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 637 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 638 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042), 639 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042), 640 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 641 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 642 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 643 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 644 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 645 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 646 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc), 647 }; 648 649 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] = 650 { 651 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff), 652 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000), 653 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382) 654 }; 655 656 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] = 657 { 658 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 659 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 660 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 661 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 662 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 663 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 664 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 665 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 666 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87), 667 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f), 668 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 669 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 670 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 671 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 672 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 673 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff) 674 }; 675 676 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] = 677 { 678 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080), 679 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 680 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 681 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041), 682 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041), 683 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 684 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107), 685 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 686 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410), 687 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 688 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 689 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 690 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 691 }; 692 693 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] = 694 { 695 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 696 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000), 697 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e), 698 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca), 699 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098), 700 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3), 701 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1), 702 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135), 703 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000), 704 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00), 705 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000) 706 }; 707 708 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = { 709 {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)}, 710 {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)}, 711 }; 712 713 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] = 714 { 715 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 716 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 717 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 718 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 719 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 720 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 721 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 722 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 723 }; 724 725 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] = 726 { 727 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0, 728 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0, 729 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0, 730 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0, 731 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0, 732 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0, 733 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0, 734 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0, 735 }; 736 737 static void gfx_v9_0_rlcg_rw(struct amdgpu_device *adev, u32 offset, u32 v, u32 flag) 738 { 739 static void *scratch_reg0; 740 static void *scratch_reg1; 741 static void *scratch_reg2; 742 static void *scratch_reg3; 743 static void *spare_int; 744 static uint32_t grbm_cntl; 745 static uint32_t grbm_idx; 746 747 scratch_reg0 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0)*4; 748 scratch_reg1 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1)*4; 749 scratch_reg2 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG2)*4; 750 scratch_reg3 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3)*4; 751 spare_int = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT)*4; 752 753 grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL; 754 grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX; 755 756 if (amdgpu_sriov_runtime(adev)) { 757 pr_err("shouldn't call rlcg write register during runtime\n"); 758 return; 759 } 760 761 if (offset == grbm_cntl || offset == grbm_idx) { 762 if (offset == grbm_cntl) 763 writel(v, scratch_reg2); 764 else if (offset == grbm_idx) 765 writel(v, scratch_reg3); 766 767 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4)); 768 } else { 769 uint32_t i = 0; 770 uint32_t retries = 50000; 771 772 writel(v, scratch_reg0); 773 writel(offset | 0x80000000, scratch_reg1); 774 writel(1, spare_int); 775 for (i = 0; i < retries; i++) { 776 u32 tmp; 777 778 tmp = readl(scratch_reg1); 779 if (!(tmp & 0x80000000)) 780 break; 781 782 udelay(10); 783 } 784 if (i >= retries) 785 pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset); 786 } 787 788 } 789 790 static void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v, u32 flag) 791 { 792 if (amdgpu_sriov_fullaccess(adev)) { 793 gfx_v9_0_rlcg_rw(adev, offset, v, flag); 794 795 return; 796 } 797 798 if (flag & AMDGPU_REGS_NO_KIQ) 799 WREG32_NO_KIQ(offset, v); 800 else 801 WREG32(offset, v); 802 } 803 804 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042 805 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041 806 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042 807 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041 808 809 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev); 810 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev); 811 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev); 812 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev); 813 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 814 struct amdgpu_cu_info *cu_info); 815 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev); 816 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring); 817 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring); 818 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 819 void *ras_error_status); 820 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 821 void *inject_if); 822 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev); 823 824 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring, 825 uint64_t queue_mask) 826 { 827 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 828 amdgpu_ring_write(kiq_ring, 829 PACKET3_SET_RESOURCES_VMID_MASK(0) | 830 /* vmid_mask:0* queue_type:0 (KIQ) */ 831 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); 832 amdgpu_ring_write(kiq_ring, 833 lower_32_bits(queue_mask)); /* queue mask lo */ 834 amdgpu_ring_write(kiq_ring, 835 upper_32_bits(queue_mask)); /* queue mask hi */ 836 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */ 837 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */ 838 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 839 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 840 } 841 842 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring, 843 struct amdgpu_ring *ring) 844 { 845 struct amdgpu_device *adev = kiq_ring->adev; 846 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 847 uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 848 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 849 850 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 851 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 852 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 853 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 854 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ 855 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 856 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 857 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) | 858 /*queue_type: normal compute queue */ 859 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | 860 /* alloc format: all_on_one_pipe */ 861 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | 862 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) | 863 /* num_queues: must be 1 */ 864 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); 865 amdgpu_ring_write(kiq_ring, 866 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); 867 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 868 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 869 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 870 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 871 } 872 873 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, 874 struct amdgpu_ring *ring, 875 enum amdgpu_unmap_queues_action action, 876 u64 gpu_addr, u64 seq) 877 { 878 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 879 880 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 881 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 882 PACKET3_UNMAP_QUEUES_ACTION(action) | 883 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 884 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) | 885 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 886 amdgpu_ring_write(kiq_ring, 887 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 888 889 if (action == PREEMPT_QUEUES_NO_UNMAP) { 890 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr)); 891 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr)); 892 amdgpu_ring_write(kiq_ring, seq); 893 } else { 894 amdgpu_ring_write(kiq_ring, 0); 895 amdgpu_ring_write(kiq_ring, 0); 896 amdgpu_ring_write(kiq_ring, 0); 897 } 898 } 899 900 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring, 901 struct amdgpu_ring *ring, 902 u64 addr, 903 u64 seq) 904 { 905 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 906 907 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5)); 908 amdgpu_ring_write(kiq_ring, 909 PACKET3_QUERY_STATUS_CONTEXT_ID(0) | 910 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) | 911 PACKET3_QUERY_STATUS_COMMAND(2)); 912 /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 913 amdgpu_ring_write(kiq_ring, 914 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) | 915 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel)); 916 amdgpu_ring_write(kiq_ring, lower_32_bits(addr)); 917 amdgpu_ring_write(kiq_ring, upper_32_bits(addr)); 918 amdgpu_ring_write(kiq_ring, lower_32_bits(seq)); 919 amdgpu_ring_write(kiq_ring, upper_32_bits(seq)); 920 } 921 922 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring, 923 uint16_t pasid, uint32_t flush_type, 924 bool all_hub) 925 { 926 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); 927 amdgpu_ring_write(kiq_ring, 928 PACKET3_INVALIDATE_TLBS_DST_SEL(1) | 929 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) | 930 PACKET3_INVALIDATE_TLBS_PASID(pasid) | 931 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type)); 932 } 933 934 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = { 935 .kiq_set_resources = gfx_v9_0_kiq_set_resources, 936 .kiq_map_queues = gfx_v9_0_kiq_map_queues, 937 .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues, 938 .kiq_query_status = gfx_v9_0_kiq_query_status, 939 .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs, 940 .set_resources_size = 8, 941 .map_queues_size = 7, 942 .unmap_queues_size = 6, 943 .query_status_size = 7, 944 .invalidate_tlbs_size = 2, 945 }; 946 947 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) 948 { 949 adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs; 950 } 951 952 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev) 953 { 954 switch (adev->asic_type) { 955 case CHIP_VEGA10: 956 soc15_program_register_sequence(adev, 957 golden_settings_gc_9_0, 958 ARRAY_SIZE(golden_settings_gc_9_0)); 959 soc15_program_register_sequence(adev, 960 golden_settings_gc_9_0_vg10, 961 ARRAY_SIZE(golden_settings_gc_9_0_vg10)); 962 break; 963 case CHIP_VEGA12: 964 soc15_program_register_sequence(adev, 965 golden_settings_gc_9_2_1, 966 ARRAY_SIZE(golden_settings_gc_9_2_1)); 967 soc15_program_register_sequence(adev, 968 golden_settings_gc_9_2_1_vg12, 969 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12)); 970 break; 971 case CHIP_VEGA20: 972 soc15_program_register_sequence(adev, 973 golden_settings_gc_9_0, 974 ARRAY_SIZE(golden_settings_gc_9_0)); 975 soc15_program_register_sequence(adev, 976 golden_settings_gc_9_0_vg20, 977 ARRAY_SIZE(golden_settings_gc_9_0_vg20)); 978 break; 979 case CHIP_ARCTURUS: 980 soc15_program_register_sequence(adev, 981 golden_settings_gc_9_4_1_arct, 982 ARRAY_SIZE(golden_settings_gc_9_4_1_arct)); 983 break; 984 case CHIP_RAVEN: 985 soc15_program_register_sequence(adev, golden_settings_gc_9_1, 986 ARRAY_SIZE(golden_settings_gc_9_1)); 987 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 988 soc15_program_register_sequence(adev, 989 golden_settings_gc_9_1_rv2, 990 ARRAY_SIZE(golden_settings_gc_9_1_rv2)); 991 else 992 soc15_program_register_sequence(adev, 993 golden_settings_gc_9_1_rv1, 994 ARRAY_SIZE(golden_settings_gc_9_1_rv1)); 995 break; 996 case CHIP_RENOIR: 997 soc15_program_register_sequence(adev, 998 golden_settings_gc_9_1_rn, 999 ARRAY_SIZE(golden_settings_gc_9_1_rn)); 1000 return; /* for renoir, don't need common goldensetting */ 1001 case CHIP_ALDEBARAN: 1002 gfx_v9_4_2_init_golden_registers(adev, 1003 adev->smuio.funcs->get_die_id(adev)); 1004 break; 1005 default: 1006 break; 1007 } 1008 1009 if ((adev->asic_type != CHIP_ARCTURUS) && 1010 (adev->asic_type != CHIP_ALDEBARAN)) 1011 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common, 1012 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common)); 1013 } 1014 1015 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev) 1016 { 1017 adev->gfx.scratch.num_reg = 8; 1018 adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0); 1019 adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1; 1020 } 1021 1022 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel, 1023 bool wc, uint32_t reg, uint32_t val) 1024 { 1025 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 1026 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) | 1027 WRITE_DATA_DST_SEL(0) | 1028 (wc ? WR_CONFIRM : 0)); 1029 amdgpu_ring_write(ring, reg); 1030 amdgpu_ring_write(ring, 0); 1031 amdgpu_ring_write(ring, val); 1032 } 1033 1034 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, 1035 int mem_space, int opt, uint32_t addr0, 1036 uint32_t addr1, uint32_t ref, uint32_t mask, 1037 uint32_t inv) 1038 { 1039 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 1040 amdgpu_ring_write(ring, 1041 /* memory (1) or register (0) */ 1042 (WAIT_REG_MEM_MEM_SPACE(mem_space) | 1043 WAIT_REG_MEM_OPERATION(opt) | /* wait */ 1044 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 1045 WAIT_REG_MEM_ENGINE(eng_sel))); 1046 1047 if (mem_space) 1048 BUG_ON(addr0 & 0x3); /* Dword align */ 1049 amdgpu_ring_write(ring, addr0); 1050 amdgpu_ring_write(ring, addr1); 1051 amdgpu_ring_write(ring, ref); 1052 amdgpu_ring_write(ring, mask); 1053 amdgpu_ring_write(ring, inv); /* poll interval */ 1054 } 1055 1056 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring) 1057 { 1058 struct amdgpu_device *adev = ring->adev; 1059 uint32_t scratch; 1060 uint32_t tmp = 0; 1061 unsigned i; 1062 int r; 1063 1064 r = amdgpu_gfx_scratch_get(adev, &scratch); 1065 if (r) 1066 return r; 1067 1068 WREG32(scratch, 0xCAFEDEAD); 1069 r = amdgpu_ring_alloc(ring, 3); 1070 if (r) 1071 goto error_free_scratch; 1072 1073 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 1074 amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START)); 1075 amdgpu_ring_write(ring, 0xDEADBEEF); 1076 amdgpu_ring_commit(ring); 1077 1078 for (i = 0; i < adev->usec_timeout; i++) { 1079 tmp = RREG32(scratch); 1080 if (tmp == 0xDEADBEEF) 1081 break; 1082 udelay(1); 1083 } 1084 1085 if (i >= adev->usec_timeout) 1086 r = -ETIMEDOUT; 1087 1088 error_free_scratch: 1089 amdgpu_gfx_scratch_free(adev, scratch); 1090 return r; 1091 } 1092 1093 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 1094 { 1095 struct amdgpu_device *adev = ring->adev; 1096 struct amdgpu_ib ib; 1097 struct dma_fence *f = NULL; 1098 1099 unsigned index; 1100 uint64_t gpu_addr; 1101 uint32_t tmp; 1102 long r; 1103 1104 r = amdgpu_device_wb_get(adev, &index); 1105 if (r) 1106 return r; 1107 1108 gpu_addr = adev->wb.gpu_addr + (index * 4); 1109 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 1110 memset(&ib, 0, sizeof(ib)); 1111 r = amdgpu_ib_get(adev, NULL, 16, 1112 AMDGPU_IB_POOL_DIRECT, &ib); 1113 if (r) 1114 goto err1; 1115 1116 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 1117 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 1118 ib.ptr[2] = lower_32_bits(gpu_addr); 1119 ib.ptr[3] = upper_32_bits(gpu_addr); 1120 ib.ptr[4] = 0xDEADBEEF; 1121 ib.length_dw = 5; 1122 1123 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 1124 if (r) 1125 goto err2; 1126 1127 r = dma_fence_wait_timeout(f, false, timeout); 1128 if (r == 0) { 1129 r = -ETIMEDOUT; 1130 goto err2; 1131 } else if (r < 0) { 1132 goto err2; 1133 } 1134 1135 tmp = adev->wb.wb[index]; 1136 if (tmp == 0xDEADBEEF) 1137 r = 0; 1138 else 1139 r = -EINVAL; 1140 1141 err2: 1142 amdgpu_ib_free(adev, &ib, NULL); 1143 dma_fence_put(f); 1144 err1: 1145 amdgpu_device_wb_free(adev, index); 1146 return r; 1147 } 1148 1149 1150 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev) 1151 { 1152 release_firmware(adev->gfx.pfp_fw); 1153 adev->gfx.pfp_fw = NULL; 1154 release_firmware(adev->gfx.me_fw); 1155 adev->gfx.me_fw = NULL; 1156 release_firmware(adev->gfx.ce_fw); 1157 adev->gfx.ce_fw = NULL; 1158 release_firmware(adev->gfx.rlc_fw); 1159 adev->gfx.rlc_fw = NULL; 1160 release_firmware(adev->gfx.mec_fw); 1161 adev->gfx.mec_fw = NULL; 1162 release_firmware(adev->gfx.mec2_fw); 1163 adev->gfx.mec2_fw = NULL; 1164 1165 kfree(adev->gfx.rlc.register_list_format); 1166 } 1167 1168 static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev) 1169 { 1170 const struct rlc_firmware_header_v2_1 *rlc_hdr; 1171 1172 rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data; 1173 adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver); 1174 adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver); 1175 adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes); 1176 adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes); 1177 adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver); 1178 adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver); 1179 adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes); 1180 adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes); 1181 adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver); 1182 adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver); 1183 adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes); 1184 adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes); 1185 adev->gfx.rlc.reg_list_format_direct_reg_list_length = 1186 le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length); 1187 } 1188 1189 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev) 1190 { 1191 adev->gfx.me_fw_write_wait = false; 1192 adev->gfx.mec_fw_write_wait = false; 1193 1194 if ((adev->asic_type != CHIP_ARCTURUS) && 1195 ((adev->gfx.mec_fw_version < 0x000001a5) || 1196 (adev->gfx.mec_feature_version < 46) || 1197 (adev->gfx.pfp_fw_version < 0x000000b7) || 1198 (adev->gfx.pfp_feature_version < 46))) 1199 DRM_WARN_ONCE("CP firmware version too old, please update!"); 1200 1201 switch (adev->asic_type) { 1202 case CHIP_VEGA10: 1203 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1204 (adev->gfx.me_feature_version >= 42) && 1205 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1206 (adev->gfx.pfp_feature_version >= 42)) 1207 adev->gfx.me_fw_write_wait = true; 1208 1209 if ((adev->gfx.mec_fw_version >= 0x00000193) && 1210 (adev->gfx.mec_feature_version >= 42)) 1211 adev->gfx.mec_fw_write_wait = true; 1212 break; 1213 case CHIP_VEGA12: 1214 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1215 (adev->gfx.me_feature_version >= 44) && 1216 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1217 (adev->gfx.pfp_feature_version >= 44)) 1218 adev->gfx.me_fw_write_wait = true; 1219 1220 if ((adev->gfx.mec_fw_version >= 0x00000196) && 1221 (adev->gfx.mec_feature_version >= 44)) 1222 adev->gfx.mec_fw_write_wait = true; 1223 break; 1224 case CHIP_VEGA20: 1225 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1226 (adev->gfx.me_feature_version >= 44) && 1227 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1228 (adev->gfx.pfp_feature_version >= 44)) 1229 adev->gfx.me_fw_write_wait = true; 1230 1231 if ((adev->gfx.mec_fw_version >= 0x00000197) && 1232 (adev->gfx.mec_feature_version >= 44)) 1233 adev->gfx.mec_fw_write_wait = true; 1234 break; 1235 case CHIP_RAVEN: 1236 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1237 (adev->gfx.me_feature_version >= 42) && 1238 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1239 (adev->gfx.pfp_feature_version >= 42)) 1240 adev->gfx.me_fw_write_wait = true; 1241 1242 if ((adev->gfx.mec_fw_version >= 0x00000192) && 1243 (adev->gfx.mec_feature_version >= 42)) 1244 adev->gfx.mec_fw_write_wait = true; 1245 break; 1246 default: 1247 adev->gfx.me_fw_write_wait = true; 1248 adev->gfx.mec_fw_write_wait = true; 1249 break; 1250 } 1251 } 1252 1253 struct amdgpu_gfxoff_quirk { 1254 u16 chip_vendor; 1255 u16 chip_device; 1256 u16 subsys_vendor; 1257 u16 subsys_device; 1258 u8 revision; 1259 }; 1260 1261 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = { 1262 /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */ 1263 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 }, 1264 /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */ 1265 { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 }, 1266 /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */ 1267 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 }, 1268 { 0, 0, 0, 0, 0 }, 1269 }; 1270 1271 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev) 1272 { 1273 const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list; 1274 1275 while (p && p->chip_device != 0) { 1276 if (pdev->vendor == p->chip_vendor && 1277 pdev->device == p->chip_device && 1278 pdev->subsystem_vendor == p->subsys_vendor && 1279 pdev->subsystem_device == p->subsys_device && 1280 pdev->revision == p->revision) { 1281 return true; 1282 } 1283 ++p; 1284 } 1285 return false; 1286 } 1287 1288 static bool is_raven_kicker(struct amdgpu_device *adev) 1289 { 1290 if (adev->pm.fw_version >= 0x41e2b) 1291 return true; 1292 else 1293 return false; 1294 } 1295 1296 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev) 1297 { 1298 if (gfx_v9_0_should_disable_gfxoff(adev->pdev)) 1299 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1300 1301 switch (adev->asic_type) { 1302 case CHIP_VEGA10: 1303 case CHIP_VEGA12: 1304 case CHIP_VEGA20: 1305 break; 1306 case CHIP_RAVEN: 1307 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) || 1308 (adev->apu_flags & AMD_APU_IS_PICASSO)) && 1309 ((!is_raven_kicker(adev) && 1310 adev->gfx.rlc_fw_version < 531) || 1311 (adev->gfx.rlc_feature_version < 1) || 1312 !adev->gfx.rlc.is_rlc_v2_1)) 1313 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1314 1315 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1316 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1317 AMD_PG_SUPPORT_CP | 1318 AMD_PG_SUPPORT_RLC_SMU_HS; 1319 break; 1320 case CHIP_RENOIR: 1321 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1322 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1323 AMD_PG_SUPPORT_CP | 1324 AMD_PG_SUPPORT_RLC_SMU_HS; 1325 break; 1326 default: 1327 break; 1328 } 1329 } 1330 1331 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev, 1332 const char *chip_name) 1333 { 1334 char fw_name[30]; 1335 int err; 1336 struct amdgpu_firmware_info *info = NULL; 1337 const struct common_firmware_header *header = NULL; 1338 const struct gfx_firmware_header_v1_0 *cp_hdr; 1339 1340 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name); 1341 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev); 1342 if (err) 1343 goto out; 1344 err = amdgpu_ucode_validate(adev->gfx.pfp_fw); 1345 if (err) 1346 goto out; 1347 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; 1348 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1349 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1350 1351 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name); 1352 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev); 1353 if (err) 1354 goto out; 1355 err = amdgpu_ucode_validate(adev->gfx.me_fw); 1356 if (err) 1357 goto out; 1358 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; 1359 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1360 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1361 1362 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name); 1363 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev); 1364 if (err) 1365 goto out; 1366 err = amdgpu_ucode_validate(adev->gfx.ce_fw); 1367 if (err) 1368 goto out; 1369 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; 1370 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1371 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1372 1373 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1374 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP]; 1375 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP; 1376 info->fw = adev->gfx.pfp_fw; 1377 header = (const struct common_firmware_header *)info->fw->data; 1378 adev->firmware.fw_size += 1379 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1380 1381 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME]; 1382 info->ucode_id = AMDGPU_UCODE_ID_CP_ME; 1383 info->fw = adev->gfx.me_fw; 1384 header = (const struct common_firmware_header *)info->fw->data; 1385 adev->firmware.fw_size += 1386 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1387 1388 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE]; 1389 info->ucode_id = AMDGPU_UCODE_ID_CP_CE; 1390 info->fw = adev->gfx.ce_fw; 1391 header = (const struct common_firmware_header *)info->fw->data; 1392 adev->firmware.fw_size += 1393 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1394 } 1395 1396 out: 1397 if (err) { 1398 dev_err(adev->dev, 1399 "gfx9: Failed to load firmware \"%s\"\n", 1400 fw_name); 1401 release_firmware(adev->gfx.pfp_fw); 1402 adev->gfx.pfp_fw = NULL; 1403 release_firmware(adev->gfx.me_fw); 1404 adev->gfx.me_fw = NULL; 1405 release_firmware(adev->gfx.ce_fw); 1406 adev->gfx.ce_fw = NULL; 1407 } 1408 return err; 1409 } 1410 1411 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev, 1412 const char *chip_name) 1413 { 1414 char fw_name[30]; 1415 int err; 1416 struct amdgpu_firmware_info *info = NULL; 1417 const struct common_firmware_header *header = NULL; 1418 const struct rlc_firmware_header_v2_0 *rlc_hdr; 1419 unsigned int *tmp = NULL; 1420 unsigned int i = 0; 1421 uint16_t version_major; 1422 uint16_t version_minor; 1423 uint32_t smu_version; 1424 1425 /* 1426 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin 1427 * instead of picasso_rlc.bin. 1428 * Judgment method: 1429 * PCO AM4: revision >= 0xC8 && revision <= 0xCF 1430 * or revision >= 0xD8 && revision <= 0xDF 1431 * otherwise is PCO FP5 1432 */ 1433 if (!strcmp(chip_name, "picasso") && 1434 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) || 1435 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF)))) 1436 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name); 1437 else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) && 1438 (smu_version >= 0x41e2b)) 1439 /** 1440 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly. 1441 */ 1442 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name); 1443 else 1444 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name); 1445 err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev); 1446 if (err) 1447 goto out; 1448 err = amdgpu_ucode_validate(adev->gfx.rlc_fw); 1449 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 1450 1451 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 1452 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 1453 if (version_major == 2 && version_minor == 1) 1454 adev->gfx.rlc.is_rlc_v2_1 = true; 1455 1456 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); 1457 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); 1458 adev->gfx.rlc.save_and_restore_offset = 1459 le32_to_cpu(rlc_hdr->save_and_restore_offset); 1460 adev->gfx.rlc.clear_state_descriptor_offset = 1461 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset); 1462 adev->gfx.rlc.avail_scratch_ram_locations = 1463 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations); 1464 adev->gfx.rlc.reg_restore_list_size = 1465 le32_to_cpu(rlc_hdr->reg_restore_list_size); 1466 adev->gfx.rlc.reg_list_format_start = 1467 le32_to_cpu(rlc_hdr->reg_list_format_start); 1468 adev->gfx.rlc.reg_list_format_separate_start = 1469 le32_to_cpu(rlc_hdr->reg_list_format_separate_start); 1470 adev->gfx.rlc.starting_offsets_start = 1471 le32_to_cpu(rlc_hdr->starting_offsets_start); 1472 adev->gfx.rlc.reg_list_format_size_bytes = 1473 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes); 1474 adev->gfx.rlc.reg_list_size_bytes = 1475 le32_to_cpu(rlc_hdr->reg_list_size_bytes); 1476 adev->gfx.rlc.register_list_format = 1477 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes + 1478 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL); 1479 if (!adev->gfx.rlc.register_list_format) { 1480 err = -ENOMEM; 1481 goto out; 1482 } 1483 1484 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1485 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); 1486 for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++) 1487 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]); 1488 1489 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i; 1490 1491 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1492 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); 1493 for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++) 1494 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); 1495 1496 if (adev->gfx.rlc.is_rlc_v2_1) 1497 gfx_v9_0_init_rlc_ext_microcode(adev); 1498 1499 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1500 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G]; 1501 info->ucode_id = AMDGPU_UCODE_ID_RLC_G; 1502 info->fw = adev->gfx.rlc_fw; 1503 header = (const struct common_firmware_header *)info->fw->data; 1504 adev->firmware.fw_size += 1505 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1506 1507 if (adev->gfx.rlc.is_rlc_v2_1 && 1508 adev->gfx.rlc.save_restore_list_cntl_size_bytes && 1509 adev->gfx.rlc.save_restore_list_gpm_size_bytes && 1510 adev->gfx.rlc.save_restore_list_srm_size_bytes) { 1511 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL]; 1512 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL; 1513 info->fw = adev->gfx.rlc_fw; 1514 adev->firmware.fw_size += 1515 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE); 1516 1517 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM]; 1518 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM; 1519 info->fw = adev->gfx.rlc_fw; 1520 adev->firmware.fw_size += 1521 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE); 1522 1523 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM]; 1524 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM; 1525 info->fw = adev->gfx.rlc_fw; 1526 adev->firmware.fw_size += 1527 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE); 1528 } 1529 } 1530 1531 out: 1532 if (err) { 1533 dev_err(adev->dev, 1534 "gfx9: Failed to load firmware \"%s\"\n", 1535 fw_name); 1536 release_firmware(adev->gfx.rlc_fw); 1537 adev->gfx.rlc_fw = NULL; 1538 } 1539 return err; 1540 } 1541 1542 static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev) 1543 { 1544 if (adev->asic_type == CHIP_ALDEBARAN || 1545 adev->asic_type == CHIP_ARCTURUS || 1546 adev->asic_type == CHIP_RENOIR) 1547 return false; 1548 1549 return true; 1550 } 1551 1552 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev, 1553 const char *chip_name) 1554 { 1555 char fw_name[30]; 1556 int err; 1557 struct amdgpu_firmware_info *info = NULL; 1558 const struct common_firmware_header *header = NULL; 1559 const struct gfx_firmware_header_v1_0 *cp_hdr; 1560 1561 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name); 1562 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev); 1563 if (err) 1564 goto out; 1565 err = amdgpu_ucode_validate(adev->gfx.mec_fw); 1566 if (err) 1567 goto out; 1568 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1569 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1570 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1571 1572 1573 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) { 1574 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name); 1575 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev); 1576 if (!err) { 1577 err = amdgpu_ucode_validate(adev->gfx.mec2_fw); 1578 if (err) 1579 goto out; 1580 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1581 adev->gfx.mec2_fw->data; 1582 adev->gfx.mec2_fw_version = 1583 le32_to_cpu(cp_hdr->header.ucode_version); 1584 adev->gfx.mec2_feature_version = 1585 le32_to_cpu(cp_hdr->ucode_feature_version); 1586 } else { 1587 err = 0; 1588 adev->gfx.mec2_fw = NULL; 1589 } 1590 } else { 1591 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version; 1592 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version; 1593 } 1594 1595 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1596 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1]; 1597 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1; 1598 info->fw = adev->gfx.mec_fw; 1599 header = (const struct common_firmware_header *)info->fw->data; 1600 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1601 adev->firmware.fw_size += 1602 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1603 1604 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT]; 1605 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT; 1606 info->fw = adev->gfx.mec_fw; 1607 adev->firmware.fw_size += 1608 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1609 1610 if (adev->gfx.mec2_fw) { 1611 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2]; 1612 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2; 1613 info->fw = adev->gfx.mec2_fw; 1614 header = (const struct common_firmware_header *)info->fw->data; 1615 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1616 adev->firmware.fw_size += 1617 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1618 1619 /* TODO: Determine if MEC2 JT FW loading can be removed 1620 for all GFX V9 asic and above */ 1621 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) { 1622 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT]; 1623 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT; 1624 info->fw = adev->gfx.mec2_fw; 1625 adev->firmware.fw_size += 1626 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, 1627 PAGE_SIZE); 1628 } 1629 } 1630 } 1631 1632 out: 1633 gfx_v9_0_check_if_need_gfxoff(adev); 1634 gfx_v9_0_check_fw_write_wait(adev); 1635 if (err) { 1636 dev_err(adev->dev, 1637 "gfx9: Failed to load firmware \"%s\"\n", 1638 fw_name); 1639 release_firmware(adev->gfx.mec_fw); 1640 adev->gfx.mec_fw = NULL; 1641 release_firmware(adev->gfx.mec2_fw); 1642 adev->gfx.mec2_fw = NULL; 1643 } 1644 return err; 1645 } 1646 1647 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev) 1648 { 1649 const char *chip_name; 1650 int r; 1651 1652 DRM_DEBUG("\n"); 1653 1654 switch (adev->asic_type) { 1655 case CHIP_VEGA10: 1656 chip_name = "vega10"; 1657 break; 1658 case CHIP_VEGA12: 1659 chip_name = "vega12"; 1660 break; 1661 case CHIP_VEGA20: 1662 chip_name = "vega20"; 1663 break; 1664 case CHIP_RAVEN: 1665 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 1666 chip_name = "raven2"; 1667 else if (adev->apu_flags & AMD_APU_IS_PICASSO) 1668 chip_name = "picasso"; 1669 else 1670 chip_name = "raven"; 1671 break; 1672 case CHIP_ARCTURUS: 1673 chip_name = "arcturus"; 1674 break; 1675 case CHIP_RENOIR: 1676 if (adev->apu_flags & AMD_APU_IS_RENOIR) 1677 chip_name = "renoir"; 1678 else 1679 chip_name = "green_sardine"; 1680 break; 1681 case CHIP_ALDEBARAN: 1682 chip_name = "aldebaran"; 1683 break; 1684 default: 1685 BUG(); 1686 } 1687 1688 /* No CPG in Arcturus */ 1689 if (adev->gfx.num_gfx_rings) { 1690 r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name); 1691 if (r) 1692 return r; 1693 } 1694 1695 r = gfx_v9_0_init_rlc_microcode(adev, chip_name); 1696 if (r) 1697 return r; 1698 1699 r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name); 1700 if (r) 1701 return r; 1702 1703 return r; 1704 } 1705 1706 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev) 1707 { 1708 u32 count = 0; 1709 const struct cs_section_def *sect = NULL; 1710 const struct cs_extent_def *ext = NULL; 1711 1712 /* begin clear state */ 1713 count += 2; 1714 /* context control state */ 1715 count += 3; 1716 1717 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 1718 for (ext = sect->section; ext->extent != NULL; ++ext) { 1719 if (sect->id == SECT_CONTEXT) 1720 count += 2 + ext->reg_count; 1721 else 1722 return 0; 1723 } 1724 } 1725 1726 /* end clear state */ 1727 count += 2; 1728 /* clear state */ 1729 count += 2; 1730 1731 return count; 1732 } 1733 1734 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev, 1735 volatile u32 *buffer) 1736 { 1737 u32 count = 0, i; 1738 const struct cs_section_def *sect = NULL; 1739 const struct cs_extent_def *ext = NULL; 1740 1741 if (adev->gfx.rlc.cs_data == NULL) 1742 return; 1743 if (buffer == NULL) 1744 return; 1745 1746 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1747 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 1748 1749 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 1750 buffer[count++] = cpu_to_le32(0x80000000); 1751 buffer[count++] = cpu_to_le32(0x80000000); 1752 1753 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { 1754 for (ext = sect->section; ext->extent != NULL; ++ext) { 1755 if (sect->id == SECT_CONTEXT) { 1756 buffer[count++] = 1757 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 1758 buffer[count++] = cpu_to_le32(ext->reg_index - 1759 PACKET3_SET_CONTEXT_REG_START); 1760 for (i = 0; i < ext->reg_count; i++) 1761 buffer[count++] = cpu_to_le32(ext->extent[i]); 1762 } else { 1763 return; 1764 } 1765 } 1766 } 1767 1768 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1769 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 1770 1771 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 1772 buffer[count++] = cpu_to_le32(0); 1773 } 1774 1775 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev) 1776 { 1777 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; 1778 uint32_t pg_always_on_cu_num = 2; 1779 uint32_t always_on_cu_num; 1780 uint32_t i, j, k; 1781 uint32_t mask, cu_bitmap, counter; 1782 1783 if (adev->flags & AMD_IS_APU) 1784 always_on_cu_num = 4; 1785 else if (adev->asic_type == CHIP_VEGA12) 1786 always_on_cu_num = 8; 1787 else 1788 always_on_cu_num = 12; 1789 1790 mutex_lock(&adev->grbm_idx_mutex); 1791 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 1792 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 1793 mask = 1; 1794 cu_bitmap = 0; 1795 counter = 0; 1796 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 1797 1798 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 1799 if (cu_info->bitmap[i][j] & mask) { 1800 if (counter == pg_always_on_cu_num) 1801 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap); 1802 if (counter < always_on_cu_num) 1803 cu_bitmap |= mask; 1804 else 1805 break; 1806 counter++; 1807 } 1808 mask <<= 1; 1809 } 1810 1811 WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap); 1812 cu_info->ao_cu_bitmap[i][j] = cu_bitmap; 1813 } 1814 } 1815 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1816 mutex_unlock(&adev->grbm_idx_mutex); 1817 } 1818 1819 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev) 1820 { 1821 uint32_t data; 1822 1823 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1824 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1825 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7); 1826 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1827 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16)); 1828 1829 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1830 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1831 1832 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1833 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500); 1834 1835 mutex_lock(&adev->grbm_idx_mutex); 1836 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1837 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1838 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1839 1840 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1841 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1842 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1843 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1844 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1845 1846 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1847 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1848 data &= 0x0000FFFF; 1849 data |= 0x00C00000; 1850 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1851 1852 /* 1853 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven), 1854 * programmed in gfx_v9_0_init_always_on_cu_mask() 1855 */ 1856 1857 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1858 * but used for RLC_LB_CNTL configuration */ 1859 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1860 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1861 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1862 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1863 mutex_unlock(&adev->grbm_idx_mutex); 1864 1865 gfx_v9_0_init_always_on_cu_mask(adev); 1866 } 1867 1868 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev) 1869 { 1870 uint32_t data; 1871 1872 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1873 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1874 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8); 1875 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1876 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16)); 1877 1878 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1879 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1880 1881 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1882 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800); 1883 1884 mutex_lock(&adev->grbm_idx_mutex); 1885 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1886 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1887 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1888 1889 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1890 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1891 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1892 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1893 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1894 1895 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1896 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1897 data &= 0x0000FFFF; 1898 data |= 0x00C00000; 1899 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1900 1901 /* 1902 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON), 1903 * programmed in gfx_v9_0_init_always_on_cu_mask() 1904 */ 1905 1906 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1907 * but used for RLC_LB_CNTL configuration */ 1908 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1909 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1910 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1911 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1912 mutex_unlock(&adev->grbm_idx_mutex); 1913 1914 gfx_v9_0_init_always_on_cu_mask(adev); 1915 } 1916 1917 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable) 1918 { 1919 WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0); 1920 } 1921 1922 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev) 1923 { 1924 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) 1925 return 5; 1926 else 1927 return 4; 1928 } 1929 1930 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev) 1931 { 1932 const struct cs_section_def *cs_data; 1933 int r; 1934 1935 adev->gfx.rlc.cs_data = gfx9_cs_data; 1936 1937 cs_data = adev->gfx.rlc.cs_data; 1938 1939 if (cs_data) { 1940 /* init clear state block */ 1941 r = amdgpu_gfx_rlc_init_csb(adev); 1942 if (r) 1943 return r; 1944 } 1945 1946 if (adev->flags & AMD_IS_APU) { 1947 /* TODO: double check the cp_table_size for RV */ 1948 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ 1949 r = amdgpu_gfx_rlc_init_cpt(adev); 1950 if (r) 1951 return r; 1952 } 1953 1954 switch (adev->asic_type) { 1955 case CHIP_RAVEN: 1956 gfx_v9_0_init_lbpw(adev); 1957 break; 1958 case CHIP_VEGA20: 1959 gfx_v9_4_init_lbpw(adev); 1960 break; 1961 default: 1962 break; 1963 } 1964 1965 /* init spm vmid with 0xf */ 1966 if (adev->gfx.rlc.funcs->update_spm_vmid) 1967 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf); 1968 1969 return 0; 1970 } 1971 1972 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev) 1973 { 1974 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 1975 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); 1976 } 1977 1978 static int gfx_v9_0_mec_init(struct amdgpu_device *adev) 1979 { 1980 int r; 1981 u32 *hpd; 1982 const __le32 *fw_data; 1983 unsigned fw_size; 1984 u32 *fw; 1985 size_t mec_hpd_size; 1986 1987 const struct gfx_firmware_header_v1_0 *mec_hdr; 1988 1989 bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 1990 1991 /* take ownership of the relevant compute queues */ 1992 amdgpu_gfx_compute_queue_acquire(adev); 1993 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE; 1994 if (mec_hpd_size) { 1995 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 1996 AMDGPU_GEM_DOMAIN_VRAM, 1997 &adev->gfx.mec.hpd_eop_obj, 1998 &adev->gfx.mec.hpd_eop_gpu_addr, 1999 (void **)&hpd); 2000 if (r) { 2001 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 2002 gfx_v9_0_mec_fini(adev); 2003 return r; 2004 } 2005 2006 memset(hpd, 0, mec_hpd_size); 2007 2008 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 2009 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 2010 } 2011 2012 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 2013 2014 fw_data = (const __le32 *) 2015 (adev->gfx.mec_fw->data + 2016 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 2017 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes); 2018 2019 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes, 2020 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 2021 &adev->gfx.mec.mec_fw_obj, 2022 &adev->gfx.mec.mec_fw_gpu_addr, 2023 (void **)&fw); 2024 if (r) { 2025 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r); 2026 gfx_v9_0_mec_fini(adev); 2027 return r; 2028 } 2029 2030 memcpy(fw, fw_data, fw_size); 2031 2032 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 2033 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 2034 2035 return 0; 2036 } 2037 2038 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address) 2039 { 2040 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 2041 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 2042 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 2043 (address << SQ_IND_INDEX__INDEX__SHIFT) | 2044 (SQ_IND_INDEX__FORCE_READ_MASK)); 2045 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 2046 } 2047 2048 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd, 2049 uint32_t wave, uint32_t thread, 2050 uint32_t regno, uint32_t num, uint32_t *out) 2051 { 2052 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 2053 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 2054 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 2055 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 2056 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) | 2057 (SQ_IND_INDEX__FORCE_READ_MASK) | 2058 (SQ_IND_INDEX__AUTO_INCR_MASK)); 2059 while (num--) 2060 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 2061 } 2062 2063 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 2064 { 2065 /* type 1 wave data */ 2066 dst[(*no_fields)++] = 1; 2067 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS); 2068 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO); 2069 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI); 2070 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO); 2071 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI); 2072 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID); 2073 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0); 2074 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1); 2075 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC); 2076 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC); 2077 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS); 2078 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS); 2079 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); 2080 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); 2081 } 2082 2083 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd, 2084 uint32_t wave, uint32_t start, 2085 uint32_t size, uint32_t *dst) 2086 { 2087 wave_read_regs( 2088 adev, simd, wave, 0, 2089 start + SQIND_WAVE_SGPRS_OFFSET, size, dst); 2090 } 2091 2092 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd, 2093 uint32_t wave, uint32_t thread, 2094 uint32_t start, uint32_t size, 2095 uint32_t *dst) 2096 { 2097 wave_read_regs( 2098 adev, simd, wave, thread, 2099 start + SQIND_WAVE_VGPRS_OFFSET, size, dst); 2100 } 2101 2102 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev, 2103 u32 me, u32 pipe, u32 q, u32 vm) 2104 { 2105 soc15_grbm_select(adev, me, pipe, q, vm); 2106 } 2107 2108 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = { 2109 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 2110 .select_se_sh = &gfx_v9_0_select_se_sh, 2111 .read_wave_data = &gfx_v9_0_read_wave_data, 2112 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 2113 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 2114 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 2115 }; 2116 2117 static const struct amdgpu_gfx_ras_funcs gfx_v9_0_ras_funcs = { 2118 .ras_late_init = amdgpu_gfx_ras_late_init, 2119 .ras_fini = amdgpu_gfx_ras_fini, 2120 .ras_error_inject = &gfx_v9_0_ras_error_inject, 2121 .query_ras_error_count = &gfx_v9_0_query_ras_error_count, 2122 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count, 2123 }; 2124 2125 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev) 2126 { 2127 u32 gb_addr_config; 2128 int err; 2129 2130 adev->gfx.funcs = &gfx_v9_0_gfx_funcs; 2131 2132 switch (adev->asic_type) { 2133 case CHIP_VEGA10: 2134 adev->gfx.config.max_hw_contexts = 8; 2135 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2136 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2137 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2138 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2139 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN; 2140 break; 2141 case CHIP_VEGA12: 2142 adev->gfx.config.max_hw_contexts = 8; 2143 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2144 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2145 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2146 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2147 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN; 2148 DRM_INFO("fix gfx.config for vega12\n"); 2149 break; 2150 case CHIP_VEGA20: 2151 adev->gfx.ras_funcs = &gfx_v9_0_ras_funcs; 2152 adev->gfx.config.max_hw_contexts = 8; 2153 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2154 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2155 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2156 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2157 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2158 gb_addr_config &= ~0xf3e777ff; 2159 gb_addr_config |= 0x22014042; 2160 /* check vbios table if gpu info is not available */ 2161 err = amdgpu_atomfirmware_get_gfx_info(adev); 2162 if (err) 2163 return err; 2164 break; 2165 case CHIP_RAVEN: 2166 adev->gfx.config.max_hw_contexts = 8; 2167 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2168 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2169 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2170 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2171 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 2172 gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN; 2173 else 2174 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN; 2175 break; 2176 case CHIP_ARCTURUS: 2177 adev->gfx.ras_funcs = &gfx_v9_4_ras_funcs; 2178 adev->gfx.config.max_hw_contexts = 8; 2179 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2180 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2181 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2182 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2183 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2184 gb_addr_config &= ~0xf3e777ff; 2185 gb_addr_config |= 0x22014042; 2186 break; 2187 case CHIP_RENOIR: 2188 adev->gfx.config.max_hw_contexts = 8; 2189 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2190 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2191 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; 2192 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2193 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2194 gb_addr_config &= ~0xf3e777ff; 2195 gb_addr_config |= 0x22010042; 2196 break; 2197 case CHIP_ALDEBARAN: 2198 adev->gfx.ras_funcs = &gfx_v9_4_2_ras_funcs; 2199 adev->gfx.config.max_hw_contexts = 8; 2200 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2201 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2202 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2203 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2204 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2205 gb_addr_config &= ~0xf3e777ff; 2206 gb_addr_config |= 0x22014042; 2207 /* check vbios table if gpu info is not available */ 2208 err = amdgpu_atomfirmware_get_gfx_info(adev); 2209 if (err) 2210 return err; 2211 break; 2212 default: 2213 BUG(); 2214 break; 2215 } 2216 2217 adev->gfx.config.gb_addr_config = gb_addr_config; 2218 2219 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << 2220 REG_GET_FIELD( 2221 adev->gfx.config.gb_addr_config, 2222 GB_ADDR_CONFIG, 2223 NUM_PIPES); 2224 2225 adev->gfx.config.max_tile_pipes = 2226 adev->gfx.config.gb_addr_config_fields.num_pipes; 2227 2228 adev->gfx.config.gb_addr_config_fields.num_banks = 1 << 2229 REG_GET_FIELD( 2230 adev->gfx.config.gb_addr_config, 2231 GB_ADDR_CONFIG, 2232 NUM_BANKS); 2233 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << 2234 REG_GET_FIELD( 2235 adev->gfx.config.gb_addr_config, 2236 GB_ADDR_CONFIG, 2237 MAX_COMPRESSED_FRAGS); 2238 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << 2239 REG_GET_FIELD( 2240 adev->gfx.config.gb_addr_config, 2241 GB_ADDR_CONFIG, 2242 NUM_RB_PER_SE); 2243 adev->gfx.config.gb_addr_config_fields.num_se = 1 << 2244 REG_GET_FIELD( 2245 adev->gfx.config.gb_addr_config, 2246 GB_ADDR_CONFIG, 2247 NUM_SHADER_ENGINES); 2248 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + 2249 REG_GET_FIELD( 2250 adev->gfx.config.gb_addr_config, 2251 GB_ADDR_CONFIG, 2252 PIPE_INTERLEAVE_SIZE)); 2253 2254 return 0; 2255 } 2256 2257 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 2258 int mec, int pipe, int queue) 2259 { 2260 unsigned irq_type; 2261 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; 2262 unsigned int hw_prio; 2263 2264 ring = &adev->gfx.compute_ring[ring_id]; 2265 2266 /* mec0 is me1 */ 2267 ring->me = mec + 1; 2268 ring->pipe = pipe; 2269 ring->queue = queue; 2270 2271 ring->ring_obj = NULL; 2272 ring->use_doorbell = true; 2273 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1; 2274 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 2275 + (ring_id * GFX9_MEC_HPD_SIZE); 2276 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 2277 2278 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 2279 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 2280 + ring->pipe; 2281 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? 2282 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 2283 /* type-2 packets are deprecated on MEC, use type-3 instead */ 2284 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 2285 hw_prio, NULL); 2286 } 2287 2288 static int gfx_v9_0_sw_init(void *handle) 2289 { 2290 int i, j, k, r, ring_id; 2291 struct amdgpu_ring *ring; 2292 struct amdgpu_kiq *kiq; 2293 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2294 2295 switch (adev->asic_type) { 2296 case CHIP_VEGA10: 2297 case CHIP_VEGA12: 2298 case CHIP_VEGA20: 2299 case CHIP_RAVEN: 2300 case CHIP_ARCTURUS: 2301 case CHIP_RENOIR: 2302 case CHIP_ALDEBARAN: 2303 adev->gfx.mec.num_mec = 2; 2304 break; 2305 default: 2306 adev->gfx.mec.num_mec = 1; 2307 break; 2308 } 2309 2310 adev->gfx.mec.num_pipe_per_mec = 4; 2311 adev->gfx.mec.num_queue_per_pipe = 8; 2312 2313 /* EOP Event */ 2314 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq); 2315 if (r) 2316 return r; 2317 2318 /* Privileged reg */ 2319 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT, 2320 &adev->gfx.priv_reg_irq); 2321 if (r) 2322 return r; 2323 2324 /* Privileged inst */ 2325 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT, 2326 &adev->gfx.priv_inst_irq); 2327 if (r) 2328 return r; 2329 2330 /* ECC error */ 2331 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR, 2332 &adev->gfx.cp_ecc_error_irq); 2333 if (r) 2334 return r; 2335 2336 /* FUE error */ 2337 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR, 2338 &adev->gfx.cp_ecc_error_irq); 2339 if (r) 2340 return r; 2341 2342 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 2343 2344 gfx_v9_0_scratch_init(adev); 2345 2346 r = gfx_v9_0_init_microcode(adev); 2347 if (r) { 2348 DRM_ERROR("Failed to load gfx firmware!\n"); 2349 return r; 2350 } 2351 2352 r = adev->gfx.rlc.funcs->init(adev); 2353 if (r) { 2354 DRM_ERROR("Failed to init rlc BOs!\n"); 2355 return r; 2356 } 2357 2358 r = gfx_v9_0_mec_init(adev); 2359 if (r) { 2360 DRM_ERROR("Failed to init MEC BOs!\n"); 2361 return r; 2362 } 2363 2364 /* set up the gfx ring */ 2365 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 2366 ring = &adev->gfx.gfx_ring[i]; 2367 ring->ring_obj = NULL; 2368 if (!i) 2369 sprintf(ring->name, "gfx"); 2370 else 2371 sprintf(ring->name, "gfx_%d", i); 2372 ring->use_doorbell = true; 2373 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 2374 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, 2375 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, 2376 AMDGPU_RING_PRIO_DEFAULT, NULL); 2377 if (r) 2378 return r; 2379 } 2380 2381 /* set up the compute queues - allocate horizontally across pipes */ 2382 ring_id = 0; 2383 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 2384 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 2385 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 2386 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j)) 2387 continue; 2388 2389 r = gfx_v9_0_compute_ring_init(adev, 2390 ring_id, 2391 i, k, j); 2392 if (r) 2393 return r; 2394 2395 ring_id++; 2396 } 2397 } 2398 } 2399 2400 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE); 2401 if (r) { 2402 DRM_ERROR("Failed to init KIQ BOs!\n"); 2403 return r; 2404 } 2405 2406 kiq = &adev->gfx.kiq; 2407 r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq); 2408 if (r) 2409 return r; 2410 2411 /* create MQD for all compute queues as wel as KIQ for SRIOV case */ 2412 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation)); 2413 if (r) 2414 return r; 2415 2416 adev->gfx.ce_ram_size = 0x8000; 2417 2418 r = gfx_v9_0_gpu_early_init(adev); 2419 if (r) 2420 return r; 2421 2422 return 0; 2423 } 2424 2425 2426 static int gfx_v9_0_sw_fini(void *handle) 2427 { 2428 int i; 2429 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2430 2431 if (adev->gfx.ras_funcs && 2432 adev->gfx.ras_funcs->ras_fini) 2433 adev->gfx.ras_funcs->ras_fini(adev); 2434 2435 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 2436 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 2437 for (i = 0; i < adev->gfx.num_compute_rings; i++) 2438 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 2439 2440 amdgpu_gfx_mqd_sw_fini(adev); 2441 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring); 2442 amdgpu_gfx_kiq_fini(adev); 2443 2444 gfx_v9_0_mec_fini(adev); 2445 amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj); 2446 if (adev->flags & AMD_IS_APU) { 2447 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 2448 &adev->gfx.rlc.cp_table_gpu_addr, 2449 (void **)&adev->gfx.rlc.cp_table_ptr); 2450 } 2451 gfx_v9_0_free_microcode(adev); 2452 2453 return 0; 2454 } 2455 2456 2457 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev) 2458 { 2459 /* TODO */ 2460 } 2461 2462 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, 2463 u32 instance) 2464 { 2465 u32 data; 2466 2467 if (instance == 0xffffffff) 2468 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); 2469 else 2470 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance); 2471 2472 if (se_num == 0xffffffff) 2473 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); 2474 else 2475 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 2476 2477 if (sh_num == 0xffffffff) 2478 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1); 2479 else 2480 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num); 2481 2482 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data); 2483 } 2484 2485 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev) 2486 { 2487 u32 data, mask; 2488 2489 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE); 2490 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE); 2491 2492 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK; 2493 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT; 2494 2495 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / 2496 adev->gfx.config.max_sh_per_se); 2497 2498 return (~data) & mask; 2499 } 2500 2501 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev) 2502 { 2503 int i, j; 2504 u32 data; 2505 u32 active_rbs = 0; 2506 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / 2507 adev->gfx.config.max_sh_per_se; 2508 2509 mutex_lock(&adev->grbm_idx_mutex); 2510 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2511 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2512 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2513 data = gfx_v9_0_get_rb_active_bitmap(adev); 2514 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * 2515 rb_bitmap_width_per_sh); 2516 } 2517 } 2518 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2519 mutex_unlock(&adev->grbm_idx_mutex); 2520 2521 adev->gfx.config.backend_enable_mask = active_rbs; 2522 adev->gfx.config.num_rbs = hweight32(active_rbs); 2523 } 2524 2525 #define DEFAULT_SH_MEM_BASES (0x6000) 2526 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev) 2527 { 2528 int i; 2529 uint32_t sh_mem_config; 2530 uint32_t sh_mem_bases; 2531 2532 /* 2533 * Configure apertures: 2534 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 2535 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 2536 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 2537 */ 2538 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16); 2539 2540 sh_mem_config = SH_MEM_ADDRESS_MODE_64 | 2541 SH_MEM_ALIGNMENT_MODE_UNALIGNED << 2542 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT; 2543 2544 mutex_lock(&adev->srbm_mutex); 2545 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2546 soc15_grbm_select(adev, 0, 0, 0, i); 2547 /* CP and shaders */ 2548 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config); 2549 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases); 2550 } 2551 soc15_grbm_select(adev, 0, 0, 0, 0); 2552 mutex_unlock(&adev->srbm_mutex); 2553 2554 /* Initialize all compute VMIDs to have no GDS, GWS, or OA 2555 acccess. These should be enabled by FW for target VMIDs. */ 2556 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2557 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0); 2558 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0); 2559 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0); 2560 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0); 2561 } 2562 } 2563 2564 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev) 2565 { 2566 int vmid; 2567 2568 /* 2569 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 2570 * access. Compute VMIDs should be enabled by FW for target VMIDs, 2571 * the driver can enable them for graphics. VMID0 should maintain 2572 * access so that HWS firmware can save/restore entries. 2573 */ 2574 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { 2575 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0); 2576 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0); 2577 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0); 2578 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0); 2579 } 2580 } 2581 2582 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev) 2583 { 2584 uint32_t tmp; 2585 2586 switch (adev->asic_type) { 2587 case CHIP_ARCTURUS: 2588 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG); 2589 tmp = REG_SET_FIELD(tmp, SQ_CONFIG, 2590 DISABLE_BARRIER_WAITCNT, 1); 2591 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp); 2592 break; 2593 default: 2594 break; 2595 } 2596 } 2597 2598 static void gfx_v9_0_constants_init(struct amdgpu_device *adev) 2599 { 2600 u32 tmp; 2601 int i; 2602 2603 WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); 2604 2605 gfx_v9_0_tiling_mode_table_init(adev); 2606 2607 gfx_v9_0_setup_rb(adev); 2608 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info); 2609 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2); 2610 2611 /* XXX SH_MEM regs */ 2612 /* where to put LDS, scratch, GPUVM in FSA64 space */ 2613 mutex_lock(&adev->srbm_mutex); 2614 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) { 2615 soc15_grbm_select(adev, 0, 0, 0, i); 2616 /* CP and shaders */ 2617 if (i == 0) { 2618 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2619 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2620 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2621 !!adev->gmc.noretry); 2622 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2623 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0); 2624 } else { 2625 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2626 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2627 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2628 !!adev->gmc.noretry); 2629 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2630 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 2631 (adev->gmc.private_aperture_start >> 48)); 2632 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE, 2633 (adev->gmc.shared_aperture_start >> 48)); 2634 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp); 2635 } 2636 } 2637 soc15_grbm_select(adev, 0, 0, 0, 0); 2638 2639 mutex_unlock(&adev->srbm_mutex); 2640 2641 gfx_v9_0_init_compute_vmid(adev); 2642 gfx_v9_0_init_gds_vmid(adev); 2643 gfx_v9_0_init_sq_config(adev); 2644 } 2645 2646 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev) 2647 { 2648 u32 i, j, k; 2649 u32 mask; 2650 2651 mutex_lock(&adev->grbm_idx_mutex); 2652 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2653 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2654 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2655 for (k = 0; k < adev->usec_timeout; k++) { 2656 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0) 2657 break; 2658 udelay(1); 2659 } 2660 if (k == adev->usec_timeout) { 2661 gfx_v9_0_select_se_sh(adev, 0xffffffff, 2662 0xffffffff, 0xffffffff); 2663 mutex_unlock(&adev->grbm_idx_mutex); 2664 DRM_INFO("Timeout wait for RLC serdes %u,%u\n", 2665 i, j); 2666 return; 2667 } 2668 } 2669 } 2670 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2671 mutex_unlock(&adev->grbm_idx_mutex); 2672 2673 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | 2674 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK | 2675 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK | 2676 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK; 2677 for (k = 0; k < adev->usec_timeout; k++) { 2678 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 2679 break; 2680 udelay(1); 2681 } 2682 } 2683 2684 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 2685 bool enable) 2686 { 2687 u32 tmp; 2688 2689 /* These interrupts should be enabled to drive DS clock */ 2690 2691 tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0); 2692 2693 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0); 2694 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0); 2695 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0); 2696 if(adev->gfx.num_gfx_rings) 2697 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0); 2698 2699 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp); 2700 } 2701 2702 static void gfx_v9_0_init_csb(struct amdgpu_device *adev) 2703 { 2704 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 2705 /* csib */ 2706 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI), 2707 adev->gfx.rlc.clear_state_gpu_addr >> 32); 2708 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO), 2709 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 2710 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH), 2711 adev->gfx.rlc.clear_state_size); 2712 } 2713 2714 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format, 2715 int indirect_offset, 2716 int list_size, 2717 int *unique_indirect_regs, 2718 int unique_indirect_reg_count, 2719 int *indirect_start_offsets, 2720 int *indirect_start_offsets_count, 2721 int max_start_offsets_count) 2722 { 2723 int idx; 2724 2725 for (; indirect_offset < list_size; indirect_offset++) { 2726 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count); 2727 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset; 2728 *indirect_start_offsets_count = *indirect_start_offsets_count + 1; 2729 2730 while (register_list_format[indirect_offset] != 0xFFFFFFFF) { 2731 indirect_offset += 2; 2732 2733 /* look for the matching indice */ 2734 for (idx = 0; idx < unique_indirect_reg_count; idx++) { 2735 if (unique_indirect_regs[idx] == 2736 register_list_format[indirect_offset] || 2737 !unique_indirect_regs[idx]) 2738 break; 2739 } 2740 2741 BUG_ON(idx >= unique_indirect_reg_count); 2742 2743 if (!unique_indirect_regs[idx]) 2744 unique_indirect_regs[idx] = register_list_format[indirect_offset]; 2745 2746 indirect_offset++; 2747 } 2748 } 2749 } 2750 2751 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev) 2752 { 2753 int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2754 int unique_indirect_reg_count = 0; 2755 2756 int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2757 int indirect_start_offsets_count = 0; 2758 2759 int list_size = 0; 2760 int i = 0, j = 0; 2761 u32 tmp = 0; 2762 2763 u32 *register_list_format = 2764 kmemdup(adev->gfx.rlc.register_list_format, 2765 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); 2766 if (!register_list_format) 2767 return -ENOMEM; 2768 2769 /* setup unique_indirect_regs array and indirect_start_offsets array */ 2770 unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs); 2771 gfx_v9_1_parse_ind_reg_list(register_list_format, 2772 adev->gfx.rlc.reg_list_format_direct_reg_list_length, 2773 adev->gfx.rlc.reg_list_format_size_bytes >> 2, 2774 unique_indirect_regs, 2775 unique_indirect_reg_count, 2776 indirect_start_offsets, 2777 &indirect_start_offsets_count, 2778 ARRAY_SIZE(indirect_start_offsets)); 2779 2780 /* enable auto inc in case it is disabled */ 2781 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL)); 2782 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK; 2783 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp); 2784 2785 /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */ 2786 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR), 2787 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET); 2788 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) 2789 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA), 2790 adev->gfx.rlc.register_restore[i]); 2791 2792 /* load indirect register */ 2793 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2794 adev->gfx.rlc.reg_list_format_start); 2795 2796 /* direct register portion */ 2797 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++) 2798 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2799 register_list_format[i]); 2800 2801 /* indirect register portion */ 2802 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) { 2803 if (register_list_format[i] == 0xFFFFFFFF) { 2804 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2805 continue; 2806 } 2807 2808 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2809 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2810 2811 for (j = 0; j < unique_indirect_reg_count; j++) { 2812 if (register_list_format[i] == unique_indirect_regs[j]) { 2813 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j); 2814 break; 2815 } 2816 } 2817 2818 BUG_ON(j >= unique_indirect_reg_count); 2819 2820 i++; 2821 } 2822 2823 /* set save/restore list size */ 2824 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; 2825 list_size = list_size >> 1; 2826 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2827 adev->gfx.rlc.reg_restore_list_size); 2828 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size); 2829 2830 /* write the starting offsets to RLC scratch ram */ 2831 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2832 adev->gfx.rlc.starting_offsets_start); 2833 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++) 2834 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2835 indirect_start_offsets[i]); 2836 2837 /* load unique indirect regs*/ 2838 for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) { 2839 if (unique_indirect_regs[i] != 0) { 2840 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0) 2841 + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i], 2842 unique_indirect_regs[i] & 0x3FFFF); 2843 2844 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0) 2845 + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i], 2846 unique_indirect_regs[i] >> 20); 2847 } 2848 } 2849 2850 kfree(register_list_format); 2851 return 0; 2852 } 2853 2854 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev) 2855 { 2856 WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1); 2857 } 2858 2859 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev, 2860 bool enable) 2861 { 2862 uint32_t data = 0; 2863 uint32_t default_data = 0; 2864 2865 default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS)); 2866 if (enable) { 2867 /* enable GFXIP control over CGPG */ 2868 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2869 if(default_data != data) 2870 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2871 2872 /* update status */ 2873 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK; 2874 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT); 2875 if(default_data != data) 2876 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2877 } else { 2878 /* restore GFXIP control over GCPG */ 2879 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2880 if(default_data != data) 2881 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2882 } 2883 } 2884 2885 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev) 2886 { 2887 uint32_t data = 0; 2888 2889 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2890 AMD_PG_SUPPORT_GFX_SMG | 2891 AMD_PG_SUPPORT_GFX_DMG)) { 2892 /* init IDLE_POLL_COUNT = 60 */ 2893 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL)); 2894 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK; 2895 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 2896 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data); 2897 2898 /* init RLC PG Delay */ 2899 data = 0; 2900 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT); 2901 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT); 2902 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT); 2903 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT); 2904 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data); 2905 2906 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2)); 2907 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK; 2908 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT); 2909 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data); 2910 2911 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3)); 2912 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK; 2913 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT); 2914 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data); 2915 2916 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL)); 2917 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK; 2918 2919 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */ 2920 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT); 2921 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data); 2922 if (adev->asic_type != CHIP_RENOIR) 2923 pwr_10_0_gfxip_control_over_cgpg(adev, true); 2924 } 2925 } 2926 2927 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev, 2928 bool enable) 2929 { 2930 uint32_t data = 0; 2931 uint32_t default_data = 0; 2932 2933 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2934 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2935 SMU_CLK_SLOWDOWN_ON_PU_ENABLE, 2936 enable ? 1 : 0); 2937 if (default_data != data) 2938 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2939 } 2940 2941 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev, 2942 bool enable) 2943 { 2944 uint32_t data = 0; 2945 uint32_t default_data = 0; 2946 2947 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2948 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2949 SMU_CLK_SLOWDOWN_ON_PD_ENABLE, 2950 enable ? 1 : 0); 2951 if(default_data != data) 2952 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2953 } 2954 2955 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev, 2956 bool enable) 2957 { 2958 uint32_t data = 0; 2959 uint32_t default_data = 0; 2960 2961 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2962 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2963 CP_PG_DISABLE, 2964 enable ? 0 : 1); 2965 if(default_data != data) 2966 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2967 } 2968 2969 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev, 2970 bool enable) 2971 { 2972 uint32_t data, default_data; 2973 2974 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2975 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2976 GFX_POWER_GATING_ENABLE, 2977 enable ? 1 : 0); 2978 if(default_data != data) 2979 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2980 } 2981 2982 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev, 2983 bool enable) 2984 { 2985 uint32_t data, default_data; 2986 2987 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2988 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2989 GFX_PIPELINE_PG_ENABLE, 2990 enable ? 1 : 0); 2991 if(default_data != data) 2992 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2993 2994 if (!enable) 2995 /* read any GFX register to wake up GFX */ 2996 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL)); 2997 } 2998 2999 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, 3000 bool enable) 3001 { 3002 uint32_t data, default_data; 3003 3004 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 3005 data = REG_SET_FIELD(data, RLC_PG_CNTL, 3006 STATIC_PER_CU_PG_ENABLE, 3007 enable ? 1 : 0); 3008 if(default_data != data) 3009 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3010 } 3011 3012 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev, 3013 bool enable) 3014 { 3015 uint32_t data, default_data; 3016 3017 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 3018 data = REG_SET_FIELD(data, RLC_PG_CNTL, 3019 DYN_PER_CU_PG_ENABLE, 3020 enable ? 1 : 0); 3021 if(default_data != data) 3022 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3023 } 3024 3025 static void gfx_v9_0_init_pg(struct amdgpu_device *adev) 3026 { 3027 gfx_v9_0_init_csb(adev); 3028 3029 /* 3030 * Rlc save restore list is workable since v2_1. 3031 * And it's needed by gfxoff feature. 3032 */ 3033 if (adev->gfx.rlc.is_rlc_v2_1) { 3034 if (adev->asic_type == CHIP_VEGA12 || 3035 (adev->apu_flags & AMD_APU_IS_RAVEN2)) 3036 gfx_v9_1_init_rlc_save_restore_list(adev); 3037 gfx_v9_0_enable_save_restore_machine(adev); 3038 } 3039 3040 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 3041 AMD_PG_SUPPORT_GFX_SMG | 3042 AMD_PG_SUPPORT_GFX_DMG | 3043 AMD_PG_SUPPORT_CP | 3044 AMD_PG_SUPPORT_GDS | 3045 AMD_PG_SUPPORT_RLC_SMU_HS)) { 3046 WREG32(mmRLC_JUMP_TABLE_RESTORE, 3047 adev->gfx.rlc.cp_table_gpu_addr >> 8); 3048 gfx_v9_0_init_gfx_power_gating(adev); 3049 } 3050 } 3051 3052 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev) 3053 { 3054 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0); 3055 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3056 gfx_v9_0_wait_for_rlc_serdes(adev); 3057 } 3058 3059 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev) 3060 { 3061 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 3062 udelay(50); 3063 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 3064 udelay(50); 3065 } 3066 3067 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev) 3068 { 3069 #ifdef AMDGPU_RLC_DEBUG_RETRY 3070 u32 rlc_ucode_ver; 3071 #endif 3072 3073 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1); 3074 udelay(50); 3075 3076 /* carrizo do enable cp interrupt after cp inited */ 3077 if (!(adev->flags & AMD_IS_APU)) { 3078 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3079 udelay(50); 3080 } 3081 3082 #ifdef AMDGPU_RLC_DEBUG_RETRY 3083 /* RLC_GPM_GENERAL_6 : RLC Ucode version */ 3084 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6); 3085 if(rlc_ucode_ver == 0x108) { 3086 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n", 3087 rlc_ucode_ver, adev->gfx.rlc_fw_version); 3088 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles, 3089 * default is 0x9C4 to create a 100us interval */ 3090 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4); 3091 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr 3092 * to disable the page fault retry interrupts, default is 3093 * 0x100 (256) */ 3094 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100); 3095 } 3096 #endif 3097 } 3098 3099 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev) 3100 { 3101 const struct rlc_firmware_header_v2_0 *hdr; 3102 const __le32 *fw_data; 3103 unsigned i, fw_size; 3104 3105 if (!adev->gfx.rlc_fw) 3106 return -EINVAL; 3107 3108 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 3109 amdgpu_ucode_print_rlc_hdr(&hdr->header); 3110 3111 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 3112 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 3113 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 3114 3115 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, 3116 RLCG_UCODE_LOADING_START_ADDRESS); 3117 for (i = 0; i < fw_size; i++) 3118 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); 3119 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); 3120 3121 return 0; 3122 } 3123 3124 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev) 3125 { 3126 int r; 3127 3128 if (amdgpu_sriov_vf(adev)) { 3129 gfx_v9_0_init_csb(adev); 3130 return 0; 3131 } 3132 3133 adev->gfx.rlc.funcs->stop(adev); 3134 3135 /* disable CG */ 3136 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0); 3137 3138 gfx_v9_0_init_pg(adev); 3139 3140 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3141 /* legacy rlc firmware loading */ 3142 r = gfx_v9_0_rlc_load_microcode(adev); 3143 if (r) 3144 return r; 3145 } 3146 3147 switch (adev->asic_type) { 3148 case CHIP_RAVEN: 3149 if (amdgpu_lbpw == 0) 3150 gfx_v9_0_enable_lbpw(adev, false); 3151 else 3152 gfx_v9_0_enable_lbpw(adev, true); 3153 break; 3154 case CHIP_VEGA20: 3155 if (amdgpu_lbpw > 0) 3156 gfx_v9_0_enable_lbpw(adev, true); 3157 else 3158 gfx_v9_0_enable_lbpw(adev, false); 3159 break; 3160 default: 3161 break; 3162 } 3163 3164 adev->gfx.rlc.funcs->start(adev); 3165 3166 return 0; 3167 } 3168 3169 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 3170 { 3171 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL); 3172 3173 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); 3174 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); 3175 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1); 3176 WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp); 3177 udelay(50); 3178 } 3179 3180 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev) 3181 { 3182 const struct gfx_firmware_header_v1_0 *pfp_hdr; 3183 const struct gfx_firmware_header_v1_0 *ce_hdr; 3184 const struct gfx_firmware_header_v1_0 *me_hdr; 3185 const __le32 *fw_data; 3186 unsigned i, fw_size; 3187 3188 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw) 3189 return -EINVAL; 3190 3191 pfp_hdr = (const struct gfx_firmware_header_v1_0 *) 3192 adev->gfx.pfp_fw->data; 3193 ce_hdr = (const struct gfx_firmware_header_v1_0 *) 3194 adev->gfx.ce_fw->data; 3195 me_hdr = (const struct gfx_firmware_header_v1_0 *) 3196 adev->gfx.me_fw->data; 3197 3198 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3199 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header); 3200 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3201 3202 gfx_v9_0_cp_gfx_enable(adev, false); 3203 3204 /* PFP */ 3205 fw_data = (const __le32 *) 3206 (adev->gfx.pfp_fw->data + 3207 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 3208 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 3209 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0); 3210 for (i = 0; i < fw_size; i++) 3211 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 3212 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); 3213 3214 /* CE */ 3215 fw_data = (const __le32 *) 3216 (adev->gfx.ce_fw->data + 3217 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 3218 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 3219 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0); 3220 for (i = 0; i < fw_size; i++) 3221 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 3222 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version); 3223 3224 /* ME */ 3225 fw_data = (const __le32 *) 3226 (adev->gfx.me_fw->data + 3227 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 3228 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 3229 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0); 3230 for (i = 0; i < fw_size; i++) 3231 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 3232 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version); 3233 3234 return 0; 3235 } 3236 3237 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev) 3238 { 3239 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; 3240 const struct cs_section_def *sect = NULL; 3241 const struct cs_extent_def *ext = NULL; 3242 int r, i, tmp; 3243 3244 /* init the CP */ 3245 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); 3246 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1); 3247 3248 gfx_v9_0_cp_gfx_enable(adev, true); 3249 3250 r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3); 3251 if (r) { 3252 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r); 3253 return r; 3254 } 3255 3256 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3257 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3258 3259 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 3260 amdgpu_ring_write(ring, 0x80000000); 3261 amdgpu_ring_write(ring, 0x80000000); 3262 3263 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 3264 for (ext = sect->section; ext->extent != NULL; ++ext) { 3265 if (sect->id == SECT_CONTEXT) { 3266 amdgpu_ring_write(ring, 3267 PACKET3(PACKET3_SET_CONTEXT_REG, 3268 ext->reg_count)); 3269 amdgpu_ring_write(ring, 3270 ext->reg_index - PACKET3_SET_CONTEXT_REG_START); 3271 for (i = 0; i < ext->reg_count; i++) 3272 amdgpu_ring_write(ring, ext->extent[i]); 3273 } 3274 } 3275 } 3276 3277 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3278 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3279 3280 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3281 amdgpu_ring_write(ring, 0); 3282 3283 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 3284 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 3285 amdgpu_ring_write(ring, 0x8000); 3286 amdgpu_ring_write(ring, 0x8000); 3287 3288 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1)); 3289 tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE | 3290 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START)); 3291 amdgpu_ring_write(ring, tmp); 3292 amdgpu_ring_write(ring, 0); 3293 3294 amdgpu_ring_commit(ring); 3295 3296 return 0; 3297 } 3298 3299 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev) 3300 { 3301 struct amdgpu_ring *ring; 3302 u32 tmp; 3303 u32 rb_bufsz; 3304 u64 rb_addr, rptr_addr, wptr_gpu_addr; 3305 3306 /* Set the write pointer delay */ 3307 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0); 3308 3309 /* set the RB to use vmid 0 */ 3310 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0); 3311 3312 /* Set ring buffer size */ 3313 ring = &adev->gfx.gfx_ring[0]; 3314 rb_bufsz = order_base_2(ring->ring_size / 8); 3315 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 3316 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 3317 #ifdef __BIG_ENDIAN 3318 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1); 3319 #endif 3320 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3321 3322 /* Initialize the ring buffer's write pointers */ 3323 ring->wptr = 0; 3324 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 3325 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 3326 3327 /* set the wb address wether it's enabled or not */ 3328 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3329 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 3330 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3331 3332 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3333 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr)); 3334 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr)); 3335 3336 mdelay(1); 3337 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3338 3339 rb_addr = ring->gpu_addr >> 8; 3340 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr); 3341 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 3342 3343 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL); 3344 if (ring->use_doorbell) { 3345 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3346 DOORBELL_OFFSET, ring->doorbell_index); 3347 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3348 DOORBELL_EN, 1); 3349 } else { 3350 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0); 3351 } 3352 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp); 3353 3354 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 3355 DOORBELL_RANGE_LOWER, ring->doorbell_index); 3356 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp); 3357 3358 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER, 3359 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 3360 3361 3362 /* start the ring */ 3363 gfx_v9_0_cp_gfx_start(adev); 3364 ring->sched.ready = true; 3365 3366 return 0; 3367 } 3368 3369 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 3370 { 3371 if (enable) { 3372 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0); 3373 } else { 3374 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 3375 (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK)); 3376 adev->gfx.kiq.ring.sched.ready = false; 3377 } 3378 udelay(50); 3379 } 3380 3381 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev) 3382 { 3383 const struct gfx_firmware_header_v1_0 *mec_hdr; 3384 const __le32 *fw_data; 3385 unsigned i; 3386 u32 tmp; 3387 3388 if (!adev->gfx.mec_fw) 3389 return -EINVAL; 3390 3391 gfx_v9_0_cp_compute_enable(adev, false); 3392 3393 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 3394 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3395 3396 fw_data = (const __le32 *) 3397 (adev->gfx.mec_fw->data + 3398 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 3399 tmp = 0; 3400 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 3401 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 3402 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp); 3403 3404 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO, 3405 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000); 3406 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI, 3407 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 3408 3409 /* MEC1 */ 3410 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3411 mec_hdr->jt_offset); 3412 for (i = 0; i < mec_hdr->jt_size; i++) 3413 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA, 3414 le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); 3415 3416 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3417 adev->gfx.mec_fw_version); 3418 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */ 3419 3420 return 0; 3421 } 3422 3423 /* KIQ functions */ 3424 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring) 3425 { 3426 uint32_t tmp; 3427 struct amdgpu_device *adev = ring->adev; 3428 3429 /* tell RLC which is KIQ queue */ 3430 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS); 3431 tmp &= 0xffffff00; 3432 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 3433 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3434 tmp |= 0x80; 3435 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3436 } 3437 3438 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd) 3439 { 3440 struct amdgpu_device *adev = ring->adev; 3441 3442 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 3443 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) { 3444 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; 3445 mqd->cp_hqd_queue_priority = 3446 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; 3447 } 3448 } 3449 } 3450 3451 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring) 3452 { 3453 struct amdgpu_device *adev = ring->adev; 3454 struct v9_mqd *mqd = ring->mqd_ptr; 3455 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 3456 uint32_t tmp; 3457 3458 mqd->header = 0xC0310800; 3459 mqd->compute_pipelinestat_enable = 0x00000001; 3460 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 3461 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 3462 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 3463 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 3464 mqd->compute_static_thread_mgmt_se4 = 0xffffffff; 3465 mqd->compute_static_thread_mgmt_se5 = 0xffffffff; 3466 mqd->compute_static_thread_mgmt_se6 = 0xffffffff; 3467 mqd->compute_static_thread_mgmt_se7 = 0xffffffff; 3468 mqd->compute_misc_reserved = 0x00000003; 3469 3470 mqd->dynamic_cu_mask_addr_lo = 3471 lower_32_bits(ring->mqd_gpu_addr 3472 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3473 mqd->dynamic_cu_mask_addr_hi = 3474 upper_32_bits(ring->mqd_gpu_addr 3475 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3476 3477 eop_base_addr = ring->eop_gpu_addr >> 8; 3478 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 3479 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 3480 3481 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3482 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL); 3483 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 3484 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1)); 3485 3486 mqd->cp_hqd_eop_control = tmp; 3487 3488 /* enable doorbell? */ 3489 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3490 3491 if (ring->use_doorbell) { 3492 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3493 DOORBELL_OFFSET, ring->doorbell_index); 3494 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3495 DOORBELL_EN, 1); 3496 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3497 DOORBELL_SOURCE, 0); 3498 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3499 DOORBELL_HIT, 0); 3500 } else { 3501 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3502 DOORBELL_EN, 0); 3503 } 3504 3505 mqd->cp_hqd_pq_doorbell_control = tmp; 3506 3507 /* disable the queue if it's active */ 3508 ring->wptr = 0; 3509 mqd->cp_hqd_dequeue_request = 0; 3510 mqd->cp_hqd_pq_rptr = 0; 3511 mqd->cp_hqd_pq_wptr_lo = 0; 3512 mqd->cp_hqd_pq_wptr_hi = 0; 3513 3514 /* set the pointer to the MQD */ 3515 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; 3516 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); 3517 3518 /* set MQD vmid to 0 */ 3519 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL); 3520 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 3521 mqd->cp_mqd_control = tmp; 3522 3523 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3524 hqd_gpu_addr = ring->gpu_addr >> 8; 3525 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 3526 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 3527 3528 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3529 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL); 3530 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 3531 (order_base_2(ring->ring_size / 4) - 1)); 3532 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 3533 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8)); 3534 #ifdef __BIG_ENDIAN 3535 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1); 3536 #endif 3537 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0); 3538 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0); 3539 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 3540 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 3541 mqd->cp_hqd_pq_control = tmp; 3542 3543 /* set the wb address whether it's enabled or not */ 3544 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3545 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 3546 mqd->cp_hqd_pq_rptr_report_addr_hi = 3547 upper_32_bits(wb_gpu_addr) & 0xffff; 3548 3549 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3550 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3551 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 3552 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 3553 3554 tmp = 0; 3555 /* enable the doorbell if requested */ 3556 if (ring->use_doorbell) { 3557 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3558 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3559 DOORBELL_OFFSET, ring->doorbell_index); 3560 3561 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3562 DOORBELL_EN, 1); 3563 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3564 DOORBELL_SOURCE, 0); 3565 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3566 DOORBELL_HIT, 0); 3567 } 3568 3569 mqd->cp_hqd_pq_doorbell_control = tmp; 3570 3571 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3572 ring->wptr = 0; 3573 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR); 3574 3575 /* set the vmid for the queue */ 3576 mqd->cp_hqd_vmid = 0; 3577 3578 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE); 3579 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53); 3580 mqd->cp_hqd_persistent_state = tmp; 3581 3582 /* set MIN_IB_AVAIL_SIZE */ 3583 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL); 3584 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 3585 mqd->cp_hqd_ib_control = tmp; 3586 3587 /* set static priority for a queue/ring */ 3588 gfx_v9_0_mqd_set_priority(ring, mqd); 3589 mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM); 3590 3591 /* map_queues packet doesn't need activate the queue, 3592 * so only kiq need set this field. 3593 */ 3594 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 3595 mqd->cp_hqd_active = 1; 3596 3597 return 0; 3598 } 3599 3600 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring) 3601 { 3602 struct amdgpu_device *adev = ring->adev; 3603 struct v9_mqd *mqd = ring->mqd_ptr; 3604 int j; 3605 3606 /* disable wptr polling */ 3607 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3608 3609 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR, 3610 mqd->cp_hqd_eop_base_addr_lo); 3611 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI, 3612 mqd->cp_hqd_eop_base_addr_hi); 3613 3614 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3615 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL, 3616 mqd->cp_hqd_eop_control); 3617 3618 /* enable doorbell? */ 3619 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3620 mqd->cp_hqd_pq_doorbell_control); 3621 3622 /* disable the queue if it's active */ 3623 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3624 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3625 for (j = 0; j < adev->usec_timeout; j++) { 3626 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3627 break; 3628 udelay(1); 3629 } 3630 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3631 mqd->cp_hqd_dequeue_request); 3632 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 3633 mqd->cp_hqd_pq_rptr); 3634 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3635 mqd->cp_hqd_pq_wptr_lo); 3636 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3637 mqd->cp_hqd_pq_wptr_hi); 3638 } 3639 3640 /* set the pointer to the MQD */ 3641 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR, 3642 mqd->cp_mqd_base_addr_lo); 3643 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI, 3644 mqd->cp_mqd_base_addr_hi); 3645 3646 /* set MQD vmid to 0 */ 3647 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL, 3648 mqd->cp_mqd_control); 3649 3650 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3651 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE, 3652 mqd->cp_hqd_pq_base_lo); 3653 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI, 3654 mqd->cp_hqd_pq_base_hi); 3655 3656 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3657 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL, 3658 mqd->cp_hqd_pq_control); 3659 3660 /* set the wb address whether it's enabled or not */ 3661 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR, 3662 mqd->cp_hqd_pq_rptr_report_addr_lo); 3663 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 3664 mqd->cp_hqd_pq_rptr_report_addr_hi); 3665 3666 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3667 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR, 3668 mqd->cp_hqd_pq_wptr_poll_addr_lo); 3669 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI, 3670 mqd->cp_hqd_pq_wptr_poll_addr_hi); 3671 3672 /* enable the doorbell if requested */ 3673 if (ring->use_doorbell) { 3674 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER, 3675 (adev->doorbell_index.kiq * 2) << 2); 3676 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3677 (adev->doorbell_index.userqueue_end * 2) << 2); 3678 } 3679 3680 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3681 mqd->cp_hqd_pq_doorbell_control); 3682 3683 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3684 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3685 mqd->cp_hqd_pq_wptr_lo); 3686 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3687 mqd->cp_hqd_pq_wptr_hi); 3688 3689 /* set the vmid for the queue */ 3690 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid); 3691 3692 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 3693 mqd->cp_hqd_persistent_state); 3694 3695 /* activate the queue */ 3696 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 3697 mqd->cp_hqd_active); 3698 3699 if (ring->use_doorbell) 3700 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1); 3701 3702 return 0; 3703 } 3704 3705 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring) 3706 { 3707 struct amdgpu_device *adev = ring->adev; 3708 int j; 3709 3710 /* disable the queue if it's active */ 3711 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3712 3713 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3714 3715 for (j = 0; j < adev->usec_timeout; j++) { 3716 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3717 break; 3718 udelay(1); 3719 } 3720 3721 if (j == AMDGPU_MAX_USEC_TIMEOUT) { 3722 DRM_DEBUG("KIQ dequeue request failed.\n"); 3723 3724 /* Manual disable if dequeue request times out */ 3725 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0); 3726 } 3727 3728 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3729 0); 3730 } 3731 3732 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0); 3733 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0); 3734 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0); 3735 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000); 3736 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0); 3737 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0); 3738 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0); 3739 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0); 3740 3741 return 0; 3742 } 3743 3744 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring) 3745 { 3746 struct amdgpu_device *adev = ring->adev; 3747 struct v9_mqd *mqd = ring->mqd_ptr; 3748 int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS; 3749 struct v9_mqd *tmp_mqd; 3750 3751 gfx_v9_0_kiq_setting(ring); 3752 3753 /* GPU could be in bad state during probe, driver trigger the reset 3754 * after load the SMU, in this case , the mqd is not be initialized. 3755 * driver need to re-init the mqd. 3756 * check mqd->cp_hqd_pq_control since this value should not be 0 3757 */ 3758 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx]; 3759 if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){ 3760 /* for GPU_RESET case , reset MQD to a clean status */ 3761 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3762 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3763 3764 /* reset ring buffer */ 3765 ring->wptr = 0; 3766 amdgpu_ring_clear_ring(ring); 3767 3768 mutex_lock(&adev->srbm_mutex); 3769 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3770 gfx_v9_0_kiq_init_register(ring); 3771 soc15_grbm_select(adev, 0, 0, 0, 0); 3772 mutex_unlock(&adev->srbm_mutex); 3773 } else { 3774 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3775 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3776 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3777 mutex_lock(&adev->srbm_mutex); 3778 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3779 gfx_v9_0_mqd_init(ring); 3780 gfx_v9_0_kiq_init_register(ring); 3781 soc15_grbm_select(adev, 0, 0, 0, 0); 3782 mutex_unlock(&adev->srbm_mutex); 3783 3784 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3785 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3786 } 3787 3788 return 0; 3789 } 3790 3791 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring) 3792 { 3793 struct amdgpu_device *adev = ring->adev; 3794 struct v9_mqd *mqd = ring->mqd_ptr; 3795 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 3796 struct v9_mqd *tmp_mqd; 3797 3798 /* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control 3799 * is not be initialized before 3800 */ 3801 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx]; 3802 3803 if (!tmp_mqd->cp_hqd_pq_control || 3804 (!amdgpu_in_reset(adev) && !adev->in_suspend)) { 3805 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3806 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3807 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3808 mutex_lock(&adev->srbm_mutex); 3809 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3810 gfx_v9_0_mqd_init(ring); 3811 soc15_grbm_select(adev, 0, 0, 0, 0); 3812 mutex_unlock(&adev->srbm_mutex); 3813 3814 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3815 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3816 } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 3817 /* reset MQD to a clean status */ 3818 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3819 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3820 3821 /* reset ring buffer */ 3822 ring->wptr = 0; 3823 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0); 3824 amdgpu_ring_clear_ring(ring); 3825 } else { 3826 amdgpu_ring_clear_ring(ring); 3827 } 3828 3829 return 0; 3830 } 3831 3832 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev) 3833 { 3834 struct amdgpu_ring *ring; 3835 int r; 3836 3837 ring = &adev->gfx.kiq.ring; 3838 3839 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3840 if (unlikely(r != 0)) 3841 return r; 3842 3843 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3844 if (unlikely(r != 0)) 3845 return r; 3846 3847 gfx_v9_0_kiq_init_queue(ring); 3848 amdgpu_bo_kunmap(ring->mqd_obj); 3849 ring->mqd_ptr = NULL; 3850 amdgpu_bo_unreserve(ring->mqd_obj); 3851 ring->sched.ready = true; 3852 return 0; 3853 } 3854 3855 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev) 3856 { 3857 struct amdgpu_ring *ring = NULL; 3858 int r = 0, i; 3859 3860 gfx_v9_0_cp_compute_enable(adev, true); 3861 3862 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3863 ring = &adev->gfx.compute_ring[i]; 3864 3865 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3866 if (unlikely(r != 0)) 3867 goto done; 3868 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3869 if (!r) { 3870 r = gfx_v9_0_kcq_init_queue(ring); 3871 amdgpu_bo_kunmap(ring->mqd_obj); 3872 ring->mqd_ptr = NULL; 3873 } 3874 amdgpu_bo_unreserve(ring->mqd_obj); 3875 if (r) 3876 goto done; 3877 } 3878 3879 r = amdgpu_gfx_enable_kcq(adev); 3880 done: 3881 return r; 3882 } 3883 3884 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev) 3885 { 3886 int r, i; 3887 struct amdgpu_ring *ring; 3888 3889 if (!(adev->flags & AMD_IS_APU)) 3890 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3891 3892 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3893 if (adev->gfx.num_gfx_rings) { 3894 /* legacy firmware loading */ 3895 r = gfx_v9_0_cp_gfx_load_microcode(adev); 3896 if (r) 3897 return r; 3898 } 3899 3900 r = gfx_v9_0_cp_compute_load_microcode(adev); 3901 if (r) 3902 return r; 3903 } 3904 3905 r = gfx_v9_0_kiq_resume(adev); 3906 if (r) 3907 return r; 3908 3909 if (adev->gfx.num_gfx_rings) { 3910 r = gfx_v9_0_cp_gfx_resume(adev); 3911 if (r) 3912 return r; 3913 } 3914 3915 r = gfx_v9_0_kcq_resume(adev); 3916 if (r) 3917 return r; 3918 3919 if (adev->gfx.num_gfx_rings) { 3920 ring = &adev->gfx.gfx_ring[0]; 3921 r = amdgpu_ring_test_helper(ring); 3922 if (r) 3923 return r; 3924 } 3925 3926 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3927 ring = &adev->gfx.compute_ring[i]; 3928 amdgpu_ring_test_helper(ring); 3929 } 3930 3931 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3932 3933 return 0; 3934 } 3935 3936 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev) 3937 { 3938 u32 tmp; 3939 3940 if (adev->asic_type != CHIP_ARCTURUS && 3941 adev->asic_type != CHIP_ALDEBARAN) 3942 return; 3943 3944 tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG); 3945 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH, 3946 adev->df.hash_status.hash_64k); 3947 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH, 3948 adev->df.hash_status.hash_2m); 3949 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH, 3950 adev->df.hash_status.hash_1g); 3951 WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp); 3952 } 3953 3954 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable) 3955 { 3956 if (adev->gfx.num_gfx_rings) 3957 gfx_v9_0_cp_gfx_enable(adev, enable); 3958 gfx_v9_0_cp_compute_enable(adev, enable); 3959 } 3960 3961 static int gfx_v9_0_hw_init(void *handle) 3962 { 3963 int r; 3964 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3965 3966 if (!amdgpu_sriov_vf(adev)) 3967 gfx_v9_0_init_golden_registers(adev); 3968 3969 gfx_v9_0_constants_init(adev); 3970 3971 gfx_v9_0_init_tcp_config(adev); 3972 3973 r = adev->gfx.rlc.funcs->resume(adev); 3974 if (r) 3975 return r; 3976 3977 r = gfx_v9_0_cp_resume(adev); 3978 if (r) 3979 return r; 3980 3981 if (adev->asic_type == CHIP_ALDEBARAN) 3982 gfx_v9_4_2_set_power_brake_sequence(adev); 3983 3984 return r; 3985 } 3986 3987 static int gfx_v9_0_hw_fini(void *handle) 3988 { 3989 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3990 3991 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); 3992 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 3993 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 3994 3995 /* DF freeze and kcq disable will fail */ 3996 if (!amdgpu_ras_intr_triggered()) 3997 /* disable KCQ to avoid CPC touch memory not valid anymore */ 3998 amdgpu_gfx_disable_kcq(adev); 3999 4000 if (amdgpu_sriov_vf(adev)) { 4001 gfx_v9_0_cp_gfx_enable(adev, false); 4002 /* must disable polling for SRIOV when hw finished, otherwise 4003 * CPC engine may still keep fetching WB address which is already 4004 * invalid after sw finished and trigger DMAR reading error in 4005 * hypervisor side. 4006 */ 4007 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 4008 return 0; 4009 } 4010 4011 /* Use deinitialize sequence from CAIL when unbinding device from driver, 4012 * otherwise KIQ is hanging when binding back 4013 */ 4014 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 4015 mutex_lock(&adev->srbm_mutex); 4016 soc15_grbm_select(adev, adev->gfx.kiq.ring.me, 4017 adev->gfx.kiq.ring.pipe, 4018 adev->gfx.kiq.ring.queue, 0); 4019 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring); 4020 soc15_grbm_select(adev, 0, 0, 0, 0); 4021 mutex_unlock(&adev->srbm_mutex); 4022 } 4023 4024 gfx_v9_0_cp_enable(adev, false); 4025 4026 /* Skip suspend with A+A reset */ 4027 if (adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) { 4028 dev_dbg(adev->dev, "Device in reset. Skipping RLC halt\n"); 4029 return 0; 4030 } 4031 4032 adev->gfx.rlc.funcs->stop(adev); 4033 return 0; 4034 } 4035 4036 static int gfx_v9_0_suspend(void *handle) 4037 { 4038 return gfx_v9_0_hw_fini(handle); 4039 } 4040 4041 static int gfx_v9_0_resume(void *handle) 4042 { 4043 return gfx_v9_0_hw_init(handle); 4044 } 4045 4046 static bool gfx_v9_0_is_idle(void *handle) 4047 { 4048 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4049 4050 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS), 4051 GRBM_STATUS, GUI_ACTIVE)) 4052 return false; 4053 else 4054 return true; 4055 } 4056 4057 static int gfx_v9_0_wait_for_idle(void *handle) 4058 { 4059 unsigned i; 4060 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4061 4062 for (i = 0; i < adev->usec_timeout; i++) { 4063 if (gfx_v9_0_is_idle(handle)) 4064 return 0; 4065 udelay(1); 4066 } 4067 return -ETIMEDOUT; 4068 } 4069 4070 static int gfx_v9_0_soft_reset(void *handle) 4071 { 4072 u32 grbm_soft_reset = 0; 4073 u32 tmp; 4074 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4075 4076 /* GRBM_STATUS */ 4077 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS); 4078 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK | 4079 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK | 4080 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK | 4081 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK | 4082 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK | 4083 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) { 4084 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4085 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4086 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4087 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1); 4088 } 4089 4090 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) { 4091 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4092 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4093 } 4094 4095 /* GRBM_STATUS2 */ 4096 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2); 4097 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY)) 4098 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4099 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 4100 4101 4102 if (grbm_soft_reset) { 4103 /* stop the rlc */ 4104 adev->gfx.rlc.funcs->stop(adev); 4105 4106 if (adev->gfx.num_gfx_rings) 4107 /* Disable GFX parsing/prefetching */ 4108 gfx_v9_0_cp_gfx_enable(adev, false); 4109 4110 /* Disable MEC parsing/prefetching */ 4111 gfx_v9_0_cp_compute_enable(adev, false); 4112 4113 if (grbm_soft_reset) { 4114 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4115 tmp |= grbm_soft_reset; 4116 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 4117 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4118 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4119 4120 udelay(50); 4121 4122 tmp &= ~grbm_soft_reset; 4123 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4124 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4125 } 4126 4127 /* Wait a little for things to settle down */ 4128 udelay(50); 4129 } 4130 return 0; 4131 } 4132 4133 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev) 4134 { 4135 signed long r, cnt = 0; 4136 unsigned long flags; 4137 uint32_t seq, reg_val_offs = 0; 4138 uint64_t value = 0; 4139 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 4140 struct amdgpu_ring *ring = &kiq->ring; 4141 4142 BUG_ON(!ring->funcs->emit_rreg); 4143 4144 spin_lock_irqsave(&kiq->ring_lock, flags); 4145 if (amdgpu_device_wb_get(adev, ®_val_offs)) { 4146 pr_err("critical bug! too many kiq readers\n"); 4147 goto failed_unlock; 4148 } 4149 amdgpu_ring_alloc(ring, 32); 4150 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 4151 amdgpu_ring_write(ring, 9 | /* src: register*/ 4152 (5 << 8) | /* dst: memory */ 4153 (1 << 16) | /* count sel */ 4154 (1 << 20)); /* write confirm */ 4155 amdgpu_ring_write(ring, 0); 4156 amdgpu_ring_write(ring, 0); 4157 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 4158 reg_val_offs * 4)); 4159 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 4160 reg_val_offs * 4)); 4161 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 4162 if (r) 4163 goto failed_undo; 4164 4165 amdgpu_ring_commit(ring); 4166 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4167 4168 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4169 4170 /* don't wait anymore for gpu reset case because this way may 4171 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 4172 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 4173 * never return if we keep waiting in virt_kiq_rreg, which cause 4174 * gpu_recover() hang there. 4175 * 4176 * also don't wait anymore for IRQ context 4177 * */ 4178 if (r < 1 && (amdgpu_in_reset(adev))) 4179 goto failed_kiq_read; 4180 4181 might_sleep(); 4182 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 4183 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 4184 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4185 } 4186 4187 if (cnt > MAX_KIQ_REG_TRY) 4188 goto failed_kiq_read; 4189 4190 mb(); 4191 value = (uint64_t)adev->wb.wb[reg_val_offs] | 4192 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL; 4193 amdgpu_device_wb_free(adev, reg_val_offs); 4194 return value; 4195 4196 failed_undo: 4197 amdgpu_ring_undo(ring); 4198 failed_unlock: 4199 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4200 failed_kiq_read: 4201 if (reg_val_offs) 4202 amdgpu_device_wb_free(adev, reg_val_offs); 4203 pr_err("failed to read gpu clock\n"); 4204 return ~0; 4205 } 4206 4207 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev) 4208 { 4209 uint64_t clock; 4210 4211 amdgpu_gfx_off_ctrl(adev, false); 4212 mutex_lock(&adev->gfx.gpu_clock_mutex); 4213 if (adev->asic_type == CHIP_VEGA10 && amdgpu_sriov_runtime(adev)) { 4214 clock = gfx_v9_0_kiq_read_clock(adev); 4215 } else { 4216 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1); 4217 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) | 4218 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 4219 } 4220 mutex_unlock(&adev->gfx.gpu_clock_mutex); 4221 amdgpu_gfx_off_ctrl(adev, true); 4222 return clock; 4223 } 4224 4225 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 4226 uint32_t vmid, 4227 uint32_t gds_base, uint32_t gds_size, 4228 uint32_t gws_base, uint32_t gws_size, 4229 uint32_t oa_base, uint32_t oa_size) 4230 { 4231 struct amdgpu_device *adev = ring->adev; 4232 4233 /* GDS Base */ 4234 gfx_v9_0_write_data_to_reg(ring, 0, false, 4235 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid, 4236 gds_base); 4237 4238 /* GDS Size */ 4239 gfx_v9_0_write_data_to_reg(ring, 0, false, 4240 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid, 4241 gds_size); 4242 4243 /* GWS */ 4244 gfx_v9_0_write_data_to_reg(ring, 0, false, 4245 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid, 4246 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 4247 4248 /* OA */ 4249 gfx_v9_0_write_data_to_reg(ring, 0, false, 4250 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid, 4251 (1 << (oa_size + oa_base)) - (1 << oa_base)); 4252 } 4253 4254 static const u32 vgpr_init_compute_shader[] = 4255 { 4256 0xb07c0000, 0xbe8000ff, 4257 0x000000f8, 0xbf110800, 4258 0x7e000280, 0x7e020280, 4259 0x7e040280, 0x7e060280, 4260 0x7e080280, 0x7e0a0280, 4261 0x7e0c0280, 0x7e0e0280, 4262 0x80808800, 0xbe803200, 4263 0xbf84fff5, 0xbf9c0000, 4264 0xd28c0001, 0x0001007f, 4265 0xd28d0001, 0x0002027e, 4266 0x10020288, 0xb8810904, 4267 0xb7814000, 0xd1196a01, 4268 0x00000301, 0xbe800087, 4269 0xbefc00c1, 0xd89c4000, 4270 0x00020201, 0xd89cc080, 4271 0x00040401, 0x320202ff, 4272 0x00000800, 0x80808100, 4273 0xbf84fff8, 0x7e020280, 4274 0xbf810000, 0x00000000, 4275 }; 4276 4277 static const u32 sgpr_init_compute_shader[] = 4278 { 4279 0xb07c0000, 0xbe8000ff, 4280 0x0000005f, 0xbee50080, 4281 0xbe812c65, 0xbe822c65, 4282 0xbe832c65, 0xbe842c65, 4283 0xbe852c65, 0xb77c0005, 4284 0x80808500, 0xbf84fff8, 4285 0xbe800080, 0xbf810000, 4286 }; 4287 4288 static const u32 vgpr_init_compute_shader_arcturus[] = { 4289 0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080, 4290 0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080, 4291 0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080, 4292 0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080, 4293 0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080, 4294 0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080, 4295 0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080, 4296 0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080, 4297 0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080, 4298 0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080, 4299 0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080, 4300 0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080, 4301 0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080, 4302 0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080, 4303 0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080, 4304 0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080, 4305 0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080, 4306 0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080, 4307 0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080, 4308 0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080, 4309 0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080, 4310 0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080, 4311 0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080, 4312 0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080, 4313 0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080, 4314 0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080, 4315 0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080, 4316 0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080, 4317 0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080, 4318 0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080, 4319 0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080, 4320 0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080, 4321 0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080, 4322 0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080, 4323 0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080, 4324 0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080, 4325 0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080, 4326 0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080, 4327 0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080, 4328 0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080, 4329 0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080, 4330 0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080, 4331 0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080, 4332 0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080, 4333 0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080, 4334 0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080, 4335 0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080, 4336 0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080, 4337 0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080, 4338 0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080, 4339 0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080, 4340 0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080, 4341 0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080, 4342 0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080, 4343 0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080, 4344 0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080, 4345 0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080, 4346 0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080, 4347 0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080, 4348 0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080, 4349 0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080, 4350 0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080, 4351 0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080, 4352 0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080, 4353 0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080, 4354 0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080, 4355 0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080, 4356 0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080, 4357 0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080, 4358 0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080, 4359 0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080, 4360 0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080, 4361 0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080, 4362 0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080, 4363 0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080, 4364 0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080, 4365 0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080, 4366 0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080, 4367 0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080, 4368 0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080, 4369 0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080, 4370 0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080, 4371 0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080, 4372 0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080, 4373 0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080, 4374 0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a, 4375 0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280, 4376 0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000, 4377 0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904, 4378 0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000, 4379 0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a, 4380 0xbf84fff8, 0xbf810000, 4381 }; 4382 4383 /* When below register arrays changed, please update gpr_reg_size, 4384 and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds, 4385 to cover all gfx9 ASICs */ 4386 static const struct soc15_reg_entry vgpr_init_regs[] = { 4387 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4388 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4389 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4390 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4391 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f }, 4392 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4393 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4394 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4395 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4396 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4397 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4398 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4399 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4400 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4401 }; 4402 4403 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = { 4404 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4405 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4406 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4407 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4408 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf }, 4409 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4410 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4411 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4412 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4413 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4414 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4415 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4416 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4417 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4418 }; 4419 4420 static const struct soc15_reg_entry sgpr1_init_regs[] = { 4421 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4422 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4423 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4424 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4425 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4426 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4427 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff }, 4428 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff }, 4429 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff }, 4430 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff }, 4431 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff }, 4432 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff }, 4433 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff }, 4434 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff }, 4435 }; 4436 4437 static const struct soc15_reg_entry sgpr2_init_regs[] = { 4438 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4439 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4440 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4441 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4442 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4443 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4444 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 }, 4445 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 }, 4446 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 }, 4447 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 }, 4448 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 }, 4449 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 }, 4450 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 }, 4451 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 }, 4452 }; 4453 4454 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = { 4455 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1}, 4456 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1}, 4457 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1}, 4458 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1}, 4459 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1}, 4460 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1}, 4461 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1}, 4462 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1}, 4463 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1}, 4464 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1}, 4465 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1}, 4466 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1}, 4467 { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1}, 4468 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6}, 4469 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16}, 4470 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16}, 4471 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16}, 4472 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16}, 4473 { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16}, 4474 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16}, 4475 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16}, 4476 { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16}, 4477 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6}, 4478 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16}, 4479 { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16}, 4480 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1}, 4481 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1}, 4482 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32}, 4483 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32}, 4484 { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72}, 4485 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16}, 4486 { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2}, 4487 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6}, 4488 }; 4489 4490 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev) 4491 { 4492 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4493 int i, r; 4494 4495 /* only support when RAS is enabled */ 4496 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4497 return 0; 4498 4499 r = amdgpu_ring_alloc(ring, 7); 4500 if (r) { 4501 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n", 4502 ring->name, r); 4503 return r; 4504 } 4505 4506 WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000); 4507 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size); 4508 4509 amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 4510 amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC | 4511 PACKET3_DMA_DATA_DST_SEL(1) | 4512 PACKET3_DMA_DATA_SRC_SEL(2) | 4513 PACKET3_DMA_DATA_ENGINE(0))); 4514 amdgpu_ring_write(ring, 0); 4515 amdgpu_ring_write(ring, 0); 4516 amdgpu_ring_write(ring, 0); 4517 amdgpu_ring_write(ring, 0); 4518 amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT | 4519 adev->gds.gds_size); 4520 4521 amdgpu_ring_commit(ring); 4522 4523 for (i = 0; i < adev->usec_timeout; i++) { 4524 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring)) 4525 break; 4526 udelay(1); 4527 } 4528 4529 if (i >= adev->usec_timeout) 4530 r = -ETIMEDOUT; 4531 4532 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000); 4533 4534 return r; 4535 } 4536 4537 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) 4538 { 4539 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4540 struct amdgpu_ib ib; 4541 struct dma_fence *f = NULL; 4542 int r, i; 4543 unsigned total_size, vgpr_offset, sgpr_offset; 4544 u64 gpu_addr; 4545 4546 int compute_dim_x = adev->gfx.config.max_shader_engines * 4547 adev->gfx.config.max_cu_per_sh * 4548 adev->gfx.config.max_sh_per_se; 4549 int sgpr_work_group_size = 5; 4550 int gpr_reg_size = adev->gfx.config.max_shader_engines + 6; 4551 int vgpr_init_shader_size; 4552 const u32 *vgpr_init_shader_ptr; 4553 const struct soc15_reg_entry *vgpr_init_regs_ptr; 4554 4555 /* only support when RAS is enabled */ 4556 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4557 return 0; 4558 4559 /* bail if the compute ring is not ready */ 4560 if (!ring->sched.ready) 4561 return 0; 4562 4563 if (adev->asic_type == CHIP_ARCTURUS) { 4564 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus; 4565 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus); 4566 vgpr_init_regs_ptr = vgpr_init_regs_arcturus; 4567 } else { 4568 vgpr_init_shader_ptr = vgpr_init_compute_shader; 4569 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader); 4570 vgpr_init_regs_ptr = vgpr_init_regs; 4571 } 4572 4573 total_size = 4574 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */ 4575 total_size += 4576 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */ 4577 total_size += 4578 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */ 4579 total_size = ALIGN(total_size, 256); 4580 vgpr_offset = total_size; 4581 total_size += ALIGN(vgpr_init_shader_size, 256); 4582 sgpr_offset = total_size; 4583 total_size += sizeof(sgpr_init_compute_shader); 4584 4585 /* allocate an indirect buffer to put the commands in */ 4586 memset(&ib, 0, sizeof(ib)); 4587 r = amdgpu_ib_get(adev, NULL, total_size, 4588 AMDGPU_IB_POOL_DIRECT, &ib); 4589 if (r) { 4590 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); 4591 return r; 4592 } 4593 4594 /* load the compute shaders */ 4595 for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++) 4596 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i]; 4597 4598 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++) 4599 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i]; 4600 4601 /* init the ib length to 0 */ 4602 ib.length_dw = 0; 4603 4604 /* VGPR */ 4605 /* write the register state for the compute dispatch */ 4606 for (i = 0; i < gpr_reg_size; i++) { 4607 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4608 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i]) 4609 - PACKET3_SET_SH_REG_START; 4610 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value; 4611 } 4612 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4613 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8; 4614 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4615 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4616 - PACKET3_SET_SH_REG_START; 4617 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4618 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4619 4620 /* write dispatch packet */ 4621 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4622 ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */ 4623 ib.ptr[ib.length_dw++] = 1; /* y */ 4624 ib.ptr[ib.length_dw++] = 1; /* z */ 4625 ib.ptr[ib.length_dw++] = 4626 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4627 4628 /* write CS partial flush packet */ 4629 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4630 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4631 4632 /* SGPR1 */ 4633 /* write the register state for the compute dispatch */ 4634 for (i = 0; i < gpr_reg_size; i++) { 4635 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4636 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i]) 4637 - PACKET3_SET_SH_REG_START; 4638 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value; 4639 } 4640 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4641 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4642 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4643 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4644 - PACKET3_SET_SH_REG_START; 4645 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4646 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4647 4648 /* write dispatch packet */ 4649 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4650 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4651 ib.ptr[ib.length_dw++] = 1; /* y */ 4652 ib.ptr[ib.length_dw++] = 1; /* z */ 4653 ib.ptr[ib.length_dw++] = 4654 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4655 4656 /* write CS partial flush packet */ 4657 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4658 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4659 4660 /* SGPR2 */ 4661 /* write the register state for the compute dispatch */ 4662 for (i = 0; i < gpr_reg_size; i++) { 4663 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4664 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i]) 4665 - PACKET3_SET_SH_REG_START; 4666 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value; 4667 } 4668 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4669 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4670 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4671 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4672 - PACKET3_SET_SH_REG_START; 4673 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4674 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4675 4676 /* write dispatch packet */ 4677 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4678 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4679 ib.ptr[ib.length_dw++] = 1; /* y */ 4680 ib.ptr[ib.length_dw++] = 1; /* z */ 4681 ib.ptr[ib.length_dw++] = 4682 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4683 4684 /* write CS partial flush packet */ 4685 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4686 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4687 4688 /* shedule the ib on the ring */ 4689 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 4690 if (r) { 4691 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r); 4692 goto fail; 4693 } 4694 4695 /* wait for the GPU to finish processing the IB */ 4696 r = dma_fence_wait(f, false); 4697 if (r) { 4698 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); 4699 goto fail; 4700 } 4701 4702 fail: 4703 amdgpu_ib_free(adev, &ib, NULL); 4704 dma_fence_put(f); 4705 4706 return r; 4707 } 4708 4709 static int gfx_v9_0_early_init(void *handle) 4710 { 4711 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4712 4713 if (adev->asic_type == CHIP_ARCTURUS || 4714 adev->asic_type == CHIP_ALDEBARAN) 4715 adev->gfx.num_gfx_rings = 0; 4716 else 4717 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS; 4718 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), 4719 AMDGPU_MAX_COMPUTE_RINGS); 4720 gfx_v9_0_set_kiq_pm4_funcs(adev); 4721 gfx_v9_0_set_ring_funcs(adev); 4722 gfx_v9_0_set_irq_funcs(adev); 4723 gfx_v9_0_set_gds_init(adev); 4724 gfx_v9_0_set_rlc_funcs(adev); 4725 4726 return 0; 4727 } 4728 4729 static int gfx_v9_0_ecc_late_init(void *handle) 4730 { 4731 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4732 int r; 4733 4734 /* 4735 * Temp workaround to fix the issue that CP firmware fails to 4736 * update read pointer when CPDMA is writing clearing operation 4737 * to GDS in suspend/resume sequence on several cards. So just 4738 * limit this operation in cold boot sequence. 4739 */ 4740 if ((!adev->in_suspend) && 4741 (adev->gds.gds_size)) { 4742 r = gfx_v9_0_do_edc_gds_workarounds(adev); 4743 if (r) 4744 return r; 4745 } 4746 4747 /* requires IBs so do in late init after IB pool is initialized */ 4748 if (adev->asic_type == CHIP_ALDEBARAN) 4749 r = gfx_v9_4_2_do_edc_gpr_workarounds(adev); 4750 else 4751 r = gfx_v9_0_do_edc_gpr_workarounds(adev); 4752 4753 if (r) 4754 return r; 4755 4756 if (adev->gfx.ras_funcs && 4757 adev->gfx.ras_funcs->ras_late_init) { 4758 r = adev->gfx.ras_funcs->ras_late_init(adev); 4759 if (r) 4760 return r; 4761 } 4762 4763 if (adev->gfx.ras_funcs && 4764 adev->gfx.ras_funcs->enable_watchdog_timer) 4765 adev->gfx.ras_funcs->enable_watchdog_timer(adev); 4766 4767 return 0; 4768 } 4769 4770 static int gfx_v9_0_late_init(void *handle) 4771 { 4772 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4773 int r; 4774 4775 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 4776 if (r) 4777 return r; 4778 4779 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 4780 if (r) 4781 return r; 4782 4783 r = gfx_v9_0_ecc_late_init(handle); 4784 if (r) 4785 return r; 4786 4787 return 0; 4788 } 4789 4790 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev) 4791 { 4792 uint32_t rlc_setting; 4793 4794 /* if RLC is not enabled, do nothing */ 4795 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL); 4796 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK)) 4797 return false; 4798 4799 return true; 4800 } 4801 4802 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev) 4803 { 4804 uint32_t data; 4805 unsigned i; 4806 4807 data = RLC_SAFE_MODE__CMD_MASK; 4808 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 4809 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4810 4811 /* wait for RLC_SAFE_MODE */ 4812 for (i = 0; i < adev->usec_timeout; i++) { 4813 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 4814 break; 4815 udelay(1); 4816 } 4817 } 4818 4819 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev) 4820 { 4821 uint32_t data; 4822 4823 data = RLC_SAFE_MODE__CMD_MASK; 4824 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4825 } 4826 4827 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev, 4828 bool enable) 4829 { 4830 amdgpu_gfx_rlc_enter_safe_mode(adev); 4831 4832 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { 4833 gfx_v9_0_enable_gfx_cg_power_gating(adev, true); 4834 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4835 gfx_v9_0_enable_gfx_pipeline_powergating(adev, true); 4836 } else { 4837 gfx_v9_0_enable_gfx_cg_power_gating(adev, false); 4838 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4839 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false); 4840 } 4841 4842 amdgpu_gfx_rlc_exit_safe_mode(adev); 4843 } 4844 4845 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev, 4846 bool enable) 4847 { 4848 /* TODO: double check if we need to perform under safe mode */ 4849 /* gfx_v9_0_enter_rlc_safe_mode(adev); */ 4850 4851 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 4852 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true); 4853 else 4854 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false); 4855 4856 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 4857 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true); 4858 else 4859 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false); 4860 4861 /* gfx_v9_0_exit_rlc_safe_mode(adev); */ 4862 } 4863 4864 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 4865 bool enable) 4866 { 4867 uint32_t data, def; 4868 4869 amdgpu_gfx_rlc_enter_safe_mode(adev); 4870 4871 /* It is disabled by HW by default */ 4872 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { 4873 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 4874 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4875 4876 if (adev->asic_type != CHIP_VEGA12) 4877 data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4878 4879 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4880 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4881 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4882 4883 /* only for Vega10 & Raven1 */ 4884 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK; 4885 4886 if (def != data) 4887 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4888 4889 /* MGLS is a global flag to control all MGLS in GFX */ 4890 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 4891 /* 2 - RLC memory Light sleep */ 4892 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { 4893 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4894 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4895 if (def != data) 4896 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4897 } 4898 /* 3 - CP memory Light sleep */ 4899 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { 4900 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4901 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4902 if (def != data) 4903 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4904 } 4905 } 4906 } else { 4907 /* 1 - MGCG_OVERRIDE */ 4908 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4909 4910 if (adev->asic_type != CHIP_VEGA12) 4911 data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4912 4913 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 4914 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4915 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4916 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4917 4918 if (def != data) 4919 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4920 4921 /* 2 - disable MGLS in RLC */ 4922 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4923 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) { 4924 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4925 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4926 } 4927 4928 /* 3 - disable MGLS in CP */ 4929 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4930 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) { 4931 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4932 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4933 } 4934 } 4935 4936 amdgpu_gfx_rlc_exit_safe_mode(adev); 4937 } 4938 4939 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev, 4940 bool enable) 4941 { 4942 uint32_t data, def; 4943 4944 if (!adev->gfx.num_gfx_rings) 4945 return; 4946 4947 amdgpu_gfx_rlc_enter_safe_mode(adev); 4948 4949 /* Enable 3D CGCG/CGLS */ 4950 if (enable) { 4951 /* write cmd to clear cgcg/cgls ov */ 4952 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4953 /* unset CGCG override */ 4954 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 4955 /* update CGCG and CGLS override bits */ 4956 if (def != data) 4957 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4958 4959 /* enable 3Dcgcg FSM(0x0000363f) */ 4960 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4961 4962 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) 4963 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4964 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 4965 else 4966 data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT; 4967 4968 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 4969 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 4970 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 4971 if (def != data) 4972 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4973 4974 /* set IDLE_POLL_COUNT(0x00900100) */ 4975 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 4976 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 4977 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 4978 if (def != data) 4979 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 4980 } else { 4981 /* Disable CGCG/CGLS */ 4982 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4983 /* disable cgcg, cgls should be disabled */ 4984 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK | 4985 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK); 4986 /* disable cgcg and cgls in FSM */ 4987 if (def != data) 4988 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4989 } 4990 4991 amdgpu_gfx_rlc_exit_safe_mode(adev); 4992 } 4993 4994 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 4995 bool enable) 4996 { 4997 uint32_t def, data; 4998 4999 amdgpu_gfx_rlc_enter_safe_mode(adev); 5000 5001 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { 5002 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 5003 /* unset CGCG override */ 5004 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 5005 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5006 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 5007 else 5008 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 5009 /* update CGCG and CGLS override bits */ 5010 if (def != data) 5011 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 5012 5013 /* enable cgcg FSM(0x0000363F) */ 5014 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 5015 5016 if (adev->asic_type == CHIP_ARCTURUS) 5017 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5018 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5019 else 5020 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5021 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5022 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5023 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 5024 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5025 if (def != data) 5026 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 5027 5028 /* set IDLE_POLL_COUNT(0x00900100) */ 5029 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 5030 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 5031 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 5032 if (def != data) 5033 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 5034 } else { 5035 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 5036 /* reset CGCG/CGLS bits */ 5037 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK); 5038 /* disable cgcg and cgls in FSM */ 5039 if (def != data) 5040 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 5041 } 5042 5043 amdgpu_gfx_rlc_exit_safe_mode(adev); 5044 } 5045 5046 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev, 5047 bool enable) 5048 { 5049 if (enable) { 5050 /* CGCG/CGLS should be enabled after MGCG/MGLS 5051 * === MGCG + MGLS === 5052 */ 5053 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 5054 /* === CGCG /CGLS for GFX 3D Only === */ 5055 gfx_v9_0_update_3d_clock_gating(adev, enable); 5056 /* === CGCG + CGLS === */ 5057 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 5058 } else { 5059 /* CGCG/CGLS should be disabled before MGCG/MGLS 5060 * === CGCG + CGLS === 5061 */ 5062 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 5063 /* === CGCG /CGLS for GFX 3D Only === */ 5064 gfx_v9_0_update_3d_clock_gating(adev, enable); 5065 /* === MGCG + MGLS === */ 5066 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 5067 } 5068 return 0; 5069 } 5070 5071 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid) 5072 { 5073 u32 reg, data; 5074 5075 reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL); 5076 if (amdgpu_sriov_is_pp_one_vf(adev)) 5077 data = RREG32_NO_KIQ(reg); 5078 else 5079 data = RREG32(reg); 5080 5081 data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK; 5082 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; 5083 5084 if (amdgpu_sriov_is_pp_one_vf(adev)) 5085 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data); 5086 else 5087 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data); 5088 } 5089 5090 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev, 5091 uint32_t offset, 5092 struct soc15_reg_rlcg *entries, int arr_size) 5093 { 5094 int i; 5095 uint32_t reg; 5096 5097 if (!entries) 5098 return false; 5099 5100 for (i = 0; i < arr_size; i++) { 5101 const struct soc15_reg_rlcg *entry; 5102 5103 entry = &entries[i]; 5104 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg; 5105 if (offset == reg) 5106 return true; 5107 } 5108 5109 return false; 5110 } 5111 5112 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset) 5113 { 5114 return gfx_v9_0_check_rlcg_range(adev, offset, 5115 (void *)rlcg_access_gc_9_0, 5116 ARRAY_SIZE(rlcg_access_gc_9_0)); 5117 } 5118 5119 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = { 5120 .is_rlc_enabled = gfx_v9_0_is_rlc_enabled, 5121 .set_safe_mode = gfx_v9_0_set_safe_mode, 5122 .unset_safe_mode = gfx_v9_0_unset_safe_mode, 5123 .init = gfx_v9_0_rlc_init, 5124 .get_csb_size = gfx_v9_0_get_csb_size, 5125 .get_csb_buffer = gfx_v9_0_get_csb_buffer, 5126 .get_cp_table_num = gfx_v9_0_cp_jump_table_num, 5127 .resume = gfx_v9_0_rlc_resume, 5128 .stop = gfx_v9_0_rlc_stop, 5129 .reset = gfx_v9_0_rlc_reset, 5130 .start = gfx_v9_0_rlc_start, 5131 .update_spm_vmid = gfx_v9_0_update_spm_vmid, 5132 .rlcg_wreg = gfx_v9_0_rlcg_wreg, 5133 .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range, 5134 }; 5135 5136 static int gfx_v9_0_set_powergating_state(void *handle, 5137 enum amd_powergating_state state) 5138 { 5139 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5140 bool enable = (state == AMD_PG_STATE_GATE); 5141 5142 switch (adev->asic_type) { 5143 case CHIP_RAVEN: 5144 case CHIP_RENOIR: 5145 if (!enable) 5146 amdgpu_gfx_off_ctrl(adev, false); 5147 5148 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { 5149 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true); 5150 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true); 5151 } else { 5152 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false); 5153 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false); 5154 } 5155 5156 if (adev->pg_flags & AMD_PG_SUPPORT_CP) 5157 gfx_v9_0_enable_cp_power_gating(adev, true); 5158 else 5159 gfx_v9_0_enable_cp_power_gating(adev, false); 5160 5161 /* update gfx cgpg state */ 5162 gfx_v9_0_update_gfx_cg_power_gating(adev, enable); 5163 5164 /* update mgcg state */ 5165 gfx_v9_0_update_gfx_mg_power_gating(adev, enable); 5166 5167 if (enable) 5168 amdgpu_gfx_off_ctrl(adev, true); 5169 break; 5170 case CHIP_VEGA12: 5171 amdgpu_gfx_off_ctrl(adev, enable); 5172 break; 5173 default: 5174 break; 5175 } 5176 5177 return 0; 5178 } 5179 5180 static int gfx_v9_0_set_clockgating_state(void *handle, 5181 enum amd_clockgating_state state) 5182 { 5183 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5184 5185 if (amdgpu_sriov_vf(adev)) 5186 return 0; 5187 5188 switch (adev->asic_type) { 5189 case CHIP_VEGA10: 5190 case CHIP_VEGA12: 5191 case CHIP_VEGA20: 5192 case CHIP_RAVEN: 5193 case CHIP_ARCTURUS: 5194 case CHIP_RENOIR: 5195 case CHIP_ALDEBARAN: 5196 gfx_v9_0_update_gfx_clock_gating(adev, 5197 state == AMD_CG_STATE_GATE); 5198 break; 5199 default: 5200 break; 5201 } 5202 return 0; 5203 } 5204 5205 static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags) 5206 { 5207 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5208 int data; 5209 5210 if (amdgpu_sriov_vf(adev)) 5211 *flags = 0; 5212 5213 /* AMD_CG_SUPPORT_GFX_MGCG */ 5214 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE)); 5215 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 5216 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5217 5218 /* AMD_CG_SUPPORT_GFX_CGCG */ 5219 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL)); 5220 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5221 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5222 5223 /* AMD_CG_SUPPORT_GFX_CGLS */ 5224 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5225 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5226 5227 /* AMD_CG_SUPPORT_GFX_RLC_LS */ 5228 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL)); 5229 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) 5230 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS; 5231 5232 /* AMD_CG_SUPPORT_GFX_CP_LS */ 5233 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL)); 5234 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) 5235 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS; 5236 5237 if (adev->asic_type != CHIP_ARCTURUS) { 5238 /* AMD_CG_SUPPORT_GFX_3D_CGCG */ 5239 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D)); 5240 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) 5241 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; 5242 5243 /* AMD_CG_SUPPORT_GFX_3D_CGLS */ 5244 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK) 5245 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS; 5246 } 5247 } 5248 5249 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) 5250 { 5251 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/ 5252 } 5253 5254 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5255 { 5256 struct amdgpu_device *adev = ring->adev; 5257 u64 wptr; 5258 5259 /* XXX check if swapping is necessary on BE */ 5260 if (ring->use_doorbell) { 5261 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]); 5262 } else { 5263 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR); 5264 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32; 5265 } 5266 5267 return wptr; 5268 } 5269 5270 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5271 { 5272 struct amdgpu_device *adev = ring->adev; 5273 5274 if (ring->use_doorbell) { 5275 /* XXX check if swapping is necessary on BE */ 5276 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5277 WDOORBELL64(ring->doorbell_index, ring->wptr); 5278 } else { 5279 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 5280 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 5281 } 5282 } 5283 5284 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5285 { 5286 struct amdgpu_device *adev = ring->adev; 5287 u32 ref_and_mask, reg_mem_engine; 5288 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg; 5289 5290 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 5291 switch (ring->me) { 5292 case 1: 5293 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe; 5294 break; 5295 case 2: 5296 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe; 5297 break; 5298 default: 5299 return; 5300 } 5301 reg_mem_engine = 0; 5302 } else { 5303 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0; 5304 reg_mem_engine = 1; /* pfp */ 5305 } 5306 5307 gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1, 5308 adev->nbio.funcs->get_hdp_flush_req_offset(adev), 5309 adev->nbio.funcs->get_hdp_flush_done_offset(adev), 5310 ref_and_mask, ref_and_mask, 0x20); 5311 } 5312 5313 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 5314 struct amdgpu_job *job, 5315 struct amdgpu_ib *ib, 5316 uint32_t flags) 5317 { 5318 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5319 u32 header, control = 0; 5320 5321 if (ib->flags & AMDGPU_IB_FLAG_CE) 5322 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 5323 else 5324 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 5325 5326 control |= ib->length_dw | (vmid << 24); 5327 5328 if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { 5329 control |= INDIRECT_BUFFER_PRE_ENB(1); 5330 5331 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid) 5332 gfx_v9_0_ring_emit_de_meta(ring); 5333 } 5334 5335 amdgpu_ring_write(ring, header); 5336 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5337 amdgpu_ring_write(ring, 5338 #ifdef __BIG_ENDIAN 5339 (2 << 0) | 5340 #endif 5341 lower_32_bits(ib->gpu_addr)); 5342 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5343 amdgpu_ring_write(ring, control); 5344 } 5345 5346 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 5347 struct amdgpu_job *job, 5348 struct amdgpu_ib *ib, 5349 uint32_t flags) 5350 { 5351 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5352 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 5353 5354 /* Currently, there is a high possibility to get wave ID mismatch 5355 * between ME and GDS, leading to a hw deadlock, because ME generates 5356 * different wave IDs than the GDS expects. This situation happens 5357 * randomly when at least 5 compute pipes use GDS ordered append. 5358 * The wave IDs generated by ME are also wrong after suspend/resume. 5359 * Those are probably bugs somewhere else in the kernel driver. 5360 * 5361 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 5362 * GDS to 0 for this ring (me/pipe). 5363 */ 5364 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 5365 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 5366 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID); 5367 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 5368 } 5369 5370 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 5371 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5372 amdgpu_ring_write(ring, 5373 #ifdef __BIG_ENDIAN 5374 (2 << 0) | 5375 #endif 5376 lower_32_bits(ib->gpu_addr)); 5377 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5378 amdgpu_ring_write(ring, control); 5379 } 5380 5381 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 5382 u64 seq, unsigned flags) 5383 { 5384 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 5385 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 5386 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY; 5387 5388 /* RELEASE_MEM - flush caches, send int */ 5389 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); 5390 amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN | 5391 EOP_TC_NC_ACTION_EN) : 5392 (EOP_TCL1_ACTION_EN | 5393 EOP_TC_ACTION_EN | 5394 EOP_TC_WB_ACTION_EN | 5395 EOP_TC_MD_ACTION_EN)) | 5396 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 5397 EVENT_INDEX(5))); 5398 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 5399 5400 /* 5401 * the address should be Qword aligned if 64bit write, Dword 5402 * aligned if only send 32bit data low (discard data high) 5403 */ 5404 if (write64bit) 5405 BUG_ON(addr & 0x7); 5406 else 5407 BUG_ON(addr & 0x3); 5408 amdgpu_ring_write(ring, lower_32_bits(addr)); 5409 amdgpu_ring_write(ring, upper_32_bits(addr)); 5410 amdgpu_ring_write(ring, lower_32_bits(seq)); 5411 amdgpu_ring_write(ring, upper_32_bits(seq)); 5412 amdgpu_ring_write(ring, 0); 5413 } 5414 5415 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 5416 { 5417 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5418 uint32_t seq = ring->fence_drv.sync_seq; 5419 uint64_t addr = ring->fence_drv.gpu_addr; 5420 5421 gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0, 5422 lower_32_bits(addr), upper_32_bits(addr), 5423 seq, 0xffffffff, 4); 5424 } 5425 5426 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 5427 unsigned vmid, uint64_t pd_addr) 5428 { 5429 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 5430 5431 /* compute doesn't have PFP */ 5432 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 5433 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 5434 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 5435 amdgpu_ring_write(ring, 0x0); 5436 } 5437 } 5438 5439 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring) 5440 { 5441 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */ 5442 } 5443 5444 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 5445 { 5446 u64 wptr; 5447 5448 /* XXX check if swapping is necessary on BE */ 5449 if (ring->use_doorbell) 5450 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]); 5451 else 5452 BUG(); 5453 return wptr; 5454 } 5455 5456 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 5457 { 5458 struct amdgpu_device *adev = ring->adev; 5459 5460 /* XXX check if swapping is necessary on BE */ 5461 if (ring->use_doorbell) { 5462 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5463 WDOORBELL64(ring->doorbell_index, ring->wptr); 5464 } else{ 5465 BUG(); /* only DOORBELL method supported on gfx9 now */ 5466 } 5467 } 5468 5469 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 5470 u64 seq, unsigned int flags) 5471 { 5472 struct amdgpu_device *adev = ring->adev; 5473 5474 /* we only allocate 32bit for each seq wb address */ 5475 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); 5476 5477 /* write fence seq to the "addr" */ 5478 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5479 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5480 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 5481 amdgpu_ring_write(ring, lower_32_bits(addr)); 5482 amdgpu_ring_write(ring, upper_32_bits(addr)); 5483 amdgpu_ring_write(ring, lower_32_bits(seq)); 5484 5485 if (flags & AMDGPU_FENCE_FLAG_INT) { 5486 /* set register to trigger INT */ 5487 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5488 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5489 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 5490 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS)); 5491 amdgpu_ring_write(ring, 0); 5492 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 5493 } 5494 } 5495 5496 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring) 5497 { 5498 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 5499 amdgpu_ring_write(ring, 0); 5500 } 5501 5502 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring) 5503 { 5504 struct v9_ce_ib_state ce_payload = {0}; 5505 uint64_t csa_addr; 5506 int cnt; 5507 5508 cnt = (sizeof(ce_payload) >> 2) + 4 - 2; 5509 csa_addr = amdgpu_csa_vaddr(ring->adev); 5510 5511 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5512 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) | 5513 WRITE_DATA_DST_SEL(8) | 5514 WR_CONFIRM) | 5515 WRITE_DATA_CACHE_POLICY(0)); 5516 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5517 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5518 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2); 5519 } 5520 5521 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring) 5522 { 5523 struct v9_de_ib_state de_payload = {0}; 5524 uint64_t csa_addr, gds_addr; 5525 int cnt; 5526 5527 csa_addr = amdgpu_csa_vaddr(ring->adev); 5528 gds_addr = csa_addr + 4096; 5529 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr); 5530 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr); 5531 5532 cnt = (sizeof(de_payload) >> 2) + 4 - 2; 5533 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5534 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 5535 WRITE_DATA_DST_SEL(8) | 5536 WR_CONFIRM) | 5537 WRITE_DATA_CACHE_POLICY(0)); 5538 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5539 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5540 amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2); 5541 } 5542 5543 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, 5544 bool secure) 5545 { 5546 uint32_t v = secure ? FRAME_TMZ : 0; 5547 5548 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); 5549 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1)); 5550 } 5551 5552 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) 5553 { 5554 uint32_t dw2 = 0; 5555 5556 if (amdgpu_sriov_vf(ring->adev)) 5557 gfx_v9_0_ring_emit_ce_meta(ring); 5558 5559 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 5560 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 5561 /* set load_global_config & load_global_uconfig */ 5562 dw2 |= 0x8001; 5563 /* set load_cs_sh_regs */ 5564 dw2 |= 0x01000000; 5565 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 5566 dw2 |= 0x10002; 5567 5568 /* set load_ce_ram if preamble presented */ 5569 if (AMDGPU_PREAMBLE_IB_PRESENT & flags) 5570 dw2 |= 0x10000000; 5571 } else { 5572 /* still load_ce_ram if this is the first time preamble presented 5573 * although there is no context switch happens. 5574 */ 5575 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags) 5576 dw2 |= 0x10000000; 5577 } 5578 5579 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 5580 amdgpu_ring_write(ring, dw2); 5581 amdgpu_ring_write(ring, 0); 5582 } 5583 5584 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring) 5585 { 5586 unsigned ret; 5587 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 5588 amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr)); 5589 amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr)); 5590 amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */ 5591 ret = ring->wptr & ring->buf_mask; 5592 amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */ 5593 return ret; 5594 } 5595 5596 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset) 5597 { 5598 unsigned cur; 5599 BUG_ON(offset > ring->buf_mask); 5600 BUG_ON(ring->ring[offset] != 0x55aa55aa); 5601 5602 cur = (ring->wptr & ring->buf_mask) - 1; 5603 if (likely(cur > offset)) 5604 ring->ring[offset] = cur - offset; 5605 else 5606 ring->ring[offset] = (ring->ring_size>>2) - offset + cur; 5607 } 5608 5609 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 5610 uint32_t reg_val_offs) 5611 { 5612 struct amdgpu_device *adev = ring->adev; 5613 5614 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 5615 amdgpu_ring_write(ring, 0 | /* src: register*/ 5616 (5 << 8) | /* dst: memory */ 5617 (1 << 20)); /* write confirm */ 5618 amdgpu_ring_write(ring, reg); 5619 amdgpu_ring_write(ring, 0); 5620 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 5621 reg_val_offs * 4)); 5622 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 5623 reg_val_offs * 4)); 5624 } 5625 5626 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 5627 uint32_t val) 5628 { 5629 uint32_t cmd = 0; 5630 5631 switch (ring->funcs->type) { 5632 case AMDGPU_RING_TYPE_GFX: 5633 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 5634 break; 5635 case AMDGPU_RING_TYPE_KIQ: 5636 cmd = (1 << 16); /* no inc addr */ 5637 break; 5638 default: 5639 cmd = WR_CONFIRM; 5640 break; 5641 } 5642 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5643 amdgpu_ring_write(ring, cmd); 5644 amdgpu_ring_write(ring, reg); 5645 amdgpu_ring_write(ring, 0); 5646 amdgpu_ring_write(ring, val); 5647 } 5648 5649 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 5650 uint32_t val, uint32_t mask) 5651 { 5652 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); 5653 } 5654 5655 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, 5656 uint32_t reg0, uint32_t reg1, 5657 uint32_t ref, uint32_t mask) 5658 { 5659 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5660 struct amdgpu_device *adev = ring->adev; 5661 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ? 5662 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait; 5663 5664 if (fw_version_ok) 5665 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, 5666 ref, mask, 0x20); 5667 else 5668 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1, 5669 ref, mask); 5670 } 5671 5672 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid) 5673 { 5674 struct amdgpu_device *adev = ring->adev; 5675 uint32_t value = 0; 5676 5677 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03); 5678 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01); 5679 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1); 5680 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid); 5681 WREG32_SOC15(GC, 0, mmSQ_CMD, value); 5682 } 5683 5684 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 5685 enum amdgpu_interrupt_state state) 5686 { 5687 switch (state) { 5688 case AMDGPU_IRQ_STATE_DISABLE: 5689 case AMDGPU_IRQ_STATE_ENABLE: 5690 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5691 TIME_STAMP_INT_ENABLE, 5692 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5693 break; 5694 default: 5695 break; 5696 } 5697 } 5698 5699 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 5700 int me, int pipe, 5701 enum amdgpu_interrupt_state state) 5702 { 5703 u32 mec_int_cntl, mec_int_cntl_reg; 5704 5705 /* 5706 * amdgpu controls only the first MEC. That's why this function only 5707 * handles the setting of interrupts for this specific MEC. All other 5708 * pipes' interrupts are set by amdkfd. 5709 */ 5710 5711 if (me == 1) { 5712 switch (pipe) { 5713 case 0: 5714 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL); 5715 break; 5716 case 1: 5717 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL); 5718 break; 5719 case 2: 5720 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL); 5721 break; 5722 case 3: 5723 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL); 5724 break; 5725 default: 5726 DRM_DEBUG("invalid pipe %d\n", pipe); 5727 return; 5728 } 5729 } else { 5730 DRM_DEBUG("invalid me %d\n", me); 5731 return; 5732 } 5733 5734 switch (state) { 5735 case AMDGPU_IRQ_STATE_DISABLE: 5736 mec_int_cntl = RREG32(mec_int_cntl_reg); 5737 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5738 TIME_STAMP_INT_ENABLE, 0); 5739 WREG32(mec_int_cntl_reg, mec_int_cntl); 5740 break; 5741 case AMDGPU_IRQ_STATE_ENABLE: 5742 mec_int_cntl = RREG32(mec_int_cntl_reg); 5743 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5744 TIME_STAMP_INT_ENABLE, 1); 5745 WREG32(mec_int_cntl_reg, mec_int_cntl); 5746 break; 5747 default: 5748 break; 5749 } 5750 } 5751 5752 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 5753 struct amdgpu_irq_src *source, 5754 unsigned type, 5755 enum amdgpu_interrupt_state state) 5756 { 5757 switch (state) { 5758 case AMDGPU_IRQ_STATE_DISABLE: 5759 case AMDGPU_IRQ_STATE_ENABLE: 5760 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5761 PRIV_REG_INT_ENABLE, 5762 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5763 break; 5764 default: 5765 break; 5766 } 5767 5768 return 0; 5769 } 5770 5771 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 5772 struct amdgpu_irq_src *source, 5773 unsigned type, 5774 enum amdgpu_interrupt_state state) 5775 { 5776 switch (state) { 5777 case AMDGPU_IRQ_STATE_DISABLE: 5778 case AMDGPU_IRQ_STATE_ENABLE: 5779 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5780 PRIV_INSTR_INT_ENABLE, 5781 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5782 break; 5783 default: 5784 break; 5785 } 5786 5787 return 0; 5788 } 5789 5790 #define ENABLE_ECC_ON_ME_PIPE(me, pipe) \ 5791 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5792 CP_ECC_ERROR_INT_ENABLE, 1) 5793 5794 #define DISABLE_ECC_ON_ME_PIPE(me, pipe) \ 5795 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5796 CP_ECC_ERROR_INT_ENABLE, 0) 5797 5798 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev, 5799 struct amdgpu_irq_src *source, 5800 unsigned type, 5801 enum amdgpu_interrupt_state state) 5802 { 5803 switch (state) { 5804 case AMDGPU_IRQ_STATE_DISABLE: 5805 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5806 CP_ECC_ERROR_INT_ENABLE, 0); 5807 DISABLE_ECC_ON_ME_PIPE(1, 0); 5808 DISABLE_ECC_ON_ME_PIPE(1, 1); 5809 DISABLE_ECC_ON_ME_PIPE(1, 2); 5810 DISABLE_ECC_ON_ME_PIPE(1, 3); 5811 break; 5812 5813 case AMDGPU_IRQ_STATE_ENABLE: 5814 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5815 CP_ECC_ERROR_INT_ENABLE, 1); 5816 ENABLE_ECC_ON_ME_PIPE(1, 0); 5817 ENABLE_ECC_ON_ME_PIPE(1, 1); 5818 ENABLE_ECC_ON_ME_PIPE(1, 2); 5819 ENABLE_ECC_ON_ME_PIPE(1, 3); 5820 break; 5821 default: 5822 break; 5823 } 5824 5825 return 0; 5826 } 5827 5828 5829 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev, 5830 struct amdgpu_irq_src *src, 5831 unsigned type, 5832 enum amdgpu_interrupt_state state) 5833 { 5834 switch (type) { 5835 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 5836 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state); 5837 break; 5838 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 5839 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 5840 break; 5841 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 5842 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 5843 break; 5844 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 5845 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 5846 break; 5847 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 5848 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 5849 break; 5850 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP: 5851 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state); 5852 break; 5853 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP: 5854 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state); 5855 break; 5856 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP: 5857 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state); 5858 break; 5859 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP: 5860 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state); 5861 break; 5862 default: 5863 break; 5864 } 5865 return 0; 5866 } 5867 5868 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev, 5869 struct amdgpu_irq_src *source, 5870 struct amdgpu_iv_entry *entry) 5871 { 5872 int i; 5873 u8 me_id, pipe_id, queue_id; 5874 struct amdgpu_ring *ring; 5875 5876 DRM_DEBUG("IH: CP EOP\n"); 5877 me_id = (entry->ring_id & 0x0c) >> 2; 5878 pipe_id = (entry->ring_id & 0x03) >> 0; 5879 queue_id = (entry->ring_id & 0x70) >> 4; 5880 5881 switch (me_id) { 5882 case 0: 5883 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); 5884 break; 5885 case 1: 5886 case 2: 5887 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5888 ring = &adev->gfx.compute_ring[i]; 5889 /* Per-queue interrupt is supported for MEC starting from VI. 5890 * The interrupt can only be enabled/disabled per pipe instead of per queue. 5891 */ 5892 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) 5893 amdgpu_fence_process(ring); 5894 } 5895 break; 5896 } 5897 return 0; 5898 } 5899 5900 static void gfx_v9_0_fault(struct amdgpu_device *adev, 5901 struct amdgpu_iv_entry *entry) 5902 { 5903 u8 me_id, pipe_id, queue_id; 5904 struct amdgpu_ring *ring; 5905 int i; 5906 5907 me_id = (entry->ring_id & 0x0c) >> 2; 5908 pipe_id = (entry->ring_id & 0x03) >> 0; 5909 queue_id = (entry->ring_id & 0x70) >> 4; 5910 5911 switch (me_id) { 5912 case 0: 5913 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); 5914 break; 5915 case 1: 5916 case 2: 5917 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5918 ring = &adev->gfx.compute_ring[i]; 5919 if (ring->me == me_id && ring->pipe == pipe_id && 5920 ring->queue == queue_id) 5921 drm_sched_fault(&ring->sched); 5922 } 5923 break; 5924 } 5925 } 5926 5927 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev, 5928 struct amdgpu_irq_src *source, 5929 struct amdgpu_iv_entry *entry) 5930 { 5931 DRM_ERROR("Illegal register access in command stream\n"); 5932 gfx_v9_0_fault(adev, entry); 5933 return 0; 5934 } 5935 5936 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev, 5937 struct amdgpu_irq_src *source, 5938 struct amdgpu_iv_entry *entry) 5939 { 5940 DRM_ERROR("Illegal instruction in command stream\n"); 5941 gfx_v9_0_fault(adev, entry); 5942 return 0; 5943 } 5944 5945 5946 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = { 5947 { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 5948 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT), 5949 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT) 5950 }, 5951 { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 5952 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT), 5953 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT) 5954 }, 5955 { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5956 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1), 5957 0, 0 5958 }, 5959 { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5960 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2), 5961 0, 0 5962 }, 5963 { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 5964 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT), 5965 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT) 5966 }, 5967 { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5968 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT), 5969 0, 0 5970 }, 5971 { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5972 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT), 5973 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT) 5974 }, 5975 { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 5976 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT), 5977 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT) 5978 }, 5979 { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 5980 SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1), 5981 0, 0 5982 }, 5983 { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 5984 SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1), 5985 0, 0 5986 }, 5987 { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 5988 SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1), 5989 0, 0 5990 }, 5991 { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 5992 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC), 5993 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED) 5994 }, 5995 { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 5996 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED), 5997 0, 0 5998 }, 5999 { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 6000 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC), 6001 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED) 6002 }, 6003 { "GDS_OA_PHY_PHY_CMD_RAM_MEM", 6004 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 6005 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC), 6006 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED) 6007 }, 6008 { "GDS_OA_PHY_PHY_DATA_RAM_MEM", 6009 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 6010 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED), 6011 0, 0 6012 }, 6013 { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM", 6014 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6015 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC), 6016 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED) 6017 }, 6018 { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM", 6019 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6020 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC), 6021 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED) 6022 }, 6023 { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM", 6024 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6025 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC), 6026 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED) 6027 }, 6028 { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM", 6029 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6030 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC), 6031 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED) 6032 }, 6033 { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 6034 SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT), 6035 0, 0 6036 }, 6037 { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6038 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT), 6039 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT) 6040 }, 6041 { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6042 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT), 6043 0, 0 6044 }, 6045 { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6046 SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT), 6047 0, 0 6048 }, 6049 { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6050 SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT), 6051 0, 0 6052 }, 6053 { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6054 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT), 6055 0, 0 6056 }, 6057 { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 6058 SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT), 6059 0, 0 6060 }, 6061 { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 6062 SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT), 6063 0, 0 6064 }, 6065 { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6066 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT), 6067 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT) 6068 }, 6069 { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6070 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT), 6071 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT) 6072 }, 6073 { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6074 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT), 6075 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT) 6076 }, 6077 { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6078 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT), 6079 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT) 6080 }, 6081 { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6082 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT), 6083 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT) 6084 }, 6085 { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6086 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT), 6087 0, 0 6088 }, 6089 { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6090 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT), 6091 0, 0 6092 }, 6093 { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6094 SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT), 6095 0, 0 6096 }, 6097 { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6098 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT), 6099 0, 0 6100 }, 6101 { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6102 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT), 6103 0, 0 6104 }, 6105 { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6106 SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT), 6107 0, 0 6108 }, 6109 { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6110 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT), 6111 0, 0 6112 }, 6113 { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6114 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT), 6115 0, 0 6116 }, 6117 { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6118 SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT), 6119 0, 0 6120 }, 6121 { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6122 SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT), 6123 0, 0 6124 }, 6125 { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6126 SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT), 6127 0, 0 6128 }, 6129 { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6130 SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT), 6131 0, 0 6132 }, 6133 { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6134 SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT), 6135 0, 0 6136 }, 6137 { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 6138 SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT), 6139 0, 0 6140 }, 6141 { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6142 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT), 6143 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT) 6144 }, 6145 { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6146 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT), 6147 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT) 6148 }, 6149 { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6150 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT), 6151 0, 0 6152 }, 6153 { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6154 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT), 6155 0, 0 6156 }, 6157 { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6158 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT), 6159 0, 0 6160 }, 6161 { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6162 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT), 6163 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT) 6164 }, 6165 { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6166 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT), 6167 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT) 6168 }, 6169 { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6170 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT), 6171 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT) 6172 }, 6173 { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6174 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT), 6175 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT) 6176 }, 6177 { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6178 SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT), 6179 0, 0 6180 }, 6181 { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6182 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT), 6183 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT) 6184 }, 6185 { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6186 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT), 6187 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT) 6188 }, 6189 { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6190 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT), 6191 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT) 6192 }, 6193 { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6194 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT), 6195 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT) 6196 }, 6197 { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6198 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT), 6199 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT) 6200 }, 6201 { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6202 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT), 6203 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT) 6204 }, 6205 { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6206 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT), 6207 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT) 6208 }, 6209 { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6210 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT), 6211 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT) 6212 }, 6213 { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6214 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT), 6215 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT) 6216 }, 6217 { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6218 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT), 6219 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT) 6220 }, 6221 { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6222 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT), 6223 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT) 6224 }, 6225 { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6226 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT), 6227 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT) 6228 }, 6229 { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6230 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT), 6231 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT) 6232 }, 6233 { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6234 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT), 6235 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT) 6236 }, 6237 { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6238 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT), 6239 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT) 6240 }, 6241 { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6242 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT), 6243 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT) 6244 }, 6245 { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6246 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT), 6247 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT) 6248 }, 6249 { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6250 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT), 6251 0, 0 6252 }, 6253 { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6254 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT), 6255 0, 0 6256 }, 6257 { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6258 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT), 6259 0, 0 6260 }, 6261 { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6262 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT), 6263 0, 0 6264 }, 6265 { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6266 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT), 6267 0, 0 6268 }, 6269 { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6270 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT), 6271 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT) 6272 }, 6273 { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6274 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT), 6275 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT) 6276 }, 6277 { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6278 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT), 6279 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT) 6280 }, 6281 { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6282 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT), 6283 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT) 6284 }, 6285 { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6286 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT), 6287 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT) 6288 }, 6289 { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6290 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT), 6291 0, 0 6292 }, 6293 { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6294 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT), 6295 0, 0 6296 }, 6297 { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6298 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT), 6299 0, 0 6300 }, 6301 { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6302 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT), 6303 0, 0 6304 }, 6305 { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6306 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT), 6307 0, 0 6308 }, 6309 { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6310 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), 6311 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT) 6312 }, 6313 { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6314 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), 6315 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT) 6316 }, 6317 { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6318 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), 6319 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT) 6320 }, 6321 { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6322 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT), 6323 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT) 6324 }, 6325 { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6326 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT), 6327 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT) 6328 }, 6329 { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6330 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), 6331 0, 0 6332 }, 6333 { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6334 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), 6335 0, 0 6336 }, 6337 { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6338 SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT), 6339 0, 0 6340 }, 6341 { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6342 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT), 6343 0, 0 6344 }, 6345 { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6346 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT), 6347 0, 0 6348 }, 6349 { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6350 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), 6351 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT) 6352 }, 6353 { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6354 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), 6355 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT) 6356 }, 6357 { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6358 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), 6359 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT) 6360 }, 6361 { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6362 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), 6363 0, 0 6364 }, 6365 { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6366 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), 6367 0, 0 6368 }, 6369 { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6370 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT), 6371 0, 0 6372 }, 6373 { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6374 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT), 6375 0, 0 6376 }, 6377 { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6378 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT), 6379 0, 0 6380 }, 6381 { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6382 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT), 6383 0, 0 6384 } 6385 }; 6386 6387 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 6388 void *inject_if) 6389 { 6390 struct ras_inject_if *info = (struct ras_inject_if *)inject_if; 6391 int ret; 6392 struct ta_ras_trigger_error_input block_info = { 0 }; 6393 6394 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6395 return -EINVAL; 6396 6397 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks)) 6398 return -EINVAL; 6399 6400 if (!ras_gfx_subblocks[info->head.sub_block_index].name) 6401 return -EPERM; 6402 6403 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type & 6404 info->head.type)) { 6405 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n", 6406 ras_gfx_subblocks[info->head.sub_block_index].name, 6407 info->head.type); 6408 return -EPERM; 6409 } 6410 6411 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type & 6412 info->head.type)) { 6413 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n", 6414 ras_gfx_subblocks[info->head.sub_block_index].name, 6415 info->head.type); 6416 return -EPERM; 6417 } 6418 6419 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block); 6420 block_info.sub_block_index = 6421 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock; 6422 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type); 6423 block_info.address = info->address; 6424 block_info.value = info->value; 6425 6426 mutex_lock(&adev->grbm_idx_mutex); 6427 ret = psp_ras_trigger_error(&adev->psp, &block_info); 6428 mutex_unlock(&adev->grbm_idx_mutex); 6429 6430 return ret; 6431 } 6432 6433 static const char *vml2_mems[] = { 6434 "UTC_VML2_BANK_CACHE_0_BIGK_MEM0", 6435 "UTC_VML2_BANK_CACHE_0_BIGK_MEM1", 6436 "UTC_VML2_BANK_CACHE_0_4K_MEM0", 6437 "UTC_VML2_BANK_CACHE_0_4K_MEM1", 6438 "UTC_VML2_BANK_CACHE_1_BIGK_MEM0", 6439 "UTC_VML2_BANK_CACHE_1_BIGK_MEM1", 6440 "UTC_VML2_BANK_CACHE_1_4K_MEM0", 6441 "UTC_VML2_BANK_CACHE_1_4K_MEM1", 6442 "UTC_VML2_BANK_CACHE_2_BIGK_MEM0", 6443 "UTC_VML2_BANK_CACHE_2_BIGK_MEM1", 6444 "UTC_VML2_BANK_CACHE_2_4K_MEM0", 6445 "UTC_VML2_BANK_CACHE_2_4K_MEM1", 6446 "UTC_VML2_BANK_CACHE_3_BIGK_MEM0", 6447 "UTC_VML2_BANK_CACHE_3_BIGK_MEM1", 6448 "UTC_VML2_BANK_CACHE_3_4K_MEM0", 6449 "UTC_VML2_BANK_CACHE_3_4K_MEM1", 6450 }; 6451 6452 static const char *vml2_walker_mems[] = { 6453 "UTC_VML2_CACHE_PDE0_MEM0", 6454 "UTC_VML2_CACHE_PDE0_MEM1", 6455 "UTC_VML2_CACHE_PDE1_MEM0", 6456 "UTC_VML2_CACHE_PDE1_MEM1", 6457 "UTC_VML2_CACHE_PDE2_MEM0", 6458 "UTC_VML2_CACHE_PDE2_MEM1", 6459 "UTC_VML2_RDIF_LOG_FIFO", 6460 }; 6461 6462 static const char *atc_l2_cache_2m_mems[] = { 6463 "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM", 6464 "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM", 6465 "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM", 6466 "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM", 6467 }; 6468 6469 static const char *atc_l2_cache_4k_mems[] = { 6470 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0", 6471 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1", 6472 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2", 6473 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3", 6474 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4", 6475 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5", 6476 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6", 6477 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7", 6478 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0", 6479 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1", 6480 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2", 6481 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3", 6482 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4", 6483 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5", 6484 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6", 6485 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7", 6486 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0", 6487 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1", 6488 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2", 6489 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3", 6490 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4", 6491 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5", 6492 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6", 6493 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7", 6494 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0", 6495 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1", 6496 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2", 6497 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3", 6498 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4", 6499 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5", 6500 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6", 6501 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7", 6502 }; 6503 6504 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev, 6505 struct ras_err_data *err_data) 6506 { 6507 uint32_t i, data; 6508 uint32_t sec_count, ded_count; 6509 6510 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6511 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6512 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6513 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6514 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6515 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6516 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6517 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6518 6519 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6520 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6521 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6522 6523 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT); 6524 if (sec_count) { 6525 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6526 "SEC %d\n", i, vml2_mems[i], sec_count); 6527 err_data->ce_count += sec_count; 6528 } 6529 6530 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT); 6531 if (ded_count) { 6532 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6533 "DED %d\n", i, vml2_mems[i], ded_count); 6534 err_data->ue_count += ded_count; 6535 } 6536 } 6537 6538 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6539 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6540 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6541 6542 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6543 SEC_COUNT); 6544 if (sec_count) { 6545 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6546 "SEC %d\n", i, vml2_walker_mems[i], sec_count); 6547 err_data->ce_count += sec_count; 6548 } 6549 6550 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6551 DED_COUNT); 6552 if (ded_count) { 6553 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6554 "DED %d\n", i, vml2_walker_mems[i], ded_count); 6555 err_data->ue_count += ded_count; 6556 } 6557 } 6558 6559 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6560 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6561 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6562 6563 sec_count = (data & 0x00006000L) >> 0xd; 6564 if (sec_count) { 6565 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6566 "SEC %d\n", i, atc_l2_cache_2m_mems[i], 6567 sec_count); 6568 err_data->ce_count += sec_count; 6569 } 6570 } 6571 6572 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6573 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6574 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6575 6576 sec_count = (data & 0x00006000L) >> 0xd; 6577 if (sec_count) { 6578 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6579 "SEC %d\n", i, atc_l2_cache_4k_mems[i], 6580 sec_count); 6581 err_data->ce_count += sec_count; 6582 } 6583 6584 ded_count = (data & 0x00018000L) >> 0xf; 6585 if (ded_count) { 6586 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6587 "DED %d\n", i, atc_l2_cache_4k_mems[i], 6588 ded_count); 6589 err_data->ue_count += ded_count; 6590 } 6591 } 6592 6593 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6594 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6595 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6596 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6597 6598 return 0; 6599 } 6600 6601 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev, 6602 const struct soc15_reg_entry *reg, 6603 uint32_t se_id, uint32_t inst_id, uint32_t value, 6604 uint32_t *sec_count, uint32_t *ded_count) 6605 { 6606 uint32_t i; 6607 uint32_t sec_cnt, ded_cnt; 6608 6609 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) { 6610 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset || 6611 gfx_v9_0_ras_fields[i].seg != reg->seg || 6612 gfx_v9_0_ras_fields[i].inst != reg->inst) 6613 continue; 6614 6615 sec_cnt = (value & 6616 gfx_v9_0_ras_fields[i].sec_count_mask) >> 6617 gfx_v9_0_ras_fields[i].sec_count_shift; 6618 if (sec_cnt) { 6619 dev_info(adev->dev, "GFX SubBlock %s, " 6620 "Instance[%d][%d], SEC %d\n", 6621 gfx_v9_0_ras_fields[i].name, 6622 se_id, inst_id, 6623 sec_cnt); 6624 *sec_count += sec_cnt; 6625 } 6626 6627 ded_cnt = (value & 6628 gfx_v9_0_ras_fields[i].ded_count_mask) >> 6629 gfx_v9_0_ras_fields[i].ded_count_shift; 6630 if (ded_cnt) { 6631 dev_info(adev->dev, "GFX SubBlock %s, " 6632 "Instance[%d][%d], DED %d\n", 6633 gfx_v9_0_ras_fields[i].name, 6634 se_id, inst_id, 6635 ded_cnt); 6636 *ded_count += ded_cnt; 6637 } 6638 } 6639 6640 return 0; 6641 } 6642 6643 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev) 6644 { 6645 int i, j, k; 6646 6647 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6648 return; 6649 6650 /* read back registers to clear the counters */ 6651 mutex_lock(&adev->grbm_idx_mutex); 6652 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6653 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6654 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6655 gfx_v9_0_select_se_sh(adev, j, 0x0, k); 6656 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6657 } 6658 } 6659 } 6660 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000); 6661 mutex_unlock(&adev->grbm_idx_mutex); 6662 6663 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6664 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6665 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6666 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6667 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6668 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6669 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6670 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6671 6672 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6673 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6674 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6675 } 6676 6677 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6678 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6679 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6680 } 6681 6682 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6683 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6684 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6685 } 6686 6687 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6688 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6689 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6690 } 6691 6692 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6693 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6694 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6695 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6696 } 6697 6698 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 6699 void *ras_error_status) 6700 { 6701 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; 6702 uint32_t sec_count = 0, ded_count = 0; 6703 uint32_t i, j, k; 6704 uint32_t reg_value; 6705 6706 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6707 return -EINVAL; 6708 6709 err_data->ue_count = 0; 6710 err_data->ce_count = 0; 6711 6712 mutex_lock(&adev->grbm_idx_mutex); 6713 6714 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6715 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6716 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6717 gfx_v9_0_select_se_sh(adev, j, 0, k); 6718 reg_value = 6719 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6720 if (reg_value) 6721 gfx_v9_0_ras_error_count(adev, 6722 &gfx_v9_0_edc_counter_regs[i], 6723 j, k, reg_value, 6724 &sec_count, &ded_count); 6725 } 6726 } 6727 } 6728 6729 err_data->ce_count += sec_count; 6730 err_data->ue_count += ded_count; 6731 6732 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 6733 mutex_unlock(&adev->grbm_idx_mutex); 6734 6735 gfx_v9_0_query_utc_edc_status(adev, err_data); 6736 6737 return 0; 6738 } 6739 6740 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring) 6741 { 6742 const unsigned int cp_coher_cntl = 6743 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) | 6744 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) | 6745 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) | 6746 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) | 6747 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1); 6748 6749 /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */ 6750 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5)); 6751 amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */ 6752 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6753 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */ 6754 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6755 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 6756 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ 6757 } 6758 6759 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring, 6760 uint32_t pipe, bool enable) 6761 { 6762 struct amdgpu_device *adev = ring->adev; 6763 uint32_t val; 6764 uint32_t wcl_cs_reg; 6765 6766 /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */ 6767 val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT; 6768 6769 switch (pipe) { 6770 case 0: 6771 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0); 6772 break; 6773 case 1: 6774 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1); 6775 break; 6776 case 2: 6777 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2); 6778 break; 6779 case 3: 6780 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3); 6781 break; 6782 default: 6783 DRM_DEBUG("invalid pipe %d\n", pipe); 6784 return; 6785 } 6786 6787 amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val); 6788 6789 } 6790 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable) 6791 { 6792 struct amdgpu_device *adev = ring->adev; 6793 uint32_t val; 6794 int i; 6795 6796 6797 /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit 6798 * number of gfx waves. Setting 5 bit will make sure gfx only gets 6799 * around 25% of gpu resources. 6800 */ 6801 val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT; 6802 amdgpu_ring_emit_wreg(ring, 6803 SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX), 6804 val); 6805 6806 /* Restrict waves for normal/low priority compute queues as well 6807 * to get best QoS for high priority compute jobs. 6808 * 6809 * amdgpu controls only 1st ME(0-3 CS pipes). 6810 */ 6811 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 6812 if (i != ring->pipe) 6813 gfx_v9_0_emit_wave_limit_cs(ring, i, enable); 6814 6815 } 6816 } 6817 6818 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = { 6819 .name = "gfx_v9_0", 6820 .early_init = gfx_v9_0_early_init, 6821 .late_init = gfx_v9_0_late_init, 6822 .sw_init = gfx_v9_0_sw_init, 6823 .sw_fini = gfx_v9_0_sw_fini, 6824 .hw_init = gfx_v9_0_hw_init, 6825 .hw_fini = gfx_v9_0_hw_fini, 6826 .suspend = gfx_v9_0_suspend, 6827 .resume = gfx_v9_0_resume, 6828 .is_idle = gfx_v9_0_is_idle, 6829 .wait_for_idle = gfx_v9_0_wait_for_idle, 6830 .soft_reset = gfx_v9_0_soft_reset, 6831 .set_clockgating_state = gfx_v9_0_set_clockgating_state, 6832 .set_powergating_state = gfx_v9_0_set_powergating_state, 6833 .get_clockgating_state = gfx_v9_0_get_clockgating_state, 6834 }; 6835 6836 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = { 6837 .type = AMDGPU_RING_TYPE_GFX, 6838 .align_mask = 0xff, 6839 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6840 .support_64bit_ptrs = true, 6841 .vmhub = AMDGPU_GFXHUB_0, 6842 .get_rptr = gfx_v9_0_ring_get_rptr_gfx, 6843 .get_wptr = gfx_v9_0_ring_get_wptr_gfx, 6844 .set_wptr = gfx_v9_0_ring_set_wptr_gfx, 6845 .emit_frame_size = /* totally 242 maximum if 16 IBs */ 6846 5 + /* COND_EXEC */ 6847 7 + /* PIPELINE_SYNC */ 6848 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6849 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6850 2 + /* VM_FLUSH */ 6851 8 + /* FENCE for VM_FLUSH */ 6852 20 + /* GDS switch */ 6853 4 + /* double SWITCH_BUFFER, 6854 the first COND_EXEC jump to the place just 6855 prior to this double SWITCH_BUFFER */ 6856 5 + /* COND_EXEC */ 6857 7 + /* HDP_flush */ 6858 4 + /* VGT_flush */ 6859 14 + /* CE_META */ 6860 31 + /* DE_META */ 6861 3 + /* CNTX_CTRL */ 6862 5 + /* HDP_INVL */ 6863 8 + 8 + /* FENCE x2 */ 6864 2 + /* SWITCH_BUFFER */ 6865 7, /* gfx_v9_0_emit_mem_sync */ 6866 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */ 6867 .emit_ib = gfx_v9_0_ring_emit_ib_gfx, 6868 .emit_fence = gfx_v9_0_ring_emit_fence, 6869 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6870 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6871 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6872 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6873 .test_ring = gfx_v9_0_ring_test_ring, 6874 .test_ib = gfx_v9_0_ring_test_ib, 6875 .insert_nop = amdgpu_ring_insert_nop, 6876 .pad_ib = amdgpu_ring_generic_pad_ib, 6877 .emit_switch_buffer = gfx_v9_ring_emit_sb, 6878 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl, 6879 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec, 6880 .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec, 6881 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl, 6882 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6883 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6884 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6885 .soft_recovery = gfx_v9_0_ring_soft_recovery, 6886 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6887 }; 6888 6889 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { 6890 .type = AMDGPU_RING_TYPE_COMPUTE, 6891 .align_mask = 0xff, 6892 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6893 .support_64bit_ptrs = true, 6894 .vmhub = AMDGPU_GFXHUB_0, 6895 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6896 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6897 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6898 .emit_frame_size = 6899 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6900 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6901 5 + /* hdp invalidate */ 6902 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6903 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6904 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6905 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6906 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */ 6907 7 + /* gfx_v9_0_emit_mem_sync */ 6908 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */ 6909 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */ 6910 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6911 .emit_ib = gfx_v9_0_ring_emit_ib_compute, 6912 .emit_fence = gfx_v9_0_ring_emit_fence, 6913 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6914 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6915 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6916 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6917 .test_ring = gfx_v9_0_ring_test_ring, 6918 .test_ib = gfx_v9_0_ring_test_ib, 6919 .insert_nop = amdgpu_ring_insert_nop, 6920 .pad_ib = amdgpu_ring_generic_pad_ib, 6921 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6922 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6923 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6924 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6925 .emit_wave_limit = gfx_v9_0_emit_wave_limit, 6926 }; 6927 6928 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = { 6929 .type = AMDGPU_RING_TYPE_KIQ, 6930 .align_mask = 0xff, 6931 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6932 .support_64bit_ptrs = true, 6933 .vmhub = AMDGPU_GFXHUB_0, 6934 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6935 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6936 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6937 .emit_frame_size = 6938 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6939 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6940 5 + /* hdp invalidate */ 6941 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6942 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6943 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6944 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6945 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 6946 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6947 .emit_fence = gfx_v9_0_ring_emit_fence_kiq, 6948 .test_ring = gfx_v9_0_ring_test_ring, 6949 .insert_nop = amdgpu_ring_insert_nop, 6950 .pad_ib = amdgpu_ring_generic_pad_ib, 6951 .emit_rreg = gfx_v9_0_ring_emit_rreg, 6952 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6953 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6954 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6955 }; 6956 6957 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev) 6958 { 6959 int i; 6960 6961 adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq; 6962 6963 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 6964 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx; 6965 6966 for (i = 0; i < adev->gfx.num_compute_rings; i++) 6967 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute; 6968 } 6969 6970 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = { 6971 .set = gfx_v9_0_set_eop_interrupt_state, 6972 .process = gfx_v9_0_eop_irq, 6973 }; 6974 6975 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = { 6976 .set = gfx_v9_0_set_priv_reg_fault_state, 6977 .process = gfx_v9_0_priv_reg_irq, 6978 }; 6979 6980 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = { 6981 .set = gfx_v9_0_set_priv_inst_fault_state, 6982 .process = gfx_v9_0_priv_inst_irq, 6983 }; 6984 6985 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = { 6986 .set = gfx_v9_0_set_cp_ecc_error_state, 6987 .process = amdgpu_gfx_cp_ecc_error_irq, 6988 }; 6989 6990 6991 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev) 6992 { 6993 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 6994 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs; 6995 6996 adev->gfx.priv_reg_irq.num_types = 1; 6997 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs; 6998 6999 adev->gfx.priv_inst_irq.num_types = 1; 7000 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs; 7001 7002 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/ 7003 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs; 7004 } 7005 7006 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev) 7007 { 7008 switch (adev->asic_type) { 7009 case CHIP_VEGA10: 7010 case CHIP_VEGA12: 7011 case CHIP_VEGA20: 7012 case CHIP_RAVEN: 7013 case CHIP_ARCTURUS: 7014 case CHIP_RENOIR: 7015 case CHIP_ALDEBARAN: 7016 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs; 7017 break; 7018 default: 7019 break; 7020 } 7021 } 7022 7023 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev) 7024 { 7025 /* init asci gds info */ 7026 switch (adev->asic_type) { 7027 case CHIP_VEGA10: 7028 case CHIP_VEGA12: 7029 case CHIP_VEGA20: 7030 adev->gds.gds_size = 0x10000; 7031 break; 7032 case CHIP_RAVEN: 7033 case CHIP_ARCTURUS: 7034 adev->gds.gds_size = 0x1000; 7035 break; 7036 case CHIP_ALDEBARAN: 7037 /* aldebaran removed all the GDS internal memory, 7038 * only support GWS opcode in kernel, like barrier 7039 * semaphore.etc */ 7040 adev->gds.gds_size = 0; 7041 break; 7042 default: 7043 adev->gds.gds_size = 0x10000; 7044 break; 7045 } 7046 7047 switch (adev->asic_type) { 7048 case CHIP_VEGA10: 7049 case CHIP_VEGA20: 7050 adev->gds.gds_compute_max_wave_id = 0x7ff; 7051 break; 7052 case CHIP_VEGA12: 7053 adev->gds.gds_compute_max_wave_id = 0x27f; 7054 break; 7055 case CHIP_RAVEN: 7056 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 7057 adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */ 7058 else 7059 adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */ 7060 break; 7061 case CHIP_ARCTURUS: 7062 adev->gds.gds_compute_max_wave_id = 0xfff; 7063 break; 7064 case CHIP_ALDEBARAN: 7065 /* deprecated for Aldebaran, no usage at all */ 7066 adev->gds.gds_compute_max_wave_id = 0; 7067 break; 7068 default: 7069 /* this really depends on the chip */ 7070 adev->gds.gds_compute_max_wave_id = 0x7ff; 7071 break; 7072 } 7073 7074 adev->gds.gws_size = 64; 7075 adev->gds.oa_size = 16; 7076 } 7077 7078 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev, 7079 u32 bitmap) 7080 { 7081 u32 data; 7082 7083 if (!bitmap) 7084 return; 7085 7086 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 7087 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 7088 7089 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data); 7090 } 7091 7092 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev) 7093 { 7094 u32 data, mask; 7095 7096 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG); 7097 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG); 7098 7099 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 7100 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 7101 7102 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); 7103 7104 return (~data) & mask; 7105 } 7106 7107 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 7108 struct amdgpu_cu_info *cu_info) 7109 { 7110 int i, j, k, counter, active_cu_number = 0; 7111 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0; 7112 unsigned disable_masks[4 * 4]; 7113 7114 if (!adev || !cu_info) 7115 return -EINVAL; 7116 7117 /* 7118 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs 7119 */ 7120 if (adev->gfx.config.max_shader_engines * 7121 adev->gfx.config.max_sh_per_se > 16) 7122 return -EINVAL; 7123 7124 amdgpu_gfx_parse_disable_cu(disable_masks, 7125 adev->gfx.config.max_shader_engines, 7126 adev->gfx.config.max_sh_per_se); 7127 7128 mutex_lock(&adev->grbm_idx_mutex); 7129 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 7130 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 7131 mask = 1; 7132 ao_bitmap = 0; 7133 counter = 0; 7134 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 7135 gfx_v9_0_set_user_cu_inactive_bitmap( 7136 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]); 7137 bitmap = gfx_v9_0_get_cu_active_bitmap(adev); 7138 7139 /* 7140 * The bitmap(and ao_cu_bitmap) in cu_info structure is 7141 * 4x4 size array, and it's usually suitable for Vega 7142 * ASICs which has 4*2 SE/SH layout. 7143 * But for Arcturus, SE/SH layout is changed to 8*1. 7144 * To mostly reduce the impact, we make it compatible 7145 * with current bitmap array as below: 7146 * SE4,SH0 --> bitmap[0][1] 7147 * SE5,SH0 --> bitmap[1][1] 7148 * SE6,SH0 --> bitmap[2][1] 7149 * SE7,SH0 --> bitmap[3][1] 7150 */ 7151 cu_info->bitmap[i % 4][j + i / 4] = bitmap; 7152 7153 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 7154 if (bitmap & mask) { 7155 if (counter < adev->gfx.config.max_cu_per_sh) 7156 ao_bitmap |= mask; 7157 counter ++; 7158 } 7159 mask <<= 1; 7160 } 7161 active_cu_number += counter; 7162 if (i < 2 && j < 2) 7163 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8)); 7164 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap; 7165 } 7166 } 7167 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 7168 mutex_unlock(&adev->grbm_idx_mutex); 7169 7170 cu_info->number = active_cu_number; 7171 cu_info->ao_cu_mask = ao_cu_mask; 7172 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 7173 7174 return 0; 7175 } 7176 7177 const struct amdgpu_ip_block_version gfx_v9_0_ip_block = 7178 { 7179 .type = AMD_IP_BLOCK_TYPE_GFX, 7180 .major = 9, 7181 .minor = 0, 7182 .rev = 0, 7183 .funcs = &gfx_v9_0_ip_funcs, 7184 }; 7185